From 5f8a369dbfc45a3c4d5fbbc0b5dcbc6405c1ade7 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:17:11 -0700 Subject: [PATCH 01/25] Use published Datasyslab JTS fork --- common/pom.xml | 2 +- pom.xml | 44 +++++++++++++++++++++++++++++++++++++++-- spark/common/pom.xml | 2 +- spark/spark-3.5/pom.xml | 2 +- spark/spark-4.0/pom.xml | 2 +- spark/spark-4.1/pom.xml | 2 +- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index e33f3f46be7..6ad6736cdba 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -70,7 +70,7 @@ gt-process-feature - org.locationtech.jts + org.datasyslab jts-core diff --git a/pom.xml b/pom.xml index 673bebf9b36..415c2d843b1 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 33.1 3.2.4 2.13.4 - 1.20.0 + 1.21.0-datasyslab-1 0.16.1 0.8 @@ -153,7 +153,7 @@ - org.locationtech.jts + org.datasyslab jts-core ${jts.version} @@ -210,6 +210,10 @@ ${geotools.version} ${geotools.scope} + + org.locationtech.jts + jts-core + com.fasterxml.jackson.core * @@ -227,6 +231,10 @@ ${geotools.version} ${geotools.scope} + + org.locationtech.jts + jts-core + com.fasterxml.jackson.core * @@ -239,6 +247,10 @@ ${geotools.version} ${geotools.scope} + + org.locationtech.jts + jts-core + com.fasterxml.jackson.core * @@ -251,30 +263,58 @@ gt-geotiff ${geotools.version} ${geotools.scope} + + + org.locationtech.jts + jts-core + + org.geotools gt-process-feature ${geotools.version} ${geotools.scope} + + + org.locationtech.jts + jts-core + + org.geotools gt-arcgrid ${geotools.version} ${geotools.scope} + + + org.locationtech.jts + jts-core + + org.geotools gt-coverage ${geotools.version} ${geotools.scope} + + + org.locationtech.jts + jts-core + + it.geosolutions.jaiext.jiffle jt-jiffle-language ${jt-jiffle.version} + + org.locationtech.jts + jts-core + javax.media * diff --git a/spark/common/pom.xml b/spark/common/pom.xml index 4e6701518ce..cde70fcef30 100644 --- a/spark/common/pom.xml +++ b/spark/common/pom.xml @@ -167,7 +167,7 @@ gt-arcgrid - org.locationtech.jts + org.datasyslab jts-core diff --git a/spark/spark-3.5/pom.xml b/spark/spark-3.5/pom.xml index f2bd9a863bd..9657d2c161f 100644 --- a/spark/spark-3.5/pom.xml +++ b/spark/spark-3.5/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.locationtech.jts + org.datasyslab jts-core diff --git a/spark/spark-4.0/pom.xml b/spark/spark-4.0/pom.xml index 09ea7df39c3..da9f1c54560 100644 --- a/spark/spark-4.0/pom.xml +++ b/spark/spark-4.0/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.locationtech.jts + org.datasyslab jts-core diff --git a/spark/spark-4.1/pom.xml b/spark/spark-4.1/pom.xml index feb9091c8cb..fd1693c28b4 100644 --- a/spark/spark-4.1/pom.xml +++ b/spark/spark-4.1/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.locationtech.jts + org.datasyslab jts-core From fa2e1429016e53cfdb04b3b99c862c93bf4bf25e Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:36:49 -0700 Subject: [PATCH 02/25] Adapt geometry contracts for JTS 1.21 --- .../org/apache/sedona/common/Functions.java | 3 ++ .../apache/sedona/common/FunctionsTest.java | 41 ++++++++++++------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index e3057e5ae12..15bb8a6eee6 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -1477,6 +1477,9 @@ public static InscribedCircle maximumInscribedCircle(Geometry geometry, double t if (Double.isNaN(tolerance)) { return null; } + if (tolerance < 0.0) { + throw new IllegalArgumentException("Tolerance must be positive"); + } // JTS requires a positive tolerance. A zero value requests the same // geometry-dependent default used by GEOS and the one-argument overload. if (tolerance == 0.0) { diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 0bd0c833090..650082b68f8 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -1011,7 +1011,7 @@ public void orientedEnvelope() throws ParseException { actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(irregularPolygon), 2)); expected = "POLYGON ((5 0, 0.29 -1.18, -0.71 2.82, 4 4, 5 0))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); Geometry point = Constructors.geomFromWKT("POINT (1 2)", 0); actual = Functions.asWKT(Functions.orientedEnvelope(point)); @@ -1039,14 +1039,14 @@ public void orientedEnvelope() throws ParseException { Geometry multiPoint = Constructors.geomFromWKT("MULTIPOINT ((0 0), (-1 -1), (3 2))", 0); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(multiPoint), 2)); expected = "POLYGON ((-1 -1, -1.12 -0.84, 2.88 2.16, 3 2, -1 -1))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); Geometry linestring = Constructors.geomFromWKT("LINESTRING (55 75, 125 150)", 0); Geometry pointGeom = Constructors.geomFromWKT("POINT (20 80)", 0); Geometry collection = linestring.union(pointGeom); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(collection), 2)); expected = "POLYGON ((125 150, 138.08 130.38, 33.08 60.38, 20 80, 125 150))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); } @Test @@ -2930,7 +2930,7 @@ public void force2D() throws ParseException { String actual = Functions.asWKT(forcedPoly); String expected = "POLYGON ((0 10, 10 10, 10 5.333333333333333, 5 7, 3 5, 5 3, 10 4.666666666666667, 10 0, 0 0, 0 10))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); geom = Constructors.geomFromWKT("POINT ZM(1 2 2 5)", 0); actual = Functions.asWKT(Functions.force2D(geom)); @@ -3640,7 +3640,7 @@ public void testBuffer() { String actual = Functions.asWKT(Functions.reducePrecision(Functions.buffer(polygon, 15), 4)); String expected = "POLYGON ((47.0736 35.2882, 44.2597 36.1418, 41.6664 37.528, 39.3934 39.3934, 37.528 41.6664, 36.1418 44.2597, 35.2882 47.0736, 35 50, 35 150, 35.2882 152.9264, 36.1418 155.7403, 37.528 158.3336, 39.3934 160.6066, 41.6664 162.472, 44.2597 163.8582, 47.0736 164.7118, 50 165, 150 165, 152.9264 164.7118, 155.7403 163.8582, 158.3336 162.472, 160.6066 160.6066, 162.472 158.3336, 163.8582 155.7403, 164.7118 152.9264, 165 150, 165 50, 164.7118 47.0736, 163.8582 44.2597, 162.472 41.6664, 160.6066 39.3934, 158.3336 37.528, 155.7403 36.1418, 152.9264 35.2882, 150 35, 50 35, 47.0736 35.2882))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 100, 100)); actual = @@ -3648,7 +3648,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "side=left"), 4)); expected = "POLYGON ((50 70, 0 0, -8.1373 5.8124, 41.8627 75.8124, 43.2167 77.3476, 44.855 78.5749, 94.855 108.5749, 100 100, 50 70))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 70, -3)); actual = @@ -3656,7 +3656,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "endcap=square"), 4)); expected = "POLYGON ((43.2156 77.3465, 44.8523 78.5733, 46.7044 79.4413, 48.6944 79.9144, 50.739 79.9727, 52.7527 79.6137, 54.6512 78.8525, 56.3552 77.7209, 57.7932 76.2663, 58.9052 74.5495, 59.6446 72.6424, 79.6446 -0.3576, 82.2869 -10.0022, 62.9978 -15.2869, 45.9128 47.0733, 8.1373 -5.8124, 2.325 -13.9497, -13.9497 -2.325, 41.8627 75.8124, 43.2156 77.3465))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(100, 90)); actual = @@ -3664,7 +3664,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(point, 10, false, "quad_segs=2"), 4)); expected = "POLYGON ((107.0711 82.9289, 100 80, 92.9289 82.9289, 90 90, 92.9289 97.0711, 100 100, 107.0711 97.0711, 110 90, 107.0711 82.9289))"; - assertEquals(expected, actual); + assertGeometryEquals(expected, actual); // Using reducePrecision in the following tests modified the geometries coordinates the // coordinates causing the comparison to fail, @@ -5230,16 +5230,21 @@ public void maximumInscribedCircle() throws ParseException { geom = Constructors.geomFromWKT("POLYGON ((0 0, 1 0, 1 1, 0 0))", 3857); actual = Functions.maximumInscribedCircle(geom, 2.0); - expected = - new InscribedCircle( - Constructors.geomFromWKT("POINT (0.75 0.5)", 3857), - Constructors.geomFromWKT("POINT (0.625 0.625)", 3857), - 0.1767766952966369); - assertEquals(expected, actual); + double triangleRadius = 1.0 - 1.0 / Math.sqrt(2.0); + assertEquals(1.0 - triangleRadius, actual.center.getCoordinate().x, 1e-15); + assertEquals(triangleRadius, actual.center.getCoordinate().y, 1e-15); + assertEquals(1.0 - triangleRadius, actual.nearest.getCoordinate().x, 1e-15); + assertEquals(0.0, actual.nearest.getCoordinate().y, 1e-15); + assertEquals(triangleRadius, actual.radius, 1e-15); assertEquals(3857, actual.center.getSRID()); assertEquals(3857, actual.nearest.getSRID()); assertEquals( Functions.maximumInscribedCircle(geom), Functions.maximumInscribedCircle(geom, 0.0)); + Geometry triangle = geom; + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(triangle, -1.0)); + assertEquals("Tolerance must be positive", error.getMessage()); geom = Constructors.geomFromWKT("POLYGON ((0 0, 0 0, 0 0, 0 0))", 3857); actual = Functions.maximumInscribedCircle(geom); @@ -5253,11 +5258,17 @@ public void maximumInscribedCircle() throws ParseException { assertEquals(expected, Functions.maximumInscribedCircle(geom, 2.0)); Geometry finalGeom = geom; - IllegalArgumentException error = + error = assertThrows( IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(finalGeom, -1.0)); assertEquals("Tolerance must be positive", error.getMessage()); + + Geometry points = Constructors.geomFromWKT("MULTIPOINT ((0 0), (1 1))", 3857); + error = + assertThrows( + IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(points, -1.0)); + assertEquals("Tolerance must be positive", error.getMessage()); } @Test From 2fb21d2fc3362327170c6ce2ca9a8ef2efa64138 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:37:58 -0700 Subject: [PATCH 03/25] Exclude Spark JTS from module classpaths --- spark/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spark/pom.xml b/spark/pom.xml index bc21727640c..963d97f12da 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -88,6 +88,12 @@ spark-sql_${scala.compat.version} ${spark.version} provided + + + org.locationtech.jts + jts-core + + io.graphframes From ae576dd49eae80895e39bc71dee1db7d4538d25a Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:39:17 -0700 Subject: [PATCH 04/25] Document Spark 4.1 JTS setup --- docs/setup/install-scala.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/setup/install-scala.md b/docs/setup/install-scala.md index 928363dce6b..ceb5d462d5b 100644 --- a/docs/setup/install-scala.md +++ b/docs/setup/install-scala.md @@ -63,6 +63,14 @@ Please refer to [Sedona Maven Central coordinates](maven-coordinates.md) to sele Please use jars with Spark major.minor versions in the filename, such as `sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}`. +### JTS dependency for Spark 4.1 + +Sedona requires `org.datasyslab:jts-core:1.21.0-datasyslab-1`. The artifact keeps the `org.locationtech.jts` Java package names, so application imports do not change. + +Spark 4.1 distributions include `jts-core-1.20.0.jar`. Before starting any driver or executor, remove that jar from every Spark installation and put the `org.datasyslab:jts-core:1.21.0-datasyslab-1` jar in the same `SPARK_HOME/jars` directory. Do not keep both JTS jars because they provide the same Java classes. + +Sedona excludes Spark's JTS Maven dependency when you build an application. That exclusion only affects dependencies resolved by the build. It cannot remove the JTS jar from a prebuilt Spark installation, so cluster images and Spark installations must be updated separately on both drivers and executors. + * Local mode: test Sedona without setting up a cluster ``` ./bin/spark-shell --jars /path/to/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar,/path/to/geotools-wrapper-{{ sedona.current_geotools }}.jar From 7e90e047f76dfe338d6da4d25b65bdf2fcb446b4 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:45:38 -0700 Subject: [PATCH 05/25] Tighten JTS compatibility checks --- .../apache/sedona/common/FunctionsTest.java | 41 +++++++++++++++---- docs/setup/install-python.md | 2 + docs/setup/install-scala.md | 16 ++++---- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 650082b68f8..77dada2ee4d 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -35,6 +35,7 @@ import org.geotools.referencing.CRS; import org.geotools.referencing.operation.projection.ProjectionException; import org.junit.Test; +import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.*; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.prep.PreparedGeometry; @@ -67,6 +68,27 @@ protected int compareCoordinate( } }; + private void assertPolygonEqualsIgnoringStartVertex(String expectedWkt, String actualWkt) { + try { + Polygon expected = (Polygon) new WKTReader().read(expectedWkt); + Polygon actual = (Polygon) new WKTReader().read(actualWkt); + assertEquals(expected.getNumInteriorRing(), actual.getNumInteriorRing()); + assertEquals( + Orientation.isCCW(expected.getExteriorRing().getCoordinates()), + Orientation.isCCW(actual.getExteriorRing().getCoordinates())); + for (int i = 0; i < expected.getNumInteriorRing(); i++) { + assertEquals( + Orientation.isCCW(expected.getInteriorRingN(i).getCoordinates()), + Orientation.isCCW(actual.getInteriorRingN(i).getCoordinates())); + } + expected.normalize(); + actual.normalize(); + assertTrue(expected.equalsExact(actual)); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } + private final WKTReader wktReader = new WKTReader(); @Test @@ -1011,7 +1033,7 @@ public void orientedEnvelope() throws ParseException { actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(irregularPolygon), 2)); expected = "POLYGON ((5 0, 0.29 -1.18, -0.71 2.82, 4 4, 5 0))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); Geometry point = Constructors.geomFromWKT("POINT (1 2)", 0); actual = Functions.asWKT(Functions.orientedEnvelope(point)); @@ -1039,14 +1061,14 @@ public void orientedEnvelope() throws ParseException { Geometry multiPoint = Constructors.geomFromWKT("MULTIPOINT ((0 0), (-1 -1), (3 2))", 0); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(multiPoint), 2)); expected = "POLYGON ((-1 -1, -1.12 -0.84, 2.88 2.16, 3 2, -1 -1))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); Geometry linestring = Constructors.geomFromWKT("LINESTRING (55 75, 125 150)", 0); Geometry pointGeom = Constructors.geomFromWKT("POINT (20 80)", 0); Geometry collection = linestring.union(pointGeom); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(collection), 2)); expected = "POLYGON ((125 150, 138.08 130.38, 33.08 60.38, 20 80, 125 150))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); } @Test @@ -2930,7 +2952,10 @@ public void force2D() throws ParseException { String actual = Functions.asWKT(forcedPoly); String expected = "POLYGON ((0 10, 10 10, 10 5.333333333333333, 5 7, 3 5, 5 3, 10 4.666666666666667, 10 0, 0 0, 0 10))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); + for (Coordinate coordinate : forcedPoly.getCoordinates()) { + assertTrue(Double.isNaN(coordinate.getZ())); + } geom = Constructors.geomFromWKT("POINT ZM(1 2 2 5)", 0); actual = Functions.asWKT(Functions.force2D(geom)); @@ -3640,7 +3665,7 @@ public void testBuffer() { String actual = Functions.asWKT(Functions.reducePrecision(Functions.buffer(polygon, 15), 4)); String expected = "POLYGON ((47.0736 35.2882, 44.2597 36.1418, 41.6664 37.528, 39.3934 39.3934, 37.528 41.6664, 36.1418 44.2597, 35.2882 47.0736, 35 50, 35 150, 35.2882 152.9264, 36.1418 155.7403, 37.528 158.3336, 39.3934 160.6066, 41.6664 162.472, 44.2597 163.8582, 47.0736 164.7118, 50 165, 150 165, 152.9264 164.7118, 155.7403 163.8582, 158.3336 162.472, 160.6066 160.6066, 162.472 158.3336, 163.8582 155.7403, 164.7118 152.9264, 165 150, 165 50, 164.7118 47.0736, 163.8582 44.2597, 162.472 41.6664, 160.6066 39.3934, 158.3336 37.528, 155.7403 36.1418, 152.9264 35.2882, 150 35, 50 35, 47.0736 35.2882))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 100, 100)); actual = @@ -3648,7 +3673,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "side=left"), 4)); expected = "POLYGON ((50 70, 0 0, -8.1373 5.8124, 41.8627 75.8124, 43.2167 77.3476, 44.855 78.5749, 94.855 108.5749, 100 100, 50 70))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 70, -3)); actual = @@ -3656,7 +3681,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "endcap=square"), 4)); expected = "POLYGON ((43.2156 77.3465, 44.8523 78.5733, 46.7044 79.4413, 48.6944 79.9144, 50.739 79.9727, 52.7527 79.6137, 54.6512 78.8525, 56.3552 77.7209, 57.7932 76.2663, 58.9052 74.5495, 59.6446 72.6424, 79.6446 -0.3576, 82.2869 -10.0022, 62.9978 -15.2869, 45.9128 47.0733, 8.1373 -5.8124, 2.325 -13.9497, -13.9497 -2.325, 41.8627 75.8124, 43.2156 77.3465))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(100, 90)); actual = @@ -3664,7 +3689,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(point, 10, false, "quad_segs=2"), 4)); expected = "POLYGON ((107.0711 82.9289, 100 80, 92.9289 82.9289, 90 90, 92.9289 97.0711, 100 100, 107.0711 97.0711, 110 90, 107.0711 82.9289))"; - assertGeometryEquals(expected, actual); + assertPolygonEqualsIgnoringStartVertex(expected, actual); // Using reducePrecision in the following tests modified the geometries coordinates the // coordinates causing the comparison to fail, diff --git a/docs/setup/install-python.md b/docs/setup/install-python.md index 1ea0e928ca0..755936cb80d 100644 --- a/docs/setup/install-python.md +++ b/docs/setup/install-python.md @@ -80,6 +80,8 @@ GeoPandas brings in `pandas` as a dependency. PyArrow is also needed for Arrow-b Sedona Python needs one additional jar file called `sedona-spark-shaded` or `sedona-spark` to work properly. Please make sure you use the correct version for Spark and Scala. +Sedona 2.0 with PySpark 4.1 also requires replacing Spark's bundled JTS jar before starting the driver and executors. Follow the [Spark 4.1 JTS setup](install-scala.md#jts-dependency-for-spark-41). + Please use Spark major.minor version number in artifact names. You can get it using one of the following methods: diff --git a/docs/setup/install-scala.md b/docs/setup/install-scala.md index ceb5d462d5b..92125d6165f 100644 --- a/docs/setup/install-scala.md +++ b/docs/setup/install-scala.md @@ -63,14 +63,6 @@ Please refer to [Sedona Maven Central coordinates](maven-coordinates.md) to sele Please use jars with Spark major.minor versions in the filename, such as `sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}`. -### JTS dependency for Spark 4.1 - -Sedona requires `org.datasyslab:jts-core:1.21.0-datasyslab-1`. The artifact keeps the `org.locationtech.jts` Java package names, so application imports do not change. - -Spark 4.1 distributions include `jts-core-1.20.0.jar`. Before starting any driver or executor, remove that jar from every Spark installation and put the `org.datasyslab:jts-core:1.21.0-datasyslab-1` jar in the same `SPARK_HOME/jars` directory. Do not keep both JTS jars because they provide the same Java classes. - -Sedona excludes Spark's JTS Maven dependency when you build an application. That exclusion only affects dependencies resolved by the build. It cannot remove the JTS jar from a prebuilt Spark installation, so cluster images and Spark installations must be updated separately on both drivers and executors. - * Local mode: test Sedona without setting up a cluster ``` ./bin/spark-shell --jars /path/to/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar,/path/to/geotools-wrapper-{{ sedona.current_geotools }}.jar @@ -81,6 +73,14 @@ Sedona excludes Spark's JTS Maven dependency when you build an application. That ./bin/spark-shell --master spark://localhost:7077 --jars /path/to/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar,/path/to/geotools-wrapper-{{ sedona.current_geotools }}.jar ``` +### JTS dependency for Spark 4.1 + +Sedona 2.0 with Spark 4.1 requires `org.datasyslab:jts-core:1.21.0-datasyslab-1`. The artifact keeps the `org.locationtech.jts` Java package names, so application imports do not change. + +Spark 4.1 distributions include `jts-core-1.20.0.jar`. Before starting any driver or executor, remove that jar from every Spark installation and put the `org.datasyslab:jts-core:1.21.0-datasyslab-1` jar in the same `SPARK_HOME/jars` directory. Do not keep both JTS jars because they provide the same Java classes. + +Sedona excludes Spark's JTS Maven dependency when you build an application. That exclusion only affects dependencies resolved by the build. It cannot remove the JTS jar from a prebuilt Spark installation, so cluster images and Spark installations must be updated separately on both drivers and executors. + ## Spark SQL shell Please see [Use Sedona in a pure SQL environment](../tutorial/sql-pure-sql.md) From c92b3fefab5c20c31fddbe3deda2884c4e918e5d Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 22:27:57 -0700 Subject: [PATCH 06/25] Preserve declared geometry dimensions across binary serialization --- .../apache/sedona/common/Constructors.java | 4 +- .../org/apache/sedona/common/Functions.java | 4 + .../ByteBufferGeometryBuffer.java | 11 +- .../DeclaredCoordinateSequence.java | 45 +++++ .../DeclaredCoordinateSequenceFactory.java | 57 ++++++ .../DeclaredGeometryFactory.java | 84 ++++++++ .../geometrySerde/GeometrySerializer.java | 7 +- .../geometrySerde/GeometryWkbReader.java | 37 ++++ .../geometrySerde/UnsafeGeometryBuffer.java | 11 +- .../sedona/common/utils/FormatUtils.java | 6 +- .../GeometryDimensionSerdeTest.java | 180 ++++++++++++++++++ python/tests/geopandas/test_geoseries.py | 35 ++++ .../sedona/sql/GeometryUdtTestScala.scala | 26 +++ 13 files changed, 487 insertions(+), 20 deletions(-) create mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java create mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java create mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java create mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java diff --git a/common/src/main/java/org/apache/sedona/common/Constructors.java b/common/src/main/java/org/apache/sedona/common/Constructors.java index e04a0a9caaa..6f3ad3b49f9 100644 --- a/common/src/main/java/org/apache/sedona/common/Constructors.java +++ b/common/src/main/java/org/apache/sedona/common/Constructors.java @@ -26,11 +26,11 @@ import org.apache.sedona.common.enums.GeometryType; import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.geometryObjects.Box3D; +import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.utils.FormatUtils; import org.apache.sedona.common.utils.GeoHashDecoder; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.gml2.GMLReader; import org.locationtech.jts.io.kml.KMLReader; @@ -74,7 +74,7 @@ public static Geometry geomFromWKB(byte[] wkb) throws ParseException { } public static Geometry geomFromWKB(byte[] wkb, int SRID) throws ParseException { - Geometry geom = new WKBReader().read(wkb); + Geometry geom = GeometryWkbReader.read(wkb); if (geom.getFactory().getSRID() != geom.getSRID() || (SRID >= 0 && geom.getSRID() != SRID)) { // Make sure that the geometry and the geometry factory have the correct SRID if (SRID < 0) { diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 15bb8a6eee6..6a03611cca5 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -34,6 +34,7 @@ import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.geometryObjects.Box3D; import org.apache.sedona.common.geometryObjects.Circle; +import org.apache.sedona.common.geometrySerde.DeclaredGeometryFactory; import org.apache.sedona.common.jts2geojson.GeoJSONWriter; import org.apache.sedona.common.sphere.Spheroid; import org.apache.sedona.common.subDivide.GeometrySubDivider; @@ -1103,6 +1104,9 @@ public static Geometry setSRID(Geometry geometry, int srid) { geometry.getPrecisionModel(), srid, geometry.getFactory().getCoordinateSequenceFactory()); + if (geometry.getFactory() instanceof DeclaredGeometryFactory) { + factory = new DeclaredGeometryFactory(geometry.getPrecisionModel(), srid); + } Geometry newGeom = factory.createGeometry(geometry); // Workaround for JTS bug: GeometryEditor.editPolygon returns the original // empty polygon without copying it to the new factory, so the SRID is not diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java index e603c67a08f..ec82f2752e6 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java @@ -25,7 +25,6 @@ import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.CoordinateXYM; import org.locationtech.jts.geom.CoordinateXYZM; -import org.locationtech.jts.geom.impl.CoordinateArraySequence; class ByteBufferGeometryBuffer implements GeometryBuffer { private CoordinateType coordinateType = CoordinateType.XY; @@ -126,20 +125,20 @@ public CoordinateSequence getCoordinate(int offset) { switch (coordinateType) { case XY: coordinates[0] = new CoordinateXY(x, y); - return new CoordinateArraySequence(coordinates, 2, 0); + return new DeclaredCoordinateSequence(coordinates, 2, 0); case XYZ: z = byteBuffer.getDouble(offset + 16); coordinates[0] = new Coordinate(x, y, z); - return new CoordinateArraySequence(coordinates, 3, 0); + return new DeclaredCoordinateSequence(coordinates, 3, 0); case XYM: m = byteBuffer.getDouble(offset + 16); coordinates[0] = new CoordinateXYM(x, y, m); - return new CoordinateArraySequence(coordinates, 3, 1); + return new DeclaredCoordinateSequence(coordinates, 3, 1); case XYZM: z = byteBuffer.getDouble(offset + 16); m = byteBuffer.getDouble(offset + 24); coordinates[0] = new CoordinateXYZM(x, y, z, m); - return new CoordinateArraySequence(coordinates, 4, 1); + return new DeclaredCoordinateSequence(coordinates, 4, 1); default: throw new IllegalStateException("coordinateType was not configured properly"); } @@ -240,7 +239,7 @@ public CoordinateSequence getCoordinates(int offset, int numCoordinates) { default: throw new IllegalStateException("coordinateType was not configured properly"); } - return new CoordinateArraySequence(coordinates, dimension, measures); + return new DeclaredCoordinateSequence(coordinates, dimension, measures); } @Override diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java new file mode 100644 index 00000000000..91e10a1b6a1 --- /dev/null +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.geometrySerde; + +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.impl.CoordinateArraySequence; + +/** Coordinates whose layout was explicitly declared by a binary geometry header. */ +final class DeclaredCoordinateSequence extends CoordinateArraySequence { + private static final long serialVersionUID = 1L; + + DeclaredCoordinateSequence(Coordinate[] coordinates, int dimension, int measures) { + super(coordinates, dimension, measures); + } + + DeclaredCoordinateSequence(int size, int dimension, int measures) { + super(size, dimension, measures); + } + + private DeclaredCoordinateSequence(CoordinateSequence sequence) { + super(sequence); + } + + @Override + public DeclaredCoordinateSequence copy() { + return new DeclaredCoordinateSequence(this); + } +} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java new file mode 100644 index 00000000000..4103cf37bcf --- /dev/null +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.geometrySerde; + +import java.io.Serializable; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.CoordinateSequenceFactory; +import org.locationtech.jts.geom.impl.CoordinateArraySequenceFactory; + +/** Retains declared layouts on explicit allocation and preserves provenance on copies. */ +final class DeclaredCoordinateSequenceFactory implements CoordinateSequenceFactory, Serializable { + private static final long serialVersionUID = 1L; + static final DeclaredCoordinateSequenceFactory INSTANCE = new DeclaredCoordinateSequenceFactory(); + + private DeclaredCoordinateSequenceFactory() {} + + @Override + public CoordinateSequence create(Coordinate[] coordinates) { + // Ordinary Coordinate values may contain a padded NaN Z, which is not a declaration. + return CoordinateArraySequenceFactory.instance().create(coordinates); + } + + @Override + public CoordinateSequence create(CoordinateSequence sequence) { + if (sequence instanceof DeclaredCoordinateSequence) { + return sequence.copy(); + } + return CoordinateArraySequenceFactory.instance().create(sequence); + } + + @Override + public CoordinateSequence create(int size, int dimension) { + return create(size, dimension, 0); + } + + @Override + public CoordinateSequence create(int size, int dimension, int measures) { + return new DeclaredCoordinateSequence(size, dimension, measures); + } +} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java new file mode 100644 index 00000000000..71725cd130a --- /dev/null +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.geometrySerde; + +import org.locationtech.jts.geom.*; + +/** + * Geometry factory that retains empty components and their declared coordinate layouts on copies. + */ +public final class DeclaredGeometryFactory extends GeometryFactory { + private static final long serialVersionUID = 1L; + + public DeclaredGeometryFactory(PrecisionModel precisionModel, int srid) { + super(precisionModel, srid, DeclaredCoordinateSequenceFactory.INSTANCE); + } + + @Override + public Geometry createGeometry(Geometry geometry) { + // GeometryEditor's default copy removes empty collection members and rebuilds empty polygons + // without their shell sequence. Copy the structure directly to preserve binary declarations. + Geometry copy; + if (geometry instanceof Point) { + copy = + createPoint( + getCoordinateSequenceFactory().create(((Point) geometry).getCoordinateSequence())); + } else if (geometry instanceof LinearRing) { + copy = + createLinearRing( + getCoordinateSequenceFactory() + .create(((LinearRing) geometry).getCoordinateSequence())); + } else if (geometry instanceof LineString) { + copy = + createLineString( + getCoordinateSequenceFactory() + .create(((LineString) geometry).getCoordinateSequence())); + } else if (geometry instanceof Polygon) { + Polygon polygon = (Polygon) geometry; + LinearRing[] holes = new LinearRing[polygon.getNumInteriorRing()]; + for (int i = 0; i < holes.length; i++) + holes[i] = (LinearRing) createGeometry(polygon.getInteriorRingN(i)); + copy = createPolygon((LinearRing) createGeometry(polygon.getExteriorRing()), holes); + } else if (geometry instanceof MultiPoint) { + Point[] points = new Point[geometry.getNumGeometries()]; + for (int i = 0; i < points.length; i++) + points[i] = (Point) createGeometry(geometry.getGeometryN(i)); + copy = createMultiPoint(points); + } else if (geometry instanceof MultiLineString) { + LineString[] lines = new LineString[geometry.getNumGeometries()]; + for (int i = 0; i < lines.length; i++) + lines[i] = (LineString) createGeometry(geometry.getGeometryN(i)); + copy = createMultiLineString(lines); + } else if (geometry instanceof MultiPolygon) { + Polygon[] polygons = new Polygon[geometry.getNumGeometries()]; + for (int i = 0; i < polygons.length; i++) + polygons[i] = (Polygon) createGeometry(geometry.getGeometryN(i)); + copy = createMultiPolygon(polygons); + } else if (geometry instanceof GeometryCollection) { + Geometry[] children = new Geometry[geometry.getNumGeometries()]; + for (int i = 0; i < children.length; i++) + children[i] = createGeometry(geometry.getGeometryN(i)); + copy = createGeometryCollection(children); + } else { + return super.createGeometry(geometry); + } + copy.setUserData(geometry.getUserData()); + return copy; + } +} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java index 1f9a82e8cba..78256e3885f 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java @@ -476,12 +476,15 @@ private static void collectCoordinateDimensions( // Measures are explicit CoordinateSequence metadata. A Z dimension is not always explicit: // JTS's default sequence factory represents ordinary XY coordinates as dimension 3 with NaN Z. - // XYZM is unambiguous, while XYZ is recoverable only when at least one Z value is finite. An + // Trusted binary layouts and XYZM are unambiguous. Unmarked XYZ is recoverable only when at + // least one Z value is non-NaN. An // ambiguous sequence does not constrain a multipart geometry whose other members establish the // shared layout. CoordinateType coordinateType = null; if (measures > 0) { coordinateType = spatialDimensions > 2 ? CoordinateType.XYZM : CoordinateType.XYM; + } else if (coordinates instanceof DeclaredCoordinateSequence && spatialDimensions == 3) { + coordinateType = CoordinateType.XYZ; } else if (spatialDimensions == 2) { coordinateType = CoordinateType.XY; } else { @@ -500,7 +503,7 @@ private static int alignedOffset(int offset) { } private static GeometryFactory createGeometryFactory(int srid) { - return new GeometryFactory(PRECISION_MODEL, srid); + return new DeclaredGeometryFactory(PRECISION_MODEL, srid); } private static Polygon createEmptyPolygon( diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java new file mode 100644 index 00000000000..723adaba282 --- /dev/null +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.geometrySerde; + +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.PrecisionModel; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; + +/** + * Reads WKB while retaining explicitly declared coordinate dimensions, including empty geometry. + */ +public final class GeometryWkbReader { + private GeometryWkbReader() {} + + public static Geometry read(byte[] bytes) throws ParseException { + GeometryFactory factory = new DeclaredGeometryFactory(new PrecisionModel(), 0); + return new WKBReader(factory).read(bytes); + } +} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java index a908a932f3d..39bfa123737 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java @@ -24,7 +24,6 @@ import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.CoordinateXYM; import org.locationtech.jts.geom.CoordinateXYZM; -import org.locationtech.jts.geom.impl.CoordinateArraySequence; import sun.misc.Unsafe; class UnsafeGeometryBuffer implements GeometryBuffer { @@ -173,20 +172,20 @@ public CoordinateSequence getCoordinate(int offset) { switch (coordinateType) { case XY: coordinates[0] = new CoordinateXY(x, y); - return new CoordinateArraySequence(coordinates, 2, 0); + return new DeclaredCoordinateSequence(coordinates, 2, 0); case XYZ: z = UNSAFE.getDouble(bytes, coordOffset + 16); coordinates[0] = new Coordinate(x, y, z); - return new CoordinateArraySequence(coordinates, 3, 0); + return new DeclaredCoordinateSequence(coordinates, 3, 0); case XYM: m = UNSAFE.getDouble(bytes, coordOffset + 16); coordinates[0] = new CoordinateXYM(x, y, m); - return new CoordinateArraySequence(coordinates, 3, 1); + return new DeclaredCoordinateSequence(coordinates, 3, 1); case XYZM: z = UNSAFE.getDouble(bytes, coordOffset + 16); m = UNSAFE.getDouble(bytes, coordOffset + 24); coordinates[0] = new CoordinateXYZM(x, y, z, m); - return new CoordinateArraySequence(coordinates, 4, 1); + return new DeclaredCoordinateSequence(coordinates, 4, 1); default: throw new IllegalStateException("coordinateType was not configured properly"); } @@ -293,7 +292,7 @@ public CoordinateSequence getCoordinates(int offset, int numCoordinates) { default: throw new IllegalStateException("coordinateType was not configured properly"); } - return new CoordinateArraySequence(coordinates, dimension, measures); + return new DeclaredCoordinateSequence(coordinates, dimension, measures); } @Override diff --git a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java index e37d05f0507..ebb78b800b0 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java @@ -25,6 +25,7 @@ import org.apache.sedona.common.Functions; import org.apache.sedona.common.enums.FileDataSplitter; import org.apache.sedona.common.enums.GeometryType; +import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.jts2geojson.GeoJSONReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; @@ -207,10 +208,7 @@ public Geometry readWkt(String line) throws ParseException { public Geometry readWkb(String line) throws ParseException { final String[] columns = line.split(splitter.getDelimiter()); final byte[] aux = WKBReader.hexToBytes(columns[this.startOffset]); - // For some unknown reasons, the wkb reader cannot be used in transient variable like the wkt - // reader. - WKBReader wkbReader = new WKBReader(); - Geometry geometry = wkbReader.read(aux); + Geometry geometry = GeometryWkbReader.read(aux); if (geometry.getSRID() != geometry.getFactory().getSRID()) { // Make sure that the geometry factory has the correct SRID when the parsed WKB // contains a non-zero SRID (EWKB) diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java index c442ac5eb97..4d90946f129 100644 --- a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java +++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java @@ -22,6 +22,10 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.sedona.common.Constructors; +import org.apache.sedona.common.Functions; import org.junit.Test; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; @@ -40,11 +44,187 @@ import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBWriter; import org.locationtech.jts.io.WKTReader; public class GeometryDimensionSerdeTest { private static final GeometryFactory FACTORY = new GeometryFactory(); + @Test + public void nestedCollectionsAndEmptyMultipartMembersSurviveFactoryCopies() + throws ParseException { + Geometry empty = + Constructors.geomFromWKB(new WKBWriter(3).write(new WKTReader().read("POINT Z EMPTY"))); + GeometryFactory factory = empty.getFactory(); + Geometry[] parts = + new Geometry[] { + factory.createMultiPoint( + new Point[] {(Point) empty, factory.createPoint(new Coordinate(1, 2, 3))}), + factory.createMultiLineString( + new LineString[] { + (LineString) + Constructors.geomFromWKB( + new WKBWriter(3).write(new WKTReader().read("LINESTRING Z EMPTY"))), + factory.createLineString( + new Coordinate[] {new Coordinate(1, 2, 3), new Coordinate(4, 5, 6)}) + }), + factory.createMultiPolygon( + new Polygon[] { + (Polygon) + Constructors.geomFromWKB( + new WKBWriter(3).write(new WKTReader().read("POLYGON Z EMPTY"))) + }) + }; + GeometryCollection nested = + factory.createGeometryCollection( + new Geometry[] { + factory.createGeometryCollection(parts), FACTORY.createPoint(new CoordinateXY(7, 8)) + }); + nested.setUserData("metadata"); + Geometry changed = Functions.setSRID(nested, 4326); + assertEquals(0, nested.getSRID()); + assertEquals(4326, changed.getSRID()); + assertEquals(4326, changed.getFactory().getSRID()); + assertEquals("metadata", changed.getUserData()); + Geometry output = roundTrip(changed); + assertEquals(2, output.getNumGeometries()); + Geometry children = output.getGeometryN(0); + for (int i = 0; i < parts.length; i++) { + Geometry part = children.getGeometryN(i); + assertEquals(parts[i].getNumGeometries(), part.getNumGeometries()); + assertTrue(part.getGeometryN(0).isEmpty()); + assertEquals(CoordinateType.XYZ, coordinateType(GeometrySerializer.serialize(part))); + } + assertEquals( + CoordinateType.XY, coordinateType(GeometrySerializer.serialize(output.getGeometryN(1)))); + } + + @Test + public void hexWkbReaderPreservesEmptyZAndInputData() throws ParseException { + byte[] bytes = new WKBWriter(3, true).write(new WKTReader().read("POLYGON Z EMPTY")); + org.apache.sedona.common.utils.FormatUtils reader = + new org.apache.sedona.common.utils.FormatUtils( + org.apache.sedona.common.enums.FileDataSplitter.WKB, true); + Geometry geometry = reader.readWkb(WKBWriter.toHex(bytes) + "\tmetadata"); + assertEquals("metadata", geometry.getUserData()); + assertEquals(CoordinateType.XYZ, coordinateType(GeometrySerializer.serialize(geometry))); + } + + @Test + public void isoAndEwkbLayoutsSurviveCopiesAndWireBuffers() throws ParseException { + for (boolean iso : new boolean[] {false, true}) { + for (int layout = 0; layout < 4; layout++) { + int dimension = 2 + (layout == 3 ? 2 : layout == 0 ? 0 : 1); + int measures = layout >= 2 ? 1 : 0; + CoordinateType expected = CoordinateType.values()[layout]; + for (int primitive = 1; primitive <= 3; primitive++) { + for (boolean empty : new boolean[] {false, true}) { + // Populated lines/polygons are covered elsewhere; this also exercises all-NaN Z/M. + if (!empty && primitive != 1) continue; + ByteBuffer wkb = ByteBuffer.allocate(64).order(ByteOrder.LITTLE_ENDIAN); + wkb.put((byte) 1); + int type = + iso + ? primitive + 1000 * layout + : primitive + | (layout == 1 || layout == 3 ? 0x80000000 : 0) + | (measures == 1 ? 0x40000000 : 0) + | 0x20000000; + wkb.putInt(type); + if (!iso) wkb.putInt(4326); + if (primitive == 1) { + wkb.putDouble(empty ? Double.NaN : 1).putDouble(empty ? Double.NaN : 2); + for (int ordinate = 2; ordinate < dimension; ordinate++) wkb.putDouble(Double.NaN); + } else { + wkb.putInt(0); + } + Geometry geometry = + Constructors.geomFromWKB(java.util.Arrays.copyOf(wkb.array(), wkb.position())); + assertEquals(iso ? 0 : 4326, geometry.getSRID()); + geometry.setUserData("retained"); + assertEquals("retained", geometry.copy().getUserData()); + for (String buffer : new String[] {"bytebuffer", "unsafe"}) { + Geometry decoded = + GeometrySerializer.deserialize( + GeometryBufferFactory.wrap(buffer, GeometrySerializer.serialize(geometry))); + for (Geometry copy : + new Geometry[] { + geometry.copy(), + geometry.reverse(), + geometry.getFactory().createGeometry(geometry), + decoded, + decoded.copy(), + decoded.reverse(), + Functions.setSRID(decoded, 3857) + }) { + byte[] bytes = GeometrySerializer.serialize(copy); + assertEquals(expected, coordinateType(bytes)); + Geometry output = GeometrySerializer.deserialize(bytes); + CoordinateSequence sequence = + output instanceof Point + ? ((Point) output).getCoordinateSequence() + : output instanceof LineString + ? ((LineString) output).getCoordinateSequence() + : ((Polygon) output).getExteriorRing().getCoordinateSequence(); + assertSequenceLayout(sequence, dimension, measures); + } + } + } + } + } + } + } + + @Test + public void declaredFactoryDoesNotPromoteOrdinaryCoordinates() throws ParseException { + Geometry source = + Constructors.geomFromWKB(new WKBWriter(3).write(new WKTReader().read("POINT Z EMPTY"))); + Geometry ordinary = source.getFactory().createPoint(new Coordinate(1, 2)); + assertEquals(CoordinateType.XY, coordinateType(GeometrySerializer.serialize(ordinary))); + assertEquals( + CoordinateType.XY, + coordinateType( + GeometrySerializer.serialize( + source.getFactory().createGeometry(FACTORY.createPoint(new Coordinate(1, 2)))))); + assertThrows(ParseException.class, () -> Constructors.geomFromWKB(new byte[] {1, 1})); + } + + @Test + public void declaredWkbZSurvivesEmptyAndNaNCoordinates() throws ParseException { + for (String wkt : + new String[] { + "POINT Z EMPTY", + "LINESTRING Z EMPTY", + "POLYGON Z EMPTY", + "POINT Z (1 2 NaN)", + "LINESTRING Z (1 2 NaN, 3 4 NaN)" + }) { + byte[] wkb = new WKBWriter(3).write(new WKTReader().read(wkt.replace("NaN", "9"))); + // WKBWriter infers populated Z from values; supply explicit NaN Z ordinates in the bytes. + if (wkt.contains("NaN")) { + ByteBuffer ordinates = ByteBuffer.wrap(wkb).order(ByteOrder.BIG_ENDIAN); + int start = wkt.startsWith("POINT") ? 5 : 9; + for (int offset = start + 16; offset < wkb.length; offset += 24) { + ordinates.putDouble(offset, Double.NaN); + } + } + Geometry input = Constructors.geomFromWKB(wkb); + assertEquals(wkt, CoordinateType.XYZ, coordinateType(GeometrySerializer.serialize(input))); + for (String buffer : new String[] {"bytebuffer", "unsafe"}) { + Geometry decoded = + GeometrySerializer.deserialize( + GeometryBufferFactory.wrap(buffer, GeometrySerializer.serialize(input))); + for (Geometry transformed : + new Geometry[] { + decoded, decoded.copy(), decoded.reverse(), Functions.setSRID(decoded, 4326) + }) { + assertEquals( + wkt, CoordinateType.XYZ, coordinateType(GeometrySerializer.serialize(transformed))); + } + } + } + } + @Test public void leadingNaNDoesNotDropLaterOrdinates() { LineString xyz = diff --git a/python/tests/geopandas/test_geoseries.py b/python/tests/geopandas/test_geoseries.py index 50197c3135d..b0985980829 100644 --- a/python/tests/geopandas/test_geoseries.py +++ b/python/tests/geopandas/test_geoseries.py @@ -748,6 +748,41 @@ def test_constructor_null_and_empty_controls(self, values): check_index_type=False, ) + @pytest.mark.parametrize( + "wkt", + [ + "POINT Z EMPTY", + "LINESTRING Z EMPTY", + "POLYGON Z EMPTY", + "POINT Z (1 2 NaN)", + "POINT Z (1 2 3)", + "POINT (1 2)", + ], + ) + def test_constructor_leading_null_preserves_declared_dimensions(self, wkt): + import shapely + + if not hasattr(shapely, "geos_version") or shapely.geos_version < (3, 12, 0): + pytest.skip("Declared NaN Z requires GEOS 3.12 or newer") + _ = self.spark + geometry = shapely.from_wkt(wkt) + local = gpd.GeoSeries( + [None, geometry, None, Point(4, 5)], + index=pd.Index([9, 2, 9, 1], name="row"), + name="shape", + ) + result = GeoSeries(local).to_geopandas() + pd.testing.assert_index_equal(result.index, local.index) + assert result.name == local.name + assert result.iloc[0] is None and result.iloc[2] is None + assert shapely.get_coordinate_dimension( + result.iloc[1] + ) == shapely.get_coordinate_dimension(geometry) + assert result.iloc[1].is_empty == geometry.is_empty + assert result.iloc[3].equals(Point(4, 5)) + if not geometry.is_empty: + assert result.iloc[1].x == geometry.x and result.iloc[1].y == geometry.y + def test_constructor_leading_null_preserves_embedded_srid(self): from shapely import wkb from sedona.spark.sql.types import GeometryType diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/GeometryUdtTestScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/GeometryUdtTestScala.scala index 2aae4ea39fb..ed566e26126 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/GeometryUdtTestScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/GeometryUdtTestScala.scala @@ -41,6 +41,32 @@ class GeometryUdtTestScala extends TestBaseScala with BeforeAndAfter { } describe("GeometryUDT Test") { + it("Should preserve declared Z through WKB and the geometry UDT") { + val wkts = + Seq("POINT Z EMPTY", "LINESTRING Z EMPTY", "POLYGON Z EMPTY", "POINT Z (1 2 NaN)") + wkts.foreach { wkt => + val wkb = new org.locationtech.jts.io.WKBWriter(3) + .write(new WKTReader().read(wkt.replace("NaN", "9"))) + if (wkt.contains("NaN")) { + java.nio.ByteBuffer.wrap(wkb).putDouble(21, Double.NaN) + } + val hex = org.locationtech.jts.io.WKBWriter.toHex(wkb) + val geometry = sparkSession + .sql(s"SELECT ST_SetSRID(ST_GeomFromWKB(unhex('$hex')), 4326)") + .collect()(0) + .getAs[Geometry](0) + val sequence = geometry match { + case point: org.locationtech.jts.geom.Point => point.getCoordinateSequence + case line: org.locationtech.jts.geom.LineString => line.getCoordinateSequence + case polygon: org.locationtech.jts.geom.Polygon => + polygon.getExteriorRing.getCoordinateSequence + } + assert(sequence.getDimension == 3, wkt) + assert(sequence.getMeasures == 0, wkt) + assert(geometry.getSRID == 4326) + } + } + it("Should write dataframe with geometry in Parquet format") { tempFolder.create() From 49de25d330dfdf73f47ab9711304b7fcf1a31a66 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 23:16:22 -0700 Subject: [PATCH 07/25] Use isolated JTS IO patch --- common/pom.xml | 6 +- .../apache/sedona/common/Constructors.java | 2 +- .../org/apache/sedona/common/Functions.java | 3 - .../common/S2Geography/WKBGeography.java | 2 +- .../sedona/common/utils/FormatUtils.java | 2 +- .../apache/sedona/common/utils/GeomUtils.java | 9 +-- .../sedona/common/ConstructorsTest.java | 14 ++++ .../apache/sedona/common/FunctionsTest.java | 66 +++++-------------- docs/setup/install-python.md | 2 - docs/setup/install-scala.md | 8 --- pom.xml | 50 +++----------- spark/common/pom.xml | 2 +- .../sedona/core/spatialRDD/SpatialRDD.java | 2 +- .../PythonGeometrySerializer.scala | 2 +- .../geopackage/transform/GeometryReader.scala | 2 +- .../geoparquet/GeoParquetRowConverter.scala | 2 +- .../io/geojson/GeoJSONJacksonGenerator.scala | 2 +- spark/pom.xml | 6 -- spark/spark-3.5/pom.xml | 2 +- spark/spark-4.0/pom.xml | 2 +- spark/spark-4.1/pom.xml | 2 +- 21 files changed, 56 insertions(+), 132 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 6ad6736cdba..0cfb6e4257b 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -70,9 +70,13 @@ gt-process-feature - org.datasyslab + org.locationtech.jts jts-core + + org.datasyslab + jts-io-patch + org.wololo jts2geojson diff --git a/common/src/main/java/org/apache/sedona/common/Constructors.java b/common/src/main/java/org/apache/sedona/common/Constructors.java index e04a0a9caaa..42f7241ebb0 100644 --- a/common/src/main/java/org/apache/sedona/common/Constructors.java +++ b/common/src/main/java/org/apache/sedona/common/Constructors.java @@ -28,9 +28,9 @@ import org.apache.sedona.common.geometryObjects.Box3D; import org.apache.sedona.common.utils.FormatUtils; import org.apache.sedona.common.utils.GeoHashDecoder; +import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.gml2.GMLReader; import org.locationtech.jts.io.kml.KMLReader; diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 15bb8a6eee6..e3057e5ae12 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -1477,9 +1477,6 @@ public static InscribedCircle maximumInscribedCircle(Geometry geometry, double t if (Double.isNaN(tolerance)) { return null; } - if (tolerance < 0.0) { - throw new IllegalArgumentException("Tolerance must be positive"); - } // JTS requires a positive tolerance. A zero value requests the same // geometry-dependent default used by GEOS and the one-argument overload. if (tolerance == 0.0) { diff --git a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java index 5adb476c9da..d30c011057d 100644 --- a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java +++ b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java @@ -130,7 +130,7 @@ public Geometry getJTSGeometry() { result = jtsGeometry; if (result == null) { try { - org.locationtech.jts.io.WKBReader reader = new org.locationtech.jts.io.WKBReader(); + org.datasyslab.jts.io.WKBReader reader = new org.datasyslab.jts.io.WKBReader(); result = reader.read(wkbBytes); result.setSRID(getSRID()); } catch (ParseException e) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java index e37d05f0507..42e4e4bc82f 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java @@ -26,9 +26,9 @@ import org.apache.sedona.common.enums.FileDataSplitter; import org.apache.sedona.common.enums.GeometryType; import org.apache.sedona.common.jts2geojson.GeoJSONReader; +import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.operation.valid.IsValidOp; import org.slf4j.Logger; diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 7648c87a6ac..606d747a94b 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -23,13 +23,13 @@ import java.nio.ByteOrder; import java.util.*; import org.apache.sedona.common.Functions; +import org.datasyslab.jts.io.WKTWriter; import org.locationtech.jts.algorithm.Angle; import org.locationtech.jts.algorithm.distance.DiscreteFrechetDistance; import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ByteOrderValues; import org.locationtech.jts.io.WKBWriter; -import org.locationtech.jts.io.WKTWriter; import org.locationtech.jts.operation.polygonize.Polygonizer; import org.locationtech.jts.operation.union.UnaryUnionOp; import org.locationtech.spatial4j.context.jts.JtsSpatialContext; @@ -186,12 +186,7 @@ public static String getWKT(Geometry geometry) { } private static String writeWKT(Geometry geometry) { - // JTS 1.20 omits the separator between a dimensional marker and EMPTY for nested empty - // geometries (for example, "MULTILINESTRING ZEMPTY"), which its own WKTReader rejects. - String wkt = new WKTWriter(4).write(geometry); - return wkt.replace("ZMEMPTY", "ZM EMPTY") - .replace("ZEMPTY", "Z EMPTY") - .replace("MEMPTY", "M EMPTY"); + return new WKTWriter(4).write(geometry); } public static String getHexEWKB(Geometry geometry, int endian) { diff --git a/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java b/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java index 66e67db4d5c..08e2472f9cf 100644 --- a/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java +++ b/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java @@ -106,6 +106,20 @@ public void geomFromWKB() throws ParseException { assertEquals(0, result.getFactory().getSRID()); } + @Test + public void geomFromWKBPreservesDeclaredDimensionForEmptyGeometry() throws ParseException { + GeometryFactory factory = new GeometryFactory(); + + Geometry result = + Constructors.geomFromWKB(new WKBWriter(3).write(factory.createPoint((Coordinate) null))); + assertTrue(result instanceof Point); + assertEquals(3, ((Point) result).getCoordinateSequence().getDimension()); + + result = Constructors.geomFromWKB(new WKBWriter(3).write(factory.createPolygon())); + assertTrue(result instanceof Polygon); + assertEquals(3, ((Polygon) result).getExteriorRing().getCoordinateSequence().getDimension()); + } + @Test public void mLineFromWKT() throws ParseException { assertNull(Constructors.mLineFromText(null, 0)); diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 77dada2ee4d..0bd0c833090 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -35,7 +35,6 @@ import org.geotools.referencing.CRS; import org.geotools.referencing.operation.projection.ProjectionException; import org.junit.Test; -import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.*; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.prep.PreparedGeometry; @@ -68,27 +67,6 @@ protected int compareCoordinate( } }; - private void assertPolygonEqualsIgnoringStartVertex(String expectedWkt, String actualWkt) { - try { - Polygon expected = (Polygon) new WKTReader().read(expectedWkt); - Polygon actual = (Polygon) new WKTReader().read(actualWkt); - assertEquals(expected.getNumInteriorRing(), actual.getNumInteriorRing()); - assertEquals( - Orientation.isCCW(expected.getExteriorRing().getCoordinates()), - Orientation.isCCW(actual.getExteriorRing().getCoordinates())); - for (int i = 0; i < expected.getNumInteriorRing(); i++) { - assertEquals( - Orientation.isCCW(expected.getInteriorRingN(i).getCoordinates()), - Orientation.isCCW(actual.getInteriorRingN(i).getCoordinates())); - } - expected.normalize(); - actual.normalize(); - assertTrue(expected.equalsExact(actual)); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } - private final WKTReader wktReader = new WKTReader(); @Test @@ -1033,7 +1011,7 @@ public void orientedEnvelope() throws ParseException { actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(irregularPolygon), 2)); expected = "POLYGON ((5 0, 0.29 -1.18, -0.71 2.82, 4 4, 5 0))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); Geometry point = Constructors.geomFromWKT("POINT (1 2)", 0); actual = Functions.asWKT(Functions.orientedEnvelope(point)); @@ -1061,14 +1039,14 @@ public void orientedEnvelope() throws ParseException { Geometry multiPoint = Constructors.geomFromWKT("MULTIPOINT ((0 0), (-1 -1), (3 2))", 0); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(multiPoint), 2)); expected = "POLYGON ((-1 -1, -1.12 -0.84, 2.88 2.16, 3 2, -1 -1))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); Geometry linestring = Constructors.geomFromWKT("LINESTRING (55 75, 125 150)", 0); Geometry pointGeom = Constructors.geomFromWKT("POINT (20 80)", 0); Geometry collection = linestring.union(pointGeom); actual = Functions.asWKT(Functions.reducePrecision(Functions.orientedEnvelope(collection), 2)); expected = "POLYGON ((125 150, 138.08 130.38, 33.08 60.38, 20 80, 125 150))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); } @Test @@ -2952,10 +2930,7 @@ public void force2D() throws ParseException { String actual = Functions.asWKT(forcedPoly); String expected = "POLYGON ((0 10, 10 10, 10 5.333333333333333, 5 7, 3 5, 5 3, 10 4.666666666666667, 10 0, 0 0, 0 10))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); - for (Coordinate coordinate : forcedPoly.getCoordinates()) { - assertTrue(Double.isNaN(coordinate.getZ())); - } + assertEquals(expected, actual); geom = Constructors.geomFromWKT("POINT ZM(1 2 2 5)", 0); actual = Functions.asWKT(Functions.force2D(geom)); @@ -3665,7 +3640,7 @@ public void testBuffer() { String actual = Functions.asWKT(Functions.reducePrecision(Functions.buffer(polygon, 15), 4)); String expected = "POLYGON ((47.0736 35.2882, 44.2597 36.1418, 41.6664 37.528, 39.3934 39.3934, 37.528 41.6664, 36.1418 44.2597, 35.2882 47.0736, 35 50, 35 150, 35.2882 152.9264, 36.1418 155.7403, 37.528 158.3336, 39.3934 160.6066, 41.6664 162.472, 44.2597 163.8582, 47.0736 164.7118, 50 165, 150 165, 152.9264 164.7118, 155.7403 163.8582, 158.3336 162.472, 160.6066 160.6066, 162.472 158.3336, 163.8582 155.7403, 164.7118 152.9264, 165 150, 165 50, 164.7118 47.0736, 163.8582 44.2597, 162.472 41.6664, 160.6066 39.3934, 158.3336 37.528, 155.7403 36.1418, 152.9264 35.2882, 150 35, 50 35, 47.0736 35.2882))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 100, 100)); actual = @@ -3673,7 +3648,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "side=left"), 4)); expected = "POLYGON ((50 70, 0 0, -8.1373 5.8124, 41.8627 75.8124, 43.2167 77.3476, 44.855 78.5749, 94.855 108.5749, 100 100, 50 70))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); lineString = GEOMETRY_FACTORY.createLineString(coordArray(0, 0, 50, 70, 70, -3)); actual = @@ -3681,7 +3656,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(lineString, 10, false, "endcap=square"), 4)); expected = "POLYGON ((43.2156 77.3465, 44.8523 78.5733, 46.7044 79.4413, 48.6944 79.9144, 50.739 79.9727, 52.7527 79.6137, 54.6512 78.8525, 56.3552 77.7209, 57.7932 76.2663, 58.9052 74.5495, 59.6446 72.6424, 79.6446 -0.3576, 82.2869 -10.0022, 62.9978 -15.2869, 45.9128 47.0733, 8.1373 -5.8124, 2.325 -13.9497, -13.9497 -2.325, 41.8627 75.8124, 43.2156 77.3465))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(100, 90)); actual = @@ -3689,7 +3664,7 @@ public void testBuffer() { Functions.reducePrecision(Functions.buffer(point, 10, false, "quad_segs=2"), 4)); expected = "POLYGON ((107.0711 82.9289, 100 80, 92.9289 82.9289, 90 90, 92.9289 97.0711, 100 100, 107.0711 97.0711, 110 90, 107.0711 82.9289))"; - assertPolygonEqualsIgnoringStartVertex(expected, actual); + assertEquals(expected, actual); // Using reducePrecision in the following tests modified the geometries coordinates the // coordinates causing the comparison to fail, @@ -5255,21 +5230,16 @@ public void maximumInscribedCircle() throws ParseException { geom = Constructors.geomFromWKT("POLYGON ((0 0, 1 0, 1 1, 0 0))", 3857); actual = Functions.maximumInscribedCircle(geom, 2.0); - double triangleRadius = 1.0 - 1.0 / Math.sqrt(2.0); - assertEquals(1.0 - triangleRadius, actual.center.getCoordinate().x, 1e-15); - assertEquals(triangleRadius, actual.center.getCoordinate().y, 1e-15); - assertEquals(1.0 - triangleRadius, actual.nearest.getCoordinate().x, 1e-15); - assertEquals(0.0, actual.nearest.getCoordinate().y, 1e-15); - assertEquals(triangleRadius, actual.radius, 1e-15); + expected = + new InscribedCircle( + Constructors.geomFromWKT("POINT (0.75 0.5)", 3857), + Constructors.geomFromWKT("POINT (0.625 0.625)", 3857), + 0.1767766952966369); + assertEquals(expected, actual); assertEquals(3857, actual.center.getSRID()); assertEquals(3857, actual.nearest.getSRID()); assertEquals( Functions.maximumInscribedCircle(geom), Functions.maximumInscribedCircle(geom, 0.0)); - Geometry triangle = geom; - IllegalArgumentException error = - assertThrows( - IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(triangle, -1.0)); - assertEquals("Tolerance must be positive", error.getMessage()); geom = Constructors.geomFromWKT("POLYGON ((0 0, 0 0, 0 0, 0 0))", 3857); actual = Functions.maximumInscribedCircle(geom); @@ -5283,17 +5253,11 @@ public void maximumInscribedCircle() throws ParseException { assertEquals(expected, Functions.maximumInscribedCircle(geom, 2.0)); Geometry finalGeom = geom; - error = + IllegalArgumentException error = assertThrows( IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(finalGeom, -1.0)); assertEquals("Tolerance must be positive", error.getMessage()); - - Geometry points = Constructors.geomFromWKT("MULTIPOINT ((0 0), (1 1))", 3857); - error = - assertThrows( - IllegalArgumentException.class, () -> Functions.maximumInscribedCircle(points, -1.0)); - assertEquals("Tolerance must be positive", error.getMessage()); } @Test diff --git a/docs/setup/install-python.md b/docs/setup/install-python.md index 755936cb80d..1ea0e928ca0 100644 --- a/docs/setup/install-python.md +++ b/docs/setup/install-python.md @@ -80,8 +80,6 @@ GeoPandas brings in `pandas` as a dependency. PyArrow is also needed for Arrow-b Sedona Python needs one additional jar file called `sedona-spark-shaded` or `sedona-spark` to work properly. Please make sure you use the correct version for Spark and Scala. -Sedona 2.0 with PySpark 4.1 also requires replacing Spark's bundled JTS jar before starting the driver and executors. Follow the [Spark 4.1 JTS setup](install-scala.md#jts-dependency-for-spark-41). - Please use Spark major.minor version number in artifact names. You can get it using one of the following methods: diff --git a/docs/setup/install-scala.md b/docs/setup/install-scala.md index 92125d6165f..928363dce6b 100644 --- a/docs/setup/install-scala.md +++ b/docs/setup/install-scala.md @@ -73,14 +73,6 @@ Please use jars with Spark major.minor versions in the filename, such as `sedona ./bin/spark-shell --master spark://localhost:7077 --jars /path/to/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar,/path/to/geotools-wrapper-{{ sedona.current_geotools }}.jar ``` -### JTS dependency for Spark 4.1 - -Sedona 2.0 with Spark 4.1 requires `org.datasyslab:jts-core:1.21.0-datasyslab-1`. The artifact keeps the `org.locationtech.jts` Java package names, so application imports do not change. - -Spark 4.1 distributions include `jts-core-1.20.0.jar`. Before starting any driver or executor, remove that jar from every Spark installation and put the `org.datasyslab:jts-core:1.21.0-datasyslab-1` jar in the same `SPARK_HOME/jars` directory. Do not keep both JTS jars because they provide the same Java classes. - -Sedona excludes Spark's JTS Maven dependency when you build an application. That exclusion only affects dependencies resolved by the build. It cannot remove the JTS jar from a prebuilt Spark installation, so cluster images and Spark installations must be updated separately on both drivers and executors. - ## Spark SQL shell Please see [Use Sedona in a pure SQL environment](../tutorial/sql-pure-sql.md) diff --git a/pom.xml b/pom.xml index 415c2d843b1..55d1c702acf 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,8 @@ 33.1 3.2.4 2.13.4 - 1.21.0-datasyslab-1 + 1.20.0 + 1.21.0-datasyslab-1 0.16.1 0.8 @@ -153,10 +154,15 @@ - org.datasyslab + org.locationtech.jts jts-core ${jts.version} + + org.datasyslab + jts-io-patch + ${jts.io.patch.version} + org.locationtech.spatial4j spatial4j @@ -210,10 +216,6 @@ ${geotools.version} ${geotools.scope} - - org.locationtech.jts - jts-core - com.fasterxml.jackson.core * @@ -231,10 +233,6 @@ ${geotools.version} ${geotools.scope} - - org.locationtech.jts - jts-core - com.fasterxml.jackson.core * @@ -247,10 +245,6 @@ ${geotools.version} ${geotools.scope} - - org.locationtech.jts - jts-core - com.fasterxml.jackson.core * @@ -263,58 +257,30 @@ gt-geotiff ${geotools.version} ${geotools.scope} - - - org.locationtech.jts - jts-core - - org.geotools gt-process-feature ${geotools.version} ${geotools.scope} - - - org.locationtech.jts - jts-core - - org.geotools gt-arcgrid ${geotools.version} ${geotools.scope} - - - org.locationtech.jts - jts-core - - org.geotools gt-coverage ${geotools.version} ${geotools.scope} - - - org.locationtech.jts - jts-core - - it.geosolutions.jaiext.jiffle jt-jiffle-language ${jt-jiffle.version} - - org.locationtech.jts - jts-core - javax.media * diff --git a/spark/common/pom.xml b/spark/common/pom.xml index cde70fcef30..4e6701518ce 100644 --- a/spark/common/pom.xml +++ b/spark/common/pom.xml @@ -167,7 +167,7 @@ gt-arcgrid - org.datasyslab + org.locationtech.jts jts-core diff --git a/spark/common/src/main/java/org/apache/sedona/core/spatialRDD/SpatialRDD.java b/spark/common/src/main/java/org/apache/sedona/core/spatialRDD/SpatialRDD.java index 5d77e147660..dbe6809dceb 100644 --- a/spark/common/src/main/java/org/apache/sedona/core/spatialRDD/SpatialRDD.java +++ b/spark/common/src/main/java/org/apache/sedona/core/spatialRDD/SpatialRDD.java @@ -45,6 +45,7 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.storage.StorageLevel; import org.apache.spark.util.random.SamplingUtils; +import org.datasyslab.jts.io.WKTWriter; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -54,7 +55,6 @@ import org.locationtech.jts.index.SpatialIndex; import org.locationtech.jts.index.strtree.STRtree; import org.locationtech.jts.io.WKBWriter; -import org.locationtech.jts.io.WKTWriter; import org.wololo.geojson.Feature; import scala.Tuple2; diff --git a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala index 46c822a917e..ee7b18674cf 100644 --- a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala +++ b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala @@ -21,7 +21,7 @@ package org.apache.sedona.python.wrapper.translation import org.apache.sedona.common.geometryObjects.Circle import org.apache.sedona.python.wrapper.SerializationException import org.locationtech.jts.geom.Geometry -import org.locationtech.jts.io.WKBReader +import org.datasyslab.jts.io.WKBReader import java.nio.ByteBuffer diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala index 07f6dd03634..3c50113121d 100644 --- a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala +++ b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala @@ -21,7 +21,7 @@ package org.apache.sedona.sql.datasources.geopackage.transform import org.apache.sedona.sql.datasources.geopackage.errors.GeopackageException import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT import org.locationtech.jts.geom.{GeometryFactory, PrecisionModel} -import org.locationtech.jts.io.WKBReader +import org.datasyslab.jts.io.WKBReader import java.nio.{ByteBuffer, ByteOrder} diff --git a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala index 67e9669dd44..38000513ef6 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.datasources.geoparquet.internal._ import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.locationtech.jts.io.WKBReader +import org.datasyslab.jts.io.WKBReader import java.math.{BigDecimal, BigInteger} import java.time.{ZoneId, ZoneOffset} diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/geojson/GeoJSONJacksonGenerator.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/geojson/GeoJSONJacksonGenerator.scala index fe792ac19d6..63f70d39d98 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/geojson/GeoJSONJacksonGenerator.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/geojson/GeoJSONJacksonGenerator.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT import org.apache.spark.sql.sedona_sql.io.geojson.GeoJSONUtils.geometryToGeoJson import org.apache.spark.sql.types._ -import org.locationtech.jts.io.WKTWriter +import org.datasyslab.jts.io.WKTWriter import java.io.Writer diff --git a/spark/pom.xml b/spark/pom.xml index 963d97f12da..bc21727640c 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -88,12 +88,6 @@ spark-sql_${scala.compat.version} ${spark.version} provided - - - org.locationtech.jts - jts-core - - io.graphframes diff --git a/spark/spark-3.5/pom.xml b/spark/spark-3.5/pom.xml index 9657d2c161f..f2bd9a863bd 100644 --- a/spark/spark-3.5/pom.xml +++ b/spark/spark-3.5/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.datasyslab + org.locationtech.jts jts-core diff --git a/spark/spark-4.0/pom.xml b/spark/spark-4.0/pom.xml index da9f1c54560..09ea7df39c3 100644 --- a/spark/spark-4.0/pom.xml +++ b/spark/spark-4.0/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.datasyslab + org.locationtech.jts jts-core diff --git a/spark/spark-4.1/pom.xml b/spark/spark-4.1/pom.xml index fd1693c28b4..feb9091c8cb 100644 --- a/spark/spark-4.1/pom.xml +++ b/spark/spark-4.1/pom.xml @@ -103,7 +103,7 @@ gt-arcgrid - org.datasyslab + org.locationtech.jts jts-core From b074b701b0a9732518969b2bb1f8a185cdca2530 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 23:19:19 -0700 Subject: [PATCH 08/25] Route shared geometry reader through IO patch --- common/src/main/java/org/apache/sedona/common/Constructors.java | 1 - .../apache/sedona/common/geometrySerde/GeometryWkbReader.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Constructors.java b/common/src/main/java/org/apache/sedona/common/Constructors.java index fe70cd97fd4..6f3ad3b49f9 100644 --- a/common/src/main/java/org/apache/sedona/common/Constructors.java +++ b/common/src/main/java/org/apache/sedona/common/Constructors.java @@ -29,7 +29,6 @@ import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.utils.FormatUtils; import org.apache.sedona.common.utils.GeoHashDecoder; -import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java index 723adaba282..56e8b711ebd 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java @@ -18,11 +18,11 @@ */ package org.apache.sedona.common.geometrySerde; +import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.PrecisionModel; import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKBReader; /** * Reads WKB while retaining explicitly declared coordinate dimensions, including empty geometry. From 6c622ffd96997dc3d20f759fed0852c5cba7b872 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 14 Sep 2026 23:26:35 -0700 Subject: [PATCH 09/25] Cover empty WKB coordinate layouts --- .../sedona/common/ConstructorsTest.java | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java b/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java index 08e2472f9cf..319ecc1e948 100644 --- a/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java +++ b/common/src/test/java/org/apache/sedona/common/ConstructorsTest.java @@ -20,10 +20,12 @@ import static org.junit.Assert.*; +import java.util.EnumSet; import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.utils.GeomUtils; import org.junit.Test; import org.locationtech.jts.geom.*; +import org.locationtech.jts.io.Ordinate; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBWriter; @@ -109,15 +111,38 @@ public void geomFromWKB() throws ParseException { @Test public void geomFromWKBPreservesDeclaredDimensionForEmptyGeometry() throws ParseException { GeometryFactory factory = new GeometryFactory(); - - Geometry result = - Constructors.geomFromWKB(new WKBWriter(3).write(factory.createPoint((Coordinate) null))); - assertTrue(result instanceof Point); - assertEquals(3, ((Point) result).getCoordinateSequence().getDimension()); - - result = Constructors.geomFromWKB(new WKBWriter(3).write(factory.createPolygon())); - assertTrue(result instanceof Polygon); - assertEquals(3, ((Polygon) result).getExteriorRing().getCoordinateSequence().getDimension()); + int[][] layouts = {{2, 0}, {3, 0}, {3, 1}, {4, 1}}; + + for (int[] layout : layouts) { + int dimension = layout[0]; + int measures = layout[1]; + EnumSet ordinates = EnumSet.of(Ordinate.X, Ordinate.Y); + if (dimension - measures > 2) { + ordinates.add(Ordinate.Z); + } + if (measures > 0) { + ordinates.add(Ordinate.M); + } + WKBWriter writer = new WKBWriter(dimension); + writer.setOutputOrdinates(ordinates); + + CoordinateSequence pointSequence = + factory.getCoordinateSequenceFactory().create(0, dimension, measures); + Geometry result = Constructors.geomFromWKB(writer.write(factory.createPoint(pointSequence))); + assertTrue(result instanceof Point); + assertEquals(dimension, ((Point) result).getCoordinateSequence().getDimension()); + assertEquals(measures, ((Point) result).getCoordinateSequence().getMeasures()); + + CoordinateSequence ringSequence = + factory.getCoordinateSequenceFactory().create(0, dimension, measures); + Polygon polygon = factory.createPolygon(factory.createLinearRing(ringSequence)); + result = Constructors.geomFromWKB(writer.write(polygon)); + assertTrue(result instanceof Polygon); + CoordinateSequence resultSequence = + ((Polygon) result).getExteriorRing().getCoordinateSequence(); + assertEquals(dimension, resultSequence.getDimension()); + assertEquals(measures, resultSequence.getMeasures()); + } } @Test From c65badc2471eaf06091aee0066625eb047bd5de9 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:07:37 -0700 Subject: [PATCH 10/25] Confine declared coordinate allocations to WKB parsing --- .../apache/sedona/common/Constructors.java | 1 - .../org/apache/sedona/common/Functions.java | 12 +- .../DeclaredCoordinateSequenceFactory.java | 6 +- .../DeclaredGeometryFactory.java | 1 - .../geometrySerde/GeometrySerializer.java | 5 +- .../geometrySerde/GeometryWkbReader.java | 89 +++++++++++- .../GeometryDimensionSerdeTest.java | 129 ++++++++++++++++-- 7 files changed, 214 insertions(+), 29 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Constructors.java b/common/src/main/java/org/apache/sedona/common/Constructors.java index fe70cd97fd4..6f3ad3b49f9 100644 --- a/common/src/main/java/org/apache/sedona/common/Constructors.java +++ b/common/src/main/java/org/apache/sedona/common/Constructors.java @@ -29,7 +29,6 @@ import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.utils.FormatUtils; import org.apache.sedona.common.utils.GeoHashDecoder; -import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 76e9b60dd91..d84cfd142a5 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -1099,13 +1099,15 @@ public static Geometry setSRID(Geometry geometry, int srid) { if (geometry == null) { return null; } - GeometryFactory factory = - new GeometryFactory( - geometry.getPrecisionModel(), - srid, - geometry.getFactory().getCoordinateSequenceFactory()); + GeometryFactory factory; if (geometry.getFactory() instanceof DeclaredGeometryFactory) { factory = new DeclaredGeometryFactory(geometry.getPrecisionModel(), srid); + } else { + factory = + new GeometryFactory( + geometry.getPrecisionModel(), + srid, + geometry.getFactory().getCoordinateSequenceFactory()); } Geometry newGeom = factory.createGeometry(geometry); // Workaround for JTS bug: GeometryEditor.editPolygon returns the original diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java index 4103cf37bcf..5d35295c13c 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java @@ -24,7 +24,7 @@ import org.locationtech.jts.geom.CoordinateSequenceFactory; import org.locationtech.jts.geom.impl.CoordinateArraySequenceFactory; -/** Retains declared layouts on explicit allocation and preserves provenance on copies. */ +/** Preserves binary layout declarations on copies without marking ordinary JTS allocations. */ final class DeclaredCoordinateSequenceFactory implements CoordinateSequenceFactory, Serializable { private static final long serialVersionUID = 1L; static final DeclaredCoordinateSequenceFactory INSTANCE = new DeclaredCoordinateSequenceFactory(); @@ -47,11 +47,11 @@ public CoordinateSequence create(CoordinateSequence sequence) { @Override public CoordinateSequence create(int size, int dimension) { - return create(size, dimension, 0); + return CoordinateArraySequenceFactory.instance().create(size, dimension); } @Override public CoordinateSequence create(int size, int dimension, int measures) { - return new DeclaredCoordinateSequence(size, dimension, measures); + return CoordinateArraySequenceFactory.instance().create(size, dimension, measures); } } diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java index 71725cd130a..c954410e211 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredGeometryFactory.java @@ -78,7 +78,6 @@ public Geometry createGeometry(Geometry geometry) { } else { return super.createGeometry(geometry); } - copy.setUserData(geometry.getUserData()); return copy; } } diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java index 78256e3885f..bbd3d3169d0 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java @@ -477,9 +477,8 @@ private static void collectCoordinateDimensions( // Measures are explicit CoordinateSequence metadata. A Z dimension is not always explicit: // JTS's default sequence factory represents ordinary XY coordinates as dimension 3 with NaN Z. // Trusted binary layouts and XYZM are unambiguous. Unmarked XYZ is recoverable only when at - // least one Z value is non-NaN. An - // ambiguous sequence does not constrain a multipart geometry whose other members establish the - // shared layout. + // least one Z value is non-NaN. An ambiguous sequence does not constrain a multipart geometry + // whose other members establish the shared layout. CoordinateType coordinateType = null; if (measures > 0) { coordinateType = spatialDimensions > 2 ? CoordinateType.XYZM : CoordinateType.XYM; diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java index 56e8b711ebd..20f5b8d33ac 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java @@ -19,9 +19,7 @@ package org.apache.sedona.common.geometrySerde; import org.datasyslab.jts.io.WKBReader; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.PrecisionModel; +import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; /** @@ -31,7 +29,88 @@ public final class GeometryWkbReader { private GeometryWkbReader() {} public static Geometry read(byte[] bytes) throws ParseException { - GeometryFactory factory = new DeclaredGeometryFactory(new PrecisionModel(), 0); - return new WKBReader(factory).read(bytes); + return read(bytes, 0); + } + + public static Geometry read(byte[] bytes, int defaultSrid) throws ParseException { + return new WKBReader(new ReaderGeometryFactory(defaultSrid)).read(bytes); + } + + /** + * WKBReader allocates sequences from this factory using dimensions from binary headers. Geometry + * construction delegates to a regular factory so that subsequent JTS operations do not mistake + * sized allocations for binary declarations. This also avoids copying the parsed coordinates or + * changing SRIDs assigned by WKBReader to individual collection members. + */ + private static final class ReaderGeometryFactory extends GeometryFactory { + private static final long serialVersionUID = 1L; + private final GeometryFactory resultFactory; + + private ReaderGeometryFactory(int srid) { + super(new PrecisionModel(), srid, new ReaderCoordinateSequenceFactory()); + resultFactory = new DeclaredGeometryFactory(getPrecisionModel(), srid); + } + + @Override + public Point createPoint(CoordinateSequence coordinates) { + return resultFactory.createPoint(coordinates); + } + + @Override + public LineString createLineString(CoordinateSequence coordinates) { + return resultFactory.createLineString(coordinates); + } + + @Override + public LinearRing createLinearRing(CoordinateSequence coordinates) { + return resultFactory.createLinearRing(coordinates); + } + + @Override + public Polygon createPolygon(LinearRing shell, LinearRing[] holes) { + return resultFactory.createPolygon(shell, holes); + } + + @Override + public MultiPoint createMultiPoint(Point[] points) { + return resultFactory.createMultiPoint(points); + } + + @Override + public MultiLineString createMultiLineString(LineString[] lines) { + return resultFactory.createMultiLineString(lines); + } + + @Override + public MultiPolygon createMultiPolygon(Polygon[] polygons) { + return resultFactory.createMultiPolygon(polygons); + } + + @Override + public GeometryCollection createGeometryCollection(Geometry[] geometries) { + return resultFactory.createGeometryCollection(geometries); + } + } + + private static final class ReaderCoordinateSequenceFactory implements CoordinateSequenceFactory { + @Override + public CoordinateSequence create(Coordinate[] coordinates) { + return DeclaredCoordinateSequenceFactory.INSTANCE.create(coordinates); + } + + @Override + public CoordinateSequence create(CoordinateSequence sequence) { + return DeclaredCoordinateSequenceFactory.INSTANCE.create(sequence); + } + + @Override + public CoordinateSequence create(int size, int dimension) { + return create(size, dimension, 0); + } + + @Override + public CoordinateSequence create(int size, int dimension, int measures) { + return new DeclaredCoordinateSequence(size, dimension, measures); + } } } diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java index 4d90946f129..ceecbe47901 100644 --- a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java +++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java @@ -19,6 +19,7 @@ package org.apache.sedona.common.geometrySerde; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -49,6 +50,29 @@ public class GeometryDimensionSerdeTest { private static final GeometryFactory FACTORY = new GeometryFactory(); + private static final WkbLayout[] WKB_LAYOUTS = { + new WkbLayout(CoordinateType.XY, 2, 0, 0, 0), + new WkbLayout(CoordinateType.XYZ, 3, 0, 1000, 0x80000000), + new WkbLayout(CoordinateType.XYM, 3, 1, 2000, 0x40000000), + new WkbLayout(CoordinateType.XYZM, 4, 1, 3000, 0xc0000000) + }; + + private static final class WkbLayout { + final CoordinateType coordinateType; + final int dimension; + final int measures; + final int isoOffset; + final int ewkbFlags; + + WkbLayout( + CoordinateType coordinateType, int dimension, int measures, int isoOffset, int ewkbFlags) { + this.coordinateType = coordinateType; + this.dimension = dimension; + this.measures = measures; + this.isoOffset = isoOffset; + this.ewkbFlags = ewkbFlags; + } + } @Test public void nestedCollectionsAndEmptyMultipartMembersSurviveFactoryCopies() @@ -85,7 +109,7 @@ public void nestedCollectionsAndEmptyMultipartMembersSurviveFactoryCopies() assertEquals(0, nested.getSRID()); assertEquals(4326, changed.getSRID()); assertEquals(4326, changed.getFactory().getSRID()); - assertEquals("metadata", changed.getUserData()); + assertNull(changed.getUserData()); Geometry output = roundTrip(changed); assertEquals(2, output.getNumGeometries()); Geometry children = output.getGeometryN(0); @@ -113,10 +137,10 @@ public void hexWkbReaderPreservesEmptyZAndInputData() throws ParseException { @Test public void isoAndEwkbLayoutsSurviveCopiesAndWireBuffers() throws ParseException { for (boolean iso : new boolean[] {false, true}) { - for (int layout = 0; layout < 4; layout++) { - int dimension = 2 + (layout == 3 ? 2 : layout == 0 ? 0 : 1); - int measures = layout >= 2 ? 1 : 0; - CoordinateType expected = CoordinateType.values()[layout]; + for (WkbLayout layout : WKB_LAYOUTS) { + int dimension = layout.dimension; + int measures = layout.measures; + CoordinateType expected = layout.coordinateType; for (int primitive = 1; primitive <= 3; primitive++) { for (boolean empty : new boolean[] {false, true}) { // Populated lines/polygons are covered elsewhere; this also exercises all-NaN Z/M. @@ -124,12 +148,7 @@ public void isoAndEwkbLayoutsSurviveCopiesAndWireBuffers() throws ParseException ByteBuffer wkb = ByteBuffer.allocate(64).order(ByteOrder.LITTLE_ENDIAN); wkb.put((byte) 1); int type = - iso - ? primitive + 1000 * layout - : primitive - | (layout == 1 || layout == 3 ? 0x80000000 : 0) - | (measures == 1 ? 0x40000000 : 0) - | 0x20000000; + iso ? primitive + layout.isoOffset : primitive | layout.ewkbFlags | 0x20000000; wkb.putInt(type); if (!iso) wkb.putInt(4326); if (primitive == 1) { @@ -189,6 +208,94 @@ public void declaredFactoryDoesNotPromoteOrdinaryCoordinates() throws ParseExcep assertThrows(ParseException.class, () -> Constructors.geomFromWKB(new byte[] {1, 1})); } + @Test + public void derivedCoordinatesFromBinaryGeometryFactoriesRemainXy() throws ParseException { + Geometry polygon = new WKTReader().read("POLYGON ((0 0, 10 0, 10 10, 0 10, 0 0))"); + for (Geometry input : + new Geometry[] { + roundTrip(polygon), Constructors.geomFromWKB(new WKBWriter().write(polygon)) + }) { + GeometryFactory factory = input.getFactory(); + Geometry points = factory.createMultiPointFromCoords(new Coordinate[] {new Coordinate(3, 4)}); + assertEquals(CoordinateType.XY, coordinateType(GeometrySerializer.serialize(points))); + assertEquals(3, points.getCoordinate().x, 0); + assertEquals(4, points.getCoordinate().y, 0); + CoordinateSequence coordinates = factory.getCoordinateSequenceFactory().create(2, 3); + coordinates.setOrdinate(0, 0, 1); + coordinates.setOrdinate(0, 1, 2); + coordinates.setOrdinate(1, 0, 3); + coordinates.setOrdinate(1, 1, 4); + assertEquals( + CoordinateType.XY, + coordinateType(GeometrySerializer.serialize(factory.createLineString(coordinates)))); + Geometry generated = Functions.generatePoints(input, 3, 100); + assertEquals(3, generated.getNumGeometries()); + assertTrue(input.covers(generated)); + assertEquals(CoordinateType.XY, coordinateType(GeometrySerializer.serialize(generated))); + } + } + + @Test + public void wkbResultsUseOrdinaryAllocationForEveryGeometryType() throws ParseException { + for (String wkt : + new String[] { + "POINT EMPTY", + "LINESTRING EMPTY", + "POLYGON EMPTY", + "POLYGON ((0 0, 4 0, 0 4, 0 0), (1 1, 2 1, 1 2, 1 1))", + "MULTIPOINT (EMPTY, (1 2))", + "MULTILINESTRING (EMPTY, (0 0, 1 1))", + "MULTIPOLYGON (EMPTY, ((0 0, 4 0, 0 4, 0 0)))", + "GEOMETRYCOLLECTION (POINT EMPTY, GEOMETRYCOLLECTION (POLYGON EMPTY))" + }) { + Geometry geometry = + Constructors.geomFromWKB(new WKBWriter().write(new WKTReader().read(wkt))); + geometry.apply( + (org.locationtech.jts.geom.GeometryComponentFilter) + component -> { + Geometry points = + component + .getFactory() + .createMultiPointFromCoords(new Coordinate[] {new Coordinate(3, 4)}); + assertEquals( + wkt, CoordinateType.XY, coordinateType(GeometrySerializer.serialize(points))); + }); + } + } + + @Test + public void wkbReaderPreservesMemberSridsAndDefaultSrid() throws ParseException { + ByteBuffer bytes = ByteBuffer.allocate(9 + 2 * 25).order(ByteOrder.LITTLE_ENDIAN); + bytes.put((byte) 1).putInt(7).putInt(2); + bytes.put((byte) 1).putInt(0x20000001).putInt(4326).putDouble(1).putDouble(2); + bytes.put((byte) 1).putInt(0x20000001).putInt(3857).putDouble(3).putDouble(4); + Geometry collection = GeometryWkbReader.read(bytes.array(), 27700); + assertEquals(27700, collection.getSRID()); + assertEquals(4326, collection.getGeometryN(0).getSRID()); + assertEquals(3857, collection.getGeometryN(1).getSRID()); + assertEquals(1, collection.getGeometryN(0).getCoordinate().x, 0); + assertEquals(4, collection.getGeometryN(1).getCoordinate().y, 0); + } + + @Test + public void factoryCopiesAndSetSridUseTheSameUserDataPolicy() throws ParseException { + Geometry point = new WKTReader().read("POINT (1 2)"); + for (Geometry input : new Geometry[] {point, roundTrip(point)}) { + input.setUserData("child metadata"); + Geometry collection = input.getFactory().createGeometryCollection(new Geometry[] {input}); + collection.setUserData("parent metadata"); + for (Geometry copy : + new Geometry[] { + collection.getFactory().createGeometry(collection), Functions.setSRID(collection, 4326) + }) { + assertNull(copy.getUserData()); + assertNull(copy.getGeometryN(0).getUserData()); + } + assertEquals("parent metadata", collection.getUserData()); + assertEquals("child metadata", input.getUserData()); + } + } + @Test public void declaredWkbZSurvivesEmptyAndNaNCoordinates() throws ParseException { for (String wkt : From 874e6082183adfc4452501073488089759a7bb8d Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:11:23 -0700 Subject: [PATCH 11/25] Document and test collection dimension requirements --- .../GeometryDimensionSerdeTest.java | 39 +++++++++++++ .../sql/Aggregate-Functions/ST_Collect_Agg.md | 6 ++ docs/api/sql/Geometry-Editors/ST_Collect.md | 6 ++ .../sql/functions/collect/TestStCollect.scala | 55 +++++++++++++++++++ 4 files changed, 106 insertions(+) diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java index ceecbe47901..17e72e24561 100644 --- a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java +++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java @@ -449,6 +449,45 @@ public void ordinaryJtsXyRemainsXy() throws ParseException { assertSequenceLayout(output.getCoordinateSequence(), 2, 0); } + @Test + public void mixedDeclaredLayoutsRequireGeometryCollections() throws ParseException { + for (String[] wkts : + new String[][] { + {"POINT (1 2)", "POINT Z EMPTY"}, + {"LINESTRING (0 0, 1 1)", "LINESTRING Z EMPTY"}, + {"POLYGON ((0 0, 4 0, 0 4, 0 0))", "POLYGON Z EMPTY"} + }) { + Geometry xy = roundTrip(new WKTReader().read(wkts[0])); + Geometry z = + roundTrip( + Constructors.geomFromWKB(new WKBWriter(3).write(new WKTReader().read(wkts[1])))); + assertMixedLayoutsRequireCollection(xy, z); + } + ByteBuffer point = ByteBuffer.allocate(29).order(ByteOrder.LITTLE_ENDIAN); + point.put((byte) 1).putInt(1001).putDouble(1).putDouble(2).putDouble(Double.NaN); + assertMixedLayoutsRequireCollection( + roundTrip(new WKTReader().read("POINT (3 4)")), + roundTrip(Constructors.geomFromWKB(point.array()))); + } + + private static void assertMixedLayoutsRequireCollection(Geometry xy, Geometry z) { + for (Geometry[] members : new Geometry[][] {{xy, z}, {z, xy}}) { + Geometry multipart = Functions.createMultiGeometry(members); + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, () -> GeometrySerializer.serialize(multipart)); + assertTrue(error.getMessage().contains("heterogeneous dimensional layouts")); + Geometry collection = roundTrip(FACTORY.createGeometryCollection(members)); + assertEquals(2, collection.getNumGeometries()); + for (int i = 0; i < members.length; i++) { + assertEquals(members[i].isEmpty(), collection.getGeometryN(i).isEmpty()); + assertEquals( + members[i] == xy ? CoordinateType.XY : CoordinateType.XYZ, + coordinateType(GeometrySerializer.serialize(collection.getGeometryN(i)))); + } + } + } + @Test public void rejectsRecoverablyHeterogeneousMultipartLayouts() { LineString xy = diff --git a/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md b/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md index 76415b03d6a..f761408dea8 100644 --- a/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md +++ b/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md @@ -41,6 +41,12 @@ All non-null `Geography` values in a group must have the same SRID; a group cont SRIDs is rejected. In contrast, scalar [`ST_Collect`](../Geometry-Editors/ST_Collect.md) uses the first non-null Geography input's SRID for the output. +For `Geometry` inputs, a multipart result must have a consistent coordinate layout (XY, XYZ, +XYM, or XYZM). Starting in 2.0.0, dimensions declared in WKB are retained even for empty geometries +or NaN Z values. Combining an XY point with a WKB `POINT Z EMPTY`, for example, raises a +heterogeneous-layout error during serialization. To preserve each member's layout in a +`GeometryCollection`, apply `ST_ForceCollection` to each input before collecting it. + SQL Example ```sql diff --git a/docs/api/sql/Geometry-Editors/ST_Collect.md b/docs/api/sql/Geometry-Editors/ST_Collect.md index 31f4b76271a..38cc3856b67 100644 --- a/docs/api/sql/Geometry-Editors/ST_Collect.md +++ b/docs/api/sql/Geometry-Editors/ST_Collect.md @@ -44,6 +44,12 @@ the first non-null input supplies the output SRID; later inputs are not required SRID. In contrast, [`ST_Collect_Agg`](../Aggregate-Functions/ST_Collect_Agg.md) rejects a group containing mixed Geography SRIDs. +For `Geometry` inputs, a multipart result must have a consistent coordinate layout (XY, XYZ, +XYM, or XYZM). Starting in 2.0.0, dimensions declared in WKB are retained even for empty geometries +or NaN Z values. Combining an XY point with a WKB `POINT Z EMPTY`, for example, raises a +heterogeneous-layout error during serialization. To preserve each member's layout in a +`GeometryCollection`, apply `ST_ForceCollection` to each input before collecting it. + SQL Example ```sql diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/functions/collect/TestStCollect.scala b/spark/common/src/test/scala/org/apache/sedona/sql/functions/collect/TestStCollect.scala index 4d7c8a1c9cf..3d63f58912d 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/functions/collect/TestStCollect.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/functions/collect/TestStCollect.scala @@ -28,6 +28,61 @@ class TestStCollect extends TestBaseScala with GeometrySample with GivenWhenThen import sparkSession.implicits._ describe("st collect workflow") { + it("should require a collection for mixed XY and declared empty or NaN Z members") { + val cases = Seq( + ("POINT (1 2)", "POINT Z EMPTY"), + ("LINESTRING (0 0, 1 1)", "LINESTRING Z EMPTY"), + ("POLYGON ((0 0, 4 0, 0 4, 0 0))", "POLYGON Z EMPTY"), + ("POINT (1 2)", "POINT Z (3 4 NaN)")) + cases.foreach { case (xyWkt, zWkt) => + val zBytes = new org.locationtech.jts.io.WKBWriter(3) + .write(wktReader.read(zWkt.replace("NaN", "9"))) + if (zWkt.contains("NaN")) java.nio.ByteBuffer.wrap(zBytes).putDouble(21, Double.NaN) + val zHex = org.locationtech.jts.io.WKBWriter.toHex(zBytes) + val inputs = Seq((xyWkt, zHex)) + .toDF("xyWkt", "zHex") + .selectExpr("ST_GeomFromWKT(xyWkt) AS xy", "ST_GeomFromWKB(unhex(zHex)) AS z") + .repartition(1) + .cache() + try { + assert(inputs.count() == 1) + val rows = inputs.selectExpr("xy AS geom").union(inputs.selectExpr("z AS geom")) + val queries = + Seq(inputs.selectExpr("ST_Collect(xy, z)"), rows.selectExpr("ST_Collect_Agg(geom)")) + queries.foreach { query => + val error = intercept[Exception] { query.collect() } + val causes = Iterator.iterate(error: Throwable)(_.getCause).takeWhile(_ != null).toSeq + assert( + causes.exists(cause => + Option(cause.getMessage) + .exists(_.contains("heterogeneous dimensional layouts"))), + error.toString) + } + val collections = Seq( + inputs.selectExpr("ST_Collect(ST_ForceCollection(xy), ST_ForceCollection(z))"), + rows.selectExpr("ST_Collect_Agg(ST_ForceCollection(geom))")) + collections.foreach { query => + val geometry = query.collect()(0).getAs[org.locationtech.jts.geom.Geometry](0) + assert(geometry.getGeometryType == "GeometryCollection") + val dimensions = (0 until geometry.getNumGeometries).map { i => + val member = geometry.getGeometryN(i).getGeometryN(0) + member match { + case point: org.locationtech.jts.geom.Point => + point.getCoordinateSequence.getDimension + case line: org.locationtech.jts.geom.LineString => + line.getCoordinateSequence.getDimension + case polygon: org.locationtech.jts.geom.Polygon => + polygon.getExteriorRing.getCoordinateSequence.getDimension + } + } + assert(dimensions.sorted == Seq(2, 3)) + } + } finally { + inputs.unpersist() + } + } + } + it("should return null when passed geometry is also null") { Given("data frame with empty geometries") val emptyGeometryDataFrame = Seq((1, null), (2, null), (3, null)).toDF("id", "geom") From 2bb61991d68870440bcc7af5235b80a693de559c Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:12:59 -0700 Subject: [PATCH 12/25] Test XY point generation after geometry decoding --- .../apache/sedona/sql/functionTestScala.scala | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 1bf7154b4a9..9f2bca6094c 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.{DataFrame, Row} import org.geotools.referencing.CRS import org.junit.Assert.{assertEquals, assertFalse, assertTrue} import org.locationtech.jts.algorithm.MinimumBoundingCircle -import org.locationtech.jts.geom.{Coordinate, Geometry, GeometryFactory, Polygon} +import org.locationtech.jts.geom.{Coordinate, Geometry, GeometryFactory, Point, Polygon} import org.locationtech.jts.io.WKTWriter import org.locationtech.jts.linearref.LengthIndexedLine import org.locationtech.jts.operation.distance3d.Distance3DOp @@ -3457,6 +3457,45 @@ class functionTestScala assertEquals(expected, actual) } + it("Should keep ST_GeneratePoints output XY for WKB and materialized polygon inputs") { + val polygonWkb = + "010300000001000000050000000000000000000000000000000000000000000000000024400000000000000000000000000000244000000000000024400000000000000000000000000000244000000000000000000000000000000000" + + def generatedPointDimensions(points: Geometry): Seq[Int] = { + assertEquals(8, points.getNumGeometries) + (0 until points.getNumGeometries).map { index => + val point = points.getGeometryN(index).asInstanceOf[Point] + assertTrue(point.getX >= 0 && point.getX <= 10) + assertTrue(point.getY >= 0 && point.getY <= 10) + point.getCoordinateSequence.getDimension + } + } + + val direct = sparkSession + .sql(s"SELECT ST_GeneratePoints(ST_GeomFromWKB(unhex('$polygonWkb')), 8, 42) AS points") + .first() + .getAs[Geometry]("points") + + val polygons = sparkSession + .sql(s"SELECT ST_GeomFromWKB(unhex('$polygonWkb')) AS polygon") + .repartition(2) + .cache() + try { + polygons.collect() + polygons.createOrReplaceTempView("materialized_xy_polygon") + val materialized = sparkSession + .sql("SELECT ST_GeneratePoints(polygon, 8, 42) AS points FROM materialized_xy_polygon") + .first() + .getAs[Geometry]("points") + assertEquals( + (Seq.fill(8)(2), Seq.fill(8)(2)), + (generatedPointDimensions(direct), generatedPointDimensions(materialized))) + assertTrue(direct.equalsExact(materialized)) + } finally { + polygons.unpersist() + } + } + it("should pass ST_NRings") { val geomTestCases = Map( ("'POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'") -> 1, From 9c3ca3288423e758bceb2b33660040224927041e Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:17:28 -0700 Subject: [PATCH 13/25] Clarify declared layout requirements in collection docs --- docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md | 4 ++-- docs/api/sql/Geometry-Editors/ST_Collect.md | 4 ++-- .../test/scala/org/apache/sedona/sql/functionTestScala.scala | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md b/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md index f761408dea8..e6bc078c887 100644 --- a/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md +++ b/docs/api/sql/Aggregate-Functions/ST_Collect_Agg.md @@ -43,8 +43,8 @@ first non-null Geography input's SRID for the output. For `Geometry` inputs, a multipart result must have a consistent coordinate layout (XY, XYZ, XYM, or XYZM). Starting in 2.0.0, dimensions declared in WKB are retained even for empty geometries -or NaN Z values. Combining an XY point with a WKB `POINT Z EMPTY`, for example, raises a -heterogeneous-layout error during serialization. To preserve each member's layout in a +or NaN Z values. Combining WKB-declared XY and XYZ components, such as an XY point and +`POINT Z EMPTY`, raises a heterogeneous-layout error during serialization. To preserve each member's layout in a `GeometryCollection`, apply `ST_ForceCollection` to each input before collecting it. SQL Example diff --git a/docs/api/sql/Geometry-Editors/ST_Collect.md b/docs/api/sql/Geometry-Editors/ST_Collect.md index 38cc3856b67..b863b5fb227 100644 --- a/docs/api/sql/Geometry-Editors/ST_Collect.md +++ b/docs/api/sql/Geometry-Editors/ST_Collect.md @@ -46,8 +46,8 @@ containing mixed Geography SRIDs. For `Geometry` inputs, a multipart result must have a consistent coordinate layout (XY, XYZ, XYM, or XYZM). Starting in 2.0.0, dimensions declared in WKB are retained even for empty geometries -or NaN Z values. Combining an XY point with a WKB `POINT Z EMPTY`, for example, raises a -heterogeneous-layout error during serialization. To preserve each member's layout in a +or NaN Z values. Combining WKB-declared XY and XYZ components, such as an XY point and +`POINT Z EMPTY`, raises a heterogeneous-layout error during serialization. To preserve each member's layout in a `GeometryCollection`, apply `ST_ForceCollection` to each input before collecting it. SQL Example diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 9f2bca6094c..bad5eda6225 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -3492,6 +3492,7 @@ class functionTestScala (generatedPointDimensions(direct), generatedPointDimensions(materialized))) assertTrue(direct.equalsExact(materialized)) } finally { + sparkSession.catalog.dropTempView("materialized_xy_polygon") polygons.unpersist() } } From 87a58cb224da69c218cf926c23345d76b8b0b820 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:18:22 -0700 Subject: [PATCH 14/25] Preserve WKB dimensions across Spark readers --- .../common/S2Geography/WKBGeography.java | 4 +- .../common/S2Geography/WKBGeographyTest.java | 32 +++++ .../PythonGeometrySerializer.scala | 7 +- .../geopackage/transform/GeometryReader.scala | 6 +- .../geoparquet/GeoParquetRowConverter.scala | 8 +- .../wrapper/TestToPythonSerialization.scala | 25 +++- .../transform/GeometryReaderTest.scala | 59 +++++++++ .../GeoParquetRowConverterTest.scala | 121 ++++++++++++++++++ 8 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala create mode 100644 spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala diff --git a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java index d30c011057d..a4ed00e2f65 100644 --- a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java +++ b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java @@ -28,6 +28,7 @@ import java.nio.ByteOrder; import java.util.Arrays; import java.util.List; +import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.PrecisionModel; import org.locationtech.jts.io.ParseException; @@ -130,8 +131,7 @@ public Geometry getJTSGeometry() { result = jtsGeometry; if (result == null) { try { - org.datasyslab.jts.io.WKBReader reader = new org.datasyslab.jts.io.WKBReader(); - result = reader.read(wkbBytes); + result = GeometryWkbReader.read(wkbBytes); result.setSRID(getSRID()); } catch (ParseException e) { throw new RuntimeException("Failed to parse WKB to JTS Geometry", e); diff --git a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java index 5073e24f9af..594b0c139db 100644 --- a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java +++ b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java @@ -25,6 +25,7 @@ import com.google.common.geometry.S2Point; import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import org.apache.sedona.common.geography.Constructors; import org.apache.sedona.common.geography.Functions; @@ -35,6 +36,7 @@ import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.PrecisionModel; import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.Ordinate; import org.locationtech.jts.io.ParseException; public class WKBGeographyTest { @@ -66,6 +68,36 @@ public void fromWKB_point_lazyParse() throws ParseException { assertEquals(4326, jts.getSRID()); } + @Test + public void fromWKB_emptyJtsGeometryRetainsDeclaredLayout() { + GeometryFactory factory = new GeometryFactory(); + int[][] layouts = {{3, 0}, {3, 1}, {4, 1}}; + for (int[] layout : layouts) { + int dimension = layout[0]; + int measures = layout[1]; + EnumSet ordinates = EnumSet.of(Ordinate.X, Ordinate.Y); + if (dimension == 4) ordinates.add(Ordinate.Z); + if (measures == 0) ordinates.add(Ordinate.Z); + if (measures == 1) ordinates.add(Ordinate.M); + org.locationtech.jts.io.WKBWriter writer = new org.locationtech.jts.io.WKBWriter(dimension); + writer.setOutputOrdinates(ordinates); + Geometry empty = + factory.createPolygon( + factory.createLinearRing( + factory.getCoordinateSequenceFactory().create(0, dimension, measures))); + + Geometry parsed = WKBGeography.fromWKB(writer.write(empty), 4326).getJTSGeometry(); + Geometry result = + org.apache.sedona.common.geometrySerde.GeometrySerializer.deserialize( + org.apache.sedona.common.geometrySerde.GeometrySerializer.serialize(parsed)); + org.locationtech.jts.geom.CoordinateSequence sequence = + ((org.locationtech.jts.geom.Polygon) result).getExteriorRing().getCoordinateSequence(); + assertEquals(dimension, sequence.getDimension()); + assertEquals(measures, sequence.getMeasures()); + assertEquals(4326, result.getSRID()); + } + } + @Test public void fromJTS_point() { GeometryFactory gf = new GeometryFactory(); diff --git a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala index ee7b18674cf..9ae458c4244 100644 --- a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala +++ b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala @@ -19,9 +19,9 @@ package org.apache.sedona.python.wrapper.translation import org.apache.sedona.common.geometryObjects.Circle +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.sedona.python.wrapper.SerializationException import org.locationtech.jts.geom.Geometry -import org.datasyslab.jts.io.WKBReader import java.nio.ByteBuffer @@ -47,13 +47,12 @@ private[python] class PythonGeometrySerializer extends Serializable { } def deserialize(isCircle: Int, values: Array[Byte], offset: Int): Geometry = { - val reader = new WKBReader() if (isCircle == 1) { - val geom = reader.read(values.slice(offset + 8, values.length)) + val geom = GeometryWkbReader.read(values.slice(offset + 8, values.length)) val radius = ByteBuffer.wrap(values.slice(offset, offset + 8)).getDouble() new Circle(geom, radius) } else if (isCircle == 0) { - reader.read(values.slice(offset, values.length)) + GeometryWkbReader.read(values.slice(offset, values.length)) } else { throw SerializationException("Can not deserialize object") } diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala index 3c50113121d..522137d31b3 100644 --- a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala +++ b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala @@ -18,10 +18,9 @@ */ package org.apache.sedona.sql.datasources.geopackage.transform +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.sedona.sql.datasources.geopackage.errors.GeopackageException import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT -import org.locationtech.jts.geom.{GeometryFactory, PrecisionModel} -import org.datasyslab.jts.io.WKBReader import java.nio.{ByteBuffer, ByteOrder} @@ -56,8 +55,7 @@ object GeometryReader { val wkb = new Array[Byte](reader.remaining()) reader.get(wkb) - val wkbReader = new WKBReader(new GeometryFactory(new PrecisionModel(), srid)) - val geom = wkbReader.read(wkb) + val geom = GeometryWkbReader.read(wkb, srid) // that needs rewriting GeometryUDT.serialize(geom) diff --git a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala index 38000513ef6..b2a15ee92f1 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala @@ -31,9 +31,9 @@ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap import org.apache.spark.sql.execution.datasources.geoparquet.internal._ import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.datasyslab.jts.io.WKBReader import java.math.{BigDecimal, BigInteger} import java.time.{ZoneId, ZoneOffset} @@ -220,8 +220,7 @@ private[geoparquet] class GeoParquetRowConverter( if (parquetType.isPrimitive) { new ParquetPrimitiveConverter(updater) { override def addBinary(value: Binary): Unit = { - val wkbReader = new WKBReader() - val geom = wkbReader.read(value.getBytes) + val geom = GeometryWkbReader.read(value.getBytes) geom.setSRID(srid) this.updater.set(GeometryUDT.serialize(geom)) } @@ -233,9 +232,8 @@ private[geoparquet] class GeoParquetRowConverter( ArrayType(ByteType, containsNull = false), updater) { override def end(): Unit = { - val wkbReader = new WKBReader() val byteArray = currentArray.map(_.asInstanceOf[Byte]).toArray - val geom = wkbReader.read(byteArray) + val geom = GeometryWkbReader.read(byteArray) geom.setSRID(srid) this.updater.set(GeometryUDT.serialize(geom)) } diff --git a/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala b/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala index ac01c4b86f7..ca655b9589a 100644 --- a/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala +++ b/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala @@ -25,11 +25,12 @@ import org.apache.sedona.python.wrapper.utils.implicits._ import org.apache.sedona.sql.TestBaseScala import org.apache.spark.api.java.JavaPairRDD import org.locationtech.jts.geom.{Geometry, GeometryFactory} -import org.locationtech.jts.io.WKTReader +import org.locationtech.jts.io.{Ordinate, WKBWriter, WKTReader} import org.scalatest.matchers.must.Matchers.contain import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import java.io.{FileInputStream, InputStream} +import java.util.EnumSet import scala.io.Source import scala.jdk.CollectionConverters._ @@ -95,6 +96,28 @@ class TestToPythonSerialization extends TestBaseScala { .flatMap(samplePoint => pythonGeometrySerializer.serialize(samplePoint))) describe("Sedona Python Wrapper Test") { + it("retains the declared layout of inbound empty WKB") { + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = geometryFactory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = geometryFactory.createPolygon(geometryFactory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + val parsed = pythonGeometrySerializer.deserialize(0, writer.write(empty), 0) + val result = org.apache.sedona.common.geometrySerde.GeometrySerializer.deserialize( + org.apache.sedona.common.geometrySerde.GeometrySerializer.serialize(parsed)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + resultSequence.getDimension shouldBe dimension + resultSequence.getMeasures shouldBe measures + } + } + it("Test Serialize To Python JavaRDD[Geometry]") { val convertedToPythonRDD = GeometryRddConverter(pointSpatialRDD, pythonGeometrySerializer).translateToPython diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala new file mode 100644 index 00000000000..c9e673b8054 --- /dev/null +++ b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.sql.datasources.geopackage.transform + +import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.locationtech.jts.geom.{GeometryFactory, Polygon} +import org.locationtech.jts.io.{Ordinate, WKBWriter} +import org.scalatest.funsuite.AnyFunSuite + +import java.nio.{ByteBuffer, ByteOrder} +import java.util.EnumSet + +class GeometryReaderTest extends AnyFunSuite { + test("extractWKB retains an empty geometry's declared layout and GeoPackage SRID") { + val factory = new GeometryFactory() + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + val wkb = writer.write(empty) + val bytes = ByteBuffer + .allocate(8 + wkb.length) + .order(ByteOrder.LITTLE_ENDIAN) + .put('G'.toByte) + .put('P'.toByte) + .put(0.toByte) + .put(1.toByte) + .putInt(4326) + .put(wkb) + .array() + + val result = GeometryUDT.deserialize(GeometryReader.extractWKB(bytes)).asInstanceOf[Polygon] + val resultSequence = result.getExteriorRing.getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + assert(result.getSRID == 4326) + } + } +} diff --git a/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala new file mode 100644 index 00000000000..33adc24d82b --- /dev/null +++ b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geoparquet + +import org.apache.parquet.io.api.Binary +import org.apache.parquet.io.api.GroupConverter +import org.apache.parquet.schema.MessageTypeParser +import org.apache.spark.sql.execution.datasources.geoparquet.internal.{LegacyBehaviorPolicy, NoopUpdater, RebaseSpec} +import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.apache.spark.sql.types.{StructField, StructType} +import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.io.{Ordinate, WKBWriter} +import org.scalatest.funsuite.AnyFunSuite + +import java.util.{Collections, EnumSet} + +class GeoParquetRowConverterTest extends AnyFunSuite { + test("primitive WKB retains declared empty coordinate layouts through GeometryUDT") { + val parquetSchema = + MessageTypeParser.parseMessageType("message root { optional binary geometry; }") + val catalystSchema = StructType(Seq(StructField("geometry", GeometryUDT(), nullable = true))) + val metadata = Collections.singletonMap( + "geo", + """{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":[],"crs":null}}}""") + val converter = new GeoParquetRowConverter( + new GeoParquetToSparkSchemaConverter(metadata, parameters = Map.empty), + parquetSchema, + catalystSchema, + None, + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + Map.empty, + NoopUpdater) + val factory = new GeometryFactory() + + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + converter.start() + converter + .getConverter(0) + .asPrimitiveConverter() + .addBinary(Binary.fromConstantByteArray(writer.write(empty))) + converter.end() + val result = GeometryUDT.deserialize(converter.currentRecord.getBinary(0)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + } + } + + test("legacy byte-array WKB retains declared empty coordinate layouts through GeometryUDT") { + val parquetSchema = MessageTypeParser.parseMessageType( + "message root { optional group geometry (LIST) { repeated int32 array (INT_8); } }") + val catalystSchema = StructType(Seq(StructField("geometry", GeometryUDT(), nullable = true))) + val metadata = Collections.singletonMap( + "geo", + """{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":[],"crs":null}}}""") + val parameters = Map("legacyMode" -> "true") + val converter = new GeoParquetRowConverter( + new GeoParquetToSparkSchemaConverter(metadata, parameters = parameters), + parquetSchema, + catalystSchema, + None, + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + parameters, + NoopUpdater) + val factory = new GeometryFactory() + + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + converter.start() + val geometryConverter = converter.getConverter(0).asInstanceOf[GroupConverter] + geometryConverter.start() + val byteConverter = geometryConverter.getConverter(0).asPrimitiveConverter() + writer.write(empty).foreach(byte => byteConverter.addInt(byte)) + geometryConverter.end() + converter.end() + val result = GeometryUDT.deserialize(converter.currentRecord.getBinary(0)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + } + } +} From 681929cf1d947ba9c86b084557a051267702fbb7 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 15 Sep 2026 01:20:07 -0700 Subject: [PATCH 15/25] Set the empty geometry flag in the GeoPackage fixture --- .../datasources/geopackage/transform/GeometryReaderTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala index c9e673b8054..70f8a05b95f 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala @@ -44,7 +44,7 @@ class GeometryReaderTest extends AnyFunSuite { .put('G'.toByte) .put('P'.toByte) .put(0.toByte) - .put(1.toByte) + .put(0x11.toByte) .putInt(4326) .put(wkb) .array() From 364ead9aec08e629d7d426e3dd566991d715768a Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 00:42:21 -0700 Subject: [PATCH 16/25] Preserve empty components when setting SRID --- .../org/apache/sedona/common/Functions.java | 10 +- .../StructurePreservingGeometryFactory.java | 85 ++++++++++++++ .../apache/sedona/common/FunctionsTest.java | 109 ++++++++++++++++++ 3 files changed, 197 insertions(+), 7 deletions(-) create mode 100644 common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index e3057e5ae12..4bbc6564245 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -34,6 +34,7 @@ import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.geometryObjects.Box3D; import org.apache.sedona.common.geometryObjects.Circle; +import org.apache.sedona.common.geometryObjects.StructurePreservingGeometryFactory; import org.apache.sedona.common.jts2geojson.GeoJSONWriter; import org.apache.sedona.common.sphere.Spheroid; import org.apache.sedona.common.subDivide.GeometrySubDivider; @@ -1099,16 +1100,11 @@ public static Geometry setSRID(Geometry geometry, int srid) { return null; } GeometryFactory factory = - new GeometryFactory( + new StructurePreservingGeometryFactory( geometry.getPrecisionModel(), srid, geometry.getFactory().getCoordinateSequenceFactory()); - Geometry newGeom = factory.createGeometry(geometry); - // Workaround for JTS bug: GeometryEditor.editPolygon returns the original - // empty polygon without copying it to the new factory, so the SRID is not - // updated for POLYGON EMPTY (and similar empty geometry types). - newGeom.setSRID(srid); - return newGeom; + return factory.createGeometry(geometry); } public static int getSRID(Geometry geometry) { diff --git a/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java b/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java new file mode 100644 index 00000000000..c8784b8fa1d --- /dev/null +++ b/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.geometryObjects; + +import org.locationtech.jts.geom.*; + +/** Geometry factory that retains empty components and coordinate layouts when copying. */ +public final class StructurePreservingGeometryFactory extends GeometryFactory { + private static final long serialVersionUID = 1L; + + public StructurePreservingGeometryFactory( + PrecisionModel precisionModel, int srid, CoordinateSequenceFactory sequenceFactory) { + super(precisionModel, srid, sequenceFactory); + } + + @Override + public Geometry createGeometry(Geometry geometry) { + if (geometry instanceof Point) { + return createPoint(copy(((Point) geometry).getCoordinateSequence())); + } + if (geometry instanceof LinearRing) { + return createLinearRing(copy(((LinearRing) geometry).getCoordinateSequence())); + } + if (geometry instanceof LineString) { + return createLineString(copy(((LineString) geometry).getCoordinateSequence())); + } + if (geometry instanceof Polygon) { + Polygon polygon = (Polygon) geometry; + LinearRing[] holes = new LinearRing[polygon.getNumInteriorRing()]; + for (int i = 0; i < holes.length; i++) { + holes[i] = (LinearRing) createGeometry(polygon.getInteriorRingN(i)); + } + return createPolygon((LinearRing) createGeometry(polygon.getExteriorRing()), holes); + } + if (geometry instanceof MultiPoint) { + Point[] points = new Point[geometry.getNumGeometries()]; + for (int i = 0; i < points.length; i++) { + points[i] = (Point) createGeometry(geometry.getGeometryN(i)); + } + return createMultiPoint(points); + } + if (geometry instanceof MultiLineString) { + LineString[] lines = new LineString[geometry.getNumGeometries()]; + for (int i = 0; i < lines.length; i++) { + lines[i] = (LineString) createGeometry(geometry.getGeometryN(i)); + } + return createMultiLineString(lines); + } + if (geometry instanceof MultiPolygon) { + Polygon[] polygons = new Polygon[geometry.getNumGeometries()]; + for (int i = 0; i < polygons.length; i++) { + polygons[i] = (Polygon) createGeometry(geometry.getGeometryN(i)); + } + return createMultiPolygon(polygons); + } + if (geometry instanceof GeometryCollection) { + Geometry[] children = new Geometry[geometry.getNumGeometries()]; + for (int i = 0; i < children.length; i++) { + children[i] = createGeometry(geometry.getGeometryN(i)); + } + return createGeometryCollection(children); + } + return super.createGeometry(geometry); + } + + private CoordinateSequence copy(CoordinateSequence sequence) { + return getCoordinateSequenceFactory().create(sequence); + } +} diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 0bd0c833090..af98e500d42 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -37,6 +37,8 @@ import org.junit.Test; import org.locationtech.jts.geom.*; import org.locationtech.jts.geom.LinearRing; +import org.locationtech.jts.geom.impl.CoordinateArraySequenceFactory; +import org.locationtech.jts.geom.impl.PackedCoordinateSequenceFactory; import org.locationtech.jts.geom.prep.PreparedGeometry; import org.locationtech.jts.geom.prep.PreparedGeometryFactory; import org.locationtech.jts.io.ParseException; @@ -4676,6 +4678,113 @@ public void setSRIDEmptyGeometries() throws ParseException { } } + @Test + public void setSRIDPreservesNestedEmptyComponentsAndCopiesStructure() { + GeometryFactory sourceFactory = + new GeometryFactory(new PrecisionModel(), 100, CoordinateArraySequenceFactory.instance()); + Point emptyPoint = + sourceFactory.createPoint(sourceFactory.getCoordinateSequenceFactory().create(0, 3, 0)); + Polygon emptyPolygon = + sourceFactory.createPolygon( + sourceFactory.createLinearRing( + sourceFactory.getCoordinateSequenceFactory().create(0, 4, 1))); + Point populatedPoint = sourceFactory.createPoint(new Coordinate(1, 2)); + GeometryCollection nested = + sourceFactory.createGeometryCollection(new Geometry[] {emptyPolygon, populatedPoint}); + GeometryCollection source = + sourceFactory.createGeometryCollection(new Geometry[] {emptyPoint, nested}); + source.setSRID(100); + emptyPoint.setSRID(101); + nested.setSRID(102); + emptyPolygon.setSRID(103); + populatedPoint.setSRID(104); + source.setUserData("root metadata"); + emptyPoint.setUserData("child metadata"); + nested.setUserData("nested metadata"); + + GeometryCollection result = (GeometryCollection) Functions.setSRID(source, 4326); + + assertEquals(2, result.getNumGeometries()); + assertEquals(2, result.getGeometryN(1).getNumGeometries()); + assertGeometryTreeUsesFactory(result, result.getFactory(), 4326); + assertNull(result.getUserData()); + assertNull(result.getGeometryN(0).getUserData()); + assertNull(result.getGeometryN(1).getUserData()); + assertEquals(3, ((Point) result.getGeometryN(0)).getCoordinateSequence().getDimension()); + assertEquals( + 4, + ((Polygon) result.getGeometryN(1).getGeometryN(0)) + .getExteriorRing() + .getCoordinateSequence() + .getDimension()); + assertEquals( + 1, + ((Polygon) result.getGeometryN(1).getGeometryN(0)) + .getExteriorRing() + .getCoordinateSequence() + .getMeasures()); + assertNotSame(source, result); + assertNotSame(source.getGeometryN(0), result.getGeometryN(0)); + Point resultPoint = (Point) result.getGeometryN(1).getGeometryN(1); + resultPoint.getCoordinateSequence().setOrdinate(0, 0, 9); + assertEquals(1, populatedPoint.getX(), 0); + assertEquals(100, source.getSRID()); + assertEquals(101, emptyPoint.getSRID()); + assertEquals(102, nested.getSRID()); + assertEquals(103, emptyPolygon.getSRID()); + assertEquals(104, populatedPoint.getSRID()); + assertEquals("root metadata", source.getUserData()); + assertEquals("child metadata", emptyPoint.getUserData()); + assertEquals("nested metadata", nested.getUserData()); + } + + @Test + public void setSRIDPreservesPackedCoordinateSequenceFactoryAndEmptyLayouts() { + GeometryFactory sourceFactory = + new GeometryFactory( + new PrecisionModel(), 7, PackedCoordinateSequenceFactory.DOUBLE_FACTORY); + Point emptyXym = + sourceFactory.createPoint(sourceFactory.getCoordinateSequenceFactory().create(0, 3, 1)); + LinearRing emptyXyzmShell = + sourceFactory.createLinearRing( + sourceFactory.getCoordinateSequenceFactory().create(0, 4, 1)); + GeometryCollection source = + sourceFactory.createGeometryCollection( + new Geometry[] {emptyXym, sourceFactory.createPolygon(emptyXyzmShell)}); + + GeometryCollection result = (GeometryCollection) Functions.setSRID(source, 3857); + + assertSame( + PackedCoordinateSequenceFactory.DOUBLE_FACTORY, + result.getFactory().getCoordinateSequenceFactory()); + CoordinateSequence pointSequence = ((Point) result.getGeometryN(0)).getCoordinateSequence(); + CoordinateSequence shellSequence = + ((Polygon) result.getGeometryN(1)).getExteriorRing().getCoordinateSequence(); + assertEquals(3, pointSequence.getDimension()); + assertEquals(1, pointSequence.getMeasures()); + assertEquals(4, shellSequence.getDimension()); + assertEquals(1, shellSequence.getMeasures()); + assertGeometryTreeUsesFactory(result, result.getFactory(), 3857); + assertEquals(7, source.getFactory().getSRID()); + } + + private static void assertGeometryTreeUsesFactory( + Geometry geometry, GeometryFactory factory, int srid) { + assertSame(factory, geometry.getFactory()); + assertEquals(srid, geometry.getSRID()); + assertEquals(srid, geometry.getFactory().getSRID()); + for (int i = 0; i < geometry.getNumGeometries(); i++) { + Geometry child = geometry.getGeometryN(i); + if (child != geometry) assertGeometryTreeUsesFactory(child, factory, srid); + } + if (geometry instanceof Polygon) { + Polygon polygon = (Polygon) geometry; + assertGeometryTreeUsesFactory(polygon.getExteriorRing(), factory, srid); + for (int i = 0; i < polygon.getNumInteriorRing(); i++) + assertGeometryTreeUsesFactory(polygon.getInteriorRingN(i), factory, srid); + } + } + @Test public void closestPoint() { Point point1 = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)); From 8a012e32635104ccdd50e64de6b25d9663c5c252 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 00:43:12 -0700 Subject: [PATCH 17/25] Cover empty members in multipart SRID copies --- .../apache/sedona/common/FunctionsTest.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index af98e500d42..5398c640bff 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -4745,23 +4745,41 @@ public void setSRIDPreservesPackedCoordinateSequenceFactoryAndEmptyLayouts() { new PrecisionModel(), 7, PackedCoordinateSequenceFactory.DOUBLE_FACTORY); Point emptyXym = sourceFactory.createPoint(sourceFactory.getCoordinateSequenceFactory().create(0, 3, 1)); + LineString emptyXyz = + sourceFactory.createLineString( + sourceFactory.getCoordinateSequenceFactory().create(0, 3, 0)); LinearRing emptyXyzmShell = sourceFactory.createLinearRing( sourceFactory.getCoordinateSequenceFactory().create(0, 4, 1)); GeometryCollection source = sourceFactory.createGeometryCollection( - new Geometry[] {emptyXym, sourceFactory.createPolygon(emptyXyzmShell)}); + new Geometry[] { + sourceFactory.createMultiPoint(new Point[] {emptyXym}), + sourceFactory.createMultiLineString(new LineString[] {emptyXyz}), + sourceFactory.createMultiPolygon( + new Polygon[] {sourceFactory.createPolygon(emptyXyzmShell)}) + }); GeometryCollection result = (GeometryCollection) Functions.setSRID(source, 3857); assertSame( PackedCoordinateSequenceFactory.DOUBLE_FACTORY, result.getFactory().getCoordinateSequenceFactory()); - CoordinateSequence pointSequence = ((Point) result.getGeometryN(0)).getCoordinateSequence(); + assertEquals(1, result.getGeometryN(0).getNumGeometries()); + assertEquals(1, result.getGeometryN(1).getNumGeometries()); + assertEquals(1, result.getGeometryN(2).getNumGeometries()); + CoordinateSequence pointSequence = + ((Point) result.getGeometryN(0).getGeometryN(0)).getCoordinateSequence(); + CoordinateSequence lineSequence = + ((LineString) result.getGeometryN(1).getGeometryN(0)).getCoordinateSequence(); CoordinateSequence shellSequence = - ((Polygon) result.getGeometryN(1)).getExteriorRing().getCoordinateSequence(); + ((Polygon) result.getGeometryN(2).getGeometryN(0)) + .getExteriorRing() + .getCoordinateSequence(); assertEquals(3, pointSequence.getDimension()); assertEquals(1, pointSequence.getMeasures()); + assertEquals(3, lineSequence.getDimension()); + assertEquals(0, lineSequence.getMeasures()); assertEquals(4, shellSequence.getDimension()); assertEquals(1, shellSequence.getMeasures()); assertGeometryTreeUsesFactory(result, result.getFactory(), 3857); From 1f4026b323b66137bd2ff1480a544dcbad79eb25 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 00:52:11 -0700 Subject: [PATCH 18/25] Preserve dimensions across WKB ingress readers --- .../common/S2Geography/WKBGeography.java | 4 +- .../common/S2Geography/WKBGeographyTest.java | 32 +++++ .../PythonGeometrySerializer.scala | 7 +- .../geopackage/transform/GeometryReader.scala | 6 +- .../geoparquet/GeoParquetRowConverter.scala | 8 +- .../wrapper/TestToPythonSerialization.scala | 25 +++- .../transform/GeometryReaderTest.scala | 59 +++++++++ .../GeoParquetRowConverterTest.scala | 121 ++++++++++++++++++ 8 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala create mode 100644 spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala diff --git a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java index d30c011057d..a4ed00e2f65 100644 --- a/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java +++ b/common/src/main/java/org/apache/sedona/common/S2Geography/WKBGeography.java @@ -28,6 +28,7 @@ import java.nio.ByteOrder; import java.util.Arrays; import java.util.List; +import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.PrecisionModel; import org.locationtech.jts.io.ParseException; @@ -130,8 +131,7 @@ public Geometry getJTSGeometry() { result = jtsGeometry; if (result == null) { try { - org.datasyslab.jts.io.WKBReader reader = new org.datasyslab.jts.io.WKBReader(); - result = reader.read(wkbBytes); + result = GeometryWkbReader.read(wkbBytes); result.setSRID(getSRID()); } catch (ParseException e) { throw new RuntimeException("Failed to parse WKB to JTS Geometry", e); diff --git a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java index 5073e24f9af..594b0c139db 100644 --- a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java +++ b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java @@ -25,6 +25,7 @@ import com.google.common.geometry.S2Point; import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import org.apache.sedona.common.geography.Constructors; import org.apache.sedona.common.geography.Functions; @@ -35,6 +36,7 @@ import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.PrecisionModel; import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.Ordinate; import org.locationtech.jts.io.ParseException; public class WKBGeographyTest { @@ -66,6 +68,36 @@ public void fromWKB_point_lazyParse() throws ParseException { assertEquals(4326, jts.getSRID()); } + @Test + public void fromWKB_emptyJtsGeometryRetainsDeclaredLayout() { + GeometryFactory factory = new GeometryFactory(); + int[][] layouts = {{3, 0}, {3, 1}, {4, 1}}; + for (int[] layout : layouts) { + int dimension = layout[0]; + int measures = layout[1]; + EnumSet ordinates = EnumSet.of(Ordinate.X, Ordinate.Y); + if (dimension == 4) ordinates.add(Ordinate.Z); + if (measures == 0) ordinates.add(Ordinate.Z); + if (measures == 1) ordinates.add(Ordinate.M); + org.locationtech.jts.io.WKBWriter writer = new org.locationtech.jts.io.WKBWriter(dimension); + writer.setOutputOrdinates(ordinates); + Geometry empty = + factory.createPolygon( + factory.createLinearRing( + factory.getCoordinateSequenceFactory().create(0, dimension, measures))); + + Geometry parsed = WKBGeography.fromWKB(writer.write(empty), 4326).getJTSGeometry(); + Geometry result = + org.apache.sedona.common.geometrySerde.GeometrySerializer.deserialize( + org.apache.sedona.common.geometrySerde.GeometrySerializer.serialize(parsed)); + org.locationtech.jts.geom.CoordinateSequence sequence = + ((org.locationtech.jts.geom.Polygon) result).getExteriorRing().getCoordinateSequence(); + assertEquals(dimension, sequence.getDimension()); + assertEquals(measures, sequence.getMeasures()); + assertEquals(4326, result.getSRID()); + } + } + @Test public void fromJTS_point() { GeometryFactory gf = new GeometryFactory(); diff --git a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala index ee7b18674cf..9ae458c4244 100644 --- a/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala +++ b/spark/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala @@ -19,9 +19,9 @@ package org.apache.sedona.python.wrapper.translation import org.apache.sedona.common.geometryObjects.Circle +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.sedona.python.wrapper.SerializationException import org.locationtech.jts.geom.Geometry -import org.datasyslab.jts.io.WKBReader import java.nio.ByteBuffer @@ -47,13 +47,12 @@ private[python] class PythonGeometrySerializer extends Serializable { } def deserialize(isCircle: Int, values: Array[Byte], offset: Int): Geometry = { - val reader = new WKBReader() if (isCircle == 1) { - val geom = reader.read(values.slice(offset + 8, values.length)) + val geom = GeometryWkbReader.read(values.slice(offset + 8, values.length)) val radius = ByteBuffer.wrap(values.slice(offset, offset + 8)).getDouble() new Circle(geom, radius) } else if (isCircle == 0) { - reader.read(values.slice(offset, values.length)) + GeometryWkbReader.read(values.slice(offset, values.length)) } else { throw SerializationException("Can not deserialize object") } diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala index 3c50113121d..522137d31b3 100644 --- a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala +++ b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala @@ -18,10 +18,9 @@ */ package org.apache.sedona.sql.datasources.geopackage.transform +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.sedona.sql.datasources.geopackage.errors.GeopackageException import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT -import org.locationtech.jts.geom.{GeometryFactory, PrecisionModel} -import org.datasyslab.jts.io.WKBReader import java.nio.{ByteBuffer, ByteOrder} @@ -56,8 +55,7 @@ object GeometryReader { val wkb = new Array[Byte](reader.remaining()) reader.get(wkb) - val wkbReader = new WKBReader(new GeometryFactory(new PrecisionModel(), srid)) - val geom = wkbReader.read(wkb) + val geom = GeometryWkbReader.read(wkb, srid) // that needs rewriting GeometryUDT.serialize(geom) diff --git a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala index 38000513ef6..b2a15ee92f1 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverter.scala @@ -31,9 +31,9 @@ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap import org.apache.spark.sql.execution.datasources.geoparquet.internal._ import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.apache.sedona.common.geometrySerde.GeometryWkbReader import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.datasyslab.jts.io.WKBReader import java.math.{BigDecimal, BigInteger} import java.time.{ZoneId, ZoneOffset} @@ -220,8 +220,7 @@ private[geoparquet] class GeoParquetRowConverter( if (parquetType.isPrimitive) { new ParquetPrimitiveConverter(updater) { override def addBinary(value: Binary): Unit = { - val wkbReader = new WKBReader() - val geom = wkbReader.read(value.getBytes) + val geom = GeometryWkbReader.read(value.getBytes) geom.setSRID(srid) this.updater.set(GeometryUDT.serialize(geom)) } @@ -233,9 +232,8 @@ private[geoparquet] class GeoParquetRowConverter( ArrayType(ByteType, containsNull = false), updater) { override def end(): Unit = { - val wkbReader = new WKBReader() val byteArray = currentArray.map(_.asInstanceOf[Byte]).toArray - val geom = wkbReader.read(byteArray) + val geom = GeometryWkbReader.read(byteArray) geom.setSRID(srid) this.updater.set(GeometryUDT.serialize(geom)) } diff --git a/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala b/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala index ac01c4b86f7..ca655b9589a 100644 --- a/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala +++ b/spark/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala @@ -25,11 +25,12 @@ import org.apache.sedona.python.wrapper.utils.implicits._ import org.apache.sedona.sql.TestBaseScala import org.apache.spark.api.java.JavaPairRDD import org.locationtech.jts.geom.{Geometry, GeometryFactory} -import org.locationtech.jts.io.WKTReader +import org.locationtech.jts.io.{Ordinate, WKBWriter, WKTReader} import org.scalatest.matchers.must.Matchers.contain import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import java.io.{FileInputStream, InputStream} +import java.util.EnumSet import scala.io.Source import scala.jdk.CollectionConverters._ @@ -95,6 +96,28 @@ class TestToPythonSerialization extends TestBaseScala { .flatMap(samplePoint => pythonGeometrySerializer.serialize(samplePoint))) describe("Sedona Python Wrapper Test") { + it("retains the declared layout of inbound empty WKB") { + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = geometryFactory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = geometryFactory.createPolygon(geometryFactory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + val parsed = pythonGeometrySerializer.deserialize(0, writer.write(empty), 0) + val result = org.apache.sedona.common.geometrySerde.GeometrySerializer.deserialize( + org.apache.sedona.common.geometrySerde.GeometrySerializer.serialize(parsed)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + resultSequence.getDimension shouldBe dimension + resultSequence.getMeasures shouldBe measures + } + } + it("Test Serialize To Python JavaRDD[Geometry]") { val convertedToPythonRDD = GeometryRddConverter(pointSpatialRDD, pythonGeometrySerializer).translateToPython diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala new file mode 100644 index 00000000000..70f8a05b95f --- /dev/null +++ b/spark/common/src/test/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReaderTest.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.sql.datasources.geopackage.transform + +import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.locationtech.jts.geom.{GeometryFactory, Polygon} +import org.locationtech.jts.io.{Ordinate, WKBWriter} +import org.scalatest.funsuite.AnyFunSuite + +import java.nio.{ByteBuffer, ByteOrder} +import java.util.EnumSet + +class GeometryReaderTest extends AnyFunSuite { + test("extractWKB retains an empty geometry's declared layout and GeoPackage SRID") { + val factory = new GeometryFactory() + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + val wkb = writer.write(empty) + val bytes = ByteBuffer + .allocate(8 + wkb.length) + .order(ByteOrder.LITTLE_ENDIAN) + .put('G'.toByte) + .put('P'.toByte) + .put(0.toByte) + .put(0x11.toByte) + .putInt(4326) + .put(wkb) + .array() + + val result = GeometryUDT.deserialize(GeometryReader.extractWKB(bytes)).asInstanceOf[Polygon] + val resultSequence = result.getExteriorRing.getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + assert(result.getSRID == 4326) + } + } +} diff --git a/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala new file mode 100644 index 00000000000..33adc24d82b --- /dev/null +++ b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geoparquet + +import org.apache.parquet.io.api.Binary +import org.apache.parquet.io.api.GroupConverter +import org.apache.parquet.schema.MessageTypeParser +import org.apache.spark.sql.execution.datasources.geoparquet.internal.{LegacyBehaviorPolicy, NoopUpdater, RebaseSpec} +import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT +import org.apache.spark.sql.types.{StructField, StructType} +import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.io.{Ordinate, WKBWriter} +import org.scalatest.funsuite.AnyFunSuite + +import java.util.{Collections, EnumSet} + +class GeoParquetRowConverterTest extends AnyFunSuite { + test("primitive WKB retains declared empty coordinate layouts through GeometryUDT") { + val parquetSchema = + MessageTypeParser.parseMessageType("message root { optional binary geometry; }") + val catalystSchema = StructType(Seq(StructField("geometry", GeometryUDT(), nullable = true))) + val metadata = Collections.singletonMap( + "geo", + """{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":[],"crs":null}}}""") + val converter = new GeoParquetRowConverter( + new GeoParquetToSparkSchemaConverter(metadata, parameters = Map.empty), + parquetSchema, + catalystSchema, + None, + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + Map.empty, + NoopUpdater) + val factory = new GeometryFactory() + + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + converter.start() + converter + .getConverter(0) + .asPrimitiveConverter() + .addBinary(Binary.fromConstantByteArray(writer.write(empty))) + converter.end() + val result = GeometryUDT.deserialize(converter.currentRecord.getBinary(0)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + } + } + + test("legacy byte-array WKB retains declared empty coordinate layouts through GeometryUDT") { + val parquetSchema = MessageTypeParser.parseMessageType( + "message root { optional group geometry (LIST) { repeated int32 array (INT_8); } }") + val catalystSchema = StructType(Seq(StructField("geometry", GeometryUDT(), nullable = true))) + val metadata = Collections.singletonMap( + "geo", + """{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":[],"crs":null}}}""") + val parameters = Map("legacyMode" -> "true") + val converter = new GeoParquetRowConverter( + new GeoParquetToSparkSchemaConverter(metadata, parameters = parameters), + parquetSchema, + catalystSchema, + None, + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED), + parameters, + NoopUpdater) + val factory = new GeometryFactory() + + Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => + val sequence = factory.getCoordinateSequenceFactory.create(0, dimension, measures) + val empty = factory.createPolygon(factory.createLinearRing(sequence)) + val writer = new WKBWriter(dimension) + val ordinates = EnumSet.of(Ordinate.X, Ordinate.Y) + if (dimension - measures > 2) ordinates.add(Ordinate.Z) + if (measures > 0) ordinates.add(Ordinate.M) + writer.setOutputOrdinates(ordinates) + + converter.start() + val geometryConverter = converter.getConverter(0).asInstanceOf[GroupConverter] + geometryConverter.start() + val byteConverter = geometryConverter.getConverter(0).asPrimitiveConverter() + writer.write(empty).foreach(byte => byteConverter.addInt(byte)) + geometryConverter.end() + converter.end() + val result = GeometryUDT.deserialize(converter.currentRecord.getBinary(0)) + val resultSequence = result + .asInstanceOf[org.locationtech.jts.geom.Polygon] + .getExteriorRing + .getCoordinateSequence + assert(resultSequence.getDimension == dimension) + assert(resultSequence.getMeasures == measures) + } + } +} From eb93164e8903bd4f88eac0bf3f8616caf960e014 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 23:14:47 -0700 Subject: [PATCH 19/25] Test empty polygon copies and interior rings when setting SRID --- .../apache/sedona/common/FunctionsTest.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 5398c640bff..3df0d1aacc1 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -4678,6 +4678,41 @@ public void setSRIDEmptyGeometries() throws ParseException { } } + @Test + public void setSRIDPreservesEmptyPolygonHoles() throws ParseException { + Polygon source = + (Polygon) Constructors.geomFromWKT("POLYGON ((0 0, 10 0, 10 10, 0 0), EMPTY)", 100); + + Polygon result = (Polygon) Functions.setSRID(source, 4326); + + assertEquals(1, result.getNumInteriorRing()); + assertTrue(result.getInteriorRingN(0).isEmpty()); + assertEquals(source.getExteriorRing(), result.getExteriorRing()); + assertNotSame(source.getInteriorRingN(0), result.getInteriorRingN(0)); + assertGeometryTreeUsesFactory(result, result.getFactory(), 4326); + assertEquals(100, source.getSRID()); + assertEquals(1, source.getNumInteriorRing()); + } + + @Test + public void setSRIDCopiesEmptyPolygonWithoutMutatingInput() throws ParseException { + Polygon source = (Polygon) Constructors.geomFromWKT("POLYGON EMPTY", 100); + source.setUserData("source metadata"); + source.getExteriorRing().setUserData("shell metadata"); + + Polygon result = (Polygon) Functions.setSRID(source, 4326); + + assertEquals(100, source.getSRID()); + assertNotSame(source, result); + assertNotSame(source.getExteriorRing(), result.getExteriorRing()); + assertTrue(result.isEmpty()); + assertGeometryTreeUsesFactory(result, result.getFactory(), 4326); + assertNull(result.getUserData()); + assertNull(result.getExteriorRing().getUserData()); + assertEquals("source metadata", source.getUserData()); + assertEquals("shell metadata", source.getExteriorRing().getUserData()); + } + @Test public void setSRIDPreservesNestedEmptyComponentsAndCopiesStructure() { GeometryFactory sourceFactory = From 7dfe3fd5af384e6b546b980d7b6c2b00895028f9 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 23:06:02 -0700 Subject: [PATCH 20/25] Test empty polygon holes through ST_SetSRID --- .../apache/sedona/sql/functionTestScala.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 1bf7154b4a9..e2a02c554b2 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1036,6 +1036,27 @@ class functionTestScala assert(df.first().get(0).asInstanceOf[Polygon].getSRID == 3021) } + it("ST_SetSRID preserves empty polygon holes") { + val polygonWithEmptyHole = + "01030000000200000005000000000000000000000000000000000000000000000000002440000000000000000000000000000024400000000000002440000000000000000000000000000024400000000000000000000000000000000000000000" + val result = sparkSession + .sql(s""" + |WITH source AS ( + | SELECT ST_GeomFromWKB(unhex('$polygonWithEmptyHole')) AS polygon + |) + |SELECT + | ST_NumInteriorRings(polygon), + | ST_NumInteriorRings(ST_SetSRID(polygon, 4326)), + | ST_SRID(ST_SetSRID(polygon, 4326)) + |FROM source + |""".stripMargin) + .first() + + assertEquals(1, result.getInt(0)) + assertEquals(1, result.getInt(1)) + assertEquals(4326, result.getInt(2)) + } + it("Passed ST_AsHEXEWKB") { val baseDf = sparkSession.sql("SELECT ST_GeomFromWKT('POINT(1 2)') as point") var actual = baseDf.selectExpr("ST_AsHEXEWKB(point)").first().get(0) From b7970fdb3e88c6448181ae5adb547547e85abb14 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 16 Sep 2026 23:25:57 -0700 Subject: [PATCH 21/25] Use isolated JTS geometry copier when setting SRID --- .../org/apache/sedona/common/Functions.java | 6 +- .../StructurePreservingGeometryFactory.java | 85 ------------------- pom.xml | 2 +- 3 files changed, 4 insertions(+), 89 deletions(-) delete mode 100644 common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 4bbc6564245..030ec563bde 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -34,11 +34,11 @@ import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.geometryObjects.Box3D; import org.apache.sedona.common.geometryObjects.Circle; -import org.apache.sedona.common.geometryObjects.StructurePreservingGeometryFactory; import org.apache.sedona.common.jts2geojson.GeoJSONWriter; import org.apache.sedona.common.sphere.Spheroid; import org.apache.sedona.common.subDivide.GeometrySubDivider; import org.apache.sedona.common.utils.*; +import org.datasyslab.jts.geom.util.GeometryCopier; import org.locationtech.jts.algorithm.Angle; import org.locationtech.jts.algorithm.MinimumAreaRectangle; import org.locationtech.jts.algorithm.MinimumBoundingCircle; @@ -1100,11 +1100,11 @@ public static Geometry setSRID(Geometry geometry, int srid) { return null; } GeometryFactory factory = - new StructurePreservingGeometryFactory( + new GeometryFactory( geometry.getPrecisionModel(), srid, geometry.getFactory().getCoordinateSequenceFactory()); - return factory.createGeometry(geometry); + return GeometryCopier.copy(geometry, factory); } public static int getSRID(Geometry geometry) { diff --git a/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java b/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java deleted file mode 100644 index c8784b8fa1d..00000000000 --- a/common/src/main/java/org/apache/sedona/common/geometryObjects/StructurePreservingGeometryFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.sedona.common.geometryObjects; - -import org.locationtech.jts.geom.*; - -/** Geometry factory that retains empty components and coordinate layouts when copying. */ -public final class StructurePreservingGeometryFactory extends GeometryFactory { - private static final long serialVersionUID = 1L; - - public StructurePreservingGeometryFactory( - PrecisionModel precisionModel, int srid, CoordinateSequenceFactory sequenceFactory) { - super(precisionModel, srid, sequenceFactory); - } - - @Override - public Geometry createGeometry(Geometry geometry) { - if (geometry instanceof Point) { - return createPoint(copy(((Point) geometry).getCoordinateSequence())); - } - if (geometry instanceof LinearRing) { - return createLinearRing(copy(((LinearRing) geometry).getCoordinateSequence())); - } - if (geometry instanceof LineString) { - return createLineString(copy(((LineString) geometry).getCoordinateSequence())); - } - if (geometry instanceof Polygon) { - Polygon polygon = (Polygon) geometry; - LinearRing[] holes = new LinearRing[polygon.getNumInteriorRing()]; - for (int i = 0; i < holes.length; i++) { - holes[i] = (LinearRing) createGeometry(polygon.getInteriorRingN(i)); - } - return createPolygon((LinearRing) createGeometry(polygon.getExteriorRing()), holes); - } - if (geometry instanceof MultiPoint) { - Point[] points = new Point[geometry.getNumGeometries()]; - for (int i = 0; i < points.length; i++) { - points[i] = (Point) createGeometry(geometry.getGeometryN(i)); - } - return createMultiPoint(points); - } - if (geometry instanceof MultiLineString) { - LineString[] lines = new LineString[geometry.getNumGeometries()]; - for (int i = 0; i < lines.length; i++) { - lines[i] = (LineString) createGeometry(geometry.getGeometryN(i)); - } - return createMultiLineString(lines); - } - if (geometry instanceof MultiPolygon) { - Polygon[] polygons = new Polygon[geometry.getNumGeometries()]; - for (int i = 0; i < polygons.length; i++) { - polygons[i] = (Polygon) createGeometry(geometry.getGeometryN(i)); - } - return createMultiPolygon(polygons); - } - if (geometry instanceof GeometryCollection) { - Geometry[] children = new Geometry[geometry.getNumGeometries()]; - for (int i = 0; i < children.length; i++) { - children[i] = createGeometry(geometry.getGeometryN(i)); - } - return createGeometryCollection(children); - } - return super.createGeometry(geometry); - } - - private CoordinateSequence copy(CoordinateSequence sequence) { - return getCoordinateSequenceFactory().create(sequence); - } -} diff --git a/pom.xml b/pom.xml index 55d1c702acf..230a4f37e37 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ 3.2.4 2.13.4 1.20.0 - 1.21.0-datasyslab-1 + 1.21.0-datasyslab-2 0.16.1 0.8 From 7d1b8d3e43bb21a0d8712b4e3963074a1f459dcc Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Thu, 17 Sep 2026 22:32:45 -0700 Subject: [PATCH 22/25] Use shared JTS coordinate layout declarations --- .../apache/sedona/common/Constructors.java | 4 +- .../ByteBufferGeometryBuffer.java | 1 + .../DeclaredCoordinateSequence.java | 45 ------------- .../DeclaredCoordinateSequenceFactory.java | 57 ---------------- .../geometrySerde/GeometrySerializer.java | 4 +- .../geometrySerde/GeometryWkbReader.java | 65 ------------------- .../geometrySerde/UnsafeGeometryBuffer.java | 1 + .../sedona/common/utils/FormatUtils.java | 3 +- .../GeometryDimensionSerdeTest.java | 3 +- 9 files changed, 10 insertions(+), 173 deletions(-) delete mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java delete mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java delete mode 100644 common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java diff --git a/common/src/main/java/org/apache/sedona/common/Constructors.java b/common/src/main/java/org/apache/sedona/common/Constructors.java index 6f3ad3b49f9..19e1550ed56 100644 --- a/common/src/main/java/org/apache/sedona/common/Constructors.java +++ b/common/src/main/java/org/apache/sedona/common/Constructors.java @@ -26,9 +26,9 @@ import org.apache.sedona.common.enums.GeometryType; import org.apache.sedona.common.geometryObjects.Box2D; import org.apache.sedona.common.geometryObjects.Box3D; -import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.utils.FormatUtils; import org.apache.sedona.common.utils.GeoHashDecoder; +import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; @@ -74,7 +74,7 @@ public static Geometry geomFromWKB(byte[] wkb) throws ParseException { } public static Geometry geomFromWKB(byte[] wkb, int SRID) throws ParseException { - Geometry geom = GeometryWkbReader.read(wkb); + Geometry geom = WKBReader.forDeclaredDimensions().read(wkb); if (geom.getFactory().getSRID() != geom.getSRID() || (SRID >= 0 && geom.getSRID() != SRID)) { // Make sure that the geometry and the geometry factory have the correct SRID if (SRID < 0) { diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java index ec82f2752e6..fd0618f5154 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.datasyslab.jts.geom.impl.DeclaredCoordinateSequence; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.CoordinateXY; diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java deleted file mode 100644 index 91e10a1b6a1..00000000000 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequence.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.sedona.common.geometrySerde; - -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.CoordinateSequence; -import org.locationtech.jts.geom.impl.CoordinateArraySequence; - -/** Coordinates whose layout was explicitly declared by a binary geometry header. */ -final class DeclaredCoordinateSequence extends CoordinateArraySequence { - private static final long serialVersionUID = 1L; - - DeclaredCoordinateSequence(Coordinate[] coordinates, int dimension, int measures) { - super(coordinates, dimension, measures); - } - - DeclaredCoordinateSequence(int size, int dimension, int measures) { - super(size, dimension, measures); - } - - private DeclaredCoordinateSequence(CoordinateSequence sequence) { - super(sequence); - } - - @Override - public DeclaredCoordinateSequence copy() { - return new DeclaredCoordinateSequence(this); - } -} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java deleted file mode 100644 index 5d35295c13c..00000000000 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/DeclaredCoordinateSequenceFactory.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.sedona.common.geometrySerde; - -import java.io.Serializable; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.CoordinateSequence; -import org.locationtech.jts.geom.CoordinateSequenceFactory; -import org.locationtech.jts.geom.impl.CoordinateArraySequenceFactory; - -/** Preserves binary layout declarations on copies without marking ordinary JTS allocations. */ -final class DeclaredCoordinateSequenceFactory implements CoordinateSequenceFactory, Serializable { - private static final long serialVersionUID = 1L; - static final DeclaredCoordinateSequenceFactory INSTANCE = new DeclaredCoordinateSequenceFactory(); - - private DeclaredCoordinateSequenceFactory() {} - - @Override - public CoordinateSequence create(Coordinate[] coordinates) { - // Ordinary Coordinate values may contain a padded NaN Z, which is not a declaration. - return CoordinateArraySequenceFactory.instance().create(coordinates); - } - - @Override - public CoordinateSequence create(CoordinateSequence sequence) { - if (sequence instanceof DeclaredCoordinateSequence) { - return sequence.copy(); - } - return CoordinateArraySequenceFactory.instance().create(sequence); - } - - @Override - public CoordinateSequence create(int size, int dimension) { - return CoordinateArraySequenceFactory.instance().create(size, dimension); - } - - @Override - public CoordinateSequence create(int size, int dimension, int measures) { - return CoordinateArraySequenceFactory.instance().create(size, dimension, measures); - } -} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java index 7816b88e1d0..b9edb3467b9 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java @@ -18,6 +18,8 @@ */ package org.apache.sedona.common.geometrySerde; +import org.datasyslab.jts.geom.impl.DeclaredCoordinateSequence; +import org.datasyslab.jts.geom.impl.DeclaredCoordinateSequenceFactory; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Geometry; @@ -502,7 +504,7 @@ private static int alignedOffset(int offset) { } private static GeometryFactory createGeometryFactory(int srid) { - return new GeometryFactory(PRECISION_MODEL, srid, DeclaredCoordinateSequenceFactory.INSTANCE); + return new GeometryFactory(PRECISION_MODEL, srid, DeclaredCoordinateSequenceFactory.instance()); } private static Polygon createEmptyPolygon( diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java deleted file mode 100644 index 93dc0d6085e..00000000000 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryWkbReader.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.sedona.common.geometrySerde; - -import org.datasyslab.jts.io.WKBReader; -import org.locationtech.jts.geom.*; -import org.locationtech.jts.io.ParseException; - -/** - * Reads WKB while retaining explicitly declared coordinate dimensions, including empty geometry. - */ -public final class GeometryWkbReader { - private GeometryWkbReader() {} - - public static Geometry read(byte[] bytes) throws ParseException { - return read(bytes, 0); - } - - public static Geometry read(byte[] bytes, int defaultSrid) throws ParseException { - GeometryFactory factory = - new GeometryFactory( - new PrecisionModel(), defaultSrid, DeclaredCoordinateSequenceFactory.INSTANCE); - // Only allocations made from WKB headers declare a layout. Later JTS operations use the - // geometry factory's ordinary allocation behavior. - return new WKBReader(factory, new ReaderCoordinateSequenceFactory()).read(bytes); - } - - private static final class ReaderCoordinateSequenceFactory implements CoordinateSequenceFactory { - @Override - public CoordinateSequence create(Coordinate[] coordinates) { - return DeclaredCoordinateSequenceFactory.INSTANCE.create(coordinates); - } - - @Override - public CoordinateSequence create(CoordinateSequence sequence) { - return DeclaredCoordinateSequenceFactory.INSTANCE.create(sequence); - } - - @Override - public CoordinateSequence create(int size, int dimension) { - return create(size, dimension, 0); - } - - @Override - public CoordinateSequence create(int size, int dimension, int measures) { - return new DeclaredCoordinateSequence(size, dimension, measures); - } - } -} diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java index 39bfa123737..2bae400f71a 100644 --- a/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java @@ -19,6 +19,7 @@ package org.apache.sedona.common.geometrySerde; import java.lang.reflect.Field; +import org.datasyslab.jts.geom.impl.DeclaredCoordinateSequence; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.CoordinateXY; diff --git a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java index 17ae36e473d..9994b8945a4 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/FormatUtils.java @@ -25,7 +25,6 @@ import org.apache.sedona.common.Functions; import org.apache.sedona.common.enums.FileDataSplitter; import org.apache.sedona.common.enums.GeometryType; -import org.apache.sedona.common.geometrySerde.GeometryWkbReader; import org.apache.sedona.common.jts2geojson.GeoJSONReader; import org.datasyslab.jts.io.WKBReader; import org.locationtech.jts.geom.*; @@ -208,7 +207,7 @@ public Geometry readWkt(String line) throws ParseException { public Geometry readWkb(String line) throws ParseException { final String[] columns = line.split(splitter.getDelimiter()); final byte[] aux = WKBReader.hexToBytes(columns[this.startOffset]); - Geometry geometry = GeometryWkbReader.read(aux); + Geometry geometry = WKBReader.forDeclaredDimensions().read(aux); if (geometry.getSRID() != geometry.getFactory().getSRID()) { // Make sure that the geometry factory has the correct SRID when the parsed WKB // contains a non-zero SRID (EWKB) diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java index 695c5eab0cc..a48f3e6e5a6 100644 --- a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java +++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryDimensionSerdeTest.java @@ -29,6 +29,7 @@ import org.apache.sedona.common.Constructors; import org.apache.sedona.common.Functions; import org.datasyslab.jts.geom.util.GeometryCopier; +import org.datasyslab.jts.io.WKBReader; import org.junit.Test; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; @@ -272,7 +273,7 @@ public void wkbReaderPreservesMemberSridsAndDefaultSrid() throws ParseException bytes.put((byte) 1).putInt(7).putInt(2); bytes.put((byte) 1).putInt(0x20000001).putInt(4326).putDouble(1).putDouble(2); bytes.put((byte) 1).putInt(0x20000001).putInt(3857).putDouble(3).putDouble(4); - Geometry collection = GeometryWkbReader.read(bytes.array(), 27700); + Geometry collection = WKBReader.forDeclaredDimensions(27700).read(bytes.array()); assertEquals(27700, collection.getSRID()); assertEquals(4326, collection.getGeometryN(0).getSRID()); assertEquals(3857, collection.getGeometryN(1).getSRID()); From 76f59c947971e06657d01229a4f940863d3e5253 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Thu, 17 Sep 2026 22:37:16 -0700 Subject: [PATCH 23/25] Preserve declared coordinate layouts in WKB output --- .../apache/sedona/common/utils/GeomUtils.java | 20 +- .../sedona/common/utils/WkbOutputTest.java | 209 ++++++++++++++++++ docs/api/sql/Geometry-Output/ST_AsBinary.md | 2 + docs/api/sql/Geometry-Output/ST_AsEWKB.md | 3 +- docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md | 2 + .../apache/sedona/sql/functionTestScala.scala | 70 ++++++ 6 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 606d747a94b..f4e2136b816 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -190,25 +190,31 @@ private static String writeWKT(Geometry geometry) { } public static String getHexEWKB(Geometry geometry, int endian) { - WKBWriter writer = - new WKBWriter(GeomUtils.getDimension(geometry), endian, geometry.getSRID() != 0); - return WKBWriter.toHex(writer.write(geometry)); + return WKBWriter.toHex( + createDimensionPreservingWKBWriter(endian, geometry.getSRID() != 0).write(geometry)); } public static byte[] getEWKB(Geometry geometry) { if (geometry == null) { return null; } - WKBWriter writer = createWKBWriter(GeomUtils.getDimension(geometry), geometry.getSRID() != 0); - return writer.write(geometry); + return createDimensionPreservingWKBWriter(NATIVE_WKB_BYTE_ORDER, geometry.getSRID() != 0) + .write(geometry); } public static byte[] getWKB(Geometry geometry) { if (geometry == null) { return null; } - WKBWriter writer = createWKBWriter(GeomUtils.getDimension(geometry), false); - return writer.write(geometry); + return createDimensionPreservingWKBWriter(NATIVE_WKB_BYTE_ORDER, false).write(geometry); + } + + private static org.datasyslab.jts.io.WKBWriter createDimensionPreservingWKBWriter( + int byteOrder, boolean includeSRID) { + org.datasyslab.jts.io.WKBWriter writer = + new org.datasyslab.jts.io.WKBWriter(4, byteOrder, includeSRID); + writer.setPreserveCoordinateDimensions(true); + return writer; } private static final int NATIVE_WKB_BYTE_ORDER = diff --git a/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java b/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java new file mode 100644 index 00000000000..153566dac83 --- /dev/null +++ b/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.common.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import org.apache.sedona.common.Constructors; +import org.apache.sedona.common.geometrySerde.GeometrySerializer; +import org.junit.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; + +public class WkbOutputTest { + private static final GeometryFactory FACTORY = new GeometryFactory(); + private static final Layout XY = new Layout(2, 0, 0); + private static final Layout XYZ = new Layout(3, 0, 0x80000000); + private static final Layout XYM = new Layout(3, 1, 0x40000000); + private static final Layout XYZM = new Layout(4, 1, 0xc0000000); + + @Test + public void emptyPrimitiveLayoutsSurviveWkbOutputAfterSerialization() throws ParseException { + for (Layout layout : new Layout[] {XY, XYZ, XYM, XYZM}) { + for (int primitive = 1; primitive <= 3; primitive++) { + assertOutputs(roundTrip(readPrimitive(primitive, layout, true)), primitive, layout, true); + } + } + } + + @Test + public void allNanZAndMLayoutsSurviveWkbOutputAfterSerialization() throws ParseException { + for (Layout layout : new Layout[] {XY, XYZ, XYM, XYZM}) { + assertOutputs(roundTrip(readPrimitive(1, layout, false)), 1, layout, false); + } + } + + @Test + public void measuredValuesAreWrittenInTheirOwnOrdinate() throws ParseException { + for (Layout layout : new Layout[] {XYM, XYZM}) { + Point source = (Point) readPrimitive(1, layout, false); + source.getCoordinateSequence().setOrdinate(0, layout.dimension - 1, 9); + if (layout == XYZM) source.getCoordinateSequence().setOrdinate(0, 2, 3); + byte[] bytes = GeomUtils.getEWKB(roundTrip(source)); + assertHeader(bytes, 1 | layout.flags | 0x20000000, 4326); + Point output = (Point) Constructors.geomFromWKB(bytes); + assertSequence(output.getCoordinateSequence(), layout); + assertEquals(9, output.getCoordinateSequence().getM(0), 0); + if (layout == XYZM) assertEquals(3, output.getCoordinateSequence().getZ(0), 0); + else assertTrue(Double.isNaN(output.getCoordinateSequence().getZ(0))); + } + } + + @Test + public void outputFindsZBeyondFirstCoordinateAndKeepsOrdinaryCoordinatesXy() + throws ParseException { + LineString source = + FACTORY.createLineString(new Coordinate[] {new Coordinate(1, 2), new Coordinate(3, 4, 5)}); + for (Geometry line : new Geometry[] {source, roundTrip(source)}) { + byte[] bytes = GeomUtils.getWKB(line); + assertHeader(bytes, 2 | XYZ.flags, 0); + CoordinateSequence sequence = + ((LineString) Constructors.geomFromWKB(bytes)).getCoordinateSequence(); + assertEquals(3, sequence.getDimension()); + assertTrue(Double.isNaN(sequence.getZ(0))); + assertEquals(5, sequence.getZ(1), 0); + } + Geometry ordinary = FACTORY.createPoint(new Coordinate(1, 2)); + assertHeader(GeomUtils.getWKB(ordinary), 1, 0); + assertHeader(GeomUtils.getEWKB(roundTrip(ordinary)), 1, 0); + } + + @Test + public void geometryCollectionMembersRetainTheirOwnLayouts() throws ParseException { + Geometry collection = + roundTrip( + FACTORY.createGeometryCollection( + new Geometry[] { + readPrimitive(1, XY, false), + readPrimitive(1, XYZ, true), + FACTORY.createGeometryCollection(new Geometry[] {readPrimitive(1, XYM, false)}) + })); + byte[] bytes = GeomUtils.getWKB(collection); + assertHeader(bytes, 7 | XYZM.flags, 0); + Geometry output = Constructors.geomFromWKB(bytes); + assertSequence(((Point) output.getGeometryN(0)).getCoordinateSequence(), XY); + assertSequence(((Point) output.getGeometryN(1)).getCoordinateSequence(), XYZ); + assertTrue(output.getGeometryN(1).isEmpty()); + assertSequence(((Point) output.getGeometryN(2).getGeometryN(0)).getCoordinateSequence(), XYM); + } + + @Test + public void nullOutputAndExplicitTwoDimensionalWriterKeepTheirBehavior() throws ParseException { + assertNull(GeomUtils.getWKB(null)); + assertNull(GeomUtils.getEWKB(null)); + WKBWriter writer = GeomUtils.createWKBWriter(2, true); + assertHeader(writer.write(readPrimitive(1, XYZM, false)), 0x20000001, 4326); + } + + private static void assertOutputs(Geometry geometry, int primitive, Layout layout, boolean empty) + throws ParseException { + byte[] wkb = GeomUtils.getWKB(geometry); + byte[] ewkb = GeomUtils.getEWKB(geometry); + int nativeOrder = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? 1 : 0; + assertEquals(nativeOrder, wkb[0]); + assertEquals(nativeOrder, ewkb[0]); + assertOutput(wkb, primitive, layout, 0, empty); + assertOutput(ewkb, primitive, layout, 4326, empty); + for (int endian : new int[] {ByteOrderValues.LITTLE_ENDIAN, ByteOrderValues.BIG_ENDIAN}) { + byte[] bytes = WKBReader.hexToBytes(GeomUtils.getHexEWKB(geometry, endian)); + assertEquals(endian == ByteOrderValues.LITTLE_ENDIAN ? 1 : 0, bytes[0]); + assertOutput(bytes, primitive, layout, 4326, empty); + } + } + + private static void assertOutput( + byte[] bytes, int primitive, Layout layout, int srid, boolean empty) throws ParseException { + assertHeader(bytes, primitive | layout.flags | (srid == 0 ? 0 : 0x20000000), srid); + assertEquals( + 5 + (srid == 0 ? 0 : 4) + (primitive == 1 ? layout.dimension * 8 : 4), bytes.length); + Geometry output = Constructors.geomFromWKB(bytes); + assertEquals(srid, output.getSRID()); + assertEquals(empty, output.isEmpty()); + CoordinateSequence sequence = + output instanceof Point + ? ((Point) output).getCoordinateSequence() + : output instanceof LineString + ? ((LineString) output).getCoordinateSequence() + : ((Polygon) output).getExteriorRing().getCoordinateSequence(); + assertSequence(sequence, layout); + if (!empty) { + assertEquals(1, sequence.getX(0), 0); + assertEquals(2, sequence.getY(0), 0); + for (int ordinate = 2; ordinate < layout.dimension; ordinate++) { + assertTrue(Double.isNaN(sequence.getOrdinate(0, ordinate))); + } + } + } + + private static void assertHeader(byte[] bytes, int type, int srid) { + ByteBuffer buffer = + ByteBuffer.wrap(bytes) + .order(bytes[0] == 1 ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); + assertEquals(type, buffer.getInt(1)); + if (srid != 0) assertEquals(srid, buffer.getInt(5)); + } + + private static void assertSequence(CoordinateSequence sequence, Layout layout) { + assertEquals(layout.dimension, sequence.getDimension()); + assertEquals(layout.measures, sequence.getMeasures()); + } + + private static Geometry readPrimitive(int primitive, Layout layout, boolean empty) + throws ParseException { + ByteBuffer buffer = ByteBuffer.allocate(64).order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) 1).putInt(primitive | layout.flags | 0x20000000).putInt(4326); + if (primitive == 1) { + buffer.putDouble(empty ? Double.NaN : 1).putDouble(empty ? Double.NaN : 2); + for (int ordinate = 2; ordinate < layout.dimension; ordinate++) buffer.putDouble(Double.NaN); + } else { + buffer.putInt(0); + } + return Constructors.geomFromWKB(Arrays.copyOf(buffer.array(), buffer.position())); + } + + private static Geometry roundTrip(Geometry geometry) { + return GeometrySerializer.deserialize(GeometrySerializer.serialize(geometry)); + } + + private static final class Layout { + final int dimension; + final int measures; + final int flags; + + Layout(int dimension, int measures, int flags) { + this.dimension = dimension; + this.measures = measures; + this.flags = flags; + } + } +} diff --git a/docs/api/sql/Geometry-Output/ST_AsBinary.md b/docs/api/sql/Geometry-Output/ST_AsBinary.md index 5e17ddbe880..9a2caa7d3a5 100644 --- a/docs/api/sql/Geometry-Output/ST_AsBinary.md +++ b/docs/api/sql/Geometry-Output/ST_AsBinary.md @@ -21,6 +21,8 @@ Introduction: Return the Well-Known Binary representation of a geometry +Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. The output does not include the SRID. + Format: `ST_AsBinary (A: Geometry)` Return type: `Binary` diff --git a/docs/api/sql/Geometry-Output/ST_AsEWKB.md b/docs/api/sql/Geometry-Output/ST_AsEWKB.md index 2189df55cfc..37e18ff313a 100644 --- a/docs/api/sql/Geometry-Output/ST_AsEWKB.md +++ b/docs/api/sql/Geometry-Output/ST_AsEWKB.md @@ -24,7 +24,8 @@ EWKB is an extended version of WKB which includes the SRID of the geometry. The format originated in PostGIS but is supported by many GIS tools. If the geometry is lacking SRID a WKB format is produced. [See ST_SetSRID](../Spatial-Reference-System/ST_SetSRID.md) -It will ignore the M coordinate if present. + +Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. Format: `ST_AsEWKB (A: Geometry)` diff --git a/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md b/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md index 9e94b9dbad6..c65e1a94d26 100644 --- a/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md +++ b/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md @@ -21,6 +21,8 @@ Introduction: This function returns the input geometry encoded to a text representation in HEXEWKB format. The HEXEWKB encoding can use either little-endian (NDR) or big-endian (XDR) byte ordering. If no encoding is explicitly specified, the function defaults to using the little-endian (NDR) format. +Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsHEXEWKB(geom: Geometry, endian: String = NDR)` Return type: `String` diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index a3d950748d2..2b04707248a 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1075,6 +1075,76 @@ class functionTestScala assert(Hex.encodeHexString(df.first().get(0).asInstanceOf[Array[Byte]]) == s) } + it("WKB output preserves declared point layouts after a shuffle") { + val nan = "000000000000F87F" + val xy = "000000000000F03F0000000000000040" + val layouts = Seq( + ("0101000000", 2, 0), + ("0101000080", 3, 0), + ("0101000040", 3, 1), + ("01010000C0", 4, 1)) + val cases = for { + (header, dimension, measures) <- layouts + empty <- Seq(false, true) + } yield ( + header + (if (empty) nan * 2 else xy) + nan * (dimension - 2), + dimension, + measures, + empty) + val geometries = cases.zipWithIndex + .map { case ((wkb, _, _, _), id) => (id, wkb) } + .toDF("id", "wkb") + .selectExpr("id", "ST_SetSRID(ST_GeomFromWKB(unhex(wkb)), 4326) AS geom") + .repartition(2) + .cache() + try { + assertEquals(cases.size.toLong, geometries.count()) + val rows = geometries + .selectExpr( + "id", + "ST_AsBinary(geom)", + "ST_AsEWKB(geom)", + "ST_AsHEXEWKB(geom)", + "ST_AsHEXEWKB(geom, 'XDR')") + .collect() + rows.foreach { row => + val (_, dimension, measures, empty) = cases(row.getInt(0)) + val outputs = Seq( + row.getAs[Array[Byte]](1) -> 0, + row.getAs[Array[Byte]](2) -> 4326, + Hex.decodeHex(row.getString(3).toCharArray) -> 4326, + Hex.decodeHex(row.getString(4).toCharArray) -> 4326) + outputs.foreach { case (bytes, srid) => + val point = + org.apache.sedona.common.Constructors.geomFromWKB(bytes).asInstanceOf[Point] + assertEquals(dimension, point.getCoordinateSequence.getDimension) + assertEquals(measures, point.getCoordinateSequence.getMeasures) + assertEquals(empty, point.isEmpty) + assertEquals(srid, point.getSRID) + assertEquals(5 + dimension * 8 + (if (srid == 0) 0 else 4), bytes.length) + if (!empty) { + assertEquals(1.0, point.getX, 0.0) + assertEquals(2.0, point.getY, 0.0) + (2 until dimension).foreach { ordinate => + assertTrue(point.getCoordinateSequence.getOrdinate(0, ordinate).isNaN) + } + } + } + assertEquals(1, outputs(2)._1(0).toInt) + assertEquals(0, outputs(3)._1(0).toInt) + } + } finally { + geometries.unpersist() + } + } + + it("WKB output functions return null for null geometry") { + val row = sparkSession + .sql("SELECT ST_AsBinary(NULL), ST_AsEWKB(NULL), ST_AsHEXEWKB(NULL)") + .first() + (0 until 3).foreach(index => assertTrue(row.isNullAt(index))) + } + it("Passed ST_Simplify") { val baseDf = sparkSession.sql("SELECT ST_Buffer(ST_GeomFromWKT('POINT (0 2)'), 10) AS geom") val actualPoints = baseDf.selectExpr("ST_NPoints(ST_Simplify(geom, 1))").first().get(0) From 0f26a29afa9583897d59856256631e5dd8087d32 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 20 Sep 2026 23:33:56 -0700 Subject: [PATCH 24/25] Cover NaN dimensions in file reader tests --- .../common/S2Geography/WKBGeographyTest.java | 5 ++- .../tutorial/files/geopackage-sedona-spark.md | 2 ++ .../tutorial/files/geoparquet-sedona-spark.md | 2 ++ .../GeoParquetRowConverterTest.scala | 36 +++++++++++++++++-- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java index 594b0c139db..b1e792fa373 100644 --- a/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java +++ b/common/src/test/java/org/apache/sedona/common/S2Geography/WKBGeographyTest.java @@ -76,9 +76,8 @@ public void fromWKB_emptyJtsGeometryRetainsDeclaredLayout() { int dimension = layout[0]; int measures = layout[1]; EnumSet ordinates = EnumSet.of(Ordinate.X, Ordinate.Y); - if (dimension == 4) ordinates.add(Ordinate.Z); - if (measures == 0) ordinates.add(Ordinate.Z); - if (measures == 1) ordinates.add(Ordinate.M); + if (dimension - measures > 2) ordinates.add(Ordinate.Z); + if (measures > 0) ordinates.add(Ordinate.M); org.locationtech.jts.io.WKBWriter writer = new org.locationtech.jts.io.WKBWriter(dimension); writer.setOutputOrdinates(ordinates); Geometry empty = diff --git a/docs/tutorial/files/geopackage-sedona-spark.md b/docs/tutorial/files/geopackage-sedona-spark.md index ab4a25b7235..4f6d2aac3d9 100644 --- a/docs/tutorial/files/geopackage-sedona-spark.md +++ b/docs/tutorial/files/geopackage-sedona-spark.md @@ -79,6 +79,8 @@ Here are the contents of the DataFrame: The geometry column can contain many different geometric objects like points, polygons, and many more. +Since Sedona 2.0.0, reads preserve declared Z/M dimensions even for empty points, lines, and polygons or NaN ordinates. If collecting mixed layouts raises a heterogeneous-layout error, see [ST_Collect](../../api/sql/Geometry-Editors/ST_Collect.md). + You can also see the metadata of the GeoPackage file: ```python diff --git a/docs/tutorial/files/geoparquet-sedona-spark.md b/docs/tutorial/files/geoparquet-sedona-spark.md index c2370afa8a5..eeae899c038 100644 --- a/docs/tutorial/files/geoparquet-sedona-spark.md +++ b/docs/tutorial/files/geoparquet-sedona-spark.md @@ -88,6 +88,8 @@ Here are the results: +---+---------------------+ ``` +Since Sedona 2.0.0, reads preserve declared Z/M dimensions even for empty points, lines, and polygons or NaN ordinates. If collecting mixed layouts raises a heterogeneous-layout error, see [ST_Collect](../../api/sql/Geometry-Editors/ST_Collect.md). + Here's how Sedona executes this query under the hood: 1. It fetches the schema from the footer of a GeoParquet file, so no schema inference is needed. diff --git a/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala index 33adc24d82b..0b5dd2cfb36 100644 --- a/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala +++ b/spark/common/src/test/scala/org/apache/spark/sql/execution/datasources/geoparquet/GeoParquetRowConverterTest.scala @@ -24,21 +24,21 @@ import org.apache.parquet.schema.MessageTypeParser import org.apache.spark.sql.execution.datasources.geoparquet.internal.{LegacyBehaviorPolicy, NoopUpdater, RebaseSpec} import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT import org.apache.spark.sql.types.{StructField, StructType} -import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.geom.{Coordinate, GeometryFactory, Point} import org.locationtech.jts.io.{Ordinate, WKBWriter} import org.scalatest.funsuite.AnyFunSuite import java.util.{Collections, EnumSet} class GeoParquetRowConverterTest extends AnyFunSuite { - test("primitive WKB retains declared empty coordinate layouts through GeometryUDT") { + private def primitiveConverter(): GeoParquetRowConverter = { val parquetSchema = MessageTypeParser.parseMessageType("message root { optional binary geometry; }") val catalystSchema = StructType(Seq(StructField("geometry", GeometryUDT(), nullable = true))) val metadata = Collections.singletonMap( "geo", """{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":[],"crs":null}}}""") - val converter = new GeoParquetRowConverter( + new GeoParquetRowConverter( new GeoParquetToSparkSchemaConverter(metadata, parameters = Map.empty), parquetSchema, catalystSchema, @@ -47,6 +47,10 @@ class GeoParquetRowConverterTest extends AnyFunSuite { RebaseSpec(LegacyBehaviorPolicy.CORRECTED), Map.empty, NoopUpdater) + } + + test("primitive WKB retains declared empty coordinate layouts through GeometryUDT") { + val converter = primitiveConverter() val factory = new GeometryFactory() Seq((3, 0), (3, 1), (4, 1)).foreach { case (dimension, measures) => @@ -74,6 +78,32 @@ class GeoParquetRowConverterTest extends AnyFunSuite { } } + test("primitive WKB retains NaN Z without carrying its layout into later XY rows") { + val converter = primitiveConverter() + val factory = new GeometryFactory() + val nanZ = new WKBWriter(3).write(factory.createPoint(new Coordinate(1, 2, 9))) + java.nio.ByteBuffer.wrap(nanZ).putDouble(21, Double.NaN) + val xy = new WKBWriter(2).write(factory.createPoint(new Coordinate(1, 2))) + + Seq((nanZ, 3), (xy, 2), (nanZ, 3)).foreach { case (wkb, dimension) => + converter.start() + converter + .getConverter(0) + .asPrimitiveConverter() + .addBinary(Binary.fromConstantByteArray(wkb)) + converter.end() + val result = + GeometryUDT.deserialize(converter.currentRecord.getBinary(0)).asInstanceOf[Point] + val sequence = result.getCoordinateSequence + assert(!result.isEmpty) + assert(sequence.getDimension == dimension) + assert(sequence.getMeasures == 0) + assert(sequence.getX(0) == 1) + assert(sequence.getY(0) == 2) + if (dimension == 3) assert(sequence.getZ(0).isNaN) + } + } + test("legacy byte-array WKB retains declared empty coordinate layouts through GeometryUDT") { val parquetSchema = MessageTypeParser.parseMessageType( "message root { optional group geometry (LIST) { repeated int32 array (INT_8); } }") From 05bbe736951d81b4b57ff1a2859677db4ded5d62 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 21 Sep 2026 01:03:23 -0700 Subject: [PATCH 25/25] Document measured WKB output across engines and cover multipart writes --- .../sedona/common/utils/WkbOutputTest.java | 89 +++++++++++++++++-- docs/api/flink/Geometry-Output/ST_AsBinary.md | 2 + docs/api/flink/Geometry-Output/ST_AsEWKB.md | 3 +- .../api/flink/Geometry-Output/ST_AsHEXEWKB.md | 2 + .../Geometry-Output/ST_AsBinary.md | 2 + .../vector-data/Geometry-Output/ST_AsEWKB.md | 2 + .../Geometry-Output/ST_AsHEXEWKB.md | 2 + docs/api/sql/Geometry-Output/ST_AsBinary.md | 2 +- docs/api/sql/Geometry-Output/ST_AsEWKB.md | 2 +- docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md | 2 +- docs/setup/release-notes.md | 3 + 11 files changed, 101 insertions(+), 10 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java b/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java index 153566dac83..888c98468df 100644 --- a/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java +++ b/common/src/test/java/org/apache/sedona/common/utils/WkbOutputTest.java @@ -39,6 +39,7 @@ import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKBReader; import org.locationtech.jts.io.WKBWriter; +import org.locationtech.jts.io.WKTReader; public class WkbOutputTest { private static final GeometryFactory FACTORY = new GeometryFactory(); @@ -79,6 +80,72 @@ public void measuredValuesAreWrittenInTheirOwnOrdinate() throws ParseException { } } + @Test + public void measuredWktRetainsMInEveryOutputFormat() throws ParseException { + String[] inputs = { + "POINT M (1 2 9)", "POINT ZM (1 2 3 9)", "LINESTRING M (1 2 9, 4 5 6)", "POINT M EMPTY" + }; + Layout[] layouts = {XYM, XYZM, XYM, XYM}; + for (int i = 0; i < inputs.length; i++) { + Geometry source = new WKTReader().read(inputs[i]); + source.setSRID(4326); + CoordinateSequence expected = coordinateSequence(source); + for (Geometry geometry : new Geometry[] {source, roundTrip(source)}) { + byte[][] outputs = outputVariants(geometry); + for (int format = 0; format < outputs.length; format++) { + int srid = format == 0 ? 0 : 4326; + int type = source instanceof Point ? 1 : 2; + assertHeader( + outputs[format], type | layouts[i].flags | (srid == 0 ? 0 : 0x20000000), srid); + Geometry output = Constructors.geomFromWKB(outputs[format]); + assertEquals(source.isEmpty(), output.isEmpty()); + CoordinateSequence actual = coordinateSequence(output); + assertSequence(actual, layouts[i]); + assertEquals(expected.size(), actual.size()); + for (int point = 0; point < expected.size(); point++) { + for (int ordinate = 0; ordinate < layouts[i].dimension; ordinate++) { + assertEquals( + expected.getOrdinate(point, ordinate), actual.getOrdinate(point, ordinate), 0); + } + } + } + } + } + } + + @Test + public void multipartOutputUsesSharedLayoutForEmptyMembers() throws ParseException { + String[] inputs = { + "MULTIPOINT Z (EMPTY, (1 2 3))", + "MULTILINESTRING Z (EMPTY, (0 0 1, 1 1 2))", + "MULTIPOLYGON Z (EMPTY, ((0 0 1, 1 0 2, 1 1 3, 0 0 1)))" + }; + for (int i = 0; i < inputs.length; i++) { + Geometry source = new WKTReader().read(inputs[i]); + source.setSRID(4326); + byte[][] outputs = outputVariants(source); + for (int format = 0; format < outputs.length; format++) { + int srid = format == 0 ? 0 : 4326; + assertHeader(outputs[format], (4 + i) | XYZ.flags | (srid == 0 ? 0 : 0x20000000), srid); + Geometry output = roundTrip(Constructors.geomFromWKB(outputs[format])); + assertEquals(srid, output.getSRID()); + assertEquals(2, output.getNumGeometries()); + assertTrue(output.getGeometryN(0).isEmpty()); + assertSequence(coordinateSequence(output.getGeometryN(0)), XYZ); + CoordinateSequence expected = coordinateSequence(source.getGeometryN(1)); + CoordinateSequence actual = coordinateSequence(output.getGeometryN(1)); + assertSequence(actual, XYZ); + assertEquals(expected.size(), actual.size()); + for (int point = 0; point < expected.size(); point++) { + for (int ordinate = 0; ordinate < 3; ordinate++) { + assertEquals( + expected.getOrdinate(point, ordinate), actual.getOrdinate(point, ordinate), 0); + } + } + } + } + } + @Test public void outputFindsZBeyondFirstCoordinateAndKeepsOrdinaryCoordinatesXy() throws ParseException { @@ -149,12 +216,7 @@ private static void assertOutput( Geometry output = Constructors.geomFromWKB(bytes); assertEquals(srid, output.getSRID()); assertEquals(empty, output.isEmpty()); - CoordinateSequence sequence = - output instanceof Point - ? ((Point) output).getCoordinateSequence() - : output instanceof LineString - ? ((LineString) output).getCoordinateSequence() - : ((Polygon) output).getExteriorRing().getCoordinateSequence(); + CoordinateSequence sequence = coordinateSequence(output); assertSequence(sequence, layout); if (!empty) { assertEquals(1, sequence.getX(0), 0); @@ -178,6 +240,21 @@ private static void assertSequence(CoordinateSequence sequence, Layout layout) { assertEquals(layout.measures, sequence.getMeasures()); } + private static CoordinateSequence coordinateSequence(Geometry geometry) { + if (geometry instanceof Point) return ((Point) geometry).getCoordinateSequence(); + if (geometry instanceof LineString) return ((LineString) geometry).getCoordinateSequence(); + return ((Polygon) geometry).getExteriorRing().getCoordinateSequence(); + } + + private static byte[][] outputVariants(Geometry geometry) { + return new byte[][] { + GeomUtils.getWKB(geometry), + GeomUtils.getEWKB(geometry), + WKBReader.hexToBytes(GeomUtils.getHexEWKB(geometry, ByteOrderValues.LITTLE_ENDIAN)), + WKBReader.hexToBytes(GeomUtils.getHexEWKB(geometry, ByteOrderValues.BIG_ENDIAN)) + }; + } + private static Geometry readPrimitive(int primitive, Layout layout, boolean empty) throws ParseException { ByteBuffer buffer = ByteBuffer.allocate(64).order(ByteOrder.LITTLE_ENDIAN); diff --git a/docs/api/flink/Geometry-Output/ST_AsBinary.md b/docs/api/flink/Geometry-Output/ST_AsBinary.md index cf9c00e8eca..11e878d4711 100644 --- a/docs/api/flink/Geometry-Output/ST_AsBinary.md +++ b/docs/api/flink/Geometry-Output/ST_AsBinary.md @@ -21,6 +21,8 @@ Introduction: Return the Well-Known Binary representation of a geometry +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsBinary (A: Geometry)` Return type: `Binary` diff --git a/docs/api/flink/Geometry-Output/ST_AsEWKB.md b/docs/api/flink/Geometry-Output/ST_AsEWKB.md index c19ab7e5df6..a7891f2f64f 100644 --- a/docs/api/flink/Geometry-Output/ST_AsEWKB.md +++ b/docs/api/flink/Geometry-Output/ST_AsEWKB.md @@ -23,7 +23,8 @@ Introduction: Return the Extended Well-Known Binary representation of a geometry EWKB is an extended version of WKB which includes the SRID of the geometry. The format originated in PostGIS but is supported by many GIS tools. If the geometry is lacking SRID a WKB format is produced. -It will ignore the M coordinate if present. + +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. Format: `ST_AsEWKB (A: Geometry)` diff --git a/docs/api/flink/Geometry-Output/ST_AsHEXEWKB.md b/docs/api/flink/Geometry-Output/ST_AsHEXEWKB.md index 9e94b9dbad6..d7ea583985f 100644 --- a/docs/api/flink/Geometry-Output/ST_AsHEXEWKB.md +++ b/docs/api/flink/Geometry-Output/ST_AsHEXEWKB.md @@ -21,6 +21,8 @@ Introduction: This function returns the input geometry encoded to a text representation in HEXEWKB format. The HEXEWKB encoding can use either little-endian (NDR) or big-endian (XDR) byte ordering. If no encoding is explicitly specified, the function defaults to using the little-endian (NDR) format. +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsHEXEWKB(geom: Geometry, endian: String = NDR)` Return type: `String` diff --git a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsBinary.md b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsBinary.md index 46b4e86e65e..d587b09a570 100644 --- a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsBinary.md +++ b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsBinary.md @@ -21,6 +21,8 @@ Introduction: Return the Well-Known Binary representation of a geometry +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsBinary (A:geometry)` Return type: `Binary` diff --git a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsEWKB.md b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsEWKB.md index abf40cc9a28..541ec026745 100644 --- a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsEWKB.md +++ b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsEWKB.md @@ -25,6 +25,8 @@ The format originated in PostGIS but is supported by many GIS tools. If the geometry is lacking SRID a WKB format is produced. [See ST_SetSRID](../Spatial-Reference-System/ST_SetSRID.md) +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsEWKB (A:geometry)` Return type: `Binary` diff --git a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsHEXEWKB.md b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsHEXEWKB.md index eda997261da..5b86e6133c5 100644 --- a/docs/api/snowflake/vector-data/Geometry-Output/ST_AsHEXEWKB.md +++ b/docs/api/snowflake/vector-data/Geometry-Output/ST_AsHEXEWKB.md @@ -21,6 +21,8 @@ Introduction: This function returns the input geometry encoded to a text representation in HEXEWKB format. The HEXEWKB encoding can use either little-endian (NDR) or big-endian (XDR) byte ordering. If no encoding is explicitly specified, the function defaults to using the little-endian (NDR) format. +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. + Format: `ST_AsHEXEWKB(geom: Geometry, endian: String = NDR)` Return type: `String` diff --git a/docs/api/sql/Geometry-Output/ST_AsBinary.md b/docs/api/sql/Geometry-Output/ST_AsBinary.md index 9a2caa7d3a5..7cc53b12ad8 100644 --- a/docs/api/sql/Geometry-Output/ST_AsBinary.md +++ b/docs/api/sql/Geometry-Output/ST_AsBinary.md @@ -21,7 +21,7 @@ Introduction: Return the Well-Known Binary representation of a geometry -Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. The output does not include the SRID. +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. The output does not include the SRID. Format: `ST_AsBinary (A: Geometry)` diff --git a/docs/api/sql/Geometry-Output/ST_AsEWKB.md b/docs/api/sql/Geometry-Output/ST_AsEWKB.md index 37e18ff313a..304715ee9ef 100644 --- a/docs/api/sql/Geometry-Output/ST_AsEWKB.md +++ b/docs/api/sql/Geometry-Output/ST_AsEWKB.md @@ -25,7 +25,7 @@ The format originated in PostGIS but is supported by many GIS tools. If the geometry is lacking SRID a WKB format is produced. [See ST_SetSRID](../Spatial-Reference-System/ST_SetSRID.md) -Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. Format: `ST_AsEWKB (A: Geometry)` diff --git a/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md b/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md index c65e1a94d26..d7ea583985f 100644 --- a/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md +++ b/docs/api/sql/Geometry-Output/ST_AsHEXEWKB.md @@ -21,7 +21,7 @@ Introduction: This function returns the input geometry encoded to a text representation in HEXEWKB format. The HEXEWKB encoding can use either little-endian (NDR) or big-endian (XDR) byte ordering. If no encoding is explicitly specified, the function defaults to using the little-endian (NDR) format. -Since v2.0.0, Z and M dimensions read from WKB are preserved in the output, including empty points, lines, and polygons and coordinates whose Z or M values are all NaN. Collections with no members have no stored coordinate layout and are written as XY. +Since v2.0.0, measured geometries retain M in the output, including those read from WKT; earlier versions dropped M. Z/M layouts declared in WKB are preserved for empty points, lines, and polygons and for NaN ordinates. Collections with no members have no stored coordinate layout and are written as XY. Format: `ST_AsHEXEWKB(geom: Geometry, endian: String = NDR)` diff --git a/docs/setup/release-notes.md b/docs/setup/release-notes.md index e41d9084517..d5d8e3f7b01 100644 --- a/docs/setup/release-notes.md +++ b/docs/setup/release-notes.md @@ -21,6 +21,9 @@ ### Breaking Changes +* **WKB output**: `ST_AsBinary`, `ST_AsEWKB`, and `ST_AsHEXEWKB` now retain M for measured + geometries in Spark, Flink, and Snowflake, including geometries read from WKT. Earlier + versions dropped M, so these inputs now produce XYM or XYZM instead of XY or XYZ. * **Flink**: The minimum supported Flink version is now 1.19 (previously 1.12). Flink 1.12 - 1.18 are no longer supported, as the Apache Flink community itself no longer supports those versions. Flink 2.2 is now supported alongside 1.19+.