diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.geometry/src/eu/esdihumboldt/hale/io/gml/geometry/GMLConstants.java b/io/plugins/eu.esdihumboldt.hale.io.gml.geometry/src/eu/esdihumboldt/hale/io/gml/geometry/GMLConstants.java index 84d24e78c4..f06b21bd2e 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml.geometry/src/eu/esdihumboldt/hale/io/gml/geometry/GMLConstants.java +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.geometry/src/eu/esdihumboldt/hale/io/gml/geometry/GMLConstants.java @@ -38,4 +38,9 @@ public interface GMLConstants { */ public static final String NS_GML_32 = "http://www.opengis.net/gml/3.2"; + /** + * Namespace URL for the WFS standard, defined by the OGC + */ + public static final String NS_WFS = "http://www.opengis.net/wfs"; + } diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/META-INF/MANIFEST.MF b/io/plugins/eu.esdihumboldt.hale.io.gml.test/META-INF/MANIFEST.MF index f48fb72418..8d2c83a05f 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml.test/META-INF/MANIFEST.MF +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/META-INF/MANIFEST.MF @@ -17,6 +17,7 @@ Import-Package: com.google.common.base;version="9.0.0", eu.esdihumboldt.hale.common.instance.model, eu.esdihumboldt.hale.common.instance.model.ext, eu.esdihumboldt.hale.common.instance.model.impl, + eu.esdihumboldt.hale.common.instance.model.ext.helper, eu.esdihumboldt.hale.common.schema.geometry, eu.esdihumboldt.hale.common.schema.io, eu.esdihumboldt.hale.common.schema.io.impl, diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter.xsd b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter.xsd new file mode 100644 index 0000000000..2dc883751a --- /dev/null +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter.xsd @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + Container für XPlanGML-Objekte + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_no_doubles.xml b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_no_doubles.xml new file mode 100644 index 0000000000..cd81b3ca01 --- /dev/null +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_no_doubles.xml @@ -0,0 +1,175 @@ + + + + + + + + 48.839927 10.086443 + 48.841809 10.088140 + + + Bohlstraße, Zeppelinstraße + III-03/3 + 0 + Bohlstraße, Zeppelinstraße + 1933-06-08 + 1933-06-08 + + + + + + + + 48.841773 10.087020 48.841648 10.087095 48.841642 10.087068 48.841452 10.087167 48.841380 10.086866 48.841288 10.086472 48.841100 10.086577 48.840931 10.086674 48.840884 10.086699 48.840847 10.086719 48.840812 10.086738 48.840777 10.086755 48.840741 10.086775 48.840700 10.086791 48.840659 10.086805 48.840618 10.086818 48.840577 10.086828 48.840549 10.086834 48.840516 10.086833 48.840465 10.086841 48.840414 10.086844 48.840323 10.086843 48.840270 10.086840 48.840218 10.086831 48.840166 10.086821 48.840163 10.086820 48.840161 10.086820 48.840158 10.086820 48.840155 10.086820 48.840153 10.086821 48.840150 10.086822 48.840147 10.086823 48.840145 10.086825 48.840142 10.086827 48.840139 10.086829 48.840137 10.086831 48.840135 10.086834 48.840133 10.086837 48.840131 10.086841 48.840129 10.086844 48.840128 10.086848 48.840127 10.086851 48.840126 10.086855 48.840125 10.086859 48.840124 10.086863 48.840124 10.086866 48.840104 10.087024 48.840023 10.087669 48.839992 10.087848 48.839988 10.087847 48.839985 10.087848 48.839983 10.087849 48.839981 10.087850 48.839979 10.087852 48.839977 10.087853 48.839975 10.087854 48.839973 10.087856 48.839971 10.087858 48.839969 10.087860 48.839967 10.087862 48.839965 10.087864 48.839963 10.087867 48.839961 10.087869 48.839960 10.087872 48.839958 10.087874 48.839957 10.087877 48.839955 10.087880 48.839954 10.087883 48.839953 10.087886 48.839952 10.087889 48.839951 10.087892 48.839927 10.088089 48.839966 10.088100 48.839992 10.087894 48.840043 10.087904 48.840088 10.087912 48.840098 10.087913 48.840169 10.087919 48.840243 10.087921 48.840349 10.087905 48.840473 10.087866 48.840537 10.087841 48.840658 10.087780 48.841365 10.087397 48.841521 10.087312 48.841691 10.087220 48.841803 10.087152 48.841797 10.087110 48.841773 10.087020 + + + + + + + + + Dokument + Legende Bebauungsplan III-03/3 + https://geodienste.komm.one/store/attachments/buckets/org/107/af127430-f0f9-44a0-9e56-3a83ef7292ed/raw/III-03-3-Legende.pdf + application/pdf + 2020-06-04 + 1020 + + + + + Dokument + Original Bebauungsplan III-03/3 + https://geodienste.komm.one/store/attachments/buckets/org/107/af127430-f0f9-44a0-9e56-3a83ef7292ed/raw/III-03-3-Bplan.pdf + application/pdf + 2020-06-04 + 1030 + + + + + 08136088 + Aalen + + + 10000 + 1000 + 4000 + 1933-06-08 + 9999-01-01 + + + + + + + + 48.878401 10.043646 + 48.879914 10.047683 + + + Änderung des Bebauungsplanes Erweiterung Gewerbegebiet Mittelfeld nördlich der Straße Im Loh, Plan Nr. 66-02/6 bezüglich der planungsrechtlichen Festsetzungen zu Gewerbegebieten + 66-02/8 + 0 + Änderung des Bebauungsplanes Erweiterung Gewerbegebiet Mittelfeld nördlich der Straße Im Loh, Plan Nr. 66-02/6 bezüglich der planungsrechtlichen Festsetzungen zu Gewerbegebieten + 1998-02-19 + + + + + + + + 48.879432 10.047267 48.879549 10.047228 48.879687 10.047181 48.879707 10.047175 48.879883 10.047115 48.879815 10.046829 48.879778 10.046670 48.879741 10.046515 48.879701 10.046344 48.879625 10.046020 48.879591 10.045873 48.879586 10.045856 48.879519 10.045617 48.879404 10.045209 48.879363 10.045062 48.879284 10.044792 48.879180 10.044433 48.879091 10.044129 48.878954 10.043657 48.878436 10.043798 48.878529 10.044115 48.878677 10.044617 48.878653 10.044703 48.878566 10.044735 48.878567 10.044742 48.878575 10.044739 48.878583 10.044737 48.878591 10.044736 48.878600 10.044735 48.878608 10.044735 48.878617 10.044735 48.878626 10.044737 48.878634 10.044739 48.878643 10.044742 48.878652 10.044745 48.878660 10.044750 48.878668 10.044755 48.878677 10.044761 48.878685 10.044767 48.878692 10.044774 48.878699 10.044782 48.878706 10.044791 48.878713 10.044799 48.878719 10.044809 48.878725 10.044819 48.878730 10.044829 48.878735 10.044839 48.878740 10.044850 48.878744 10.044861 48.878748 10.044871 48.878811 10.045082 48.878869 10.045276 48.878925 10.045470 48.878980 10.045664 48.879084 10.046057 48.879157 10.046339 48.879186 10.046452 48.879234 10.046650 48.879285 10.046847 48.879328 10.047055 48.879382 10.047284 48.879444 10.047601 48.879445 10.047607 48.879447 10.047612 48.879449 10.047618 48.879451 10.047624 48.879453 10.047630 48.879456 10.047635 48.879459 10.047641 48.879462 10.047646 48.879465 10.047650 48.879469 10.047655 48.879473 10.047659 48.879477 10.047662 48.879481 10.047665 48.879485 10.047668 48.879489 10.047670 48.879493 10.047672 48.879498 10.047674 48.879502 10.047675 48.879506 10.047675 48.879511 10.047675 48.879515 10.047675 48.879432 10.047267 + + + + + + + + + Dokument + Begründung Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Begruendung.pdf + application/pdf + 2008-03-06 + 1010 + + + + + Dokument + Legende Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Legende.pdf + application/pdf + 2008-03-06 + 1020 + + + + + Dokument + Textteil Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Textteil.pdf + application/pdf + 2008-03-06 + 9998 + + + + + Dokument + Original Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Bplan.pdf + application/pdf + 2008-03-06 + 1030 + + + + + 08136088 + Aalen + + + 10001 + 1000 + 4000 + 1996-07-04 + 1998-04-27 + 1998-05-27 + 1996-07-11 + 2000-01-20 + 1998-02-19 + 2000-02-02 + 2000-02-02 + + + + + + + + 0 + III-03/3 + 9999-01-01 + Andere Gesetzliche Bestimmung + + + + + + 0 + 66-02/8 + 1993-04-22 + BauNVO 1990 zul. geändert 22.04.1993 + + + + + + \ No newline at end of file diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_with_doubles.xml b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_with_doubles.xml new file mode 100644 index 0000000000..0cf1afe0dd --- /dev/null +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_with_doubles.xml @@ -0,0 +1,184 @@ + + + + + + + + 48.839927 10.086443 + 48.841809 10.088140 + + + Bohlstraße, Zeppelinstraße + III-03/3 + 0 + Bohlstraße, Zeppelinstraße + 1933-06-08 + 1933-06-08 + + + + + + + + 48.841773 10.087020 48.841648 10.087095 48.841642 10.087068 48.841452 10.087167 48.841380 10.086866 48.841288 10.086472 48.841100 10.086577 48.840931 10.086674 48.840884 10.086699 48.840847 10.086719 48.840812 10.086738 48.840777 10.086755 48.840741 10.086775 48.840700 10.086791 48.840659 10.086805 48.840618 10.086818 48.840577 10.086828 48.840549 10.086834 48.840516 10.086833 48.840465 10.086841 48.840414 10.086844 48.840323 10.086843 48.840270 10.086840 48.840218 10.086831 48.840166 10.086821 48.840163 10.086820 48.840161 10.086820 48.840158 10.086820 48.840155 10.086820 48.840153 10.086821 48.840150 10.086822 48.840147 10.086823 48.840145 10.086825 48.840142 10.086827 48.840139 10.086829 48.840137 10.086831 48.840135 10.086834 48.840133 10.086837 48.840131 10.086841 48.840129 10.086844 48.840128 10.086848 48.840127 10.086851 48.840126 10.086855 48.840125 10.086859 48.840124 10.086863 48.840124 10.086866 48.840104 10.087024 48.840023 10.087669 48.839992 10.087848 48.839988 10.087847 48.839985 10.087848 48.839983 10.087849 48.839981 10.087850 48.839979 10.087852 48.839977 10.087853 48.839975 10.087854 48.839973 10.087856 48.839971 10.087858 48.839969 10.087860 48.839967 10.087862 48.839965 10.087864 48.839963 10.087867 48.839961 10.087869 48.839960 10.087872 48.839958 10.087874 48.839957 10.087877 48.839955 10.087880 48.839954 10.087883 48.839953 10.087886 48.839952 10.087889 48.839951 10.087892 48.839927 10.088089 48.839966 10.088100 48.839992 10.087894 48.840043 10.087904 48.840088 10.087912 48.840098 10.087913 48.840169 10.087919 48.840243 10.087921 48.840349 10.087905 48.840473 10.087866 48.840537 10.087841 48.840658 10.087780 48.841365 10.087397 48.841521 10.087312 48.841691 10.087220 48.841803 10.087152 48.841797 10.087110 48.841773 10.087020 + + + + + + + + + Dokument + Legende Bebauungsplan III-03/3 + https://geodienste.komm.one/store/attachments/buckets/org/107/af127430-f0f9-44a0-9e56-3a83ef7292ed/raw/III-03-3-Legende.pdf + application/pdf + 2020-06-04 + 1020 + + + + + Dokument + Original Bebauungsplan III-03/3 + https://geodienste.komm.one/store/attachments/buckets/org/107/af127430-f0f9-44a0-9e56-3a83ef7292ed/raw/III-03-3-Bplan.pdf + application/pdf + 2020-06-04 + 1030 + + + + + 08136088 + Aalen + + + 10000 + 1000 + 4000 + 1933-06-08 + 9999-01-01 + + + + + + + + 48.878401 10.043646 + 48.879914 10.047683 + + + Änderung des Bebauungsplanes Erweiterung Gewerbegebiet Mittelfeld nördlich der Straße Im Loh, Plan Nr. 66-02/6 bezüglich der planungsrechtlichen Festsetzungen zu Gewerbegebieten + 66-02/8 + 0 + Änderung des Bebauungsplanes Erweiterung Gewerbegebiet Mittelfeld nördlich der Straße Im Loh, Plan Nr. 66-02/6 bezüglich der planungsrechtlichen Festsetzungen zu Gewerbegebieten + 1998-02-19 + + + + + + + + 48.879432 10.047267 48.879549 10.047228 48.879687 10.047181 48.879707 10.047175 48.879883 10.047115 48.879815 10.046829 48.879778 10.046670 48.879741 10.046515 48.879701 10.046344 48.879625 10.046020 48.879591 10.045873 48.879586 10.045856 48.879519 10.045617 48.879404 10.045209 48.879363 10.045062 48.879284 10.044792 48.879180 10.044433 48.879091 10.044129 48.878954 10.043657 48.878436 10.043798 48.878529 10.044115 48.878677 10.044617 48.878653 10.044703 48.878566 10.044735 48.878567 10.044742 48.878575 10.044739 48.878583 10.044737 48.878591 10.044736 48.878600 10.044735 48.878608 10.044735 48.878617 10.044735 48.878626 10.044737 48.878634 10.044739 48.878643 10.044742 48.878652 10.044745 48.878660 10.044750 48.878668 10.044755 48.878677 10.044761 48.878685 10.044767 48.878692 10.044774 48.878699 10.044782 48.878706 10.044791 48.878713 10.044799 48.878719 10.044809 48.878725 10.044819 48.878730 10.044829 48.878735 10.044839 48.878740 10.044850 48.878744 10.044861 48.878748 10.044871 48.878811 10.045082 48.878869 10.045276 48.878925 10.045470 48.878980 10.045664 48.879084 10.046057 48.879157 10.046339 48.879186 10.046452 48.879234 10.046650 48.879285 10.046847 48.879328 10.047055 48.879382 10.047284 48.879444 10.047601 48.879445 10.047607 48.879447 10.047612 48.879449 10.047618 48.879451 10.047624 48.879453 10.047630 48.879456 10.047635 48.879459 10.047641 48.879462 10.047646 48.879465 10.047650 48.879469 10.047655 48.879473 10.047659 48.879477 10.047662 48.879481 10.047665 48.879485 10.047668 48.879489 10.047670 48.879493 10.047672 48.879498 10.047674 48.879502 10.047675 48.879506 10.047675 48.879511 10.047675 48.879515 10.047675 48.879432 10.047267 + + + + + + + + + Dokument + Begründung Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Begruendung.pdf + application/pdf + 2008-03-06 + 1010 + + + + + Dokument + Legende Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Legende.pdf + application/pdf + 2008-03-06 + 1020 + + + + + Dokument + Textteil Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Textteil.pdf + application/pdf + 2008-03-06 + 9998 + + + + + Dokument + Original Bebauungsplan 66-02/8 + https://geodienste.komm.one/store/attachments/buckets/org/107/3ea9dbb7-05ba-45e9-9eac-50a563230e45/raw/66-02-8-Bplan.pdf + application/pdf + 2008-03-06 + 1030 + + + + + 08136088 + Aalen + + + 10001 + 1000 + 4000 + 1996-07-04 + 1998-04-27 + 1998-05-27 + 1996-07-11 + 2000-01-20 + 1998-02-19 + 2000-02-02 + 2000-02-02 + + + + + + + + 0 + III-03/3 + 9999-01-01 + Andere Gesetzliche Bestimmung + + + + + + 0 + 66-02/8 + 1993-04-22 + BauNVO 1990 zul. geändert 22.04.1993 + + + + + + 0 + 66-02/8 + 1993-04-22 + BauNVO 1990 zul. geändert 22.04.1993 + + + + + + \ No newline at end of file diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollectionTest.java b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollectionTest.java index 49bd7cf475..84063cfad5 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollectionTest.java +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollectionTest.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.net.URI; +import java.net.URISyntaxException; import java.util.Collection; import javax.xml.namespace.QName; @@ -38,10 +39,12 @@ import eu.esdihumboldt.hale.common.instance.model.Instance; import eu.esdihumboldt.hale.common.instance.model.InstanceCollection; import eu.esdihumboldt.hale.common.instance.model.ResourceIterator; +import eu.esdihumboldt.hale.common.instance.model.ext.helper.FullInstanceIteratorSupport; import eu.esdihumboldt.hale.common.schema.io.SchemaReader; import eu.esdihumboldt.hale.common.schema.model.Schema; import eu.esdihumboldt.hale.common.schema.model.TypeDefinition; import eu.esdihumboldt.hale.common.test.TestUtil; +import eu.esdihumboldt.hale.io.gml.reader.internal.wfs.DuplicateIDsFilterIterator; import eu.esdihumboldt.hale.io.xsd.constraint.XmlElements; import eu.esdihumboldt.hale.io.xsd.model.XmlElement; import eu.esdihumboldt.hale.io.xsd.reader.XmlSchemaReader; @@ -266,6 +269,58 @@ public void testLoadShiporder() throws Exception { } } + /** + * Test loading a simple XML file with one instance + * + * @throws Exception if an error occurs + */ + @Test + public void testLoadFilterNoDuplicates() throws Exception { + + int uniqueInstanceCount = extractedUniqueInstances( + "/data/filter_doubles/filter_with_doubles.xml"); + System.out.println( + "There are " + uniqueInstanceCount + " unique instances in the file with doubles."); + assertEquals(3, uniqueInstanceCount); + + uniqueInstanceCount = extractedUniqueInstances( + "/data/filter_doubles/filter_no_doubles.xml"); + + System.out.println("There are " + uniqueInstanceCount + + " unique instances in the file with unique instances."); + assertEquals(4, uniqueInstanceCount); + } + + /** + * @param pathToXML + * @return how many unique instances are present + * @throws IOException + * @throws IOProviderConfigurationException + * @throws URISyntaxException + */ + private int extractedUniqueInstances(String pathToXML) + throws IOException, IOProviderConfigurationException, URISyntaxException { + GmlInstanceCollection instances = loadInstances( + getClass().getResource("/data/filter_doubles/filter.xsd").toURI(), + getClass().getResource(pathToXML).toURI(), false); + + FullInstanceIteratorSupport fullInstance = new FullInstanceIteratorSupport( + instances.iterator()); + + int uniqueInstanceCount = 0; + try (DuplicateIDsFilterIterator myIterator = new DuplicateIDsFilterIterator(fullInstance)) { + while (myIterator.hasNext()) { + myIterator.skip(); + uniqueInstanceCount++; + System.out.println("Start " + uniqueInstanceCount + " uniqueInstanceCount"); + } + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println(uniqueInstanceCount + " uniqueInstanceCount"); + return uniqueInstanceCount; + } + /** * Test loading a simple XML file with one instance including a choice. * diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/StreamGmlReaderTest.groovy b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/StreamGmlReaderTest.groovy index 2b6885b68b..1ba84ccafd 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/StreamGmlReaderTest.groovy +++ b/io/plugins/eu.esdihumboldt.hale.io.gml.test/src/eu/esdihumboldt/hale/io/gml/reader/internal/StreamGmlReaderTest.groovy @@ -137,6 +137,39 @@ class StreamGmlReaderTest { assertEquals(expected, count) } + // this test might not work anymore in the future. At that moment please ignore it + // @Ignore + @Test + public void testWfsPagination() { + /* + * FIXME relies on external resources that are not guaranteed to exist and is thus not enabled for automated testing. + * Better would be a test that could mock the WFS responses (e.g. a mock service running w/ testcontainers) + */ + def schemaUrl = 'https://geodienste.komm.one/ows/services/org.107.7e499bca-5e63-4595-b3c4-eaece8b68608_wfs?SERVICE=WFS&VERSION=2.0.0&REQUEST=DescribeFeatureType' + def dataUrl = 'https://geodienste.komm.one/ows/services/org.107.7e499bca-5e63-4595-b3c4-eaece8b68608_wfs?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature&typenames=xplan:BP_Plan&resolvedepth=*' + def paging = 100 + def expected = 2262 + + def schema = loadSchema(URI.create(schemaUrl)) + + Map params = [ + (StreamGmlReader.PARAM_FEATURES_PER_WFS_REQUEST): paging as String, + (StreamGmlReader.PARAM_PAGINATE_REQUEST): 'true' + ] + + def instances = loadGml(URI.create(dataUrl), schema, params) + + int count = 0 + instances.iterator().withCloseable { it -> + while (it.hasNext()) { + ((InstanceIterator) it).skip() + count++ + } + } + + assertEquals(expected, count) + } + // helpers Schema loadSchema(URI schemaLocation) throws Exception { diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml/META-INF/MANIFEST.MF b/io/plugins/eu.esdihumboldt.hale.io.gml/META-INF/MANIFEST.MF index ec7a5c5d4a..7ca7bd9f3d 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml/META-INF/MANIFEST.MF +++ b/io/plugins/eu.esdihumboldt.hale.io.gml/META-INF/MANIFEST.MF @@ -63,6 +63,7 @@ Export-Package: eu.esdihumboldt.hale.io.gml, eu.esdihumboldt.hale.io.gml.internal.simpletype.converters;x-internal:=true, eu.esdihumboldt.hale.io.gml.reader.internal;x-internal:=true, eu.esdihumboldt.hale.io.gml.reader.internal.instance;x-internal:=true, + eu.esdihumboldt.hale.io.gml.reader.internal.wfs, eu.esdihumboldt.hale.io.gml.writer, eu.esdihumboldt.hale.io.gml.writer.internal;x-internal:=true, eu.esdihumboldt.hale.io.gml.writer.internal.geometry;x-internal:=true, diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollection.java b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollection.java index 338aae6c81..44d02e6255 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollection.java +++ b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/GmlInstanceCollection.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Stack; import javax.annotation.Nullable; import javax.xml.XMLConstants; @@ -50,6 +51,7 @@ import eu.esdihumboldt.hale.common.instance.model.InstanceCollection; import eu.esdihumboldt.hale.common.instance.model.InstanceReference; import eu.esdihumboldt.hale.common.instance.model.InstanceResolver; +import eu.esdihumboldt.hale.common.instance.model.MutableInstance; import eu.esdihumboldt.hale.common.instance.model.ResourceIterator; import eu.esdihumboldt.hale.common.instance.model.ext.InstanceIterator; import eu.esdihumboldt.hale.common.instance.model.impl.FilteredInstanceCollection; @@ -61,6 +63,7 @@ import eu.esdihumboldt.hale.common.schema.model.TypeDefinition; import eu.esdihumboldt.hale.common.schema.model.TypeIndex; import eu.esdihumboldt.hale.common.schema.model.constraint.type.MappingRelevantFlag; +import eu.esdihumboldt.hale.io.gml.geometry.GMLConstants; import eu.esdihumboldt.hale.io.gml.reader.internal.instance.StreamGmlHelper; import eu.esdihumboldt.hale.io.gml.reader.internal.instance.StreamGmlInstance; import eu.esdihumboldt.hale.io.xsd.constraint.XmlAttributeFlag; @@ -75,6 +78,8 @@ */ public class GmlInstanceCollection implements InstanceCollection, LogAware { + public static final String ADDITIONAL_OBJECTS = "ADDITIONAL_OBJECTS"; + /** * Iterates over {@link Instance}s in an XML/GML stream */ @@ -90,6 +95,7 @@ public class GmlInstanceIterator implements InstanceIterator { private Map allElements; private TypeDefinition nextType; + private boolean isAnAdditionalObject = false; /** * The index in the stream for the element returned next with @@ -109,6 +115,7 @@ public class GmlInstanceIterator implements InstanceIterator { * Uses a linked list to allow null items. */ private final Deque typeStack = new LinkedList<>(); + private final Stack elementStack = new Stack<>(); /** * Default constructor @@ -205,6 +212,16 @@ private void proceedToNext() throws XMLStreamException { } } typeStack.push(def); + elementStack.push(elementName); + + // Now you want to find the parent element + for (int i = elementStack.size() - 1; i >= 0; i--) { + if (elementStack.get(i).getNamespaceURI().startsWith(GMLConstants.NS_WFS) + && elementStack.get(i).getLocalPart().equals("additionalObjects")) { + isAnAdditionalObject = true; + break; + } + } if (!rootEncountered) { rootEncountered = true; @@ -224,6 +241,7 @@ private void proceedToNext() throws XMLStreamException { } else if (event == XMLStreamConstants.END_ELEMENT) { typeStack.pop(); + elementStack.pop(); } } } @@ -568,13 +586,20 @@ public synchronized Instance next() { } try { - return StreamGmlHelper.parseInstance(reader, nextType, elementIndex++, strict, null, - crsProvider, nextType, null, false, ignoreNamespaces, ioProvider); + Instance instance = StreamGmlHelper.parseInstance(reader, nextType, elementIndex++, + strict, null, crsProvider, nextType, null, false, ignoreNamespaces, + ioProvider); + + if (isAnAdditionalObject) { + ((MutableInstance) instance).setMetaData(ADDITIONAL_OBJECTS, true); + } + return instance; } catch (XMLStreamException e) { throw new IllegalStateException(e); } finally { nextType = null; typeStack.pop(); // parseInstance consumes END_ELEMENT + elementStack.pop(); } } diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/instance/StreamGmlHelper.java b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/instance/StreamGmlHelper.java index 46e2909740..0c3ed00d04 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/instance/StreamGmlHelper.java +++ b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/instance/StreamGmlHelper.java @@ -34,9 +34,10 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import com.google.common.collect.Iterables; import org.locationtech.jts.geom.Geometry; +import com.google.common.collect.Iterables; + import de.fhg.igd.slf4jplus.ALogger; import de.fhg.igd.slf4jplus.ALoggerFactory; import eu.esdihumboldt.hale.common.core.io.IOProvider; diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/DuplicateIDsFilterIterator.java b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/DuplicateIDsFilterIterator.java new file mode 100644 index 0000000000..d93b34f692 --- /dev/null +++ b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/DuplicateIDsFilterIterator.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2024 wetransform GmbH + * + * All rights reserved. This program and the accompanying materials are made + * available under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution. If not, see . + * + * Contributors: + * wetransform GmbH + */ + +package eu.esdihumboldt.hale.io.gml.reader.internal.wfs; + +import java.util.HashSet; +import java.util.NoSuchElementException; + +import javax.xml.namespace.QName; + +import eu.esdihumboldt.hale.common.instance.model.Instance; +import eu.esdihumboldt.hale.common.instance.model.ext.InstanceIterator; +import eu.esdihumboldt.hale.common.schema.model.TypeDefinition; +import eu.esdihumboldt.hale.io.gml.geometry.GMLConstants; + +/** + * Filter the instances by storing only those with unique IDs. + */ +public class DuplicateIDsFilterIterator implements InstanceIterator { + + // HashSet to store unique instance IDs, ensuring no duplicates are + // processed. + private final HashSet uniqueIDInstances = new HashSet(); + + // Iterator for traversing through instances. + private final InstanceIterator iterator; + + // Holds the next instance to be processed, which should be filled if the + // next() or skip() have already been called for the current instance + private Instance nextInstance; + + // Flag to indicate if the current instance has already been returned by the + // iterator. + private boolean instanceAlreadyReturned = false; + + /** + * @param instanceIterator InstanceIterator + */ + public DuplicateIDsFilterIterator(InstanceIterator instanceIterator) { + this.iterator = instanceIterator; + } + + /** + * + * @see eu.esdihumboldt.hale.common.instance.model.ResourceIterator#close() + */ + @Override + public void close() { + iterator.close(); + nextInstance = null; + instanceAlreadyReturned = true; + } + + /** + * @return TypeDefinition + * @see eu.esdihumboldt.hale.common.instance.model.ext.InstanceIterator#typePeek() + */ + @Override + public TypeDefinition typePeek() { + hasNext(); + if (nextInstance == null) { + return null; + } + return nextInstance.getDefinition(); + } + + /** + * @return boolean + * @see eu.esdihumboldt.hale.common.instance.model.ext.InstanceIterator#supportsTypePeek() + */ + @Override + public boolean supportsTypePeek() { + return true; + } + + /** + * + * @see eu.esdihumboldt.hale.common.instance.model.ext.InstanceIterator#skip() + */ + @Override + public void skip() { + next(); + } + + /** + * Determines if there is a next instance available for iteration. + * + * @return boolean - true if there is another instance to process, false if + * the iteration is complete. + * + * This method checks if the current instance has already been + * returned. If it has, or if the next instance has not been + * initialized, it attempts to fetch the next unique instance. + * + * The iteration continues until a non-duplicate instance is found, + * which is then stored in `nextInstance`. If no more instances are + * available, the method returns false. + * + * @see java.util.Iterator#hasNext() + */ + @Override + public boolean hasNext() { + if (nextInstance == null || instanceAlreadyReturned) { + + while (true) { + if (iterator != null && !iterator.hasNext()) { + return false; + } + else { + Instance instance = iterator.next(); + if (!isTheInstancePresent(instance)) { + nextInstance = instance; + instanceAlreadyReturned = false; + return true; + } + } + } + } + else { + return true; + } + } + + /** + * @return Instance + * @see java.util.Iterator#next() + */ + @Override + public Instance next() { + hasNext(); + if (nextInstance == null) { + for (String string : uniqueIDInstances) { + System.out.println(string); + } + throw new NoSuchElementException(); + } + instanceAlreadyReturned = true; + return nextInstance; + } + + /** + * @param instance Instance to be checked if it has been already given or + * should be returned for further investigation + * @return true if the instance to be checked has been returned already, + * false in contrary case + */ + private boolean isTheInstancePresent(Instance instance) { + for (QName propertyName : instance.getPropertyNames()) { + if (isGmlIdProperty(propertyName)) { + Object[] gmlID = instance.getProperty(propertyName); + if (gmlID[0] != null) { + String gmlIDToCheck = (String) gmlID[0]; + + if (!uniqueIDInstances.contains(gmlIDToCheck)) { + uniqueIDInstances.add(gmlIDToCheck); + return false; + } + else { + return true; + } + + } + } + } + return true; + } + + private boolean isGmlIdProperty(QName propertyName) { + return (propertyName.getNamespaceURI().startsWith(GMLConstants.NS_WFS) + || propertyName.getNamespaceURI().startsWith(GMLConstants.GML_NAMESPACE_CORE)) + && "id".equals(propertyName.getLocalPart()); + } + +} \ No newline at end of file diff --git a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java index 90a00d58af..df74f41b8b 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java +++ b/io/plugins/eu.esdihumboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java @@ -228,6 +228,7 @@ public WfsBackedGmlInstanceCollection(LocatableInputSupplier= maxNumberOfFeatures) - || (size != UNKNOWN_SIZE && totalFeaturesProcessed >= size); + || (size != UNKNOWN_SIZE && totalFeaturesProcessed >= size + && !iterator.hasNext()); } /** * @see java.util.Iterator#next() + * + * Condition: if the instance is part of additional objects then + * that instance is it added but is not counted for the + * totalFeaturesProcessed */ @Override public Instance next() { @@ -571,7 +587,16 @@ public Instance next() { } Instance instance = iterator.next(); - return new StreamGmlInstance(instance, totalFeaturesProcessed++); + boolean hasAdditionalObjects = instance + .getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS) != null + && !instance.getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS).isEmpty(); + + if (primordialQueryParams.containsKey("RESOLVEDEPTH") && hasAdditionalObjects) { + return new StreamGmlInstance(instance, totalFeaturesProcessed); + } + else { + return new StreamGmlInstance(instance, totalFeaturesProcessed++); + } } /** @@ -635,4 +660,4 @@ public void skip() { } -} +} \ No newline at end of file