From 9c16dcbc0c7b4c2461e029df7e2fe278e6ee34e7 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Wed, 15 Feb 2023 10:23:43 +1100 Subject: [PATCH 1/6] Remove shapefile reader, conda GDAL dependency, move cmake gdal dependency to cpp tests only --- README.md | 11 +- .../all_cuda-118_arch-x86_64.yaml | 1 - .../recipes/cuspatial/conda_build_config.yaml | 3 - conda/recipes/cuspatial/meta.yaml | 1 - .../libcuspatial/conda_build_config.yaml | 3 - conda/recipes/libcuspatial/meta.yaml | 2 - cpp/CMakeLists.txt | 11 +- cpp/include/cuspatial/shapefile_reader.hpp | 61 ---- cpp/include/doxygen_groups.h | 5 - cpp/src/io/shp/polygon_shapefile_reader.cpp | 180 ----------- cpp/src/io/shp/polygon_shapefile_reader.cu | 100 ------ cpp/tests/CMakeLists.txt | 13 +- .../io/shp/polygon_shapefile_reader_test.cpp | 133 -------- dependencies.yaml | 1 - docs/source/api_docs/io.rst | 4 - .../user_guide/cuspatial_api_examples.ipynb | 285 +++--------------- python/cuspatial/benchmarks/api/bench_api.py | 9 - python/cuspatial/cuspatial/__init__.py | 1 - .../cuspatial/cuspatial/_lib/CMakeLists.txt | 1 - .../cuspatial/_lib/cpp/shapefile_reader.pxd | 19 -- .../cuspatial/_lib/shapefile_reader.pyx | 37 --- .../cuspatial/core/_column/geocolumn.py | 85 +----- python/cuspatial/cuspatial/core/geoseries.py | 24 -- python/cuspatial/cuspatial/io/shapefile.py | 54 ---- .../cuspatial/tests/test_geoseries.py | 39 --- .../cuspatial/tests/test_shapefile_reader.py | 137 --------- 26 files changed, 66 insertions(+), 1154 deletions(-) delete mode 100644 cpp/include/cuspatial/shapefile_reader.hpp delete mode 100644 cpp/src/io/shp/polygon_shapefile_reader.cpp delete mode 100644 cpp/src/io/shp/polygon_shapefile_reader.cu delete mode 100644 cpp/tests/io/shp/polygon_shapefile_reader_test.cpp delete mode 100644 python/cuspatial/cuspatial/_lib/cpp/shapefile_reader.pxd delete mode 100644 python/cuspatial/cuspatial/_lib/shapefile_reader.pyx delete mode 100644 python/cuspatial/cuspatial/io/shapefile.py delete mode 100644 python/cuspatial/cuspatial/tests/test_shapefile_reader.py diff --git a/README.md b/README.md index 30766dd62..12b47e74e 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,7 @@ conda env update --file conda/environments/all_cuda-115_arch-x86_64.yaml pre-generated polygon shapefiles that contain 0, 1 and 2 polygons, respectively. They are available at `$CUSPATIAL_HOME/test_fixtures/shapefiles`
-**NOTE:** Currently, cuSpatial supports reading point/polyline/polygon data using -Structure of Array (SoA) format and a [shapefile reader](./cpp/src/io/shp) -to read polygon data from a shapefile. -Alternatively, python users can read any point/polyline/polygon data using -existing python packages, e.g., [Shapely](https://pypi.org/project/Shapely/) -and [Fiona](https://github.com/Toblerity/Fiona),to generate numpy arrays and feed them to -[cuSpatial Python APIs](https://docs.rapids.ai/api/cuspatial/stable/). +Python users can read any point/polyline/polygon data using existing python +packages, e.g., [Shapely](https://pypi.org/project/Shapely/) and +[Fiona](https://github.com/Toblerity/Fiona), to generate numpy arrays and feed +them to [cuSpatial Python APIs](https://docs.rapids.ai/api/cuspatial/stable/). diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 634ecfb34..230e0cf55 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,7 +13,6 @@ dependencies: - cxx-compiler - cython>=0.29,<0.30 - gcc_linux-64=9.* -- gdal>3.5.0,<3.6.0 - geopandas>=0.11.0 - gmock=1.10.0 - gtest=1.10.0 diff --git a/conda/recipes/cuspatial/conda_build_config.yaml b/conda/recipes/cuspatial/conda_build_config.yaml index f5584cc01..322fe6faa 100644 --- a/conda/recipes/cuspatial/conda_build_config.yaml +++ b/conda/recipes/cuspatial/conda_build_config.yaml @@ -9,6 +9,3 @@ cuda_compiler: sysroot_version: - "2.17" - -gdal_version: - - ">3.5.0,<3.6.0" diff --git a/conda/recipes/cuspatial/meta.yaml b/conda/recipes/cuspatial/meta.yaml index 7f3de0711..756311aeb 100644 --- a/conda/recipes/cuspatial/meta.yaml +++ b/conda/recipes/cuspatial/meta.yaml @@ -50,7 +50,6 @@ requirements: - setuptools run: - cudf ={{ minor_version }} - - gdal {{ gdal_version }} - geopandas >=0.11.0 - python - rmm ={{ minor_version }} diff --git a/conda/recipes/libcuspatial/conda_build_config.yaml b/conda/recipes/libcuspatial/conda_build_config.yaml index 5cc0c3ea7..99969d85b 100644 --- a/conda/recipes/libcuspatial/conda_build_config.yaml +++ b/conda/recipes/libcuspatial/conda_build_config.yaml @@ -10,9 +10,6 @@ cuda_compiler: cmake_version: - ">=3.23.1,!=3.25.0" -gdal_version: - - ">3.5.0,<3.6.0" - gtest_version: - "1.10.0" diff --git a/conda/recipes/libcuspatial/meta.yaml b/conda/recipes/libcuspatial/meta.yaml index f1767043f..c09201ac6 100644 --- a/conda/recipes/libcuspatial/meta.yaml +++ b/conda/recipes/libcuspatial/meta.yaml @@ -38,7 +38,6 @@ requirements: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit ={{ cuda_version }} - - gdal {{ gdal_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - libcudf ={{ minor_version }} @@ -63,7 +62,6 @@ outputs: - ninja run: - cudatoolkit {{ cuda_spec }} - - gdal {{ gdal_version }} - libcudf ={{ minor_version }} - libcusparse {{ libcusparse_run_version }} - libcusparse-dev {{ libcusparse_run_version }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ae06a6699..0e873e1df 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -99,13 +99,6 @@ include(cmake/Modules/ConfigureCUDA.cmake) ################################################################################################### # - dependencies ---------------------------------------------------------------------------------- -# find gdal -rapids_find_package( - GDAL REQUIRED - GLOBAL_TARGETS GDAL::GDAL - BUILD_EXPORT_SET cuspatial-exports - INSTALL_EXPORT_SET cuspatial-exports -) # add third party dependencies using CPM rapids_cpm_init() # find or add cuDF @@ -118,8 +111,6 @@ include(cmake/thirdparty/CUSPATIAL_GetCUDF.cmake) add_library(cuspatial src/indexing/construction/point_quadtree.cu src/interpolate/cubic_spline.cu - src/io/shp/polygon_shapefile_reader.cpp - src/io/shp/polygon_shapefile_reader.cu src/join/quadtree_point_in_polygon.cu src/join/quadtree_point_to_nearest_linestring.cu src/join/quadtree_bbox_filtering.cu @@ -195,7 +186,7 @@ endif() target_compile_definitions(cuspatial PUBLIC "SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_${RMM_LOGGING_LEVEL}") # Specify the target module library dependencies -target_link_libraries(cuspatial PUBLIC GDAL::GDAL cudf::cudf CUDA::cusparse${_ctk_static_suffix}) +target_link_libraries(cuspatial PUBLIC cudf::cudf CUDA::cusparse${_ctk_static_suffix}) add_library(cuspatial::cuspatial ALIAS cuspatial) diff --git a/cpp/include/cuspatial/shapefile_reader.hpp b/cpp/include/cuspatial/shapefile_reader.hpp deleted file mode 100644 index 4280de6bb..000000000 --- a/cpp/include/cuspatial/shapefile_reader.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include - -namespace cuspatial { - -typedef enum class winding_order : bool { CLOCKWISE, COUNTER_CLOCKWISE } winding_order; - -/** - * @addtogroup io - * @{ - */ - -/** - * @brief read polygon data from an ESRI Shapefile. - * - * @param[in] filename: ESRI Shapefile file path (usually ends in .shp) - * @param[in] outer_ring_winding: the ordering of the outer ring of polygon vertices; clockwise or - * counter-clockwise - * @param[in] mr: Optional, The resource to use to allocate the returned data - * - * @return Vector of 4 columns representing one or more polygons: - * - Column 0, INT32: beginning index of the first ring in each polygon - * - Column 1, INT32: beginning index of the first point in each ring - * - Column 2, FLOAT64: x component of polygon points - * - Column 3, FLOAT64: y component of polygon points - * - * @note The number of polygons is equal to the length of the first column - * - **/ -[[deprecated( - "Use Python libraries or GDAL to load shapefiles.")]] std::vector> -read_polygon_shapefile( - std::string const& filename, - const winding_order outer_ring_winding = winding_order::COUNTER_CLOCKWISE, - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -/** - * @} // end of doxygen group - */ - -} // namespace cuspatial diff --git a/cpp/include/doxygen_groups.h b/cpp/include/doxygen_groups.h index 3c16b0a6c..7aee576fb 100644 --- a/cpp/include/doxygen_groups.h +++ b/cpp/include/doxygen_groups.h @@ -123,11 +123,6 @@ * @file multipoint_range.cuh * @file multilinestring_range.cuh * @} - * @defgroup io I/O - * @{ - * @brief APIs for spatial data I/O - * @file shapefile_reader.hpp - * @} * @defgroup exception Exception * @{ * @brief cuSpatial exception types diff --git a/cpp/src/io/shp/polygon_shapefile_reader.cpp b/cpp/src/io/shp/polygon_shapefile_reader.cpp deleted file mode 100644 index bdb6f746d..000000000 --- a/cpp/src/io/shp/polygon_shapefile_reader.cpp +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -#include - -#include -#include -#include - -namespace { - -// TODO: a better approach is to precompute the total size needed by xs and ys -// as a preprocess and preallocate xs, ys. `read_ring` should simply accept -// an iterator range and return the pass-the-end iterator. -cudf::size_type read_ring(OGRLinearRing const& ring, - std::vector& xs, - std::vector& ys, - const cuspatial::winding_order ring_order) -{ - auto num_vertices = ring.getNumPoints(); - auto prev_num_vertices = xs.size(); - xs.resize(xs.size() + num_vertices); - ys.resize(ys.size() + num_vertices); - - auto output_it = thrust::make_zip_iterator( - thrust::make_tuple(xs.begin() + prev_num_vertices, ys.begin() + prev_num_vertices)); - if (ring_order == cuspatial::winding_order::COUNTER_CLOCKWISE) { - auto ring_it = cuspatial::detail::make_counting_transform_iterator( - 0, [&ring](auto i) { return thrust::make_tuple(ring.getX(i), ring.getY(i)); }); - - std::copy_n(ring_it, num_vertices, output_it); - } else { - auto ring_it = - cuspatial::detail::make_counting_transform_iterator(0, [&ring, &num_vertices](auto i) { - return thrust::make_tuple(ring.getX(num_vertices - i - 1), ring.getY(num_vertices - i - 1)); - }); - std::copy_n(ring_it, num_vertices, output_it); - } - - return num_vertices; -} - -cudf::size_type read_polygon(OGRPolygon const& polygon, - std::vector& ring_lengths, - std::vector& xs, - std::vector& ys, - const cuspatial::winding_order ring_order) -{ - auto num_vertices = read_ring(*(polygon.getExteriorRing()), xs, ys, ring_order); - ring_lengths.push_back(num_vertices); - - cudf::size_type num_interior_rings = polygon.getNumInteriorRings(); - - for (cudf::size_type i = 0; i < num_interior_rings; i++) { - auto num_vertices = read_ring(*(polygon.getInteriorRing(i)), xs, ys, ring_order); - ring_lengths.push_back(num_vertices); - } - - return 1 + num_interior_rings; -} - -cudf::size_type read_geometry_feature(OGRGeometry const* geometry, - std::vector& ring_lengths, - std::vector& xs, - std::vector& ys, - const cuspatial::winding_order ring_order) -{ - OGRwkbGeometryType geometry_type = wkbFlatten(geometry->getGeometryType()); - - if (geometry_type == wkbPolygon) { - auto polygon = dynamic_cast(geometry); - if (polygon == nullptr) { CUSPATIAL_FAIL("Can't cast `wkbPolygon` to `OGRPolygon&`"); } - return read_polygon(*polygon, ring_lengths, xs, ys, ring_order); - } - - if (geometry_type == wkbMultiPolygon || geometry_type == wkbGeometryCollection) { - auto* geometry_collection = (OGRGeometryCollection*)geometry; - - int num_rings = 0; - - for (int i = 0; i < geometry_collection->getNumGeometries(); i++) { - num_rings += read_geometry_feature( - geometry_collection->getGeometryRef(i), ring_lengths, xs, ys, ring_order); - } - - return num_rings; - } - - CUSPATIAL_FAIL("Shapefile reader supports polygon geometry only"); -} - -cudf::size_type read_layer(const OGRLayerH layer, - std::vector& feature_lengths, - std::vector& ring_lengths, - std::vector& xs, - std::vector& ys, - const cuspatial::winding_order ring_order) -{ - cudf::size_type num_features = 0; - - OGR_L_ResetReading(layer); - - OGRFeatureH feature; - - while ((feature = OGR_L_GetNextFeature(layer)) != nullptr) { - auto geometry = static_cast(OGR_F_GetGeometryRef(feature)); - - CUSPATIAL_EXPECTS(geometry != nullptr, "Invalid Shape"); - - auto num_rings = read_geometry_feature(geometry, ring_lengths, xs, ys, ring_order); - - feature_lengths.push_back(num_rings); - - OGR_F_Destroy(feature); - - num_features++; - } - - return num_features; -} - -} // namespace - -namespace cuspatial { -namespace detail { - -std::tuple, - std::vector, - std::vector, - std::vector> -read_polygon_shapefile(std::string const& filename, cuspatial::winding_order outer_ring_winding) -{ - GDALAllRegister(); - - GDALDatasetH dataset = GDALOpenEx(filename.c_str(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr); - - CUSPATIAL_EXPECTS(dataset != nullptr, "ESRI Shapefile: Failed to open file"); - - OGRLayerH dataset_layer = GDALDatasetGetLayer(dataset, 0); - - CUSPATIAL_EXPECTS(dataset_layer != nullptr, "ESRI Shapefile: Failed to read first layer"); - - std::vector feature_lengths; - std::vector ring_lengths; - std::vector xs; - std::vector ys; - - read_layer(dataset_layer, feature_lengths, ring_lengths, xs, ys, outer_ring_winding); - feature_lengths.shrink_to_fit(); - ring_lengths.shrink_to_fit(); - xs.shrink_to_fit(); - ys.shrink_to_fit(); - - return std::make_tuple( - std::move(feature_lengths), std::move(ring_lengths), std::move(xs), std::move(ys)); -} - -} // namespace detail -} // namespace cuspatial diff --git a/cpp/src/io/shp/polygon_shapefile_reader.cu b/cpp/src/io/shp/polygon_shapefile_reader.cu deleted file mode 100644 index 5972ea2b1..000000000 --- a/cpp/src/io/shp/polygon_shapefile_reader.cu +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2020, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include - -namespace cuspatial { -namespace detail { - -template -std::unique_ptr make_column(std::vector source, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - auto tid = cudf::type_to_id(); - auto type = cudf::data_type{tid}; - auto buffer = rmm::device_buffer(source.data(), sizeof(T) * source.size(), stream, mr); - return std::make_unique(type, source.size(), std::move(buffer)); -} - -std::tuple, - std::vector, - std::vector, - std::vector> -read_polygon_shapefile(std::string const& filename, cuspatial::winding_order outer_ring_winding); - -std::vector> read_polygon_shapefile( - std::string const& filename, - cuspatial::winding_order outer_ring_winding, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUSPATIAL_EXPECTS(not filename.empty(), "Filename cannot be empty."); - - auto poly_vectors = detail::read_polygon_shapefile(filename, outer_ring_winding); - - auto polygon_offsets = make_column(std::get<0>(poly_vectors), stream, mr); - auto ring_offsets = make_column(std::get<1>(poly_vectors), stream, mr); - auto xs = make_column(std::get<2>(poly_vectors), stream, mr); - auto ys = make_column(std::get<3>(poly_vectors), stream, mr); - - // transform polygon lengths to polygon offsets - thrust::exclusive_scan(rmm::exec_policy(stream), - polygon_offsets->view().begin(), - polygon_offsets->view().end(), - polygon_offsets->mutable_view().begin()); - - // transform ring lengths to ring offsets - thrust::exclusive_scan(rmm::exec_policy(stream), - ring_offsets->view().begin(), - ring_offsets->view().end(), - ring_offsets->mutable_view().begin()); - - std::vector> poly_columns{}; - poly_columns.reserve(4); - poly_columns.push_back(std::move(polygon_offsets)); - poly_columns.push_back(std::move(ring_offsets)); - poly_columns.push_back(std::move(xs)); - poly_columns.push_back(std::move(ys)); - return poly_columns; -} - -} // namespace detail - -std::vector> read_polygon_shapefile( - std::string const& filename, - cuspatial::winding_order outer_ring_winding, - rmm::mr::device_memory_resource* mr) -{ - return detail::read_polygon_shapefile(filename, outer_ring_winding, rmm::cuda_stream_default, mr); -} - -} // namespace cuspatial diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index ac20558b9..b29d2b6c7 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -17,6 +17,14 @@ ################################################################################################### # - compiler function ----------------------------------------------------------------------------- +# find gdal +rapids_find_package( + GDAL REQUIRED + GLOBAL_TARGETS GDAL::GDAL + BUILD_EXPORT_SET cuspatial-exports + INSTALL_EXPORT_SET cuspatial-exports +) + function(ConfigureTest CMAKE_TEST_NAME) add_executable(${CMAKE_TEST_NAME} ${ARGN}) target_compile_options(${CMAKE_TEST_NAME} @@ -30,7 +38,7 @@ function(ConfigureTest CMAKE_TEST_NAME) PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" INSTALL_RPATH "\$ORIGIN/../../../lib" ) - target_link_libraries(${CMAKE_TEST_NAME} GTest::gtest_main GTest::gmock_main cudf::cudftestutil cuspatial) + target_link_libraries(${CMAKE_TEST_NAME} GDAL::GDAL GTest::gtest_main GTest::gmock_main cudf::cudftestutil cuspatial) add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME}) install( TARGETS ${CMAKE_TEST_NAME} @@ -92,9 +100,6 @@ ConfigureTest(LINESTRING_DISTANCE_TEST ConfigureTest(POINT_LINESTRING_NEAREST_POINT_TEST spatial/point_linestring_nearest_points_test.cpp) -ConfigureTest(SHAPEFILE_READER_TEST - io/shp/polygon_shapefile_reader_test.cpp) - ConfigureTest(QUADTREE_POLYGON_FILTERING_TEST join/quadtree_polygon_filtering_test.cu) diff --git a/cpp/tests/io/shp/polygon_shapefile_reader_test.cpp b/cpp/tests/io/shp/polygon_shapefile_reader_test.cpp deleted file mode 100644 index 5cbd418f9..000000000 --- a/cpp/tests/io/shp/polygon_shapefile_reader_test.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2020, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include - -using namespace cudf::test; - -constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; - -std::string get_shapefile_path(std::string filename) -{ - const char* cuspatial_home = std::getenv("CUSPATIAL_HOME"); - CUSPATIAL_EXPECTS(cuspatial_home != nullptr, "CUSPATIAL_HOME environmental variable must be set"); - return std::string(cuspatial_home) + std::string("/test_fixtures/shapefiles/") + - std::string(filename); -} - -template -using wrapper = fixed_width_column_wrapper; - -void test(std::string const& shapefile_name, - std::vector poly_offsets, - std::vector ring_offsets, - std::vector xs, - std::vector ys) -{ - auto shape_filename = get_shapefile_path(shapefile_name); - auto polygon_columns = - cuspatial::read_polygon_shapefile(shape_filename, cuspatial::winding_order::COUNTER_CLOCKWISE); - - auto expected_poly_offsets = wrapper(poly_offsets.begin(), poly_offsets.end()); - auto expected_ring_offsets = wrapper(ring_offsets.begin(), ring_offsets.end()); - auto expected_poly_point_xs = wrapper(xs.begin(), xs.end()); - auto expected_poly_point_ys = wrapper(ys.begin(), ys.end()); - - expect_columns_equivalent(expected_poly_offsets, polygon_columns.at(0)->view(), verbosity); - expect_columns_equivalent(expected_ring_offsets, polygon_columns.at(1)->view(), verbosity); - expect_columns_equivalent(expected_poly_point_xs, polygon_columns.at(2)->view(), verbosity); - expect_columns_equivalent(expected_poly_point_ys, polygon_columns.at(3)->view(), verbosity); -} - -void test_reverse(std::string const& shapefile_name, - std::vector poly_offsets, - std::vector ring_offsets, - std::vector xs, - std::vector ys) -{ - auto shape_filename = get_shapefile_path(shapefile_name); - auto polygon_columns = - cuspatial::read_polygon_shapefile(shape_filename, cuspatial::winding_order::CLOCKWISE); - - auto expected_poly_offsets = wrapper(poly_offsets.begin(), poly_offsets.end()); - auto expected_ring_offsets = wrapper(ring_offsets.begin(), ring_offsets.end()); - auto expected_poly_point_xs = wrapper(xs.begin(), xs.end()); - auto expected_poly_point_ys = wrapper(ys.begin(), ys.end()); - - expect_columns_equivalent(expected_poly_offsets, polygon_columns.at(0)->view(), verbosity); - expect_columns_equivalent(expected_ring_offsets, polygon_columns.at(1)->view(), verbosity); - expect_columns_equivalent(expected_poly_point_xs, polygon_columns.at(2)->view(), verbosity); - expect_columns_equivalent(expected_poly_point_ys, polygon_columns.at(3)->view(), verbosity); -} - -struct PolygonShapefileReaderTest : public BaseFixture { -}; - -TEST_F(PolygonShapefileReaderTest, NonExistentFile) -{ - auto shape_filename = get_shapefile_path("non_exist.shp"); - EXPECT_THROW(cuspatial::read_polygon_shapefile(shape_filename), cuspatial::logic_error); -} - -TEST_F(PolygonShapefileReaderTest, ZeroPolygons) { test("empty_poly.shp", {}, {}, {}, {}); } - -TEST_F(PolygonShapefileReaderTest, OnePolygonReversed) -{ - test_reverse("one_poly.shp", {0}, {0}, {-10, 5, 5, -10, -10}, {-10, -10, 5, 5, -10}); -} - -TEST_F(PolygonShapefileReaderTest, OnePolygon) -{ - test("one_poly.shp", {0}, {0}, {-10, -10, 5, 5, -10}, {-10, 5, 5, -10, -10}); -} - -TEST_F(PolygonShapefileReaderTest, TwoPolygons) -{ - test("two_polys.shp", - {0, 1}, - {0, 5}, - {-10, -10, 5, 5, -10, 0, 0, 10, 10, 0}, - {-10, 5, 5, -10, -10, 0, 10, 10, 0, 0}); -} - -TEST_F(PolygonShapefileReaderTest, OnePointInPolygon) -{ - auto shape_filename = get_shapefile_path("one_poly.shp"); - auto polygon_columns = cuspatial::read_polygon_shapefile(shape_filename); - - auto polygons = polygon_columns.at(0)->view(); - auto rings = polygon_columns.at(1)->view(); - auto xs = polygon_columns.at(2)->view(); - auto ys = polygon_columns.at(3)->view(); - fixed_width_column_wrapper test_xs({0.0}); - fixed_width_column_wrapper test_ys({0.0}); - fixed_width_column_wrapper expected({true}); - - auto ret = cuspatial::point_in_polygon(test_xs, test_ys, polygons, rings, xs, ys); - - expect_columns_equivalent(ret->view(), expected); -} diff --git a/dependencies.yaml b/dependencies.yaml index 911ed3978..072f87062 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -42,7 +42,6 @@ dependencies: - &cmake_ver cmake>=3.23.1,!=3.25.0 - c-compiler - cxx-compiler - - gdal>3.5.0,<3.6.0 - gmock=1.10.0 - gtest=1.10.0 - libcudf=23.04 diff --git a/docs/source/api_docs/io.rst b/docs/source/api_docs/io.rst index 2e112580b..1da899090 100644 --- a/docs/source/api_docs/io.rst +++ b/docs/source/api_docs/io.rst @@ -1,12 +1,8 @@ IO -- -cuSpatial offers limited shapefile reading (polygons only), but this is deprecated and will be -removed in a future release. - Any host-side GeoPandas DataFrame can be copied into GPU memory for use with cuSpatial algorithms. .. currentmodule:: cuspatial -.. autofunction:: cuspatial.read_polygon_shapefile .. autofunction:: cuspatial.from_geopandas diff --git a/docs/source/user_guide/cuspatial_api_examples.ipynb b/docs/source/user_guide/cuspatial_api_examples.ipynb index e2a26e9de..f57e98b8d 100644 --- a/docs/source/user_guide/cuspatial_api_examples.ipynb +++ b/docs/source/user_guide/cuspatial_api_examples.ipynb @@ -49,13 +49,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "7265f9d2-9203-4da2-bbb2-b35c7f933641", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [], "source": [ "# !conda create -n rapids-22.08 -c rapidsai -c conda-forge -c nvidia \\ \n", @@ -93,39 +89,11 @@ "https://geopandas.org/en/stable/docs/reference/api/geopandas.GeoSeries.html)." ] }, - { - "cell_type": "markdown", - "id": "4b1251d1-558a-4899-8e7a-8066db0ad091", - "metadata": {}, - "source": [ - "## Input / Output\n", - "\n", - "The primary method of loading features into cuSpatial is using [cuspatial.from_geopandas](\n", - "https://docs.rapids.ai/api/cuspatial/stable/api_docs/io.html?highlight=from_geopandas#cuspatial.from_geopandas).\n", - "\n", - "One can also create feature geometries directly using any Python buffer that supports \n", - "`__array_interface__` for coordinates and their feature offsets.\n", - "\n", - "Note: cuSpatial's built-in but limited shapefile loading support is now deprecated and will be\n", - "removed in a future release.\n", - "\n", - "### [cuspatial.read_polygon_shapefile](https://docs.rapids.ai/api/cuspatial/stable/api_docs/io.html#cuspatial.read_polygon_shapefile) \n", - "\n", - "`cuspatial.read_polygon_shapefile` loads a `Polygon`-only shapefile from disk. It uses GPU acceleration and \n", - "can read hundreds of megabytes of `Polygon` information in milliseconds.\n", - "\n", - "Examples of cuSpatial's I/O functionality in the below cells:" - ] - }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "88d05bb9-c924-4d0b-8736-cd5183602d76", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Imports used throughout this notebook.\n", @@ -137,96 +105,18 @@ "from shapely.geometry import *" ] }, - { - "cell_type": "code", - "execution_count": 3, - "id": "e8f38524-e89f-4d86-b58a-63057cab6b60", - "metadata": { - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 0\n", - "1 3\n", - "2 4\n", - "3 5\n", - "4 35\n", - "Name: f_pos, dtype: int32\n", - "0 0\n", - "1 8\n", - "2 17\n", - "3 22\n", - "4 74\n", - "Name: r_pos, dtype: int32\n", - " x y\n", - "0 180.000000 -16.067133\n", - "1 180.000000 -16.555217\n", - "2 179.364143 -16.801354\n", - "3 178.725059 -17.012042\n", - "4 178.596839 -16.639150\n" - ] - } - ], - "source": [ - "# The following formula provides easy to access data for this demo notebook, you \n", - "# will see it again.\n", - "host_dataframe = geopandas.read_file(geopandas.datasets.get_path(\n", - " \"naturalearth_lowres\"\n", - "))\n", - "# Write the file to a shapefile.\n", - "host_dataframe[\"geometry\"].to_file(\"test_file\")\n", - "\n", - "geometry_offsets, ring_offsets, points = cuspatial.read_polygon_shapefile(\n", - " \"test_file\"\n", - ")\n", - "print(geometry_offsets.head())\n", - "print(ring_offsets.head())\n", - "print(points.head())" - ] - }, { "cell_type": "markdown", - "id": "4281cb89-3a31-4b99-9dc4-be4f8b0fde69", + "id": "4b1251d1-558a-4899-8e7a-8066db0ad091", "metadata": {}, "source": [ - "The result of `cuspatial.read_polygon_shapefile` is a `Tuple` of GeoArrow buffers that can be \n", - "converted into a `cuspatial.GeoSeries` or used directly with other interface methods. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "614d508f-0e6b-4f46-ae58-1877fa1c91d1", - "metadata": { - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0 POLYGON ((180.00000 -16.06713, 180.00000 -16.5...\n", - "1 POLYGON ((33.90371 -0.95000, 34.07262 -1.05982...\n", - "2 POLYGON ((-8.66559 27.65643, -8.66512 27.58948...\n", - "3 POLYGON ((-122.84000 49.00000, -122.97421 49.0...\n", - "4 POLYGON ((-122.84000 49.00000, -120.00000 49.0...\n", - "dtype: geometry" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "gpu_series = cuspatial.GeoSeries((geometry_offsets, ring_offsets, points))\n", - "gpu_series.head()" + "## Input / Output\n", + "\n", + "The primary method of loading features into cuSpatial is using [cuspatial.from_geopandas](\n", + "https://docs.rapids.ai/api/cuspatial/stable/api_docs/io.html?highlight=from_geopandas#cuspatial.from_geopandas).\n", + "\n", + "One can also create feature geometries directly using any Python buffer that supports \n", + "`__array_interface__` for coordinates and their feature offsets." ] }, { @@ -236,19 +126,14 @@ "source": [ "### [cuspatial.from_geopandas](https://docs.rapids.ai/api/cuspatial/stable/api_docs/io.html?highlight=from_geopandas#cuspatial.from_geopandas)\n", "\n", - "If you need other geometry types, the easiest way to get data into cuSpatial is via\n", - "`cuspatial.from_geopandas`." + "The easiest way to get data into cuSpatial is via `cuspatial.from_geopandas`." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "255fbfbe-8be1-498c-9a26-f4a3f31bdded", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -304,13 +189,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "956451e2-a520-441d-a939-575ed179917b", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -370,19 +251,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "cd8d1c39-b44f-4d06-9e11-04c2dca4cf15", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/svg+xml": [ - "" - ], + "image/svg+xml": "", "text/plain": [ "" ] @@ -422,13 +297,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "cb5acdad-53aa-418f-9948-8445515bd2b2", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -488,13 +359,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "03b75847-090d-40f8-8147-cb10b900d6ec", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -556,13 +423,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "452f60cb-28cc-4ad8-8aa2-9d73e3d56ec6", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -601,13 +464,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9266aac4-f925-4fb7-b287-5f0b795d5756", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -649,13 +508,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "15b5bb38-702f-4360-b48c-2e49ffd650d7", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -698,13 +553,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "a7a870dd-c0ae-41c1-a66c-cff4bd2db0ec", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -766,13 +617,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "e75b0352-0f80-404d-a113-f301601cd5a3", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -833,13 +680,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "70f66319-c4d2-4a93-ab98-0debcce4a719", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "data": { @@ -898,13 +741,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "35dfb7c9-1914-488a-b22e-8d0067ea7a8b", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -958,13 +797,9 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "40e9a41e-21af-47cc-a142-b19a67941f7f", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1031,13 +866,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "d1ade9da-c9e2-45c4-9685-dffeda3fd358", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1099,13 +930,9 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "bf7b2256", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "data": { @@ -1206,13 +1033,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "e3a0a9a3-0bdd-4f05-bcb5-7db4b99a44a3", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1272,13 +1095,9 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "023bd25a-35be-435d-ab0b-ecbd7a47e147", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1345,13 +1164,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "5a2ca191-b1a4-49fe-8deb-9a4c133b1ac6", - "metadata": { - "vscode": { - "languageId": "python" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", diff --git a/python/cuspatial/benchmarks/api/bench_api.py b/python/cuspatial/benchmarks/api/bench_api.py index b50f54b7a..161057aac 100644 --- a/python/cuspatial/benchmarks/api/bench_api.py +++ b/python/cuspatial/benchmarks/api/bench_api.py @@ -8,15 +8,6 @@ import cuspatial -def bench_io_read_polygon_shapefile(benchmark, shapefile): - benchmark(cuspatial.read_polygon_shapefile, shapefile) - - -def bench_io_geoseries_from_offsets(benchmark, shapefile): - shapefile_data = cuspatial.read_polygon_shapefile(shapefile) - benchmark(cuspatial.core.geoseries.GeoSeries, shapefile_data) - - def bench_io_from_geopandas(benchmark, host_dataframe): benchmark(cuspatial.from_geopandas, host_dataframe) diff --git a/python/cuspatial/cuspatial/__init__.py b/python/cuspatial/cuspatial/__init__.py index 1e73c0c9e..492cb73c0 100644 --- a/python/cuspatial/cuspatial/__init__.py +++ b/python/cuspatial/cuspatial/__init__.py @@ -26,7 +26,6 @@ trajectory_distances_and_speeds, ) from .io.geopandas import from_geopandas -from .io.shapefile import read_polygon_shapefile __version__ = get_versions()["version"] del get_versions diff --git a/python/cuspatial/cuspatial/_lib/CMakeLists.txt b/python/cuspatial/cuspatial/_lib/CMakeLists.txt index 375aa3f12..5d362a089 100644 --- a/python/cuspatial/cuspatial/_lib/CMakeLists.txt +++ b/python/cuspatial/cuspatial/_lib/CMakeLists.txt @@ -22,7 +22,6 @@ set(cython_sources polygon_bounding_boxes.pyx linestring_bounding_boxes.pyx quadtree.pyx - shapefile_reader.pyx spatial.pyx spatial_join.pyx spatial_window.pyx diff --git a/python/cuspatial/cuspatial/_lib/cpp/shapefile_reader.pxd b/python/cuspatial/cuspatial/_lib/cpp/shapefile_reader.pxd deleted file mode 100644 index 5cfb2232b..000000000 --- a/python/cuspatial/cuspatial/_lib/cpp/shapefile_reader.pxd +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. - -from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string -from libcpp.vector cimport vector - -from cudf._lib.cpp.column.column cimport column - -ctypedef bool winding_order_type_t - -cdef extern from "cuspatial/shapefile_reader.hpp" namespace "cuspatial" nogil: - cdef vector[unique_ptr[column]] \ - read_polygon_shapefile( - const string filename, winding_order outer_ring_winding - ) except + - ctypedef enum winding_order "cuspatial::winding_order": - COUNTER_CLOCKWISE "cuspatial::winding_order::COUNTER_CLOCKWISE", - CLOCKWISE "cuspatial::winding_order::CLOCKWISE" diff --git a/python/cuspatial/cuspatial/_lib/shapefile_reader.pyx b/python/cuspatial/cuspatial/_lib/shapefile_reader.pyx deleted file mode 100644 index 5592eb734..000000000 --- a/python/cuspatial/cuspatial/_lib/shapefile_reader.pyx +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2019-2020, NVIDIA CORPORATION. - -from enum import IntEnum - -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string -from libcpp.utility cimport move -from libcpp.vector cimport vector - -from cudf._lib.column cimport Column, column - -from cuspatial._lib.cpp.shapefile_reader cimport ( - read_polygon_shapefile as cpp_read_polygon_shapefile, - winding_order, - winding_order_type_t, -) - - -class WindingOrder(IntEnum): - COUNTER_CLOCKWISE = winding_order.COUNTER_CLOCKWISE, - CLOCKWISE = winding_order.CLOCKWISE - - -cpdef read_polygon_shapefile(object filepath, object winding): - cdef string c_string = str(filepath).encode() - cdef vector[unique_ptr[column]] c_result - cdef winding_order winding_value = ( - (winding.value) - ) - with nogil: - c_result = move(cpp_read_polygon_shapefile(c_string, winding_value)) - return ( - Column.from_unique_ptr(move(c_result[0])), - Column.from_unique_ptr(move(c_result[1])), - Column.from_unique_ptr(move(c_result[2])), - Column.from_unique_ptr(move(c_result[3])), - ) diff --git a/python/cuspatial/cuspatial/core/_column/geocolumn.py b/python/cuspatial/cuspatial/core/_column/geocolumn.py index 371932703..3c3fe211a 100644 --- a/python/cuspatial/cuspatial/core/_column/geocolumn.py +++ b/python/cuspatial/cuspatial/core/_column/geocolumn.py @@ -32,90 +32,9 @@ def __init__( self, data: Tuple, meta: GeoMeta = None, - shuffle_order: cudf.Index = None, - from_read_polygon_shapefile=False, + shuffle_order: cudf.Index = None ): - if from_read_polygon_shapefile: - """ - A cudf.ListSeries needs four levels of nesting to represent - a polygon shapefile. The rings_offsets and polygons_offsets - have already been computed in the `read_polygon_shapefile` - function. In order to convert it into an arrow list<...> - we need a set of offsets buffers for each point tuple, and - an offsets buffer for the 1-offset multipolygons. - - Coordinates: List of length 2 offsets: [0, 2, 4, ... n/2] - Rings: List of polygon ring offsets - Polygons: Offset into rings of each polygon - Multipolygons: List of length 1 offsets: No multipolygons - - Finally, each set of offsets must have the length of the - array appended to the end, as Arrow offset lists are length - n + 1 but our original shapefile code offset lists are only - length n. - """ - polygons_col = data[0].astype("int32") - rings_col = data[1].astype("int32") - coordinates = ( - data[2].stack().astype("float64").reset_index(drop=True) - ) - """ - Store a fixed-size offsets buffer of even numbers: - 0 0 - 1 2 - 2 4 - ... - Up to the size of the original input. - """ - coordinate_offsets = as_column( - cp.arange(len(coordinates) + 1, step=2), dtype="int32" - ) - rings_offsets = cudf.concat( - [ - cudf.Series(rings_col), - cudf.Series([len(coordinate_offsets) - 1], dtype="int32"), - ] - ).reset_index(drop=True) - polygons_offsets = cudf.concat( - [ - cudf.Series(polygons_col), - cudf.Series([len(polygons_col)], dtype="int32"), - ] - ).reset_index(drop=True) - coords = cudf.core.column.ListColumn( - size=len(coordinate_offsets) - 1, - dtype=cudf.ListDtype(coordinates.dtype), - children=(coordinate_offsets, coordinates._column), - ) - rings = cudf.core.column.ListColumn( - size=len(rings_offsets) - 1, - dtype=cudf.ListDtype(coords.dtype), - children=(rings_offsets._column, coords), - ) - polygons = cudf.core.column.ListColumn( - size=len(polygons_offsets) - 1, - dtype=cudf.ListDtype(rings.dtype), - children=(polygons_offsets._column, rings), - ) - mpolygons = cudf.core.column.ListColumn( - size=len(polygons_offsets) - 1, - dtype=cudf.ListDtype(polygons.dtype), - children=( - as_column(cp.arange(len(polygons) + 1), dtype="int32"), - polygons, - ), - ) - self.points = cudf.Series([]) - self.points.name = "points" - self.mpoints = cudf.Series([]) - self.mpoints.name = "mpoints" - self.lines = cudf.Series([]) - self.lines.name = "lines" - self.polygons = cudf.Series(mpolygons) - self.polygons.name = "polygons" - self._meta = meta - - elif ( + if ( isinstance(data[0], cudf.Series) and isinstance(data[1], cudf.Series) and isinstance(data[2], cudf.Series) diff --git a/python/cuspatial/cuspatial/core/geoseries.py b/python/cuspatial/cuspatial/core/geoseries.py index 0753d57bb..cc1c04231 100644 --- a/python/cuspatial/cuspatial/core/geoseries.py +++ b/python/cuspatial/cuspatial/core/geoseries.py @@ -81,30 +81,6 @@ def __init__( adapter = GeoPandasReader(data) pandas_meta = GeoMeta(adapter.get_geopandas_meta()) column = GeoColumn(adapter._get_geotuple(), pandas_meta) - elif isinstance(data, Tuple): - # This must be a Polygon Tuple returned by - # cuspatial.read_polygon_shapefile - # TODO: If an index is passed in, it needs to be reflected - # in the column, because otherwise .iloc indexing will ignore - # it. - column = GeoColumn( - data, - GeoMeta( - { - "input_types": cp.repeat( - cp.array( - [Feature_Enum.POLYGON.value], dtype="int8" - ), - len(data[0]), - ), - "union_offsets": cp.arange( - len(data[0]), dtype="int32" - ), - } - ), - from_read_polygon_shapefile=True, - ) - else: raise TypeError( f"Incompatible object passed to GeoSeries ctor {type(data)}" diff --git a/python/cuspatial/cuspatial/io/shapefile.py b/python/cuspatial/cuspatial/io/shapefile.py deleted file mode 100644 index 98c1894a2..000000000 --- a/python/cuspatial/cuspatial/io/shapefile.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. - -import warnings - -from cudf import DataFrame, Series - -from cuspatial._lib.shapefile_reader import ( - WindingOrder, - read_polygon_shapefile as cpp_read_polygon_shapefile, -) - - -def read_polygon_shapefile( - filename, outer_ring_order=WindingOrder.COUNTER_CLOCKWISE -): - """ - Reads polygon geometry from an ESRI shapefile into GPU memory. - - Parameters - ---------- - filename : str, pathlike - ESRI Shapefile file path (usually ends in ``.shp``) - winding_order : WindingOrder(Enum) - COUNTER_CLOCKWISE: ESRI Format, or CLOCKWISE: Simple Feature - - Returns - ------- - result : tuple (cudf.Series, cudf.Series, cudf.DataFrame) - poly_offsets : cudf.Series(dtype=np.int32) - Offsets of the first ring in each polygon - ring_offsets : cudf.Series(dtype=np.int32) - Offsets of the first point in each ring - points : cudf.DataFrame - DataFrame of all points in the shapefile - x : cudf.Series(dtype=np.float64) - x-components of each polygon's points - y : cudf.Series(dtype=np.float64) - y-components of each polygon's points - - Notes - ----- - This function is deprecated and will be removed in a future release. - """ - warning_msg = ( - "read_polygon_shapefile is deprecated and will be removed in a " - "future release. Polygon data can be loaded using other libraries " - "such as GeoPandas or PyShp." - ) - warnings.warn(warning_msg, DeprecationWarning) - - result = cpp_read_polygon_shapefile(filename, outer_ring_order) - f_pos = Series(result[0], name="f_pos") - r_pos = Series(result[1], name="r_pos") - return (f_pos, r_pos, DataFrame({"x": result[2], "y": result[3]})) diff --git a/python/cuspatial/cuspatial/tests/test_geoseries.py b/python/cuspatial/cuspatial/tests/test_geoseries.py index bb4d01b40..3d94531b8 100644 --- a/python/cuspatial/cuspatial/tests/test_geoseries.py +++ b/python/cuspatial/cuspatial/tests/test_geoseries.py @@ -22,7 +22,6 @@ import cudf import cuspatial -from cuspatial.io.shapefile import WindingOrder np.random.seed(0) @@ -545,44 +544,6 @@ def test_construction_from_foreign_object(data): assert_geoseries_equal(cugs.to_geopandas(), gps) - -def test_shapefile_constructor(): - host_dataframe = gpd.read_file( - gpd.datasets.get_path("naturalearth_lowres") - ) - # Shapefile reader only works with Polygons so we drop - # all the MultiPolygons by slicing the first 19 out. - gs = host_dataframe["geometry"][ - host_dataframe["geometry"].type == "Polygon" - ][19:] - gs.to_file("naturalearth_lowres_polygon") - data = cuspatial.read_polygon_shapefile("naturalearth_lowres_polygon") - cus = cuspatial.GeoSeries(data) - - assert_eq_geo(gs.reset_index(drop=True), cus.to_geopandas()) - - -def test_shapefile_constructor_reversed(): - host_dataframe = gpd.read_file( - gpd.datasets.get_path("naturalearth_lowres") - ) - # Shapefile reader only works with Polygons so we drop - # all the MultiPolygons by slicing the first 19 out. - gs = host_dataframe["geometry"][ - host_dataframe["geometry"].type == "Polygon" - ][19:] - gs.to_file("naturalearth_lowres_polygon") - data = cuspatial.read_polygon_shapefile( - "naturalearth_lowres_polygon", outer_ring_order=WindingOrder.CLOCKWISE - ) - cus = cuspatial.GeoSeries(data) - - from shapely.geometry import polygon - - reversed_gs = gpd.GeoSeries([polygon.orient(p, 1) for p in gs]) - assert_eq_geo(reversed_gs, cus.to_geopandas()) - - def test_memory_usage_simple(gs): cugs = cuspatial.from_geopandas(gs) assert cugs.memory_usage() == 1616 diff --git a/python/cuspatial/cuspatial/tests/test_shapefile_reader.py b/python/cuspatial/cuspatial/tests/test_shapefile_reader.py deleted file mode 100644 index a799185a9..000000000 --- a/python/cuspatial/cuspatial/tests/test_shapefile_reader.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. - -import os - -import numpy as np -import pytest - -import cudf - -import cuspatial -from cuspatial.io.shapefile import WindingOrder - -shapefiles_path = os.path.join( - os.environ["CUSPATIAL_HOME"], "test_fixtures", "shapefiles" -) - - -def test_non_existent_file(): - with pytest.raises(RuntimeError): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - "non_exist.shp" - ) - - -def test_zero_polygons(): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - os.path.join(shapefiles_path, "empty_poly.shp") - ) - cudf.testing.assert_series_equal( - f_pos, cudf.Series(dtype=np.int32, name="f_pos") - ) - cudf.testing.assert_series_equal( - r_pos, cudf.Series(dtype=np.int32, name="r_pos") - ) - cudf.testing.assert_frame_equal( - points, - cudf.DataFrame( - { - "x": cudf.Series(dtype=np.float64), - "y": cudf.Series(dtype=np.float64), - } - ), - ) - - -def test_one_polygon_reversed(): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - os.path.join(shapefiles_path, "one_poly.shp"), - outer_ring_order=WindingOrder.CLOCKWISE, - ) - cudf.testing.assert_series_equal( - f_pos, cudf.Series([0], dtype=np.int32, name="f_pos") - ) - cudf.testing.assert_series_equal( - r_pos, cudf.Series([0], dtype=np.int32, name="r_pos") - ) - cudf.testing.assert_frame_equal( - points, - cudf.DataFrame( - { - "x": cudf.Series([-10, 5, 5, -10, -10], dtype=np.float64), - "y": cudf.Series([-10, -10, 5, 5, -10], dtype=np.float64), - } - ), - ) - - -def test_one_polygon(): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - os.path.join(shapefiles_path, "one_poly.shp") - ) - cudf.testing.assert_series_equal( - f_pos, cudf.Series([0], dtype=np.int32, name="f_pos") - ) - cudf.testing.assert_series_equal( - r_pos, cudf.Series([0], dtype=np.int32, name="r_pos") - ) - cudf.testing.assert_frame_equal( - points, - cudf.DataFrame( - { - "x": cudf.Series([-10, -10, 5, 5, -10], dtype=np.float64), - "y": cudf.Series([-10, 5, 5, -10, -10], dtype=np.float64), - } - ), - ) - - -def test_two_polygons_reversed(): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - os.path.join(shapefiles_path, "two_polys.shp"), - outer_ring_order=WindingOrder.CLOCKWISE, - ) - cudf.testing.assert_series_equal( - f_pos, cudf.Series([0, 1], dtype=np.int32, name="f_pos") - ) - cudf.testing.assert_series_equal( - r_pos, cudf.Series([0, 5], dtype=np.int32, name="r_pos") - ) - cudf.testing.assert_frame_equal( - points, - cudf.DataFrame( - { - "x": cudf.Series( - [-10, 5, 5, -10, -10, 0, 10, 10, 0, 0], dtype=np.float64 - ), - "y": cudf.Series( - [-10, -10, 5, 5, -10, 0, 0, 10, 10, 0], dtype=np.float64 - ), - } - ), - ) - - -def test_two_polygons(): - f_pos, r_pos, points = cuspatial.read_polygon_shapefile( - os.path.join(shapefiles_path, "two_polys.shp") - ) - cudf.testing.assert_series_equal( - f_pos, cudf.Series([0, 1], dtype=np.int32, name="f_pos") - ) - cudf.testing.assert_series_equal( - r_pos, cudf.Series([0, 5], dtype=np.int32, name="r_pos") - ) - cudf.testing.assert_frame_equal( - points, - cudf.DataFrame( - { - "x": cudf.Series( - [-10, -10, 5, 5, -10, 0, 0, 10, 10, 0], dtype=np.float64 - ), - "y": cudf.Series( - [-10, 5, 5, -10, -10, 0, 10, 10, 0, 0], dtype=np.float64 - ), - } - ), - ) From f4eb3bc57bb20d8bc21383e3dadb8dca746eb67a Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Wed, 15 Feb 2023 10:39:16 +1100 Subject: [PATCH 2/6] style --- python/cuspatial/cuspatial/core/_column/geocolumn.py | 2 +- python/cuspatial/cuspatial/tests/test_geoseries.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/cuspatial/cuspatial/core/_column/geocolumn.py b/python/cuspatial/cuspatial/core/_column/geocolumn.py index 28f233823..b17bfa483 100644 --- a/python/cuspatial/cuspatial/core/_column/geocolumn.py +++ b/python/cuspatial/cuspatial/core/_column/geocolumn.py @@ -32,7 +32,7 @@ def __init__( self, data: Tuple, meta: GeoMeta = None, - shuffle_order: cudf.Index = None + shuffle_order: cudf.Index = None, ): if ( isinstance(data[0], cudf.Series) diff --git a/python/cuspatial/cuspatial/tests/test_geoseries.py b/python/cuspatial/cuspatial/tests/test_geoseries.py index 3db7b9671..8f00e7f5b 100644 --- a/python/cuspatial/cuspatial/tests/test_geoseries.py +++ b/python/cuspatial/cuspatial/tests/test_geoseries.py @@ -544,6 +544,7 @@ def test_construction_from_foreign_object(data): assert_geoseries_equal(cugs.to_geopandas(), gps) + def test_memory_usage_simple(gs): cugs = cuspatial.from_geopandas(gs) assert cugs.memory_usage() == 1616 From cf5dd192fa25804a4c6f612a9e54c690b5ab1ee5 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Wed, 15 Feb 2023 16:12:38 +1100 Subject: [PATCH 3/6] Re-add gdal dependency for CI? --- conda/recipes/libcuspatial/conda_build_config.yaml | 3 +++ conda/recipes/libcuspatial/meta.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/conda/recipes/libcuspatial/conda_build_config.yaml b/conda/recipes/libcuspatial/conda_build_config.yaml index 99969d85b..5cc0c3ea7 100644 --- a/conda/recipes/libcuspatial/conda_build_config.yaml +++ b/conda/recipes/libcuspatial/conda_build_config.yaml @@ -10,6 +10,9 @@ cuda_compiler: cmake_version: - ">=3.23.1,!=3.25.0" +gdal_version: + - ">3.5.0,<3.6.0" + gtest_version: - "1.10.0" diff --git a/conda/recipes/libcuspatial/meta.yaml b/conda/recipes/libcuspatial/meta.yaml index c09201ac6..e6c6d04bf 100644 --- a/conda/recipes/libcuspatial/meta.yaml +++ b/conda/recipes/libcuspatial/meta.yaml @@ -38,6 +38,7 @@ requirements: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit ={{ cuda_version }} + - gdal {{ gdal_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - libcudf ={{ minor_version }} From 2ad0948d30008df45ac72964f5ddcaca947fc227 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Wed, 15 Feb 2023 10:18:30 -0500 Subject: [PATCH 4/6] add `gdal` as `run` dep for `libcuspatial-tests` --- conda/recipes/libcuspatial/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/libcuspatial/meta.yaml b/conda/recipes/libcuspatial/meta.yaml index e6c6d04bf..d9e363e52 100644 --- a/conda/recipes/libcuspatial/meta.yaml +++ b/conda/recipes/libcuspatial/meta.yaml @@ -90,5 +90,6 @@ outputs: run: - {{ pin_subpackage('libcuspatial', exact=True) }} - cudatoolkit {{ cuda_spec }} + - gdal {{ gdal_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} From 4b5419038420fb8ff648f0d9b08a41fe48074143 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Thu, 16 Feb 2023 10:26:48 +1100 Subject: [PATCH 5/6] Update README.md Co-authored-by: Bradley Dice --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12b47e74e..16ac2bbaa 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ conda env update --file conda/environments/all_cuda-115_arch-x86_64.yaml pre-generated polygon shapefiles that contain 0, 1 and 2 polygons, respectively. They are available at `$CUSPATIAL_HOME/test_fixtures/shapefiles`
-Python users can read any point/polyline/polygon data using existing python +Python users can read any point/polyline/polygon data using existing Python packages, e.g., [Shapely](https://pypi.org/project/Shapely/) and [Fiona](https://github.com/Toblerity/Fiona), to generate numpy arrays and feed them to [cuSpatial Python APIs](https://docs.rapids.ai/api/cuspatial/stable/). From 4d72fa2733dd9f5bc88d0cb52ee5f5c4443da203 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 21 Feb 2023 15:27:06 -0500 Subject: [PATCH 6/6] re-add `gdal` to `dependencies.yaml` `gdal` is still required for building `libcuspatial` tests, therefore it is necessary in `dependencies.yaml` --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 230e0cf55..634ecfb34 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,6 +13,7 @@ dependencies: - cxx-compiler - cython>=0.29,<0.30 - gcc_linux-64=9.* +- gdal>3.5.0,<3.6.0 - geopandas>=0.11.0 - gmock=1.10.0 - gtest=1.10.0 diff --git a/dependencies.yaml b/dependencies.yaml index 072f87062..911ed3978 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -42,6 +42,7 @@ dependencies: - &cmake_ver cmake>=3.23.1,!=3.25.0 - c-compiler - cxx-compiler + - gdal>3.5.0,<3.6.0 - gmock=1.10.0 - gtest=1.10.0 - libcudf=23.04