feat: replace PDAL with Laspy so that make dependencies simpler (#2)

HideBa · Feb 23, 2024 · ce94544 · ce94544
1 parent cbb1e05
commit ce94544
Show file tree

Hide file tree

Showing 17 changed files with 527 additions and 1,022 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,8 +8,6 @@ on:
       - "requirements.txt"
       - "requirements.dev.txt"
   pull_request:
-    branches:
-      - main
 
 jobs:
   ci:
@@ -24,49 +22,6 @@ jobs:
         with:
           python-version: 3.11.1
 
-      - name: Install PDAL
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-          g++ \
-          cmake \
-          git \
-          libgdal-dev \
-          libgeotiff-dev \
-          liblaszip-dev \
-          libboost-all-dev \
-          libboost-program-options-dev \
-          libboost-filesystem-dev \
-          libboost-system-dev \
-          libsqlite3-dev \
-          sqlite3 \
-          libproj-dev \
-          libcurl4-openssl-dev \
-          libjsoncpp-dev \
-          libxml2-dev \
-          libflann-dev \
-          libpcl-dev \
-          libeigen3-dev \
-          libnitro-dev \
-          libqhull-dev \
-          libtbb-dev
-      - name: Clone PDAL
-        run: |
-          git clone https://github.com/PDAL/PDAL.git
-          cd PDAL
-          git checkout 2.5-maintenance
-          mkdir build
-          cd build
-          cmake ..
-          make
-          sudo make install
-          sudo ldconfig
-          pdal --version
-      - name: Check pdal
-        run: |
-          echo "PDAL version -------------"
-          pdal --version
-
       - name: Cache pip packages
         uses: actions/cache@v2
         with:
@@ -84,36 +39,27 @@ jobs:
       - name: Install dependencies
         run: |
           poetry install
-      # - name: Install dependencies excluding PDAL
-      #   run: |
-      #     poetry add pdal --optional
-      #     poetry install --no-root
-
-      # - name: Install PDAL Python package
-      #   run: |
-      #     source .venv/bin/activate
-      #     pip install pdal==3.2.3
 
       - name: Linting
         run: |
-          poetry run pylint ./ahn_cli/**/*.py
+          make lint
 
       - name: Type checking
         run: |
-          poetry run mypy ./ahn_cli/**/*.py
+          make type
 
       - name: Testing
-        run: poetry run pytest ./ahn_cli/**/*.py --cov=./ahn_cli
+        run: make test
 
-      - name: Export requirements
-        run: |
-          poetry export --dev --format requirements.txt --output requirements.txt
-          poetry export --dev --format requirements.txt --output requirements.dev.txt
+      # - name: Export requirements
+      #   run: |
+      #     poetry export --dev --format requirements.txt --output requirements.txt
+      #     poetry export --dev --format requirements.txt --output requirements.dev.txt
 
-      - name: Commit exported requirements
-        run: |
-          git config --global user.name "${{ secrets.GH_USER_NAME }}"
-          git config --global user.email "${{ secrets.GH_USER_EMAIL }}"
-          git add requirements.txt requirements.dev.txt
-          git commit -m "Update requirements"
-          git push origin main
+      # - name: Commit exported requirements
+      #   run: |
+      #     git config --global user.name "${{ secrets.GH_USER_NAME }}"
+      #     git config --global user.email "${{ secrets.GH_USER_EMAIL }}"
+      #     git add requirements.txt requirements.dev.txt
+      #     git commit -m "Update requirements"
+      #     git push origin main
diff --git a/.gitignore b/.gitignore
@@ -164,3 +164,5 @@ cython_debug/
 *.qmd
 /testdata
 /out
+.polyscope.ini
+imgui.ini
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,11 @@
+# Changelog
+## [0.1.6] - 2024-02-23
+### Changed
+
+### Added
+This is the first release of AHN CLI. There are a couple of features which helps users to easily download AHN point cloud data they need.
+* Validation of user input
+* Multi-thread download to speed up downloading time
+* Rasterization of city polygon to reduce time complexity
+* Filter points out by parameters such as classification classes, decimate, bounding box, etc
+* Preview of downloaded data
diff --git a/README.md b/README.md
@@ -36,7 +36,8 @@ Options:
  -ncc, --no-clip-city          Avoid clipping the point cloud data to the city boundary.
  -cf, --clip-file <file>       Provide a file path for a clipping boundary file to clip
                                the point cloud data to a specified area.
- -r, --radius <radius>         Define a radius (in meters) to clip the point cloud data,
+ -e, --epsg <epsg>             Set the EPSG code for user's clip file.
+ -b, --bbox <bbox>             Specify a bounding box to clip the point cloud data. It should be comma-separated list with minx,miny,maxx,maxy
                                centered on the city polygon.
  -p, --preview                 Preview the point cloud data in a 3D viewer.
  -h, --help [category]         Show help information. Optionally specify a category for
@@ -70,9 +71,13 @@ ahn_cli -c delft -o ./delft.laz -i 1,2 -ncc
 ahn_cli -c delft -o ./delft.laz -i 1,2 -d 2
 ```
 
-**Specify a Radius for Clipping:**
+**Specify a Bounding box for clipping:**
+
+If you specify a `b`, it will clip the point cloud data with specified bounding box.
+```
+ahn_cli -c delft -o ./delft.laz -i 1,2 -d 2 -b 194198.302994,443461.343994,194594.109009,443694.838989
+```
 
-If you specify a `radius` (in meters), it will clip the point cloud data from the center of the city polygon to the specified radius.
 
 ## Reporting Issues
 

diff --git a/ahn_cli/fetcher/request.py b/ahn_cli/fetcher/request.py
@@ -7,6 +7,7 @@
 from urllib.parse import urlparse
 
 import requests
+from tqdm import tqdm
 
 from ahn_cli.fetcher.geotiles import ahn_subunit_indicies_of_city
 
@@ -23,23 +24,29 @@ def fetch(self) -> dict:
         logging.info("Start fetching AHN data")
         logging.info(f"Fetching {len(self.urls)} tiles")
 
-        def req(url: str, nth: int, results: dict, lock: Lock) -> None:
-            logging.info(f"Fetching tile {nth + 1}/{len(self.urls)}")
-            print(f"Fetching tile {nth + 1}/{len(self.urls)}")
+        def req(
+            url: str, nth: int, results: dict, lock: Lock, pbar: tqdm
+        ) -> None:
             res = requests.get(url, stream=True)
             with tempfile.NamedTemporaryFile(
                 delete=False, mode="w+b", suffix=".laz"
             ) as temp_file:
-                for chunk in res.iter_content(chunk_size=1024 * 1024):
+                for chunk in tqdm(
+                    res.iter_content(chunk_size=1024 * 1024),
+                    desc="writing a file",
+                ):
                     temp_file.write(chunk)
                 with lock:
                     results[url] = temp_file.name
+            pbar.update(1)
 
         results: dict = {}
         lock = threading.Lock()
-        with ThreadPoolExecutor(max_workers=8) as executor:
-            for i, url in enumerate(self.urls):
-                executor.submit(req, url, i, results, lock)
+        with tqdm(total=len(self.urls)) as pbar:
+            pbar.set_description("Fetching AHN data")
+            with ThreadPoolExecutor(max_workers=8) as executor:
+                for i, url in enumerate(self.urls):
+                    executor.submit(req, url, i, results, lock, pbar)
         return results
 
     def _check_valid_url(self, url: str) -> bool:

diff --git a/ahn_cli/kwargs.py b/ahn_cli/kwargs.py
@@ -8,6 +8,7 @@ class CLIArgs(TypedDict):
     exclude_class: str | None
     no_clip_city: bool
     clip_file: str | None
+    epsg: int | None
     decimate: int | None
-    radius: int | None
+    bbox: list[float] | None
     preview: bool
diff --git a/ahn_cli/main.py b/ahn_cli/main.py
@@ -19,8 +19,8 @@
  -ncc --no-clip-city           Do not clip the point cloud data to the city boundary.
  -cf, --clip-file <file>       Specify a file path to a clipping boundary file. The tool will
                                use this file to clip the point cloud data to a specific area.
+ -e, --epsg <epsg>             Set the EPSG code for user's clip file.
  -b, --bbox <bbox>             Specify a bounding box to clip the point cloud data. It should be comma-separated list with minx,miny,maxx,maxy
- -r, --radius <radius>         Set the radius of the circle to clip the point cloud data to. Unit is in meter.
  -p, --preview                 Preview the point cloud data in a 3D viewer.
  -h, --help [category]         Display help information. Optionally, specify a category to get
                                more detailed help for a specific command.
@@ -29,7 +29,7 @@
 
 
 @click.command()
-@click.version_option(version="0.1.5", prog_name="ahn_cli")
+@click.version_option(version="0.1.6", prog_name="ahn_cli")
 @click.option(
     "-o",
     "--output",
@@ -69,6 +69,12 @@
     type=str,
     help="Specify a file path to a clipping boundary file. The tool will use this file to clip the point cloud data to a specific area.",
 )
+@click.option(
+    "-e",
+    "--epsg",
+    type=int,
+    help="Set the EPSG code for user's clip file.",
+)
 @click.option(
     "-d",
     "--decimate",
@@ -81,12 +87,6 @@
     type=str,
     help="Specify a bounding box to clip the point cloud data. It should be comma-separated list with minx,miny,maxx,maxy",
 )
-@click.option(
-    "-r",
-    "--radius",
-    type=int,
-    help="Set the radius of the circle to clip the point cloud data to. Unit is in meter.",
-)
 @click.option(
     "-p",
     "--preview",
@@ -110,13 +110,13 @@ def main(**kwargs: Any) -> None:
     )
     no_clip_city = params.get("no_clip_city")
     clip_file = params.get("clip_file")
+    epsg = params.get("epsg")
     decimate = params.get("decimate")
     bbox = (
         [float(x) for x in str(params.get("bbox", "")).split(",")]
         if params.get("bbox", "")
         else None
     )
-    radius = params.get("radius")
     preview = params.get("preview")
     if validate_all(
         cfg,
@@ -126,9 +126,9 @@ def main(**kwargs: Any) -> None:
         exclude_classes,
         no_clip_city,
         clip_file,
+        epsg,
         decimate,
         bbox,
-        radius,
     ):
         process(
             cfg.geotiles_base_url,
@@ -139,8 +139,9 @@ def main(**kwargs: Any) -> None:
             exclude_classes,
             no_clip_city,
             clip_file,
+            epsg,
+            decimate,
             bbox,
-            radius,
             preview,
         )