Merge pull request #127 from 3DBAG/uv-setup

Uv setup

.github/hooks/pre-commit

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+version=$(bumpver show --no-fetch | cut -d' ' -f3)
+if ! grep -q "\[${version}\]" CHANGELOG.md; then
+    echo "Error: CHANGELOG.md has not been updated for version ${version}"
+    exit 1
+fi

.github/scripts/parse_changelog.py

@@ -0,0 +1,76 @@
+import re
+import sys
+from typing import Optional
+
+
+def parse_changelog(version: str) -> Optional[str]:
+    """Parse CHANGELOG.md and return the content for the specified version.
+
+    Args:
+        version: Version string to find in the changelog (e.g. "2024.12.16")
+
+    Returns:
+        String containing the changelog entry for the specified version,
+        or None if version not found
+    """
+    with open("CHANGELOG.md", "r") as f:
+        content = f.read()
+
+    # Pattern to match version headers
+    version_header_pattern = r"## \[([\d.]+)\]"
+
+    # Find all version headers and their positions
+    version_matches = list(re.finditer(version_header_pattern, content))
+
+    # Find the index of our target version
+    target_index = None
+    for i, match in enumerate(version_matches):
+        if match.group(1) == version:
+            target_index = i
+            break
+
+    if target_index is None:
+        print(f"No changelog entry found for version {version}")
+        return None
+
+    # Get the start position (right after the version header)
+    start_pos = version_matches[target_index].end()
+
+    # Get the end position (start of next version or end of file)
+    if target_index + 1 < len(version_matches):
+        end_pos = version_matches[target_index + 1].start()
+    else:
+        end_pos = len(content)
+
+    # Extract the content between these positions
+    changelog_content = content[start_pos:end_pos]
+
+    # Clean up the content
+    # Remove leading/trailing whitespace and empty lines while preserving internal formatting
+    cleaned_lines = []
+    for line in changelog_content.split('\n'):
+        if line.strip() or cleaned_lines:  # Keep empty lines only after we've started collecting content
+            cleaned_lines.append(line)
+
+    # Remove trailing empty lines
+    while cleaned_lines and not cleaned_lines[-1].strip():
+        cleaned_lines.pop()
+
+    return '\n'.join(cleaned_lines)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python parse_changelog.py VERSION")
+        sys.exit(1)
+
+    version = sys.argv[1]
+    changelog_content = parse_changelog(version)
+
+    if changelog_content:
+        with open("RELEASE_NOTES.md", "w") as f:
+            f.write(changelog_content)
+        print("Successfully extracted changelog content")
+    else:
+        print("Failed to extract changelog content")
+        sys.exit(1)

.github/workflows/documentation.yaml

@@ -25,7 +25,7 @@ jobs:
           path: .cache
           restore-keys: |
             mkdocs-material-
-      - run: |
-          pip install -r requirements-dev.txt
-          pip install --no-deps packages/common
-      - run: mkdocs gh-deploy --force
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Run mkdocs
+        run: uv run mkdocs gh-deploy --force

.github/workflows/lint-and-formatting.yaml

@@ -18,9 +18,7 @@ jobs:
           python-version: "3.11"
       - name: Install uv
         uses: astral-sh/setup-uv@v5
-      - name: Install dependencies
-        run: uv pip install ruff --system
       - name: Lint
-        run: ruff check
+        run: uv tool run ruff check
       - name: Format
-        run: ruff format --check
+        run: uv tool run ruff format --check

.github/workflows/release.yaml

@@ -0,0 +1,74 @@
+name: Release
+description: Create a new release for the 3dbag-pipeline with the specified version, use the release notes from the CHANGELOG.md file and create a pull request to merge the changes back to the develop branch.
+
+permissions:
+  contents: write
+  pull-requests: write
+
+on:
+  workflow_dispatch:
+    branches:
+      - master
+    inputs:
+      version:
+        description: '3dbag-pipeline release version (YYYY.MM.DD)'
+        required: true
+        type: string
+
+jobs:
+  release:
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install bumpver
+        run: pip install bumpver
+
+      - name: Configure Git
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+
+      - name: Update version
+        run: |
+          bumpver update --set-version=${{ github.event.inputs.version }} --no-fetch
+
+      - name: Parse Changelog
+        run: |
+          python .github/scripts/parse_changelog.py ${{ github.event.inputs.version }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ github.event.inputs.version }}
+          name: 3DBAG ${{ github.event.inputs.version }}
+          body_path: RELEASE_NOTES.md
+          draft: false
+          prerelease: false
+
+      - name: Create Pull Request to develop
+        uses: peter-evans/create-pull-request@v7
+        with:
+          branch: merge-release-back-to-develop
+          base: develop
+          title: 'Merge release ${{ github.event.inputs.version }} back to develop'
+          body: |
+            This PR merges the changes from release ${{ github.event.inputs.version }} back to the develop branch.
+
+            Changes included in this release:
+
+            ```
+            $(cat RELEASE_NOTES.md)
+            ```
+          labels: |
+            release
+            automated pr
+          delete-branch: true

.gitignore

@@ -34,4 +34,5 @@ docs/reference
 # Tools
 .tox
 .coverage
-*.ruff_cache
+*.ruff_cache
+*.python-version

CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+All notable changes to the 3dbag-pipeline are documented in this file.
+For the changes in the 3DBAG data set, see the [3DBAG release notes](https://docs.3dbag.nl/en/overview/release_notes/).
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [2024.12.16]
+
+Release that produced the 3DBAG data set version 2024.12.16.
+
+### Added
+- Documentation for deploying the 3dbag-pipeline, contributor guidelines and the project layout.
+- Docker-based deployment.
+- CI pipeline for testing and docker image builds.
+
+### Changed
+- Major refactoring of the project structure for easier maintenance and deployment.
+
+### Docker images
+
+The docker images for this release:
+- `3dgi/3dbag-pipeline-dagster:2024.12.16`
+- `3dgi/3dbag-pipeline-core:2024.12.16`
+- `3dgi/3dbag-pipeline-floors-estimation:2024.12.16`
+- `3dgi/3dbag-pipeline-party-walls:2024.12.16`

README.md

@@ -43,6 +43,19 @@ You can read about the details on how can you deploy it in the [deployment secti
 
 ## Production
 
+## Integration as a library
+
+The 3dbag-pipeline can be used as a library in other projects.
+The packages can be installed directly from GitHub using specific release versions:
+
+```bash
+# Install specific release version of the common package
+pip install "bag3d-common @ git+https://github.com/3DBAG/3dbag-pipeline.git@v2024.12.16#egg=bag3d-common&subdirectory=packages/common"
+
+# Install specific commit of the common package
+pip install "bag3d-common @ git+https://github.com/3DBAG/3dbag-pipeline.git@<commit-hash>#egg=bag3d-common&subdirectory=packages/common"
+```
+
 ## License
 
 Licensed under either of

docker/compose.yaml

@@ -89,7 +89,7 @@ services:
     # dagster's DockerRunLauncher, it always starts a new container from the image, for
     # each run. While, watch:action:sync (or sync+restart)copies the changed local code into the
     # 3dbag-pipeline-core-develop container (not the image). But since DockerRunLauncher
-    # doesn't actaully use this container, but always starts a new one, we have to build
+    # doesn't actually use this container, but always starts a new one, we have to build
     # a new image on each change.
     develop:
       watch:

docker/pipeline/bag3d-core.dockerfile

@@ -12,20 +12,28 @@ LABEL org.opencontainers.image.licenses="(MIT OR Apache-2.0)"
 WORKDIR $BAG3D_PIPELINE_LOCATION
 
 ENV UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV
-# Install dependencies
+
+# Install only dependencies except the bag3d-common package
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=./packages/core/uv.lock,target=$BAG3D_PIPELINE_LOCATION/packages/core/uv.lock \
     --mount=type=bind,source=./packages/core/pyproject.toml,target=$BAG3D_PIPELINE_LOCATION/packages/core/pyproject.toml \
     uv sync \
+    --frozen \
+    --all-extras \
     --no-install-project \
+    --no-install-package bag3d-common\
     --project $BAG3D_PIPELINE_LOCATION/packages/core \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
 
-# Install the workflow package
+# Install the workflow package and the bag3d-common package in editable mode
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/core/.[dev] && \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/common/.[dev]
+    uv sync \
+    --frozen \
+    --all-extras \
+    --project $BAG3D_PIPELINE_LOCATION/packages/core \
+    --python $VIRTUAL_ENV/bin/python
 
 # Run dagster gRPC server on port 4000
 EXPOSE 4000

docker/pipeline/bag3d-floors-estimation.dockerfile

@@ -12,20 +12,28 @@ LABEL org.opencontainers.image.licenses="(MIT OR Apache-2.0)"
 WORKDIR $BAG3D_PIPELINE_LOCATION
 
 ENV UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV
-# Install dependencies
+
+# Install only dependencies except the bag3d-common package
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=./packages/floors_estimation/uv.lock,target=$BAG3D_PIPELINE_LOCATION/packages/floors_estimation/uv.lock \
     --mount=type=bind,source=./packages/floors_estimation/pyproject.toml,target=$BAG3D_PIPELINE_LOCATION/packages/floors_estimation/pyproject.toml \
     uv sync \
+    --frozen \
+    --all-extras \
     --no-install-project \
+    --no-install-package bag3d-common\
     --project $BAG3D_PIPELINE_LOCATION/packages/floors_estimation \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
 
-# Install the workflow package
+# Install the workflow package and the bag3d-common package in editable mode
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/floors_estimation/.[dev] && \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/common/.[dev]
+    uv sync \
+    --frozen \
+    --all-extras \
+    --project $BAG3D_PIPELINE_LOCATION/packages/floors_estimation \
+    --python $VIRTUAL_ENV/bin/python
 
 # Run dagster gRPC server on port 4001
 EXPOSE 4001

docker/pipeline/bag3d-party-walls.dockerfile

@@ -13,7 +13,7 @@ RUN rm -rf $VIRTUAL_ENV
 RUN uv venv --python 3.11 $VIRTUAL_ENV
 ENV UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV
 # Install packages into the virtual environment
-COPY --link docker/tools/requirements.txt .
+COPY docker/tools/requirements.txt .
 RUN --mount=type=cache,mode=0755,target=/root/.cache/uv uv pip install -r requirements.txt
 RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
     --mount=target=/var/cache/apt,type=cache,sharing=locked \
@@ -25,20 +25,28 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
 
 WORKDIR $BAG3D_PIPELINE_LOCATION
 
-# Install dependencies
+
+# Install only dependencies except the bag3d-common package
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=./packages/party_walls/uv.lock,target=$BAG3D_PIPELINE_LOCATION/packages/party_walls/uv.lock \
     --mount=type=bind,source=./packages/party_walls/pyproject.toml,target=$BAG3D_PIPELINE_LOCATION/packages/party_walls/pyproject.toml \
     uv sync \
+    --frozen \
+    --all-extras \
     --no-install-project \
+    --no-install-package bag3d-common\
     --project $BAG3D_PIPELINE_LOCATION/packages/party_walls \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
 
-# Install the workflow package
+# Install the workflow package and the bag3d-common package in editable mode
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/party_walls/.[dev] && \
-    uv pip install -e $BAG3D_PIPELINE_LOCATION/packages/common/.[dev]
+    uv sync \
+    --frozen \
+    --all-extras \
+    --project $BAG3D_PIPELINE_LOCATION/packages/party_walls \
+    --python $VIRTUAL_ENV/bin/python
 
 # Run dagster gRPC server on port 4002
 EXPOSE 4002

docs/development/code.md

@@ -101,17 +101,9 @@ To run a specific test, set up a run configuration with the python interpreter i
 
 For further details, see the [PyCharm documentation](https://www.jetbrains.com/help/pycharm/using-docker-compose-as-a-remote-interpreter.html#run).
 
-### Local setup
+### Code formatting
 
-For development purposes, you should also create a local virtual environment and install the required packages from `requirements-dev.txt`.
-
-You can do this in one step with:
-
-```bash
-make local_venv
-```
-
-Then you can format you code with:
+In you have a local installation of `uv`, you can format you code with:
 
 ```
 make format
@@ -226,6 +218,16 @@ For example `Returns a collection type, storing the...`
 Assets are usually some results of computations, therefore their names are nouns, not verbs.
 
 
+## Release process
+
+Release always happens from the `master` branch, after merging the successful production candidate branch into `master`.
+See the [branches](#branches) section for more information.
+
+1. Update the CHANGELOG.md file with the new version and the changes. It must include the new version number that you are releasing, e.g. `## [2024.10.24]`.
+2. On GitHub, create a new pull request from the current production candidate branch to the `master` branch and merge it.
+3. Manually trigger the release workflow on GitHub Actions. You'll need to input the new version number that you added to the CHANGELOG, e.g. `2024.10.24`. This will create a new release on GitHub and add the contents of the CHANGELOG to the release notes.
+4. The workflow will automatically open a pull request from `master` to `develop` to merge back the changes from the release. This is done to keep the `develop` branch up to date with the latest changes from the `master` branch. You can merge this pull request after the release is done.
+
 ## Dagster
 
 #### Terminate all in the queue