Merge pull request #5 from bneijt/monitor-directory-contents

Monitor directory contents

.github/workflows/main.yml

@@ -11,8 +11,14 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - run: pip install poetry==1.1.7
+      - run: poetry install
+      - run: poetry run pytest tests
+      - run: poetry run poetry-lock-package --build --no-root
 
       - name: Log in to Docker Hub
         uses: docker/login-action@v1

.gitignore

@@ -1 +1,4 @@
 playbook.retry
+.vscode
+dist
+__pycache__

Dockerfile

@@ -6,17 +6,26 @@ RUN set -e; \
     pacman --noconfirm -Su; \
     pacman-db-upgrade; \
     trust extract-compat; \
-    pacman --noconfirm -S go-ipfs python nginx; \
+    pacman --noconfirm -S go-ipfs python python-pip nginx; \
     useradd -G daemon -s /bin/nologin -d /ipfs ipfs;
 
 WORKDIR /ipfs
 
-ADD src/start.py /
-ADD src/nginx.conf /etc/nginx/nginx.conf
-ADD src/pinner /srv/http/pinner
+ADD static/nginx.conf /etc/nginx/nginx.conf
+ADD static/pinner /srv/http/pinner
+
+# Copy and install base requirements
+COPY dist/*_lock*.whl /
+RUN  pip install --no-cache-dir /*.whl \
+    && rm /*.whl
+
+# Copy and install package
+COPY dist/*.whl /
+RUN  pip install --no-cache-dir /*.whl \
+    && rm /*.whl
 
 VOLUME /ipfs
 
 EXPOSE 80
 
-CMD ["/usr/sbin/python", "/start.py"]
+CMD ["/usr/sbin/python", "-m", "ipfs_video_gateway.start"]

build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+poetry install
+poetry build
+poetry run poetry-lock-package --build --no-root
+docker build . -t ipfs-gateway

pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "ipfs-video-gateway"
+version = "1.0.0"
+description = "Docker based IPFS pinning front-end"
+authors = ["Bram Neijt <bram@neijt.nl>"]
+packages = [{ include = "ipfs_video_gateway", from = "src" }]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.5"
+black = "^21.10b0"
+poetry-lock-package = "^0.4.3"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

src/start.py → src/ipfs_video_gateway/start.py

@@ -7,6 +7,7 @@
 import shutil
 
 IPFS_HOME = "/ipfs"
+FOLDER_STATUS_CACHE = {}
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -73,32 +74,67 @@ def contains_files(path: str, matcher: Callable[[str], bool]) -> bool:
     return matcher(os.path.basename(path))
 
 
+def ipfs_add(file_path: str):
+    logger.info(f"Adding '{file_path}'")
+    sys.stdout.flush()
+    checked_run(
+        as_ipfs(
+            f"ipfs add --recursive --wrap-with-directory --silent --pin '{file_path}'"
+        )
+    )
+
+
+def is_stable(path: str) -> bool:
+    global FOLDER_STATUS_CACHE
+    current_status = "#".join(
+        [
+            "#".join(
+                [
+                    f"{name}={os.path.getsize(os.path.join(root, name))}"
+                    for name in sorted(files)
+                ]
+            )
+            for root, _, files in os.walk(path)
+        ]
+    )
+    old_state = FOLDER_STATUS_CACHE.get(path)
+    FOLDER_STATUS_CACHE[path] = current_status
+    return old_state is not None and old_state == current_status
+
+
 def check_for_new_files() -> None:
     for name in os.listdir(IPFS_HOME):
         if name.startswith(".") or name.endswith(".part"):
             continue
         file_path = os.path.join(IPFS_HOME, name)
 
-        # We ignore the directory if it contains hidden files
-        # and example of this is partial rsync uploads
-        if contains_files(name, is_hidden_or_part_file):
-            logger.info(
-                f"Ignoring '{file_path}' because it contains hidden or partial files"
-            )
-            sys.stdout.flush()
-            continue
-
-        logger.info(f"Adding '{file_path}'")
-        sys.stdout.flush()
-        checked_run(
-            as_ipfs(
-                f"ipfs add --recursive --wrap-with-directory --silent --pin '{file_path}'"
-            )
-        )
         if os.path.isdir(file_path):
-            shutil.rmtree(file_path)
+            if not contains_files(name, is_hidden_or_part_file):
+                if is_stable(file_path):
+                    ipfs_add(file_path)
+                    shutil.rmtree(file_path)
+                else:
+                    logger.info(
+                        f"Ignoring '{file_path}' because it has changing content"
+                    )
+
+            else:
+                logger.info(
+                    f"Ignoring '{file_path}' because it contains hidden or partial files"
+                )
+                sys.stdout.flush()
+                continue
+
         else:
-            os.unlink(file_path)
+            if not is_hidden_or_part_file(file_path):
+                ipfs_add(file_path)
+                os.unlink(file_path)
+            else:
+                logger.info(
+                    f"Ignoring '{file_path}' because it contains hidden or partial files"
+                )
+                sys.stdout.flush()
+                continue
 
 
 def main():

test.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-docker build -t ipfs-gateway .
+./build.sh
 docker run  --publish 4000:80 --volume /tmp/ipfs:/ipfs --rm -it ipfs-gateway

tests/test_start.py

@@ -0,0 +1,6 @@
+
+from ipfs_video_gateway.start import is_stable
+
+def test_is_stable():
+    assert is_stable("src") == False, "First time should be unstable"
+    assert is_stable("src") == True, "Second time we see nothing changed"