Asset packs index (#1)

* Initial repo copy

* Some progress on packs index structure

* Implement pack parser

* Differenciate API types

* Formatting

* Better differenciate API endpoints

* Format

* Description in meta

* Include preview videos

* Jpeg too

* Gzipped packs

* Change title to name

* Rebuild packs on reindex

* Format

* Refactor pack parsing with pathlib

* Update asset packer from fw repo

* Compute pack stats

* Better cleanup

* Fix pack filetypes

* Add some sanity checks and logging

* Warn about too many previews too

* Format

* Update asset packer

* Improve warning messages

* Offload repacking to separate repo, only parsing and indexing here

* Rename to download

* Include files dir skeleton structure

* Cleanup repacking related things

* Add asset packs submodule

* Fix for having files in indexer dirs

* Fixes for new FastAPI

* No None fields

.gitignore

@@ -1,5 +1,6 @@
 __pycache__
 *.swp
 venv
-files
+files/*/*
+!files/*/.gitkeep
 .env

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "files/asset-packs"]
+	path = files/asset-packs
+	url = https://github.com/Next-Flip/Asset-Packs.git

indexer/main.py

@@ -28,6 +28,11 @@ async def lifespan(app: FastAPI):
             os.makedirs(dir_path, exist_ok=True)
         except Exception:
             logging.exception(f"Failed to create {dir_path}")
+    logger = logging.getLogger()
+    prev_level = logger.level
+    logger.setLevel(logging.INFO)
+    logging.info("Startup complete")
+    logger.setLevel(prev_level)
 
     yield
 

indexer/src/directories.py

@@ -3,7 +3,7 @@
 from fastapi import APIRouter
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, FileResponse
 
-from .repository import indexes
+from .repository import indexes, RepositoryIndex, PacksCatalog
 
 
 router = APIRouter()
@@ -18,123 +18,136 @@ async def root_request():
     return RedirectResponse("/firmware", status_code=303)
 
 
-@router.get("/{directory}/directory.json")
-@router.get("/{directory}")
-async def directory_request(directory):
-    """
-    Method for obtaining indices
-    Args:
-        directory: Repository name
-
-    Returns:
-        Indices in json
-    """
-    if directory not in indexes:
-        return JSONResponse(f"{directory} not found!", status_code=404)
-    return indexes.get(directory).index
-
-
-@router.get(
-    "/{directory}/{channel}/{target}/{file_type}",
-    response_class=RedirectResponse,
-    status_code=302,
-)
-async def latest_request(directory, channel, target, file_type):
-    """
-    A method for retrieving a file from the repository
-    of a specific version
-    Args:
-        directory: Repository name
-        channel: Channel type (release, dev)
-        target: Operating System (linux, mac, win)
-        file_type: File Type
-
-    Returns:
-        Artifact file
-    """
-    if directory not in indexes:
-        return JSONResponse(f"{directory} not found!", status_code=404)
-    index = indexes.get(directory)
-    if len(index.index["channels"]) == 0:
-        return JSONResponse("No channels found!", status_code=404)
-    try:
-        return index.get_file_from_latest_version(channel, target, file_type)
-    except Exception as e:
-        return JSONResponse(str(e), status_code=404)
-
-
-@router.get("/{directory}/{channel}/{file_name}")
-async def file_request(directory, channel, file_name):
-    """
-    A method for retrieving a file from the repository
-    of a specific version
-    Args:
-        directory: Repository name
-        channel: Channel type (release, dev)
-        file_name: File Name
-
-    Returns:
-        Artifact file
-    """
-    if directory not in indexes:
-        return JSONResponse(f"{directory} not found!", status_code=404)
-    index = indexes.get(directory)
-    if len(index.index["channels"]) == 0:
-        return JSONResponse("No channels found!", status_code=404)
-    try:
-        return FileResponse(
-            index.get_file_path(channel, file_name),
-            media_type="application/octet-stream",
-            status_code=200,
-        )
-    except Exception as e:
-        return JSONResponse(str(e), status_code=404)
+def setup_routes(prefix: str, index):
+    @router.get(prefix + "/directory.json")
+    @router.get(prefix)
+    async def directory_request():
+        """
+        Method for obtaining indices
+        Args:
+            Nothing
 
+        Returns:
+            Indices in json
+        """
+        return index.index
 
-@router.get("/{directory}/reindex")
-async def reindex_request(directory):
-    """
-    Method for starting reindexing
-    Args:
-        directory: Repository name
+    if isinstance(index, RepositoryIndex):
 
-    Returns:
-        Reindex status
-    """
-    if directory not in indexes:
-        return JSONResponse(f"{directory} not found!", status_code=404)
-    async with lock:
-        try:
-            indexes.get(directory).reindex()
-            return JSONResponse("Reindexing is done!")
-        except Exception as e:
-            logging.exception(e)
-            return JSONResponse("Reindexing is failed!", status_code=500)
-
-
-@router.get("/{directory}/{branch}")
-async def repository_branch_request(directory, branch):
-    """
-    A method for retrieving the list of files from a specific branch
-    Made for support of `ufbt update --index-url {base_url}/firmware --branch {branch}`
-    Args:
-        directory: Repository name
-        branch: Branch name
-
-    Returns:
-        HTML links in format that ufbt understands
-    """
-    if directory not in indexes:
-        return JSONResponse(f"{directory} not found!", status_code=404)
-    index = indexes.get(directory)
-    if len(index.index["channels"]) == 0:
-        return JSONResponse("No channels found!", status_code=404)
-    try:
-        branch_files = index.get_branch_file_names(branch)
-        response = "\n".join(f'<a href="{file}"></a>' for file in branch_files)
-        return HTMLResponse(
-            response,
-            status_code=200,
+        @router.get(
+            prefix + "/{channel}/{target}/{file_type}",
+            response_class=RedirectResponse,
+            status_code=302,
         )
-    except Exception as e:
-        return JSONResponse(str(e), status_code=404)
+        async def repository_latest_request(channel, target, file_type):
+            """
+            A method for retrieving a file from the repository
+            of a specific version
+            Args:
+                channel: Channel type (release, dev)
+                target: Operating System (linux, mac, win)
+                file_type: File Type
+
+            Returns:
+                Artifact file
+            """
+            if len(index.index["channels"]) == 0:
+                return JSONResponse("No channels found!", status_code=404)
+            try:
+                return index.get_file_from_latest_version(channel, target, file_type)
+            except Exception as e:
+                return JSONResponse(str(e), status_code=404)
+
+        @router.get(prefix + "/{channel}/{file_name}")
+        async def repository_file_request(channel, file_name):
+            """
+            A method for retrieving a file from a specific version
+            Args:
+                channel: Channel type (release, dev)
+                file_name: File Name
+
+            Returns:
+                Artifact file
+            """
+            if len(index.index["channels"]) == 0:
+                return JSONResponse("No channels found!", status_code=404)
+            try:
+                return FileResponse(
+                    index.get_file_path(channel, file_name),
+                    media_type="application/octet-stream",
+                    status_code=200,
+                )
+            except Exception as e:
+                return JSONResponse(str(e), status_code=404)
+
+    elif isinstance(index, PacksCatalog):
+
+        @router.get(prefix + "/{pack}/{file_type}/{file_name}")
+        async def pack_file_request(pack, file_type, file_name):
+            """
+            A method for retrieving a file from a specific pack
+            Args:
+                pack: Pack id
+                file_type: File Type (download, preview)
+                file_name: File Name
+
+            Returns:
+                Artifact file
+            """
+            if len(index.index["packs"]) == 0:
+                return JSONResponse("No packs found!", status_code=404)
+            try:
+                return FileResponse(
+                    index.get_file_path(pack, file_type, file_name),
+                    media_type="application/octet-stream",
+                    status_code=200,
+                )
+            except Exception as e:
+                return JSONResponse(str(e), status_code=404)
+
+    @router.get(prefix + "/reindex")
+    async def reindex_request():
+        """
+        Method for starting reindexing
+        Args:
+            Nothing
+
+        Returns:
+            Reindex status
+        """
+        async with lock:
+            try:
+                index.reindex()
+                return JSONResponse("Reindexing is done!")
+            except Exception as e:
+                logging.exception(e)
+                return JSONResponse("Reindexing is failed!", status_code=500)
+
+    if isinstance(index, RepositoryIndex):
+
+        @router.get(prefix + "/{branch}")
+        async def repository_branch_request(branch):
+            """
+            A method for retrieving the list of files from a specific branch
+            Made for support of `ufbt update --index-url {base_url}/firmware --branch {branch}`
+            Args:
+                branch: Branch name
+
+            Returns:
+                HTML links in format that ufbt understands
+            """
+            if len(index.index["channels"]) == 0:
+                return JSONResponse("No channels found!", status_code=404)
+            try:
+                branch_files = index.get_branch_file_names(branch)
+                response = "\n".join(f'<a href="{file}"></a>' for file in branch_files)
+                return HTMLResponse(
+                    response,
+                    status_code=200,
+                )
+            except Exception as e:
+                return JSONResponse(str(e), status_code=404)
+
+
+for directory, index in indexes.items():
+    setup_routes(f"/{directory}", index)