Merge pull request #176 from arabcoders/dev

.github/workflows/main.yml

@@ -92,7 +92,7 @@ jobs:
           cache-to: type=gha, scope=${{ github.workflow }}
 
       - name: Version tag
-        if: github.event_name != 'pull_request'
+        if: github.event_name == 'disabled_now'
         uses: arabcoders/action-python-autotagger@master
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -102,7 +102,7 @@ jobs:
 
   dockerhub-sync-readme:
     needs: push-build
-    if: github.event_name != 'pull_request'
+    if: github.event_name == 'push'
     runs-on: ubuntu-latest
     steps:
       - name: Sync README
@@ -116,3 +116,72 @@ jobs:
         with:
           entrypoint: node
           args: /opt/docker-readme-sync/sync
+
+  create_release:
+    needs: push-build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push' && success()
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # so we can git log the full history
+
+      - name: Get commits between old and new
+        id: commits
+        run: |
+          PREVIOUS_SHA="${{ github.event.before }}"
+          CURRENT_SHA="${{ github.event.after }}"
+
+          echo "Previous SHA: $PREVIOUS_SHA"
+          echo "Current  SHA: $CURRENT_SHA"
+
+          # If "before" is empty or all zeros, log all commits
+          if [ -z "$PREVIOUS_SHA" ] || [ "$PREVIOUS_SHA" = "0000000000000000000000000000000000000000" ]; then
+            LOG=$(git log --pretty=format:"- %h %s by %an")
+          else
+            LOG=$(git log "$PREVIOUS_SHA".."$CURRENT_SHA" --pretty=format:"- %h %s by %an")
+          fi
+
+          echo "LOG<<EOF" >> "$GITHUB_ENV"
+          echo "$LOG" >> "$GITHUB_ENV"
+          echo "EOF" >> "$GITHUB_ENV"
+
+      - name: Create local tag matching Docker's raw pattern
+        id: createtag
+        run: |
+          # We'll replicate raw format: branch + base_ref + date + short sha
+          # If base_ref is empty (normal push to master, no PR?), it won't appear.
+          BRANCH_NAME="${{ github.ref_name }}"
+          BASE_REF="${{ github.base_ref }}"
+          CURRENT_SHA="${{ github.event.after }}"
+          DATE=$(date +%Y%m%d)
+          SHORT_SHA=$(echo "$CURRENT_SHA" | cut -c1-7)
+
+          TAG="${BRANCH_NAME}${BASE_REF}-${DATE}-${SHORT_SHA}"
+
+          echo "Using tag: $TAG"
+
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+
+          # Create a lightweight local tag
+          git tag "$TAG" "$CURRENT_SHA"
+
+          # Push the tag back to GitHub
+          git push origin "$TAG"
+
+          # Expose as an output
+          echo "TAG=$TAG" >> "$GITHUB_OUTPUT"
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@master
+        with:
+          tag_name: ${{ steps.createtag.outputs.TAG }}
+          name: "${{ steps.createtag.outputs.TAG }}"
+          body: ${{ env.LOG }}
+          draft: false
+          prerelease: false
+          generate_release_notes: true
+          append_body: true
+          make_latest: true
+          token: ${{ secrets.GITHUB_TOKEN }}

app/library/Download.py

@@ -95,13 +95,13 @@ def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False):
     def _progress_hook(self, data: dict):
         dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
 
-        if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", False):
+        if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", None):
             dataDict["filename"] = data["info_dict"]["filename"]
 
         self.status_queue.put({"id": self.id, **dataDict})
 
     def _postprocessor_hook(self, data: dict):
-        if data.get("postprocessor") != "MoveFiles" or data.get("status") != "finished":
+        if "MoveFiles" != data.get("postprocessor") or "finished" != data.get("status"):
             return
 
         if self.debug:
@@ -116,24 +116,22 @@ def _postprocessor_hook(self, data: dict):
 
     def _download(self):
         try:
-            params: dict = {
-                "color": "no_color",
-                "paths": {
-                    "home": self.download_dir,
-                    "temp": self.tempPath,
-                },
-                "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter},
-                "noprogress": True,
-                "break_on_existing": True,
-                "progress_hooks": [self._progress_hook],
-                "postprocessor_hooks": [self._postprocessor_hook],
-                **get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts)),
-            }
+            params: dict = get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts))
+            params.update(
+                {
+                    "color": "no_color",
+                    "paths": {"home": self.download_dir, "temp": self.tempPath},
+                    "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter},
+                    "noprogress": True,
+                    "break_on_existing": True,
+                    "progress_hooks": [self._progress_hook],
+                    "postprocessor_hooks": [self._postprocessor_hook],
+                    "ignoreerrors": False,
+                }
+            )
 
             if "format" not in params and self.default_ytdl_opts.get("format", None):
-                params["format"] = self.default_ytdl_opts["format"]
-
-            params["ignoreerrors"] = False
+                params["format"] = "best"
 
             if self.debug:
                 params["verbose"] = True
@@ -151,7 +149,7 @@ def _download(self):
 
                     params["cookiefile"] = f.name
                 except ValueError as e:
-                    LOG.error(f"Invalid cookies: was provided for {self.info.title} - {str(e)}")
+                    LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{str(e)}'.")
 
             if self.is_live or self.is_manifestless:
                 hasDeletedOptions = False
@@ -176,7 +174,7 @@ def _download(self):
             cls = yt_dlp.YoutubeDL(params=params)
 
             if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
-                LOG.debug("Downloading using pre-info.")
+                LOG.debug(f"Downloading '{self.info.url}' using pre-info.")
                 cls.process_ie_result(self.info_dict, download=True)
                 ret = cls._download_retcode
             else:
@@ -297,7 +295,7 @@ def delete_temp(self):
 
         if self.info.status != "finished" and self.is_live:
             LOG.warning(
-                f"Keeping live temp folder [{self.tempPath}], as the reported status is not finished [{self.info.status}]."
+                f"Keeping live temp folder '{self.tempPath}', as the reported status is not finished '{self.info.status}'."
             )
             return
 
@@ -310,7 +308,7 @@ def delete_temp(self):
             )
             return
 
-        LOG.info(f"Deleting Temp folder: {self.tempPath}")
+        LOG.info(f"Deleting Temp folder '{self.tempPath}'.")
         shutil.rmtree(self.tempPath, ignore_errors=True)
 
     async def progress_update(self):

app/library/DownloadQueue.py

@@ -80,8 +80,8 @@ async def __add_entry(
         live_in: str | None = None
 
         eventType = entry.get("_type") or "video"
-        if eventType == "playlist":
-            entries = entry["entries"]
+        if "playlist" == eventType:
+            entries = entry.get("entries", [])
             playlist_index_digits = len(str(len(entries)))
             results = []
             for i, etr in enumerate(entries, start=1):
@@ -106,16 +106,16 @@ async def __add_entry(
                     )
                 )
 
-            if any(res["status"] == "error" for res in results):
+            if any("error" == res["status"] for res in results):
                 return {
                     "status": "error",
                     "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
                 }
 
             return {"status": "ok"}
-        elif (eventType == "video" or eventType.startswith("url")) and "id" in entry and "title" in entry:
+        elif ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
             # check if the video is live stream.
-            if "live_status" in entry and entry.get("live_status") == "is_upcoming":
+            if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
                 if "release_timestamp" in entry and entry.get("release_timestamp"):
                     live_in = formatdate(entry.get("release_timestamp"))
                 else:
@@ -230,10 +230,12 @@ async def add(
         already=None,
     ):
         ytdlp_config = ytdlp_config if ytdlp_config else {}
-        folder = str(folder)
+        folder = str(folder) if folder else ""
+
+        filePath = calcDownloadPath(basePath=self.config.download_path, folder=folder)
 
         LOG.info(
-            f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'."
+            f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'."
         )
 
         if isinstance(ytdlp_config, str):
@@ -279,13 +281,13 @@ async def add(
                 f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'."
             )
         except yt_dlp.utils.ExistingVideoReached as exc:
-            LOG.error(f"Video has been downloaded already and recorded in archive.log file. {str(exc)}")
+            LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{str(exc)}'.")
             return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
         except yt_dlp.utils.YoutubeDLError as exc:
-            LOG.error(f"YoutubeDLError: Unable to extract info. {str(exc)}")
+            LOG.error(f"YoutubeDLError: Unable to extract info. '{str(exc)}'.")
             return {"status": "error", "msg": str(exc)}
         except asyncio.exceptions.TimeoutError as exc:
-            LOG.error(f"TimeoutError: Unable to extract info. {str(exc)}")
+            LOG.error(f"TimeoutError: Unable to extract info. '{str(exc)}'.")
             return {
                 "status": "error",
                 "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.",
@@ -366,7 +368,7 @@ async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, st
                         os.remove(realFile)
                         fileDeleted = True
                     else:
-                        LOG.warning(f"File '{filename}' does not exist.")
+                        LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.")
                 except Exception as e:
                     LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}")
 
@@ -448,8 +450,9 @@ async def __download_pool(self):
                 await asyncio.sleep(1)
 
     async def __downloadFile(self, id: str, entry: Download):
+        filePath = calcDownloadPath(basePath=self.config.download_path, folder=entry.info.folder)
         LOG.info(
-            f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {entry.info.folder}'."
+            f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'."
         )
 
         try:

app/library/HttpAPI.py

@@ -222,24 +222,32 @@ async def add_url(self, request: Request) -> Response:
         post = await request.json()
 
         url: str = post.get("url")
-        preset: str = post.get("preset", "default")
 
         if not url:
             return web.json_response(data={"error": "url is required."}, status=web.HTTPBadRequest.status_code)
 
-        folder: str = post.get("folder")
-        ytdlp_cookies: str = post.get("ytdlp_cookies")
-        ytdlp_config: dict | None = post.get("ytdlp_config")
-        output_template: str = post.get("output_template")
-        if ytdlp_config is None:
-            ytdlp_config = {}
+        preset: str = str(post.get("preset", self.config.default_preset))
+        folder: str = str(post.get("folder")) if post.get("folder") else ""
+        ytdlp_cookies: str = str(post.get("ytdlp_cookies")) if post.get("ytdlp_cookies") else ""
+        output_template: str = str(post.get("output_template")) if post.get("output_template") else ""
+
+        ytdlp_config = post.get("ytdlp_config")
+        if isinstance(ytdlp_config, str) and ytdlp_config:
+            try:
+                ytdlp_config = json.loads(ytdlp_config)
+            except Exception as e:
+                LOG.error(f"Failed to parse json yt-dlp config for '{url}'. {str(e)}")
+                return web.json_response(
+                    data={"error": f"Failed to parse json yt-dlp config. {str(e)}"},
+                    status=web.HTTPBadRequest.status_code,
+                )
 
         status = await self.add(
             url=url,
             preset=preset,
             folder=folder,
             ytdlp_cookies=ytdlp_cookies,
-            ytdlp_config=ytdlp_config,
+            ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {},
             output_template=output_template,
         )
 

app/library/HttpSocket.py

@@ -1,12 +1,13 @@
 import asyncio
 import errno
 import functools
+import json
 import logging
 import os
 import pty
 import shlex
-from datetime import datetime
 import time
+from datetime import datetime
 
 import socketio
 from aiohttp import web
@@ -161,20 +162,25 @@ async def add_url(self, sid: str, data: dict):
             await self.emitter.warning("No URL provided.", to=sid)
             return
 
-        preset: str = str(data.get("preset", "default"))
-        folder: str = str(data.get("folder"))
-        ytdlp_cookies: str = str(data.get("ytdlp_cookies"))
-        ytdlp_config: dict | None = data.get("ytdlp_config")
-        output_template: str = data.get("output_template")
-        if ytdlp_config is None:
-            ytdlp_config = {}
+        preset: str = str(data.get("preset", self.config.default_preset))
+        folder: str = str(data.get("folder")) if data.get("folder") else ""
+        ytdlp_cookies: str = str(data.get("ytdlp_cookies")) if data.get("ytdlp_cookies") else ""
+        output_template: str = str(data.get("output_template")) if data.get("output_template") else ""
+
+        ytdlp_config = data.get("ytdlp_config")
+        if isinstance(ytdlp_config, str) and ytdlp_config:
+            try:
+                ytdlp_config = json.loads(ytdlp_config)
+            except Exception as e:
+                await self.emitter.warning(f"Failed to parse json yt-dlp config. {str(e)}", to=sid)
+                return
 
         status = await self.add(
             url=url,
             preset=preset,
             folder=folder,
             ytdlp_cookies=ytdlp_cookies,
-            ytdlp_config=ytdlp_config,
+            ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {},
             output_template=output_template,
         )
 

app/library/Utils.py

@@ -109,7 +109,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
     return yt_dlp.YoutubeDL().extract_info(url, download=False)
 
 
-def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) -> str:
+def calcDownloadPath(basePath: str, folder: str|None = None, createPath: bool = True) -> str:
     """Calculates download path and prevents folder traversal.
 
     Returns:

app/library/common.py

@@ -22,9 +22,6 @@ def __init__(self, queue: DownloadQueue, encoder: Encoder):
     async def add(
         self, url: str, preset: str, folder: str, ytdlp_cookies: str, ytdlp_config: dict, output_template: str
     ) -> dict[str, str]:
-        if ytdlp_config is None:
-            ytdlp_config = {}
-
         if ytdlp_cookies and isinstance(ytdlp_cookies, dict):
             ytdlp_cookies = self.encoder.encode(ytdlp_cookies)
 
@@ -33,7 +30,7 @@ async def add(
             preset=preset if preset else "default",
             folder=folder,
             ytdlp_cookies=ytdlp_cookies,
-            ytdlp_config=ytdlp_config,
+            ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {},
             output_template=output_template,
         )
 

ui/stores/SocketStore.js

@@ -79,8 +79,8 @@ export const useSocketStore = defineStore('socket', () => {
     socket.value.on("updated", stream => {
       const data = JSON.parse(stream);
 
-      if (true === stateStore.has('history', item._id)) {
-        stateStore.update('history', item._id, item);
+      if (true === stateStore.has('history', data._id)) {
+        stateStore.update('history', data._id, data);
         return;
       }