Merge pull request #406 from TheExistingOne/conflictmode-deduplicate

Add `deduplicate` conflict mode

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## [Unreleased]
+- Added a new conflict mode `deduplicate` which skips duplicate files amd renames non-duplicates
 
 ## v3.2.5 (2024-07-09)
 

docs/actions.md

@@ -58,6 +58,21 @@ rules:
           on_conflict: overwrite
 ```
 
+Use a placeholder to copy all .pdf files into a "PDF" folder and all .jpg files into a "JPG" folder. If two files share the same file name and are duplicates, the duplicate will be skipped. If they aren't duplicates, the second file will be renamed.
+
+```yaml
+rules:
+  - locations: ~/Desktop
+    filters:
+      - extension:
+          - pdf
+          - jpg
+    actions:
+      - copy:
+          dest: "~/Desktop/{extension.upper()}/"
+          on_conflict: deduplicate
+```
+
 Copy into the folder `Invoices`. Keep the filename but do not overwrite existing files.
 To prevent overwriting files, an index is added to the filename, so `somefile.jpg` becomes `somefile 2.jpg`.
 The counter separator is `' '` by default, but can be changed using the `counter_separator` property.

organize/actions/common/conflict.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import filecmp
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal, NamedTuple
 
@@ -11,7 +12,9 @@
     from jinja2 import Template
 
 # TODO: keep_newer, keep_older, keep_bigger, keep_smaller
-ConflictMode = Literal["skip", "overwrite", "trash", "rename_new", "rename_existing"]
+ConflictMode = Literal[
+    "skip", "overwrite", "deduplicate", "trash", "rename_new", "rename_existing"
+]
 
 
 class ConflictResult(NamedTuple):
@@ -104,6 +107,17 @@ def _print(msg: str):
             delete(path=dst)
         return ConflictResult(skip_action=False, use_dst=dst)
 
+    elif conflict_mode == "deduplicate":
+        if filecmp.cmp(res.path, dst, shallow=True):
+            _print("Duplicate skipped.")
+            return ConflictResult(skip_action=True, use_dst=res.path)
+        else:
+            new_path = next_free_name(
+                dst=dst,
+                template=rename_template,
+            )
+            return ConflictResult(skip_action=False, use_dst=new_path)
+
     elif conflict_mode == "rename_new":
         new_path = next_free_name(
             dst=dst,

tests/actions/test_copy.py

@@ -137,6 +137,46 @@ def test_copy_conflict(fs, mode, result):
     assert read_files("test") == result
 
 
+def test_copy_deduplicate_conflict(fs):
+    files = {
+        "src.txt": "src",
+        "duplicate": {
+            "src.txt": "src",
+        },
+        "nonduplicate": {
+            "src.txt": "src2",
+        },
+    }
+
+    config = """
+    rules:
+      - locations: "/test"
+        subfolders: true
+        filters:
+          - name: src
+        actions:
+          - copy:
+              dest: "/test/dst.txt"
+              on_conflict: deduplicate
+    """
+    make_files(files, "test")
+
+    Config.from_string(config).execute(simulate=False)
+    result = read_files("test")
+
+    assert result == {
+        "src.txt": "src",
+        "duplicate": {
+            "src.txt": "src",
+        },
+        "nonduplicate": {
+            "src.txt": "src2",
+        },
+        "dst.txt": "src",
+        "dst 2.txt": "src2",
+    }
+
+
 def test_does_not_create_folder_in_simulation(fs):
     config = """
         rules:

tests/actions/test_move.py

@@ -56,6 +56,43 @@ def test_move_conflict(fs, mode, result):
     assert read_files("test") == result
 
 
+def test_move_deduplicate_conflict(fs):
+    files = {
+        "src.txt": "src",
+        "duplicate": {
+            "src.txt": "src",
+        },
+        "nonduplicate": {
+            "src.txt": "src2",
+        },
+    }
+
+    config = """
+    rules:
+      - locations: "/test"
+        subfolders: true
+        filters:
+          - name: src
+        actions:
+          - move:
+              dest: "/test/dst.txt"
+              on_conflict: deduplicate
+    """
+    make_files(files, "test")
+
+    Config.from_string(config).execute(simulate=False)
+    result = read_files("test")
+
+    assert result == {
+        "duplicate": {
+            "src.txt": "src",
+        },
+        "nonduplicate": {},
+        "dst.txt": "src",
+        "dst 2.txt": "src2",
+    }
+
+
 def test_move_folder_conflict(fs):
     make_files(
         {