Add an admin api to delete local media.

changelog.d/8519.feature

@@ -0,0 +1 @@
+Add an admin api `DELETE /_synapse/admin/v1/media/<server_name>/<media_id>` to delete a single file from server. Contributed by @dklimpel.

docs/admin_api/media_admin_api.md

@@ -100,3 +100,27 @@ Response:
   "num_quarantined": 10  # The number of media items successfully quarantined
 }
 ```
+
+# Draft: Delete local media
+This API deletes the *local* media from the disc of your own server.
+
+Request:
+
+```
+DELETE /_synapse/admin/v1/media/<server_name>/<media_id>
+
+{}
+```
+
+URL Parameters
+
+* `server_name` - The name of your local server (e.g `matrix.org`)
+* `media_id` - The ID of the media (e.g `abcdefghijklmnopqrstuvwx`)
+
+Response:
+
+```
+{
+  "deleted": 1  # The number of media items successfully deleted
+}
+```

synapse/rest/admin/media.py

@@ -16,9 +16,10 @@
 
 import logging
 
-from synapse.api.errors import AuthError
+from synapse.api.errors import AuthError, NotFoundError, SynapseError
 from synapse.http.servlet import RestServlet, parse_integer
 from synapse.rest.admin._base import (
+    admin_patterns,
     assert_requester_is_admin,
     assert_user_is_admin,
     historical_admin_path_patterns,
@@ -150,6 +151,33 @@ async def on_POST(self, request):
         return 200, ret
 
 
+class DeleteMediaByID(RestServlet):
+    """Delete local media by a given ID. Removes it from this server.
+    """
+
+    PATTERNS = admin_patterns("/media/(?P<server_name>[^/]+)/(?P<media_id>[^/]+)", "v1")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+        self.server_name = hs.hostname
+        self.media_repository = hs.get_media_repository()
+
+    async def on_DELETE(self, request, server_name: str, media_id: str):
+        await assert_requester_is_admin(self.auth, request)
+
+        if self.server_name != server_name:
+            raise SynapseError(400, "Can only delete local media")
+
+        if await self.store.get_local_media(media_id) is None:
+            raise NotFoundError("Unknown media")
+
+        logging.info("Deleting local media by ID: %s", media_id)
+
+        ret = await self.media_repository.delete_local_media(media_id)
+        return 200, {"deleted": ret}
+
+
 def register_servlets_for_media_repo(hs, http_server):
     """
     Media repo specific APIs.
@@ -159,3 +187,4 @@ def register_servlets_for_media_repo(hs, http_server):
     QuarantineMediaByID(hs).register(http_server)
     QuarantineMediaByUser(hs).register(http_server)
     ListMediaInRoom(hs).register(http_server)
+    DeleteMediaByID(hs).register(http_server)

synapse/rest/media/v1/filepath.py

@@ -69,6 +69,23 @@ def local_media_thumbnail_rel(self, media_id, width, height, content_type, metho
 
     local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel)
 
+    def local_media_thumbnail_dir(self, media_id: str) -> str:
+        """
+        Retrieve the local store path of thumbnails of a given media_id
+
+        Args:
+            media_id: The media ID to query.
+        Returns:
+            Path of local_thumbnails from media_id
+        """
+        return os.path.join(
+            self.base_path,
+            "local_thumbnails",
+            media_id[0:2],
+            media_id[2:4],
+            media_id[4:],
+        )
+
     def remote_media_filepath_rel(self, server_name, file_id):
         return os.path.join(
             "remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:]

synapse/rest/media/v1/media_repository.py

@@ -767,6 +767,36 @@ async def delete_old_remote_media(self, before_ts):
 
         return {"deleted": deleted}
 
+    async def delete_local_media(self, media_id: str) -> int:
+        """
+        Delete the given media_id from this server
+
+        Args:
+            media_id: The media ID to delete.
+        Returns:
+            Number of deleted files.
+            In this case 1 or 0
+        """
+        logger.info("Deleting local media: %s", media_id)
+
+        full_path = self.filepaths.local_media_filepath(media_id)
+        try:
+            os.remove(full_path)
+        except OSError as e:
+            logger.warning("Failed to remove file: %r: %s", full_path, e)
+            if e.errno != errno.ENOENT:
+                return 0
+
+        thumbnail_dir = self.filepaths.local_media_thumbnail_dir(media_id)
+        shutil.rmtree(thumbnail_dir, ignore_errors=True)
+
+        await self.store.delete_remote_media(self.server_name, media_id)
+
+        await self.store.delete_url_cache((media_id,))
+        await self.store.delete_url_cache_media((media_id,))
+
+        return 1
+
 
 class MediaRepositoryResource(Resource):
     """File uploading and downloading.

tests/rest/admin/test_media.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 Dirk Klimpel
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from binascii import unhexlify
+
+import synapse.rest.admin
+from synapse.api.errors import Codes
+from synapse.rest.client.v1 import login
+from synapse.rest.media.v1.filepath import MediaFilePaths
+
+from tests import unittest
+
+
+class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_media_repo,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.handler = hs.get_device_handler()
+        self.media_repo = hs.get_media_repository_resource()
+        self.server_name = hs.hostname
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.filepaths = MediaFilePaths(hs.config.media_store_path)
+
+    def test_no_auth(self):
+        """
+        Try to delete media without authentication.
+        """
+        url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
+
+        request, channel = self.make_request("DELETE", url, b"{}")
+        self.render(request)
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+    def test_requester_is_no_admin(self):
+        """
+        If the user is not a server admin, an error is returned.
+        """
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+        url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
+
+        request, channel = self.make_request(
+            "DELETE", url, access_token=self.other_user_token,
+        )
+        self.render(request)
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+    def test_media_does_not_exist(self):
+        """
+        Tests that a lookup for a media that does not exist returns a 404
+        """
+        url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
+
+        request, channel = self.make_request(
+            "DELETE", url, access_token=self.admin_user_tok,
+        )
+        self.render(request)
+
+        self.assertEqual(404, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+    def test_media_is_not_local(self):
+        """
+        Tests that a lookup for a media that is not a local returns a 400
+        """
+        url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345")
+
+        request, channel = self.make_request(
+            "DELETE", url, access_token=self.admin_user_tok,
+        )
+        self.render(request)
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual("Can only delete local media", channel.json_body["error"])
+
+    def test_delete_media(self):
+        """
+        Tests that delete a media is successfully
+        """
+
+        download_resource = self.media_repo.children[b"download"]
+        upload_resource = self.media_repo.children[b"upload"]
+        image_data = unhexlify(
+            b"89504e470d0a1a0a0000000d4948445200000001000000010806"
+            b"0000001f15c4890000000a49444154789c63000100000500010d"
+            b"0a2db40000000049454e44ae426082"
+        )
+
+        # Upload some media into the room
+        response = self.helper.upload_media(
+            upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
+        )
+        # Extract media ID from the response
+        server_and_media_id = response["content_uri"][6:]  # Cut off 'mxc://'
+        server_name, media_id = server_and_media_id.split("/")
+
+        self.assertEqual(server_name, self.server_name)
+
+        # Attempt to access media
+        request, channel = self.make_request(
+            "GET",
+            server_and_media_id,
+            shorthand=False,
+            access_token=self.admin_user_tok,
+        )
+        request.render(download_resource)
+        self.pump(1.0)
+
+        # Should be successful
+        self.assertEqual(
+            200,
+            channel.code,
+            msg=(
+                "Expected to receive a 200 on accessing media: %s" % server_and_media_id
+            ),
+        )
+
+        # Test if the file exists
+        local_path = self.filepaths.local_media_filepath(media_id)
+        self.assertTrue(os.path.exists(local_path))
+
+        url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id)
+
+        # Delete media
+        request, channel = self.make_request(
+            "DELETE", url, access_token=self.admin_user_tok,
+        )
+        self.render(request)
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(1, channel.json_body["deleted"])
+
+        # Attempt to access media
+        request, channel = self.make_request(
+            "GET",
+            server_and_media_id,
+            shorthand=False,
+            access_token=self.admin_user_tok,
+        )
+        request.render(download_resource)
+        self.pump(1.0)
+        self.assertEqual(
+            404,
+            channel.code,
+            msg=(
+                "Expected to receive a 404 on accessing deleted media: %s"
+                % server_and_media_id
+            ),
+        )
+
+        # Test if the file is deleted
+        self.assertFalse(os.path.exists(local_path))