Remove files after build

readthedocs/core/utils/__init__.py

@@ -2,22 +2,21 @@
 
 """Common utilty functions."""
 
-from __future__ import absolute_import
-
 import errno
 import logging
 import os
 import re
 
+from celery import chain, chord, group
 from django.conf import settings
 from django.utils.functional import keep_lazy
 from django.utils.safestring import SafeText, mark_safe
 from django.utils.text import slugify as slugify_base
-from celery import group, chord
 
 from readthedocs.builds.constants import BUILD_STATE_TRIGGERED
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
 
+
 log = logging.getLogger(__name__)
 
 
@@ -81,7 +80,7 @@ def prepare_build(
     # Avoid circular import
     from readthedocs.builds.models import Build
     from readthedocs.projects.models import Project
-    from readthedocs.projects.tasks import update_docs_task
+    from readthedocs.projects.tasks import clean_build_task, update_docs_task
 
     build = None
 
@@ -130,11 +129,17 @@ def prepare_build(
     options['time_limit'] = int(time_limit * 1.2)
 
     return (
-        update_docs_task.signature(
-            args=(project.pk,),
-            kwargs=kwargs,
-            options=options,
-            immutable=True,
+        chain(
+            update_docs_task.signature(
+                args=(project.pk,),
+                kwargs=kwargs,
+                options=options,
+                immutable=True,
+            ),
+            clean_build_task.signature(
+                args=(version.pk,),
+                immutable=True,
+            )
         ),
         build,
     )

readthedocs/projects/tasks.py

@@ -1315,6 +1315,20 @@ def warn(self, msg):
     delete_queryset.delete()
 
 
+@app.task(max_retries=5, default_retry_delay=7 * 60)
+def clean_build_task(version_id):
+    """Clean the files used in the build of the given version."""
+    version = Version.objects.get_object_or_log(pk=version_id)
+    if not version:
+        return
+    del_dirs = [
+        os.path.join(version.project.doc_path, dir_, version.slug)
+        for dir_ in ('checkouts', 'envs', 'conda')
+    ]
+    for del_dir in del_dirs:
+        broadcast(type='build', task=remove_dirs, args=[(del_dir,)])
+
+
 def _manage_imported_files(version, path, commit):
     """
     Update imported files for version.

readthedocs/rtd_tests/tests/test_core_utils.py

@@ -1,20 +1,19 @@
-# -*- coding: utf-8 -*-
 """Test core util functions."""
 
 import os
-import mock
 
-from mock import call
+import mock
 from django.http import Http404
 from django.test import TestCase
 from django_dynamic_fixture import get
+from mock import call
 
+from readthedocs.builds.constants import LATEST
 from readthedocs.builds.models import Version
-from readthedocs.core.utils.general import wipe_version_via_slugs
-from readthedocs.projects.tasks import remove_dirs
 from readthedocs.core.utils import slugify, trigger_build
+from readthedocs.core.utils.general import wipe_version_via_slugs
 from readthedocs.projects.models import Project
-from readthedocs.builds.constants import LATEST
+from readthedocs.projects.tasks import remove_dirs
 
 
 class CoreUtilTests(TestCase):
@@ -64,7 +63,7 @@ def test_trigger_build_when_version_not_provided_default_version_exist(self, upd
                 immutable=True,
             ),
         ])
-    
+
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
     def test_trigger_build_when_version_not_provided_default_version_doesnt_exist(self, update_docs_task):
 
@@ -90,8 +89,10 @@ def test_trigger_build_when_version_not_provided_default_version_doesnt_exist(se
             ),
         ])
 
+    @mock.patch('readthedocs.projects.tasks.clean_build_task')
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
-    def test_trigger_custom_queue(self, update_docs):
+    @mock.patch('readthedocs.core.utils.chain')
+    def test_trigger_custom_queue(self, chain, update_docs, clean_build_task):
         """Use a custom queue when routing the task."""
         self.project.build_queue = 'build03'
         trigger_build(project=self.project, version=self.version)
@@ -106,18 +107,23 @@ def test_trigger_custom_queue(self, update_docs):
             'time_limit': 720,
             'soft_time_limit': 600,
         }
-        update_docs.signature.assert_has_calls([
-            mock.call(
+        chain.assert_called_with(
+            update_docs.signature(
                 args=(self.project.pk,),
                 kwargs=kwargs,
                 options=options,
                 immutable=True,
             ),
-        ])
-        update_docs.signature().apply_async.assert_called()
+            clean_build_task.signature(
+                args=(self.version.pk,),
+                immutable=True,
+            ),
+        )
 
+    @mock.patch('readthedocs.projects.tasks.clean_build_task')
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
-    def test_trigger_build_time_limit(self, update_docs):
+    @mock.patch('readthedocs.core.utils.chain')
+    def test_trigger_build_time_limit(self, chain, update_docs, clean_build_task):
         """Pass of time limit."""
         trigger_build(project=self.project, version=self.version)
         kwargs = {
@@ -139,10 +145,23 @@ def test_trigger_build_time_limit(self, update_docs):
                 immutable=True,
             ),
         ])
-        update_docs.signature().apply_async.assert_called()
+        chain.assert_called_with(
+            update_docs.signature(
+                args=(self.project.pk,),
+                kwargs=kwargs,
+                options=options,
+                immutable=True,
+            ),
+            clean_build_task.signature(
+                args=(self.version.pk,),
+                immutable=True,
+            ),
+        )
 
+    @mock.patch('readthedocs.projects.tasks.clean_build_task')
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
-    def test_trigger_build_invalid_time_limit(self, update_docs):
+    @mock.patch('readthedocs.core.utils.chain')
+    def test_trigger_build_invalid_time_limit(self, chain, update_docs, clean_build_task):
         """Time limit as string."""
         self.project.container_time_limit = '200s'
         trigger_build(project=self.project, version=self.version)
@@ -165,10 +184,23 @@ def test_trigger_build_invalid_time_limit(self, update_docs):
                 immutable=True,
             ),
         ])
-        update_docs.signature().apply_async.assert_called()
+        chain.assert_called_with(
+            update_docs.signature(
+                args=(self.project.pk,),
+                kwargs=kwargs,
+                options=options,
+                immutable=True,
+            ),
+            clean_build_task.signature(
+                args=(self.version.pk,),
+                immutable=True,
+            ),
+        )
 
+    @mock.patch('readthedocs.projects.tasks.clean_build_task')
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
-    def test_trigger_build_rounded_time_limit(self, update_docs):
+    @mock.patch('readthedocs.core.utils.chain')
+    def test_trigger_build_rounded_time_limit(self, chain, update_docs, clean_build_task):
         """Time limit should round down."""
         self.project.container_time_limit = 3
         trigger_build(project=self.project, version=self.version)
@@ -191,7 +223,18 @@ def test_trigger_build_rounded_time_limit(self, update_docs):
                 immutable=True,
             ),
         ])
-        update_docs.signature().apply_async.assert_called()
+        chain.assert_called_with(
+            update_docs.signature(
+                args=(self.project.pk,),
+                kwargs=kwargs,
+                options=options,
+                immutable=True,
+            ),
+            clean_build_task.signature(
+                args=(self.version.pk,),
+                immutable=True,
+            ),
+        )
 
     def test_slugify(self):
         """Test additional slugify."""

readthedocs/rtd_tests/tests/test_privacy.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-import json
 import logging
 
 import mock
@@ -9,14 +7,15 @@
 
 from readthedocs.builds.constants import LATEST
 from readthedocs.builds.models import Build, Version
-from readthedocs.projects import tasks
 from readthedocs.projects.forms import UpdateProjectForm
 from readthedocs.projects.models import Project
 
 
 log = logging.getLogger(__name__)
 
 
+@mock.patch('readthedocs.projects.tasks.clean_build_task.signature', new=mock.MagicMock)
+@mock.patch('readthedocs.projects.tasks.update_docs_task.signature', new=mock.MagicMock)
 class PrivacyTests(TestCase):
 
     def setUp(self):
@@ -28,8 +27,6 @@ def setUp(self):
         self.tester.set_password('test')
         self.tester.save()
 
-        tasks.update_docs_task.delay = mock.Mock()
-
     def _create_kong(
         self, privacy_level='private',
         version_privacy_level='private',