Limit concurrent builds

readthedocs/api/v2/views/model_views.py

@@ -10,7 +10,12 @@
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import BRANCH, TAG, INTERNAL
+from readthedocs.builds.constants import (
+    BRANCH,
+    TAG,
+    INTERNAL,
+    BUILD_STATE_FINISHED,
+)
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
@@ -276,6 +281,20 @@ class BuildViewSetBase(UserSelectViewSet):
     model = Build
     filterset_fields = ('project__slug', 'commit')
 
+    @decorators.action(
+        detail=False,
+        permission_classes=[permissions.IsAdminUser],
+        methods=['get'],
+    )
+    def running(self, request, **kwargs):
+        project_slug = request.GET.get('project__slug')
+        queryset = (
+            self.get_queryset()
+            .filter(project__slug=project_slug)
+            .exclude(state__in=[BUILD_STATE_FINISHED])
+        )
+        return Response({'count': queryset.count()})
+
 
 class BuildViewSet(SettingsOverrideObject):
 

readthedocs/doc_builder/exceptions.py

@@ -53,6 +53,10 @@ class BuildTimeoutError(BuildEnvironmentError):
     message = ugettext_noop('Build exited due to time out')
 
 
+class BuildMaxConcurrencyError(BuildEnvironmentError):
+    message = ugettext_noop('Concurrency limit reached ({limit}), retrying in 5 minutes.')
+
+
 class BuildEnvironmentWarning(BuildEnvironmentException):
     pass
 

readthedocs/projects/models.py

@@ -1515,6 +1515,7 @@ def add_features(sender, **kwargs):
     SKIP_SYNC_TAGS = 'skip_sync_tags'
     SKIP_SYNC_BRANCHES = 'skip_sync_branches'
     CACHED_ENVIRONMENT = 'cached_environment'
+    LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1585,6 +1586,10 @@ def add_features(sender, **kwargs):
             CACHED_ENVIRONMENT,
             _('Cache the environment (virtualenv, conda, pip cache, repository) in storage'),
         ),
+        (
+            LIMIT_CONCURRENT_BUILDS,
+            _('Limit the amount of concurrent builds'),
+        ),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks.py

@@ -57,6 +57,7 @@
 from readthedocs.doc_builder.exceptions import (
     BuildEnvironmentError,
     BuildEnvironmentWarning,
+    BuildMaxConcurrencyError,
     BuildTimeoutError,
     MkDocsYAMLParseError,
     ProjectBuildsSkippedError,
@@ -510,6 +511,30 @@ def run(
             self.commit = commit
             self.config = None
 
+            if self.project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS):
+                response = api_v2.build.running.get(project__slug=self.project.slug)
+                if response.get('count', 0) >= settings.RTD_MAX_CONCURRENT_BUILDS:
+                    log.warning(
+                        'Delaying tasks due to concurrency limit. project=%s version=%s',
+                        self.project.slug,
+                        self.version.slug,
+                    )
+
+                    # This is done automatically on the environment context, but
+                    # we are executing this code before creating one
+                    api_v2.build(self.build['id']).patch({
+                        'error': BuildMaxConcurrencyError.message.format(
+                            limit=settings.RTD_MAX_CONCURRENT_BUILDS,
+                        ),
+                    })
+                    self.task.retry(
+                        exc=BuildMaxConcurrencyError,
+                        throw=False,
+                        # We want to retry this build more times
+                        max_retries=25,
+                    )
+                    return False
+
             # Build process starts here
             setup_successful = self.run_setup(record=record)
             if not setup_successful:

readthedocs/rtd_tests/tests/test_privacy_urls.py

@@ -374,6 +374,7 @@ def setUp(self):
             'api_webhook': {'integration_pk': self.integration.pk},
         }
         self.response_data = {
+            'build-running': {'status_code': 403},
             'project-sync-versions': {'status_code': 403},
             'project-token': {'status_code': 403},
             'emailhook-list': {'status_code': 403},

readthedocs/settings/base.py

@@ -98,6 +98,7 @@ class CommunityBaseSettings(Settings):
     RTD_STABLE = 'stable'
     RTD_STABLE_VERBOSE_NAME = 'stable'
     RTD_CLEAN_AFTER_BUILD = False
+    RTD_MAX_CONCURRENT_BUILDS = 4
 
     # Database and API hitting settings
     DONT_HIT_API = False