Limit concurrent builds

readthedocs/api/v2/views/model_views.py

@@ -10,7 +10,13 @@
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import BRANCH, TAG, INTERNAL
+from readthedocs.builds.constants import (
+    BRANCH,
+    TAG,
+    INTERNAL,
+    BUILD_STATE_QUEUED,
+    BUILD_STATE_FINISHED,
+)
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
@@ -276,6 +282,19 @@ class BuildViewSetBase(UserSelectViewSet):
     model = Build
     filterset_fields = ('project__slug', 'commit')
 
+    @decorators.action(
+        detail=False,
+        permission_classes=[permissions.IsAdminUser],
+        methods=['get'],
+    )
+    def running(self, request, **kwargs):
+        project_slug = request.GET.get('project__slug')
+        queryset = (
+            self.get_queryset()
+            .filter(project__slug=project_slug)
+            .exclude(state__in=[BUILD_STATE_FINISHED, BUILD_STATE_QUEUED])
+        )
+        return Response({'count': queryset.count()})
 
 class BuildViewSet(SettingsOverrideObject):
 

readthedocs/builds/constants.py

@@ -5,6 +5,7 @@
 
 
 BUILD_STATE_TRIGGERED = 'triggered'
+BUILD_STATE_QUEUED = 'queued'
 BUILD_STATE_CLONING = 'cloning'
 BUILD_STATE_INSTALLING = 'installing'
 BUILD_STATE_BUILDING = 'building'
@@ -13,6 +14,7 @@
 
 BUILD_STATE = (
     (BUILD_STATE_TRIGGERED, _('Triggered')),
+    (BUILD_STATE_QUEUED, _('Queued')),
     (BUILD_STATE_CLONING, _('Cloning')),
     (BUILD_STATE_INSTALLING, _('Installing')),
     (BUILD_STATE_BUILDING, _('Building')),

readthedocs/builds/migrations/0016_add-queued-state.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-04-01 20:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0015_uploading_build_state'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='state',
+            field=models.CharField(choices=[('triggered', 'Triggered'), ('queued', 'Queued'), ('cloning', 'Cloning'), ('installing', 'Installing'), ('building', 'Building'), ('uploading', 'Uploading'), ('finished', 'Finished')], default='finished', max_length=55, verbose_name='State'),
+        ),
+    ]

readthedocs/doc_builder/exceptions.py

@@ -53,6 +53,10 @@ class BuildTimeoutError(BuildEnvironmentError):
     message = ugettext_noop('Build exited due to time out')
 
 
+class BuildMaxConcurrencyError(BuildEnvironmentError):
+    message = ugettext_noop('Concurrent limit reached ({limit}), retrying in 5 minutes.')
+
+
 class BuildEnvironmentWarning(BuildEnvironmentException):
     pass
 

readthedocs/projects/models.py

@@ -1515,6 +1515,7 @@ def add_features(sender, **kwargs):
     SKIP_SYNC_TAGS = 'skip_sync_tags'
     SKIP_SYNC_BRANCHES = 'skip_sync_branches'
     CACHED_ENVIRONMENT = 'cached_environment'
+    LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1585,6 +1586,10 @@ def add_features(sender, **kwargs):
             CACHED_ENVIRONMENT,
             _('Cache the environment (virtualenv, conda, pip cache, repository) in storage'),
         ),
+        (
+            LIMIT_CONCURRENT_BUILDS,
+            _('Limit the amount of concurrent builds'),
+        ),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks.py

@@ -29,6 +29,7 @@
 
 from readthedocs.api.v2.client import api as api_v2
 from readthedocs.builds.constants import (
+    BUILD_STATE_QUEUED,
     BUILD_STATE_BUILDING,
     BUILD_STATE_CLONING,
     BUILD_STATE_FINISHED,
@@ -57,6 +58,7 @@
 from readthedocs.doc_builder.exceptions import (
     BuildEnvironmentError,
     BuildEnvironmentWarning,
+    BuildMaxConcurrencyError,
     BuildTimeoutError,
     MkDocsYAMLParseError,
     ProjectBuildsSkippedError,
@@ -510,6 +512,26 @@ def run(
             self.commit = commit
             self.config = None
 
+            if self.project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS):
+                response = api_v2.build.running.get(project__slug=self.project.slug)
+                if response.get('count', 0) >= settings.RTD_MAX_CONCURRENT_BUILDS:
+                    log.warning(
+                        'Delaying tasks due to concurrency limit. project=%s version=%s',
+                        self.project.slug,
+                        self.version.slug,
+                    )
+
+                    # This is done automatically on the environment context, but
+                    # we are executing this code before creating one
+                    api_v2.build(self.build['id']).patch({
+                        'error': BuildMaxConcurrencyError.message.format(
+                            limit=settings.RTD_MAX_CONCURRENT_BUILDS,
+                        ),
+                        'state': BUILD_STATE_QUEUED,
+                    })
+                    self.task.retry(exc=BuildMaxConcurrencyError, throw=False)
+                    return False
+
             # Build process starts here
             setup_successful = self.run_setup(record=record)
             if not setup_successful:

readthedocs/settings/base.py

@@ -98,6 +98,7 @@ class CommunityBaseSettings(Settings):
     RTD_STABLE = 'stable'
     RTD_STABLE_VERBOSE_NAME = 'stable'
     RTD_CLEAN_AFTER_BUILD = False
+    RTD_MAX_CONCURRENT_BUILDS = 4
 
     # Database and API hitting settings
     DONT_HIT_API = False