Limit concurrent builds

Add a Feature flag to be able to limit concurrent builds for specific
projects. This will allow us to protect ourselves for projects
triggering a lot of builds via the API or for a few very long builds
of the same project blocking all the rest of users.

If the limit is reached, the Build is marked as "Queue" and an error
message is shown in the build's detail page saying that the concurrent
limit was reached and the build is re-triggered to be executed in 5
minutes again.

readthedocs/api/v2/views/model_views.py

@@ -10,7 +10,13 @@
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import BRANCH, TAG, INTERNAL
+from readthedocs.builds.constants import (
+    BRANCH,
+    TAG,
+    INTERNAL,
+    BUILD_STATE_QUEUED,
+    BUILD_STATE_FINISHED,
+)
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
@@ -276,6 +282,19 @@ class BuildViewSetBase(UserSelectViewSet):
     model = Build
     filterset_fields = ('project__slug', 'commit')
 
+    @decorators.action(
+        detail=False,
+        permission_classes=[permissions.IsAdminUser],
+        methods=['get'],
+    )
+    def running(self, request, **kwargs):
+        project_slug = request.GET.get('project__slug')
+        queryset = (
+            self.get_queryset()
+            .filter(project__slug=project_slug)
+            .exclude(state__in=[BUILD_STATE_FINISHED, BUILD_STATE_QUEUED])
+        )
+        return Response({'count': queryset.count()})
 
 class BuildViewSet(SettingsOverrideObject):
 

readthedocs/builds/constants.py

@@ -5,6 +5,7 @@
 
 
 BUILD_STATE_TRIGGERED = 'triggered'
+BUILD_STATE_QUEUED = 'queued'
 BUILD_STATE_CLONING = 'cloning'
 BUILD_STATE_INSTALLING = 'installing'
 BUILD_STATE_BUILDING = 'building'
@@ -13,6 +14,7 @@
 
 BUILD_STATE = (
     (BUILD_STATE_TRIGGERED, _('Triggered')),
+    (BUILD_STATE_QUEUED, _('Queued')),
     (BUILD_STATE_CLONING, _('Cloning')),
     (BUILD_STATE_INSTALLING, _('Installing')),
     (BUILD_STATE_BUILDING, _('Building')),

readthedocs/builds/migrations/0016_add-queued-state.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-04-01 20:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0015_uploading_build_state'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='state',
+            field=models.CharField(choices=[('triggered', 'Triggered'), ('queued', 'Queued'), ('cloning', 'Cloning'), ('installing', 'Installing'), ('building', 'Building'), ('uploading', 'Uploading'), ('finished', 'Finished')], default='finished', max_length=55, verbose_name='State'),
+        ),
+    ]

readthedocs/doc_builder/exceptions.py

@@ -53,6 +53,10 @@ class BuildTimeoutError(BuildEnvironmentError):
     message = ugettext_noop('Build exited due to time out')
 
 
+class BuildMaxConcurrencyError(BuildEnvironmentError):
+    message = ugettext_noop('Concurrent limit reached ({limit}), retrying in 5 minutes.')
+
+
 class BuildEnvironmentWarning(BuildEnvironmentException):
     pass
 

readthedocs/projects/models.py

@@ -1515,6 +1515,7 @@ def add_features(sender, **kwargs):
     SKIP_SYNC_TAGS = 'skip_sync_tags'
     SKIP_SYNC_BRANCHES = 'skip_sync_branches'
     CACHED_ENVIRONMENT = 'cached_environment'
+    LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1585,6 +1586,10 @@ def add_features(sender, **kwargs):
             CACHED_ENVIRONMENT,
             _('Cache the environment (virtualenv, conda, pip cache, repository) in storage'),
         ),
+        (
+            LIMIT_CONCURRENT_BUILDS,
+            _('Limit the amount of concurrent builds'),
+        ),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks.py

@@ -29,6 +29,7 @@
 
 from readthedocs.api.v2.client import api as api_v2
 from readthedocs.builds.constants import (
+    BUILD_STATE_QUEUED,
     BUILD_STATE_BUILDING,
     BUILD_STATE_CLONING,
     BUILD_STATE_FINISHED,
@@ -57,6 +58,7 @@
 from readthedocs.doc_builder.exceptions import (
     BuildEnvironmentError,
     BuildEnvironmentWarning,
+    BuildMaxConcurrencyError,
     BuildTimeoutError,
     MkDocsYAMLParseError,
     ProjectBuildsSkippedError,
@@ -510,6 +512,26 @@ def run(
             self.commit = commit
             self.config = None
 
+            if self.project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS):
+                response = api_v2.build.running.get(project__slug=self.project.slug)
+                if response.get('count', 0) >= settings.RTD_MAX_CONCURRENT_BUILDS:
+                    log.warning(
+                        'Delaying tasks due to concurrency limit. project=%s version=%s',
+                        self.project.slug,
+                        self.version.slug,
+                    )
+
+                    # This is done automatically on the environment context, but
+                    # we are executing this code before creating one
+                    api_v2.build(self.build['id']).patch({
+                        'error': BuildMaxConcurrencyError.message.format(
+                            limit=settings.RTD_MAX_CONCURRENT_BUILDS,
+                        ),
+                        'state': BUILD_STATE_QUEUED,
+                    })
+                    self.task.retry(exc=BuildMaxConcurrencyError, throw=False)
+                    return False
+
             # Build process starts here
             setup_successful = self.run_setup(record=record)
             if not setup_successful:

readthedocs/settings/base.py

@@ -98,6 +98,7 @@ class CommunityBaseSettings(Settings):
     RTD_STABLE = 'stable'
     RTD_STABLE_VERBOSE_NAME = 'stable'
     RTD_CLEAN_AFTER_BUILD = False
+    RTD_MAX_CONCURRENT_BUILDS = 4
 
     # Database and API hitting settings
     DONT_HIT_API = False