diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index b7152295faa..d70def61b3c 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,13 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL +from readthedocs.builds.constants import ( + BRANCH, + TAG, + INTERNAL, + BUILD_STATE_QUEUED, + BUILD_STATE_FINISHED, +) from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -276,6 +282,19 @@ class BuildViewSetBase(UserSelectViewSet): model = Build filterset_fields = ('project__slug', 'commit') + @decorators.action( + detail=False, + permission_classes=[permissions.IsAdminUser], + methods=['get'], + ) + def running(self, request, **kwargs): + project_slug = request.GET.get('project__slug') + queryset = ( + self.get_queryset() + .filter(project__slug=project_slug) + .exclude(state__in=[BUILD_STATE_FINISHED, BUILD_STATE_QUEUED]) + ) + return Response({'count': queryset.count()}) class BuildViewSet(SettingsOverrideObject): diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 9c067a3b5e1..8fa64911990 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -5,6 +5,7 @@ BUILD_STATE_TRIGGERED = 'triggered' +BUILD_STATE_QUEUED = 'queued' BUILD_STATE_CLONING = 'cloning' BUILD_STATE_INSTALLING = 'installing' BUILD_STATE_BUILDING = 'building' @@ -13,6 +14,7 @@ BUILD_STATE = ( (BUILD_STATE_TRIGGERED, _('Triggered')), + (BUILD_STATE_QUEUED, _('Queued')), (BUILD_STATE_CLONING, _('Cloning')), (BUILD_STATE_INSTALLING, _('Installing')), (BUILD_STATE_BUILDING, _('Building')), diff --git a/readthedocs/builds/migrations/0016_add-queued-state.py b/readthedocs/builds/migrations/0016_add-queued-state.py new file mode 100644 index 00000000000..bf2dc400764 --- /dev/null +++ b/readthedocs/builds/migrations/0016_add-queued-state.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.11 on 2020-04-01 20:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0015_uploading_build_state'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='state', + field=models.CharField(choices=[('triggered', 'Triggered'), ('queued', 'Queued'), ('cloning', 'Cloning'), ('installing', 'Installing'), ('building', 'Building'), ('uploading', 'Uploading'), ('finished', 'Finished')], default='finished', max_length=55, verbose_name='State'), + ), + ] diff --git a/readthedocs/doc_builder/exceptions.py b/readthedocs/doc_builder/exceptions.py index ee0a24616ad..c15e97a4664 100644 --- a/readthedocs/doc_builder/exceptions.py +++ b/readthedocs/doc_builder/exceptions.py @@ -53,6 +53,10 @@ class BuildTimeoutError(BuildEnvironmentError): message = ugettext_noop('Build exited due to time out') +class BuildMaxConcurrencyError(BuildEnvironmentError): + message = ugettext_noop('Concurrent limit reached ({limit}), retrying in 5 minutes.') + + class BuildEnvironmentWarning(BuildEnvironmentException): pass diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 6a88363f382..0c321aa7217 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1515,6 +1515,7 @@ def add_features(sender, **kwargs): SKIP_SYNC_TAGS = 'skip_sync_tags' SKIP_SYNC_BRANCHES = 'skip_sync_branches' CACHED_ENVIRONMENT = 'cached_environment' + LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1585,6 +1586,10 @@ def add_features(sender, **kwargs): CACHED_ENVIRONMENT, _('Cache the environment (virtualenv, conda, pip cache, repository) in storage'), ), + ( + LIMIT_CONCURRENT_BUILDS, + _('Limit the amount of concurrent builds'), + ), ) projects = models.ManyToManyField( diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 70082612835..49d5647639d 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -29,6 +29,7 @@ from readthedocs.api.v2.client import api as api_v2 from readthedocs.builds.constants import ( + BUILD_STATE_QUEUED, BUILD_STATE_BUILDING, BUILD_STATE_CLONING, BUILD_STATE_FINISHED, @@ -57,6 +58,7 @@ from readthedocs.doc_builder.exceptions import ( BuildEnvironmentError, BuildEnvironmentWarning, + BuildMaxConcurrencyError, BuildTimeoutError, MkDocsYAMLParseError, ProjectBuildsSkippedError, @@ -510,6 +512,26 @@ def run( self.commit = commit self.config = None + if self.project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS): + response = api_v2.build.running.get(project__slug=self.project.slug) + if response.get('count', 0) >= settings.RTD_MAX_CONCURRENT_BUILDS: + log.warning( + 'Delaying tasks due to concurrency limit. project=%s version=%s', + self.project.slug, + self.version.slug, + ) + + # This is done automatically on the environment context, but + # we are executing this code before creating one + api_v2.build(self.build['id']).patch({ + 'error': BuildMaxConcurrencyError.message.format( + limit=settings.RTD_MAX_CONCURRENT_BUILDS, + ), + 'state': BUILD_STATE_QUEUED, + }) + self.task.retry(exc=BuildMaxConcurrencyError, throw=False) + return False + # Build process starts here setup_successful = self.run_setup(record=record) if not setup_successful: diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index e83d18c74ab..247f3cd3530 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -98,6 +98,7 @@ class CommunityBaseSettings(Settings): RTD_STABLE = 'stable' RTD_STABLE_VERBOSE_NAME = 'stable' RTD_CLEAN_AFTER_BUILD = False + RTD_MAX_CONCURRENT_BUILDS = 4 # Database and API hitting settings DONT_HIT_API = False