diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index b7152295faa..2c55596a672 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,12 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL +from readthedocs.builds.constants import ( + BRANCH, + TAG, + INTERNAL, + BUILD_STATE_FINISHED, +) from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -276,6 +281,20 @@ class BuildViewSetBase(UserSelectViewSet): model = Build filterset_fields = ('project__slug', 'commit') + @decorators.action( + detail=False, + permission_classes=[permissions.IsAdminUser], + methods=['get'], + ) + def running(self, request, **kwargs): + project_slug = request.GET.get('project__slug') + queryset = ( + self.get_queryset() + .filter(project__slug=project_slug) + .exclude(state__in=[BUILD_STATE_FINISHED]) + ) + return Response({'count': queryset.count()}) + class BuildViewSet(SettingsOverrideObject): diff --git a/readthedocs/doc_builder/exceptions.py b/readthedocs/doc_builder/exceptions.py index ee0a24616ad..d7a511430a4 100644 --- a/readthedocs/doc_builder/exceptions.py +++ b/readthedocs/doc_builder/exceptions.py @@ -53,6 +53,10 @@ class BuildTimeoutError(BuildEnvironmentError): message = ugettext_noop('Build exited due to time out') +class BuildMaxConcurrencyError(BuildEnvironmentError): + message = ugettext_noop('Concurrency limit reached ({limit}), retrying in 5 minutes.') + + class BuildEnvironmentWarning(BuildEnvironmentException): pass diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 6a88363f382..0c321aa7217 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1515,6 +1515,7 @@ def add_features(sender, **kwargs): SKIP_SYNC_TAGS = 'skip_sync_tags' SKIP_SYNC_BRANCHES = 'skip_sync_branches' CACHED_ENVIRONMENT = 'cached_environment' + LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1585,6 +1586,10 @@ def add_features(sender, **kwargs): CACHED_ENVIRONMENT, _('Cache the environment (virtualenv, conda, pip cache, repository) in storage'), ), + ( + LIMIT_CONCURRENT_BUILDS, + _('Limit the amount of concurrent builds'), + ), ) projects = models.ManyToManyField( diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 70082612835..06f968376df 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -57,6 +57,7 @@ from readthedocs.doc_builder.exceptions import ( BuildEnvironmentError, BuildEnvironmentWarning, + BuildMaxConcurrencyError, BuildTimeoutError, MkDocsYAMLParseError, ProjectBuildsSkippedError, @@ -510,6 +511,30 @@ def run( self.commit = commit self.config = None + if self.project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS): + response = api_v2.build.running.get(project__slug=self.project.slug) + if response.get('count', 0) >= settings.RTD_MAX_CONCURRENT_BUILDS: + log.warning( + 'Delaying tasks due to concurrency limit. project=%s version=%s', + self.project.slug, + self.version.slug, + ) + + # This is done automatically on the environment context, but + # we are executing this code before creating one + api_v2.build(self.build['id']).patch({ + 'error': BuildMaxConcurrencyError.message.format( + limit=settings.RTD_MAX_CONCURRENT_BUILDS, + ), + }) + self.task.retry( + exc=BuildMaxConcurrencyError, + throw=False, + # We want to retry this build more times + max_retries=25, + ) + return False + # Build process starts here setup_successful = self.run_setup(record=record) if not setup_successful: diff --git a/readthedocs/rtd_tests/tests/test_privacy_urls.py b/readthedocs/rtd_tests/tests/test_privacy_urls.py index 317db6e6dab..79788cf3514 100644 --- a/readthedocs/rtd_tests/tests/test_privacy_urls.py +++ b/readthedocs/rtd_tests/tests/test_privacy_urls.py @@ -374,6 +374,7 @@ def setUp(self): 'api_webhook': {'integration_pk': self.integration.pk}, } self.response_data = { + 'build-running': {'status_code': 403}, 'project-sync-versions': {'status_code': 403}, 'project-token': {'status_code': 403}, 'emailhook-list': {'status_code': 403}, diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index e83d18c74ab..247f3cd3530 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -98,6 +98,7 @@ class CommunityBaseSettings(Settings): RTD_STABLE = 'stable' RTD_STABLE_VERBOSE_NAME = 'stable' RTD_CLEAN_AFTER_BUILD = False + RTD_MAX_CONCURRENT_BUILDS = 4 # Database and API hitting settings DONT_HIT_API = False