Merge branch 'main' into state-auth-sentry

README.md

@@ -40,7 +40,7 @@ This service will startup when you run codecov.io normally. It is under that `ap
 The easiest way to run tests (that doesn't require installing postgres and other dependencies) is to run inside of docker:
 
     docker-compose up
-    docker exec -it codecov-api_api_1 pytest -rf
+    docker exec -it codecov-api_api_1 pytest -rf --no-migrations
 
 ### Testing standalone
 

codecov_auth/authentication/repo_auth.py

@@ -1,4 +1,5 @@
 import re
+from datetime import datetime
 from typing import List
 from uuid import UUID
 
@@ -8,7 +9,9 @@
 from django.db.models import QuerySet
 from django.utils import timezone
 from rest_framework import authentication, exceptions
-from shared.torngit.exceptions import TorngitObjectNotFoundError
+from sentry_sdk import metrics as sentry_metrics
+from shared.metrics import metrics
+from shared.torngit.exceptions import TorngitObjectNotFoundError, TorngitRateLimitError
 
 from codecov_auth.authentication.types import RepositoryAsUser, RepositoryAuthInterface
 from codecov_auth.models import (
@@ -211,6 +214,7 @@ class TokenlessAuthentication(authentication.TokenAuthentication):
     """
 
     auth_failed_message = "Not valid tokenless upload"
+    rate_limit_failed_message = "Tokenless has reached GitHub rate limit. Please upload using a token: https://docs.codecov.com/docs/adding-the-codecov-token."
 
     def _get_repo_info_from_request_path(self, request) -> Repository:
         path_info = request.get_full_path_info()
@@ -245,6 +249,17 @@ async def get_pull_request_info(self, repository_service, fork_pr: str):
             return await repository_service.get_pull_request(fork_pr)
         except TorngitObjectNotFoundError:
             raise exceptions.AuthenticationFailed(self.auth_failed_message)
+        except TorngitRateLimitError as e:
+            metrics.incr("auth.get_pr_info.rate_limit_hit")
+            sentry_metrics.incr("auth.get_pr_info.rate_limit_hit")
+            if e.reset:
+                now_timestamp = datetime.now().timestamp()
+                retry_after = int(e.reset) - int(now_timestamp)
+            elif e.retry_after:
+                retry_after = int(e.retry_after)
+            raise exceptions.Throttled(
+                wait=retry_after, detail=self.rate_limit_failed_message
+            )
 
     def authenticate(self, request):
         fork_slug = request.headers.get("X-Tokenless", None)

codecov_auth/tests/unit/test_repo_authentication.py

@@ -8,7 +8,7 @@
 from django.utils import timezone
 from rest_framework import exceptions
 from rest_framework.test import APIRequestFactory
-from shared.torngit.exceptions import TorngitObjectNotFoundError
+from shared.torngit.exceptions import TorngitObjectNotFoundError, TorngitRateLimitError
 
 from codecov_auth.authentication.repo_auth import (
     GlobalTokenAuthentication,
@@ -532,3 +532,44 @@ def test_tokenless_success(self, mock_repo_provider, db, mocker):
         assert repo_as_user.is_authenticated()
         assert isinstance(auth_class, TokenlessAuth)
         mock_adapter.get_pull_request.assert_called_with("15")
+
+    @patch("codecov_auth.authentication.repo_auth.RepoProviderService")
+    def test_tokenless_rate_limit(self, mock_repo_provider, db, mocker):
+        repo = RepositoryFactory(private=False)
+        err = TorngitRateLimitError(
+            "error", "err msg", int(datetime.now().timestamp()) + 20, None
+        )
+        mock_adapter = MagicMock(
+            name="mock_provider_adapter",
+            get_pull_request=AsyncMock(name="mock_get_pr", side_effect=err),
+        )
+        mock_repo_provider.return_value.get_adapter.return_value = mock_adapter
+
+        request = APIRequestFactory().post(
+            f"/upload/github/{repo.author.username}::::{repo.name}/commits/commit_sha/reports/report_code/uploads",
+            headers={"X-Tokenless": f"some-user/{repo.name}", "X-Tokenless-PR": "15"},
+        )
+        authentication = TokenlessAuthentication()
+
+        with pytest.raises(exceptions.Throttled):
+            res = authentication.authenticate(request)
+        mock_adapter.get_pull_request.assert_called_with("15")
+
+    @patch("codecov_auth.authentication.repo_auth.RepoProviderService")
+    def test_tokenless_rate_limit_retry_after(self, mock_repo_provider, db, mocker):
+        repo = RepositoryFactory(private=False)
+        err = TorngitRateLimitError("error", "err msg", None, 20)
+        mock_adapter = MagicMock(
+            name="mock_provider_adapter",
+            get_pull_request=AsyncMock(name="mock_get_pr", side_effect=err),
+        )
+        mock_repo_provider.return_value.get_adapter.return_value = mock_adapter
+
+        request = APIRequestFactory().post(
+            f"/upload/github/{repo.author.username}::::{repo.name}/commits/commit_sha/reports/report_code/uploads",
+            headers={"X-Tokenless": f"some-user/{repo.name}", "X-Tokenless-PR": "15"},
+        )
+        authentication = TokenlessAuthentication()
+        with pytest.raises(exceptions.Throttled):
+            res = authentication.authenticate(request)
+        mock_adapter.get_pull_request.assert_called_with("15")

dev.sh

@@ -17,6 +17,7 @@ _start_gunicorn() {
   fi
   if [[ "$RUN_ENV" == "enterprise" ]] || [[ "$RUN_ENV" == "DEV" ]]; then
     python manage.py migrate
+    python manage.py migrate --database "timeseries"
   fi
   gunicorn codecov.wsgi:application --reload --bind 0.0.0.0:8000 --access-logfile '-' --timeout "${GUNICORN_TIMEOUT:-600}" $suffix
 }

graphql_api/helpers/mutation.py

@@ -1,4 +1,5 @@
 from codecov.commands import exceptions
+from codecov_auth.helpers import current_user_part_of_org
 
 
 class WrappedException:
@@ -57,5 +58,22 @@ def authenticated_resolver(instance, info, *args, **kwargs):
     return authenticated_resolver
 
 
+def require_part_of_org(resolver):
+    def authenticated_resolver(queried_owner, info, *args, **kwargs):
+        current_user = info.context["request"].user
+        current_owner = info.context["request"].current_owner
+        if (
+            not current_user
+            or not current_user.is_authenticated
+            or not current_owner
+            or not current_user_part_of_org(current_owner, queried_owner)
+        ):
+            return None
+
+        return resolver(queried_owner, info, *args, **kwargs)
+
+    return authenticated_resolver
+
+
 def resolve_union_error_type(error, *_):
     return error.get_graphql_type()

graphql_api/tests/test_owner.py

@@ -2,7 +2,6 @@
 from datetime import timedelta
 from unittest.mock import patch
 
-import pytest
 from django.test import TransactionTestCase
 from django.utils import timezone
 from freezegun import freeze_time
@@ -279,8 +278,8 @@ def test_is_current_user_not_an_admin(self):
         user = OwnerFactory(username="random_org_user", service="github")
         owner = OwnerFactory(username="random_org_test", service="github")
         query = query_current_user_is_admin % (owner.username)
-        data = self.gql_request(query, owner=user)
-        assert data["owner"]["isAdmin"] is False
+        data = self.gql_request(query, owner=user, with_errors=True)
+        assert data["data"]["owner"]["isAdmin"] is None
 
     @patch(
         "codecov_auth.commands.owner.interactors.get_is_current_user_an_admin.get_provider"
@@ -296,6 +295,8 @@ def test_is_current_user_an_admin(self, mocked_get_adapter):
         owner = OwnerFactory(
             username="random_org_test", service="github", admins=[user.ownerid]
         )
+        user.organizations = [owner.ownerid]
+        user.save()
         mocked_get_adapter.return_value = GetAdminProviderAdapter()
         query = query_current_user_is_admin % (owner.username)
         data = self.gql_request(query, owner=user)
@@ -381,6 +382,8 @@ def test_owner_without_owner_profile_returns_no_default_org(self):
 
     def test_is_current_user_not_activated(self):
         owner = OwnerFactory(username="sample-owner", service="github")
+        self.owner.organizations = [owner.ownerid]
+        self.owner.save()
         query = """{
             owner(username: "%s") {
                 isCurrentUserActivated
@@ -411,6 +414,8 @@ def test_is_current_user_activated(self):
         owner = OwnerFactory(
             username="sample-owner", plan_activated_users=[user.ownerid]
         )
+        user.organizations = [owner.ownerid]
+        user.save()
         query = """{
             owner(username: "%s") {
                 isCurrentUserActivated
@@ -425,6 +430,8 @@ def test_is_current_user_activated(self):
     def test_is_current_user_activated_when_plan_activated_users_is_none(self):
         user = OwnerFactory(username="sample-user")
         owner = OwnerFactory(username="sample-owner", plan_activated_users=None)
+        user.organizations = [owner.ownerid]
+        user.save()
         query = """{
             owner(username: "%s") {
                 isCurrentUserActivated
@@ -449,8 +456,12 @@ def test_is_current_user_activated_anonymous(self):
         data = self.gql_request(query)
         assert data["owner"]["isCurrentUserActivated"] == False
 
-    def test_admin_is_current_user_activated(self):
-        owner = OwnerFactory(username="sample-owner", admins=[self.owner.ownerid])
+    def test_admin_is_current_user_activated_authorized(self):
+        owner = OwnerFactory(
+            username="sample-owner-authorized", admins=[self.owner.ownerid]
+        )
+        self.owner.organizations = [owner.ownerid]
+        self.owner.save()
         query = """{
             owner(username: "%s") {
                 isCurrentUserActivated
@@ -622,3 +633,19 @@ def test_owner_query_with_public_repos(self):
 
         data = self.gql_request(query, owner=current_org)
         assert data["owner"]["hasPrivateRepos"] == False
+
+    def test_owner_hash_owner_id(self):
+        user = OwnerFactory(username="sample-user")
+        owner = OwnerFactory(username="sample-owner", plan_activated_users=None)
+        user.organizations = [owner.ownerid]
+        user.save()
+        query = """{
+            owner(username: "%s") {
+                hashOwnerid
+            }
+        }
+        """ % (
+            owner.username
+        )
+        data = self.gql_request(query, owner=user)
+        assert data["owner"]["hashOwnerid"] is not None

graphql_api/types/owner/owner.graphql

@@ -15,21 +15,21 @@ type Owner {
   repository(name: String!): RepositoryResult!
   repositoryDeprecated(name: String!): Repository
   numberOfUploads: Int
-  hasPrivateRepos: Boolean!
-  isAdmin: Boolean!
-  hashOwnerid: String!
-  ownerid: Int!
+  hasPrivateRepos: Boolean
+  isAdmin: Boolean
+  hashOwnerid: String
+  ownerid: Int
   plan: Plan
   pretrialPlan: PlanRepresentation
-  availablePlans: [PlanRepresentation!]!
+  availablePlans: [PlanRepresentation!]
   orgUploadToken: String
   defaultOrgUsername: String
-  isCurrentUserActivated: Boolean!
+  isCurrentUserActivated: Boolean
   measurements(
     interval: MeasurementInterval!
     after: DateTime
     before: DateTime
     repos: [String!]
     isPublic: Boolean
-  ): [Measurement!]!
+  ): [Measurement!]
 }

graphql_api/types/owner/owner.py

@@ -17,6 +17,7 @@
     build_connection_graphql,
     queryset_to_connection,
 )
+from graphql_api.helpers.mutation import require_part_of_org
 from graphql_api.types.enums import OrderingDirection, RepositoryOrdering
 from graphql_api.types.errors.errors import NotFoundError, OwnerNotActivatedError
 from plan.constants import FREE_PLAN_REPRESENTATIONS, PlanData, PlanName
@@ -68,19 +69,22 @@ def resolve_yaml(owner, info):
 
 
 @owner_bindable.field("plan")
+@require_part_of_org
 def resolve_plan(owner: Owner, info) -> PlanService:
     return PlanService(current_org=owner)
 
 
 @owner_bindable.field("pretrialPlan")
 @convert_kwargs_to_snake_case
+@require_part_of_org
 def resolve_plan_representation(owner: Owner, info) -> PlanData:
     info.context["plan_service"] = PlanService(current_org=owner)
     return FREE_PLAN_REPRESENTATIONS[PlanName.BASIC_PLAN_NAME.value]
 
 
 @owner_bindable.field("availablePlans")
 @convert_kwargs_to_snake_case
+@require_part_of_org
 def resolve_available_plans(owner: Owner, info) -> List[PlanData]:
     plan_service = PlanService(current_org=owner)
     info.context["plan_service"] = plan_service
@@ -90,10 +94,17 @@ def resolve_available_plans(owner: Owner, info) -> List[PlanData]:
 
 @owner_bindable.field("hasPrivateRepos")
 @sync_to_async
+@require_part_of_org
 def resolve_has_private_repos(owner: Owner, info) -> List[PlanData]:
     return owner.has_private_repos
 
 
+@owner_bindable.field("ownerid")
+@require_part_of_org
+def resolve_ownerid(owner, info) -> int:
+    return owner.ownerid
+
+
 @owner_bindable.field("repository")
 async def resolve_repository(owner, info, name):
     command = info.context["executor"].get_command("repository")
@@ -132,32 +143,37 @@ async def resolve_repository_deprecated(owner, info, name):
 
 
 @owner_bindable.field("numberOfUploads")
+@require_part_of_org
 async def resolve_number_of_uploads(owner, info, **kwargs):
     command = info.context["executor"].get_command("owner")
     return await command.get_uploads_number_per_user(owner)
 
 
 @owner_bindable.field("isAdmin")
+@require_part_of_org
 def resolve_is_current_user_an_admin(owner, info):
     current_owner = info.context["request"].current_owner
     command = info.context["executor"].get_command("owner")
     return command.get_is_current_user_an_admin(owner, current_owner)
 
 
 @owner_bindable.field("hashOwnerid")
+@require_part_of_org
 def resolve_hash_ownerid(owner, info):
     hash_ownerid = sha1(str(owner.ownerid).encode())
     return hash_ownerid.hexdigest()
 
 
 @owner_bindable.field("orgUploadToken")
+@require_part_of_org
 def resolve_org_upload_token(owner, info, **kwargs):
     command = info.context["executor"].get_command("owner")
     return command.get_org_upload_token(owner)
 
 
 @owner_bindable.field("defaultOrgUsername")
 @sync_to_async
+@require_part_of_org
 def resolve_org_default_org_username(owner: Owner, info, **kwargs) -> int:
     return None if owner.default_org is None else owner.default_org.username
 

upload/serializers.py

@@ -22,6 +22,7 @@ class UploadSerializer(serializers.ModelSerializer):
     version = serializers.CharField(write_only=True, required=False)
     url = serializers.SerializerMethodField()
     storage_path = serializers.CharField(write_only=True, required=False)
+    ci_service = serializers.CharField(write_only=True, required=False)
 
     class Meta:
         read_only_fields = (
@@ -41,6 +42,7 @@ class Meta:
             "job_code",
             "version",
             "storage_path",
+            "ci_service",
         )
         model = ReportSession
 
@@ -67,11 +69,12 @@ def create(self, validated_data):
         flag_names = (
             validated_data.pop("flags") if "flags" in validated_data.keys() else []
         )
-        _ = (
-            validated_data.pop("version")
-            if "version" in validated_data.keys()
-            else None
-        )
+
+        # default is necessary here, or else if the key is not in the dict
+        # the below will throw a KeyError
+        validated_data.pop("version", None)
+        validated_data.pop("ci_service", None)
+
         upload = ReportSession.objects.create(**validated_data)
         flags = []