Sync PSF sponsors with logo placement API (pypi#10766)

* Add new config variables to integrate with pythondotorg

* Add new fields with data from pythondotorg logo placement API

* Register task to update sponsors table

* Code linter

* Fix typo

* Minimal working code to integrate with logo palcement endpoint

* First unit test to figure out how to mock requests

* Create a new sponsor

* Add logic to update existing sponsors

* Reformat code

* Leave HTTP vs HTTPS configuration to be handle by env variable

* Fix linter errors

* Create sponsor directly from the exception handler

* Run task every 10 minutes

* There's no need for the if condition

* Only schedule cron job if there's a token in the env

* Remove pythondotorg env variables from dev

* Run linter

Co-authored-by: Ee Durbin <ewdurbin@gmail.com>

tests/common/db/sponsors.py

@@ -35,3 +35,8 @@ class Meta:
     infra_sponsor = False
     one_time = False
     sidebar = True
+
+    origin = "manual"
+    level_name = ""
+    level_order = 0
+    slug = factory.Faker("slug")

tests/unit/sponsors/test_init.py

@@ -12,24 +12,48 @@
 
 import pretend
 
+from celery.schedules import crontab
 from sqlalchemy import true
 
 from warehouse import sponsors
 from warehouse.sponsors.models import Sponsor
+from warehouse.sponsors.tasks import update_pypi_sponsors
 
 from ...common.db.sponsors import SponsorFactory
 
 
 def test_includeme():
+    settings = {"pythondotorg.api_token": "test-token"}
     config = pretend.stub(
-        add_request_method=pretend.call_recorder(lambda f, name, reify: None)
+        add_request_method=pretend.call_recorder(lambda f, name, reify: None),
+        add_periodic_task=pretend.call_recorder(lambda crontab, task: None),
+        registry=pretend.stub(settings=settings),
     )
 
     sponsors.includeme(config)
 
     assert config.add_request_method.calls == [
         pretend.call(sponsors._sponsors, name="sponsors", reify=True),
     ]
+    assert config.add_periodic_task.calls == [
+        pretend.call(crontab(minute=10), update_pypi_sponsors),
+    ]
+
+
+def test_do_not_schedule_sponsor_api_integration_if_no_token():
+    settings = {}
+    config = pretend.stub(
+        add_request_method=pretend.call_recorder(lambda f, name, reify: None),
+        add_periodic_task=pretend.call_recorder(lambda crontab, task: None),
+        registry=pretend.stub(settings=settings),
+    )
+
+    sponsors.includeme(config)
+
+    assert config.add_request_method.calls == [
+        pretend.call(sponsors._sponsors, name="sponsors", reify=True),
+    ]
+    assert not config.add_periodic_task.calls
 
 
 def test_list_sponsors(db_request):

tests/unit/sponsors/test_tasks.py

@@ -0,0 +1,225 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from urllib.parse import urlencode
+
+import pretend
+import pytest
+
+from requests.exceptions import HTTPError
+
+from warehouse.sponsors import tasks
+from warehouse.sponsors.models import Sponsor
+
+from ...common.db.sponsors import SponsorFactory
+
+
+@pytest.fixture
+def fake_task_request():
+    cfg = {
+        "pythondotorg.host": "https://API_HOST",
+        "pythondotorg.api_token": "API_TOKEN",
+    }
+    request = pretend.stub(registry=pretend.stub(settings=cfg))
+    return request
+
+
+@pytest.fixture
+def sponsor_api_data():
+    return [
+        {
+            "publisher": "pypi",
+            "flight": "sponsors",
+            "sponsor": "Sponsor Name",
+            "sponsor_slug": "sponsor-name",
+            "description": "Sponsor description",
+            "logo": "https://logourl.com",
+            "start_date": "2021-02-17",
+            "end_date": "2022-02-17",
+            "sponsor_url": "https://sponsor.example.com/",
+            "level_name": "Partner",
+            "level_order": 5,
+        }
+    ]
+
+
+def test_raise_error_if_invalid_response(monkeypatch, db_request, fake_task_request):
+    response = pretend.stub(
+        status_code=418,
+        text="I'm a teapot",
+        raise_for_status=pretend.raiser(HTTPError),
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+
+    with pytest.raises(HTTPError):
+        tasks.update_pypi_sponsors(fake_task_request)
+
+    qs = urlencode({"publisher": "pypi", "flight": "sponsors"})
+    headers = {"Authorization": "Token API_TOKEN"}
+    expected_url = f"https://API_HOST/api/v2/sponsors/logo-placement/?{qs}"
+    assert requests.get.calls == [pretend.call(expected_url, headers=headers)]
+
+
+def test_create_new_sponsor_if_no_matching(
+    monkeypatch, db_request, fake_task_request, sponsor_api_data
+):
+    response = pretend.stub(
+        raise_for_status=lambda: None, json=lambda: sponsor_api_data
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+    assert 0 == len(db_request.db.query(Sponsor).all())
+
+    fake_task_request.db = db_request.db
+    tasks.update_pypi_sponsors(fake_task_request)
+
+    db_sponsor = db_request.db.query(Sponsor).one()
+    assert "sponsor-name" == db_sponsor.slug
+    assert "Sponsor Name" == db_sponsor.name
+    assert "Sponsor description" == db_sponsor.service
+    assert "https://sponsor.example.com/" == db_sponsor.link_url
+    assert "https://logourl.com" == db_sponsor.color_logo_url
+    assert db_sponsor.activity_markdown is None
+    assert db_sponsor.white_logo_url is None
+    assert db_sponsor.is_active is True
+    assert db_sponsor.psf_sponsor is True
+    assert db_sponsor.footer is False
+    assert db_sponsor.infra_sponsor is False
+    assert db_sponsor.one_time is False
+    assert db_sponsor.sidebar is False
+    assert "remote" == db_sponsor.origin
+    assert "Partner" == db_sponsor.level_name
+    assert 5 == db_sponsor.level_order
+
+
+def test_update_remote_sponsor_with_same_name_with_new_logo(
+    monkeypatch, db_request, fake_task_request, sponsor_api_data
+):
+    response = pretend.stub(
+        raise_for_status=lambda: None, json=lambda: sponsor_api_data
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+    created_sponsor = SponsorFactory.create(
+        name=sponsor_api_data[0]["sponsor"],
+        psf_sponsor=True,
+        footer=False,
+        sidebar=False,
+        one_time=False,
+        origin="manual",
+    )
+
+    fake_task_request.db = db_request.db
+    tasks.update_pypi_sponsors(fake_task_request)
+
+    assert 1 == len(db_request.db.query(Sponsor).all())
+    db_sponsor = db_request.db.query(Sponsor).one()
+    assert db_sponsor.id == created_sponsor.id
+    assert "sponsor-name" == db_sponsor.slug
+    assert "Sponsor description" == db_sponsor.service
+    assert "https://sponsor.example.com/" == db_sponsor.link_url
+    assert "https://logourl.com" == db_sponsor.color_logo_url
+    assert db_sponsor.activity_markdown is created_sponsor.activity_markdown
+    assert db_sponsor.white_logo_url is created_sponsor.white_logo_url
+    assert db_sponsor.is_active is True
+    assert db_sponsor.psf_sponsor is True
+    assert db_sponsor.footer is False
+    assert db_sponsor.infra_sponsor is False
+    assert db_sponsor.one_time is False
+    assert db_sponsor.sidebar is False
+    assert "remote" == db_sponsor.origin
+    assert "Partner" == db_sponsor.level_name
+    assert 5 == db_sponsor.level_order
+
+
+def test_do_not_update_if_not_psf_sponsor(
+    monkeypatch, db_request, fake_task_request, sponsor_api_data
+):
+    response = pretend.stub(
+        raise_for_status=lambda: None, json=lambda: sponsor_api_data
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+    infra_sponsor = SponsorFactory.create(
+        name=sponsor_api_data[0]["sponsor"],
+        psf_sponsor=False,
+        infra_sponsor=True,
+        one_time=False,
+        origin="manual",
+    )
+
+    fake_task_request.db = db_request.db
+    tasks.update_pypi_sponsors(fake_task_request)
+
+    assert 1 == len(db_request.db.query(Sponsor).all())
+    db_sponsor = db_request.db.query(Sponsor).one()
+    assert db_sponsor.id == infra_sponsor.id
+    assert "manual" == db_sponsor.origin
+    assert "sponsor-name" != db_sponsor.slug
+
+
+def test_update_remote_sponsor_with_same_slug_with_new_logo(
+    monkeypatch, db_request, fake_task_request, sponsor_api_data
+):
+    response = pretend.stub(
+        raise_for_status=lambda: None, json=lambda: sponsor_api_data
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+    created_sponsor = SponsorFactory.create(
+        slug=sponsor_api_data[0]["sponsor_slug"],
+        psf_sponsor=True,
+        footer=False,
+        sidebar=False,
+        one_time=False,
+        origin="manual",
+    )
+
+    fake_task_request.db = db_request.db
+    tasks.update_pypi_sponsors(fake_task_request)
+
+    assert 1 == len(db_request.db.query(Sponsor).all())
+    db_sponsor = db_request.db.query(Sponsor).one()
+    assert db_sponsor.id == created_sponsor.id
+    assert "Sponsor Name" == db_sponsor.name
+    assert "Sponsor description" == db_sponsor.service
+
+
+def test_flag_existing_psf_sponsor_to_false_if_not_present_in_api_response(
+    monkeypatch, db_request, fake_task_request, sponsor_api_data
+):
+    response = pretend.stub(
+        raise_for_status=lambda: None, json=lambda: sponsor_api_data
+    )
+    requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response))
+    monkeypatch.setattr(tasks, "requests", requests)
+    created_sponsor = SponsorFactory.create(
+        slug="other-slug",
+        name="Other Sponsor",
+        psf_sponsor=True,
+        footer=True,
+        sidebar=True,
+        origin="manual",
+    )
+
+    fake_task_request.db = db_request.db
+    tasks.update_pypi_sponsors(fake_task_request)
+
+    assert 2 == len(db_request.db.query(Sponsor).all())
+    created_sponsor = (
+        db_request.db.query(Sponsor).filter(Sponsor.id == created_sponsor.id).one()
+    )
+    # no longer PSF sponsor but stay active as sidebar/footer sponsor
+    assert created_sponsor.psf_sponsor is False
+    assert created_sponsor.sidebar is True
+    assert created_sponsor.footer is True
+    assert created_sponsor.is_active is True

tests/unit/test_config.py

@@ -233,6 +233,7 @@ def __init__(self):
         "site.name": "Warehouse",
         "token.two_factor.max_age": 300,
         "token.default.max_age": 21600,
+        "pythondotorg.host": "python.org",
         "warehouse.xmlrpc.client.ratelimit_string": "3600 per hour",
         "warehouse.xmlrpc.search.enabled": True,
         "github.token_scanning_meta_api.url": (

warehouse/config.py

@@ -235,6 +235,10 @@ def configure(settings=None):
     maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS")
     maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND")
 
+    # Pythondotorg integration settings
+    maybe_set(settings, "pythondotorg.host", "PYTHONDOTORG_HOST", default="python.org")
+    maybe_set(settings, "pythondotorg.api_token", "PYTHONDOTORG_API_TOKEN")
+
     # Configure our ratelimiters
     maybe_set(
         settings,

warehouse/migrations/versions/19cf76d2d459_new_sponsor_columns_to_save_data_from_.py

@@ -0,0 +1,52 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+New Sponsor columns to save data from pythondotorg API
+
+Revision ID: 19cf76d2d459
+Revises: 29a8901a4635
+Create Date: 2022-02-13 14:31:18.366248
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "19cf76d2d459"
+down_revision = "29a8901a4635"
+
+# Note: It is VERY important to ensure that a migration does not lock for a
+#       long period of time and to ensure that each individual migration does
+#       not break compatibility with the *previous* version of the code base.
+#       This is because the migrations will be ran automatically as part of the
+#       deployment process, but while the previous version of the code is still
+#       up and running. Thus backwards incompatible changes must be broken up
+#       over multiple migrations inside of multiple pull requests in order to
+#       phase them in over multiple deploys.
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("sponsors", sa.Column("origin", sa.String(), nullable=True))
+    op.add_column("sponsors", sa.Column("level_name", sa.String(), nullable=True))
+    op.add_column("sponsors", sa.Column("level_order", sa.Integer(), nullable=True))
+    op.add_column("sponsors", sa.Column("slug", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("sponsors", "slug")
+    op.drop_column("sponsors", "level_order")
+    op.drop_column("sponsors", "level_name")
+    op.drop_column("sponsors", "origin")
+    # ### end Alembic commands ###

warehouse/sponsors/__init__.py

@@ -10,9 +10,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from celery.schedules import crontab
 from sqlalchemy import true
 
 from warehouse.sponsors.models import Sponsor
+from warehouse.sponsors.tasks import update_pypi_sponsors
 
 
 def _sponsors(request):
@@ -22,3 +24,7 @@ def _sponsors(request):
 def includeme(config):
     # Add a request method which will allow to list sponsors
     config.add_request_method(_sponsors, name="sponsors", reify=True)
+
+    # Add a periodic task to update sponsors table
+    if config.registry.settings.get("pythondotorg.api_token"):
+        config.add_periodic_task(crontab(minute=10), update_pypi_sponsors)

warehouse/sponsors/models.py

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from sqlalchemy import Boolean, Column, String, Text
+from sqlalchemy import Boolean, Column, Integer, String, Text
 from sqlalchemy_utils.types.url import URLType
 
 from warehouse import db
@@ -38,6 +38,12 @@ class Sponsor(db.Model):
     one_time = Column(Boolean, default=False, nullable=False)
     sidebar = Column(Boolean, default=False, nullable=False)
 
+    # pythondotorg integration
+    origin = Column(String, default="manual")
+    level_name = Column(String)
+    level_order = Column(Integer, default=0)
+    slug = Column(String)
+
     @property
     def color_logo_img(self):
         return f'<img src="{ self.color_logo_url }" alt="{ self.name }">'