diff --git a/tests/common/db/sponsors.py b/tests/common/db/sponsors.py index 102d7978d140..fa21303f3ac8 100644 --- a/tests/common/db/sponsors.py +++ b/tests/common/db/sponsors.py @@ -35,3 +35,8 @@ class Meta: infra_sponsor = False one_time = False sidebar = True + + origin = "manual" + level_name = "" + level_order = 0 + slug = factory.Faker("slug") diff --git a/tests/unit/sponsors/test_init.py b/tests/unit/sponsors/test_init.py index 993b1d69578a..d874241bbb84 100644 --- a/tests/unit/sponsors/test_init.py +++ b/tests/unit/sponsors/test_init.py @@ -12,17 +12,22 @@ import pretend +from celery.schedules import crontab from sqlalchemy import true from warehouse import sponsors from warehouse.sponsors.models import Sponsor +from warehouse.sponsors.tasks import update_pypi_sponsors from ...common.db.sponsors import SponsorFactory def test_includeme(): + settings = {"pythondotorg.api_token": "test-token"} config = pretend.stub( - add_request_method=pretend.call_recorder(lambda f, name, reify: None) + add_request_method=pretend.call_recorder(lambda f, name, reify: None), + add_periodic_task=pretend.call_recorder(lambda crontab, task: None), + registry=pretend.stub(settings=settings), ) sponsors.includeme(config) @@ -30,6 +35,25 @@ def test_includeme(): assert config.add_request_method.calls == [ pretend.call(sponsors._sponsors, name="sponsors", reify=True), ] + assert config.add_periodic_task.calls == [ + pretend.call(crontab(minute=10), update_pypi_sponsors), + ] + + +def test_do_not_schedule_sponsor_api_integration_if_no_token(): + settings = {} + config = pretend.stub( + add_request_method=pretend.call_recorder(lambda f, name, reify: None), + add_periodic_task=pretend.call_recorder(lambda crontab, task: None), + registry=pretend.stub(settings=settings), + ) + + sponsors.includeme(config) + + assert config.add_request_method.calls == [ + pretend.call(sponsors._sponsors, name="sponsors", reify=True), + ] + assert not config.add_periodic_task.calls def test_list_sponsors(db_request): diff --git a/tests/unit/sponsors/test_tasks.py b/tests/unit/sponsors/test_tasks.py new file mode 100644 index 000000000000..0c6588796e13 --- /dev/null +++ b/tests/unit/sponsors/test_tasks.py @@ -0,0 +1,225 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from urllib.parse import urlencode + +import pretend +import pytest + +from requests.exceptions import HTTPError + +from warehouse.sponsors import tasks +from warehouse.sponsors.models import Sponsor + +from ...common.db.sponsors import SponsorFactory + + +@pytest.fixture +def fake_task_request(): + cfg = { + "pythondotorg.host": "https://API_HOST", + "pythondotorg.api_token": "API_TOKEN", + } + request = pretend.stub(registry=pretend.stub(settings=cfg)) + return request + + +@pytest.fixture +def sponsor_api_data(): + return [ + { + "publisher": "pypi", + "flight": "sponsors", + "sponsor": "Sponsor Name", + "sponsor_slug": "sponsor-name", + "description": "Sponsor description", + "logo": "https://logourl.com", + "start_date": "2021-02-17", + "end_date": "2022-02-17", + "sponsor_url": "https://sponsor.example.com/", + "level_name": "Partner", + "level_order": 5, + } + ] + + +def test_raise_error_if_invalid_response(monkeypatch, db_request, fake_task_request): + response = pretend.stub( + status_code=418, + text="I'm a teapot", + raise_for_status=pretend.raiser(HTTPError), + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + + with pytest.raises(HTTPError): + tasks.update_pypi_sponsors(fake_task_request) + + qs = urlencode({"publisher": "pypi", "flight": "sponsors"}) + headers = {"Authorization": "Token API_TOKEN"} + expected_url = f"https://API_HOST/api/v2/sponsors/logo-placement/?{qs}" + assert requests.get.calls == [pretend.call(expected_url, headers=headers)] + + +def test_create_new_sponsor_if_no_matching( + monkeypatch, db_request, fake_task_request, sponsor_api_data +): + response = pretend.stub( + raise_for_status=lambda: None, json=lambda: sponsor_api_data + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + assert 0 == len(db_request.db.query(Sponsor).all()) + + fake_task_request.db = db_request.db + tasks.update_pypi_sponsors(fake_task_request) + + db_sponsor = db_request.db.query(Sponsor).one() + assert "sponsor-name" == db_sponsor.slug + assert "Sponsor Name" == db_sponsor.name + assert "Sponsor description" == db_sponsor.service + assert "https://sponsor.example.com/" == db_sponsor.link_url + assert "https://logourl.com" == db_sponsor.color_logo_url + assert db_sponsor.activity_markdown is None + assert db_sponsor.white_logo_url is None + assert db_sponsor.is_active is True + assert db_sponsor.psf_sponsor is True + assert db_sponsor.footer is False + assert db_sponsor.infra_sponsor is False + assert db_sponsor.one_time is False + assert db_sponsor.sidebar is False + assert "remote" == db_sponsor.origin + assert "Partner" == db_sponsor.level_name + assert 5 == db_sponsor.level_order + + +def test_update_remote_sponsor_with_same_name_with_new_logo( + monkeypatch, db_request, fake_task_request, sponsor_api_data +): + response = pretend.stub( + raise_for_status=lambda: None, json=lambda: sponsor_api_data + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + created_sponsor = SponsorFactory.create( + name=sponsor_api_data[0]["sponsor"], + psf_sponsor=True, + footer=False, + sidebar=False, + one_time=False, + origin="manual", + ) + + fake_task_request.db = db_request.db + tasks.update_pypi_sponsors(fake_task_request) + + assert 1 == len(db_request.db.query(Sponsor).all()) + db_sponsor = db_request.db.query(Sponsor).one() + assert db_sponsor.id == created_sponsor.id + assert "sponsor-name" == db_sponsor.slug + assert "Sponsor description" == db_sponsor.service + assert "https://sponsor.example.com/" == db_sponsor.link_url + assert "https://logourl.com" == db_sponsor.color_logo_url + assert db_sponsor.activity_markdown is created_sponsor.activity_markdown + assert db_sponsor.white_logo_url is created_sponsor.white_logo_url + assert db_sponsor.is_active is True + assert db_sponsor.psf_sponsor is True + assert db_sponsor.footer is False + assert db_sponsor.infra_sponsor is False + assert db_sponsor.one_time is False + assert db_sponsor.sidebar is False + assert "remote" == db_sponsor.origin + assert "Partner" == db_sponsor.level_name + assert 5 == db_sponsor.level_order + + +def test_do_not_update_if_not_psf_sponsor( + monkeypatch, db_request, fake_task_request, sponsor_api_data +): + response = pretend.stub( + raise_for_status=lambda: None, json=lambda: sponsor_api_data + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + infra_sponsor = SponsorFactory.create( + name=sponsor_api_data[0]["sponsor"], + psf_sponsor=False, + infra_sponsor=True, + one_time=False, + origin="manual", + ) + + fake_task_request.db = db_request.db + tasks.update_pypi_sponsors(fake_task_request) + + assert 1 == len(db_request.db.query(Sponsor).all()) + db_sponsor = db_request.db.query(Sponsor).one() + assert db_sponsor.id == infra_sponsor.id + assert "manual" == db_sponsor.origin + assert "sponsor-name" != db_sponsor.slug + + +def test_update_remote_sponsor_with_same_slug_with_new_logo( + monkeypatch, db_request, fake_task_request, sponsor_api_data +): + response = pretend.stub( + raise_for_status=lambda: None, json=lambda: sponsor_api_data + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + created_sponsor = SponsorFactory.create( + slug=sponsor_api_data[0]["sponsor_slug"], + psf_sponsor=True, + footer=False, + sidebar=False, + one_time=False, + origin="manual", + ) + + fake_task_request.db = db_request.db + tasks.update_pypi_sponsors(fake_task_request) + + assert 1 == len(db_request.db.query(Sponsor).all()) + db_sponsor = db_request.db.query(Sponsor).one() + assert db_sponsor.id == created_sponsor.id + assert "Sponsor Name" == db_sponsor.name + assert "Sponsor description" == db_sponsor.service + + +def test_flag_existing_psf_sponsor_to_false_if_not_present_in_api_response( + monkeypatch, db_request, fake_task_request, sponsor_api_data +): + response = pretend.stub( + raise_for_status=lambda: None, json=lambda: sponsor_api_data + ) + requests = pretend.stub(get=pretend.call_recorder(lambda url, headers: response)) + monkeypatch.setattr(tasks, "requests", requests) + created_sponsor = SponsorFactory.create( + slug="other-slug", + name="Other Sponsor", + psf_sponsor=True, + footer=True, + sidebar=True, + origin="manual", + ) + + fake_task_request.db = db_request.db + tasks.update_pypi_sponsors(fake_task_request) + + assert 2 == len(db_request.db.query(Sponsor).all()) + created_sponsor = ( + db_request.db.query(Sponsor).filter(Sponsor.id == created_sponsor.id).one() + ) + # no longer PSF sponsor but stay active as sidebar/footer sponsor + assert created_sponsor.psf_sponsor is False + assert created_sponsor.sidebar is True + assert created_sponsor.footer is True + assert created_sponsor.is_active is True diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 5ee76cce8c69..212269d7b9cc 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -233,6 +233,7 @@ def __init__(self): "site.name": "Warehouse", "token.two_factor.max_age": 300, "token.default.max_age": 21600, + "pythondotorg.host": "python.org", "warehouse.xmlrpc.client.ratelimit_string": "3600 per hour", "warehouse.xmlrpc.search.enabled": True, "github.token_scanning_meta_api.url": ( diff --git a/warehouse/config.py b/warehouse/config.py index 5af6481bc764..25242b310ee3 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -235,6 +235,10 @@ def configure(settings=None): maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS") maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND") + # Pythondotorg integration settings + maybe_set(settings, "pythondotorg.host", "PYTHONDOTORG_HOST", default="python.org") + maybe_set(settings, "pythondotorg.api_token", "PYTHONDOTORG_API_TOKEN") + # Configure our ratelimiters maybe_set( settings, diff --git a/warehouse/migrations/versions/19cf76d2d459_new_sponsor_columns_to_save_data_from_.py b/warehouse/migrations/versions/19cf76d2d459_new_sponsor_columns_to_save_data_from_.py new file mode 100644 index 000000000000..53217670e9b4 --- /dev/null +++ b/warehouse/migrations/versions/19cf76d2d459_new_sponsor_columns_to_save_data_from_.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +New Sponsor columns to save data from pythondotorg API + +Revision ID: 19cf76d2d459 +Revises: 29a8901a4635 +Create Date: 2022-02-13 14:31:18.366248 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "19cf76d2d459" +down_revision = "29a8901a4635" + +# Note: It is VERY important to ensure that a migration does not lock for a +# long period of time and to ensure that each individual migration does +# not break compatibility with the *previous* version of the code base. +# This is because the migrations will be ran automatically as part of the +# deployment process, but while the previous version of the code is still +# up and running. Thus backwards incompatible changes must be broken up +# over multiple migrations inside of multiple pull requests in order to +# phase them in over multiple deploys. + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("sponsors", sa.Column("origin", sa.String(), nullable=True)) + op.add_column("sponsors", sa.Column("level_name", sa.String(), nullable=True)) + op.add_column("sponsors", sa.Column("level_order", sa.Integer(), nullable=True)) + op.add_column("sponsors", sa.Column("slug", sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("sponsors", "slug") + op.drop_column("sponsors", "level_order") + op.drop_column("sponsors", "level_name") + op.drop_column("sponsors", "origin") + # ### end Alembic commands ### diff --git a/warehouse/sponsors/__init__.py b/warehouse/sponsors/__init__.py index 1aefa75f02e2..8faaf88347ac 100644 --- a/warehouse/sponsors/__init__.py +++ b/warehouse/sponsors/__init__.py @@ -10,9 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from celery.schedules import crontab from sqlalchemy import true from warehouse.sponsors.models import Sponsor +from warehouse.sponsors.tasks import update_pypi_sponsors def _sponsors(request): @@ -22,3 +24,7 @@ def _sponsors(request): def includeme(config): # Add a request method which will allow to list sponsors config.add_request_method(_sponsors, name="sponsors", reify=True) + + # Add a periodic task to update sponsors table + if config.registry.settings.get("pythondotorg.api_token"): + config.add_periodic_task(crontab(minute=10), update_pypi_sponsors) diff --git a/warehouse/sponsors/models.py b/warehouse/sponsors/models.py index df11e0b10df8..f1b61dfbcf83 100644 --- a/warehouse/sponsors/models.py +++ b/warehouse/sponsors/models.py @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sqlalchemy import Boolean, Column, String, Text +from sqlalchemy import Boolean, Column, Integer, String, Text from sqlalchemy_utils.types.url import URLType from warehouse import db @@ -38,6 +38,12 @@ class Sponsor(db.Model): one_time = Column(Boolean, default=False, nullable=False) sidebar = Column(Boolean, default=False, nullable=False) + # pythondotorg integration + origin = Column(String, default="manual") + level_name = Column(String) + level_order = Column(Integer, default=0) + slug = Column(String) + @property def color_logo_img(self): return f'{ self.name }' diff --git a/warehouse/sponsors/tasks.py b/warehouse/sponsors/tasks.py new file mode 100644 index 000000000000..e0aa26fe75fa --- /dev/null +++ b/warehouse/sponsors/tasks.py @@ -0,0 +1,68 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from urllib.parse import urlencode + +import requests + +from sqlalchemy import or_, true +from sqlalchemy.exc import NoResultFound + +from warehouse import tasks +from warehouse.sponsors.models import Sponsor + + +@tasks.task(ignore_result=True, acks_late=True) +def update_pypi_sponsors(request): + """ + Read data from pythondotorg's logo placement API and update Sponsors + table to mirror it. + """ + host = request.registry.settings["pythondotorg.host"] + token = request.registry.settings["pythondotorg.api_token"] + headers = {"Authorization": f"Token {token}"} + + qs = urlencode({"publisher": "pypi", "flight": "sponsors"}) + url = f"{host}/api/v2/sponsors/logo-placement/?{qs}" + response = requests.get(url, headers=headers) + response.raise_for_status() + + # deactivate current PSF sponsors to keep it up to date with API + request.db.query(Sponsor).filter(Sponsor.psf_sponsor == true()).update( + {"psf_sponsor": False} + ) + + for sponsor_info in response.json(): + name = sponsor_info["sponsor"] + slug = sponsor_info["sponsor_slug"] + query = request.db.query(Sponsor) + try: + sponsor = query.filter( + or_(Sponsor.name == name, Sponsor.slug == slug) + ).one() + if sponsor.infra_sponsor or sponsor.one_time: + continue + except NoResultFound: + sponsor = Sponsor() + request.db.add(sponsor) + + sponsor.name = name + sponsor.slug = slug + sponsor.service = sponsor_info["description"] + sponsor.link_url = sponsor_info["sponsor_url"] + sponsor.color_logo_url = sponsor_info["logo"] + sponsor.level_name = sponsor_info["level_name"] + sponsor.level_order = sponsor_info["level_order"] + sponsor.is_active = True + sponsor.psf_sponsor = True + sponsor.origin = "remote" + + request.db.commit()