From c70cfcf1e2da6e99e27c9599678ae45e5f26285c Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 21 Jun 2024 10:50:10 +0200 Subject: [PATCH 01/15] feat: Implement Flask-Limiter on the search module --- canonicalwebteam/search/views.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 77981fb..01a771b 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -3,6 +3,8 @@ # Packages import flask +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address # Local from canonicalwebteam.search.models import get_search_results @@ -22,7 +24,7 @@ def build_search_view( template_path="search.html", search_engine_id="009048213575199080868:i3zoqdwqk8o", site_restricted_search=False, - request_limit="500/day", + request_limit="2000/day", ): """ Build and return a view function that will query the @@ -46,6 +48,14 @@ def build_search_view( ) """ + app = flask.Flask(__name__) + limiter = Limiter( + app, + key_func=get_remote_address, + default_limits=[request_limit] + ) + + @limiter.limit(request_limit) def search_view(): """ Get search results from Google Custom Search @@ -53,10 +63,10 @@ def search_view(): # Rate limit requests to protect from spamming # To adjust this rate visit # https://limits.readthedocs.io/en/latest/quickstart.html#examples - limit = parse(request_limit) - rate_limit = fixed_window.hit(limit) - if not rate_limit: - return flask.abort(429, f"The rate limit is: {request_limit}") + # limit = parse(request_limit) + # rate_limit = fixed_window.hit(limit) + # if not rate_limit: + # return flask.abort(429, f"The rate limit is: {request_limit}") # API key should always be provided as an environment variable search_api_key = os.getenv("SEARCH_API_KEY") From faba23ffc33f2bbcc3939b78ef340e84c7e53cc5 Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 21 Jun 2024 11:08:12 +0200 Subject: [PATCH 02/15] chore: Clean up code and remove unneeded dependencies --- canonicalwebteam/search/views.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 01a771b..960d502 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -8,10 +8,6 @@ # Local from canonicalwebteam.search.models import get_search_results -from limits import storage, strategies, parse - -memory_storage = storage.MemoryStorage() -fixed_window = strategies.MovingWindowRateLimiter(memory_storage) class NoAPIKeyError(Exception): @@ -60,14 +56,6 @@ def search_view(): """ Get search results from Google Custom Search """ - # Rate limit requests to protect from spamming - # To adjust this rate visit - # https://limits.readthedocs.io/en/latest/quickstart.html#examples - # limit = parse(request_limit) - # rate_limit = fixed_window.hit(limit) - # if not rate_limit: - # return flask.abort(429, f"The rate limit is: {request_limit}") - # API key should always be provided as an environment variable search_api_key = os.getenv("SEARCH_API_KEY") From 33a5c243650e88ecc3043240ba162ee8b3320d03 Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 21 Jun 2024 13:01:41 +0200 Subject: [PATCH 03/15] chore: Bump version to 1.3.1 and update CHANGES.txt --- CHANGES.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index e64cba8..73938d7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,3 +11,4 @@ v1.2.5, 2022-07-12 -- Block some more bot useragents v1.2.6, 2022-07-13 -- Block one more useragent - "gh" v1.2.7, 2022-07-15 -- Block more user agents - "Petalbot" v1.3.0, 2023-02-20 -- Add rate limits +v1.3.1, 2-24-06-21 -- Migrate to Flask.Limiter for rate limits diff --git a/setup.py b/setup.py index 131cbf5..83c897b 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="canonicalwebteam.search", - version="1.3.0", + version="1.3.1", author="Canonical webteam", author_email="webteam@canonical.com", url="https://github.com/canonical/canonicalwebteam.search", From fa89afeb7762eee0de24e7ed698d5e395bfa603c Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 21 Jun 2024 13:06:51 +0200 Subject: [PATCH 04/15] style: Format Python with Black --- canonicalwebteam/search/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 960d502..62d1890 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -46,9 +46,7 @@ def build_search_view( app = flask.Flask(__name__) limiter = Limiter( - app, - key_func=get_remote_address, - default_limits=[request_limit] + app, key_func=get_remote_address, default_limits=[request_limit] ) @limiter.limit(request_limit) From 53aa9a836f78bff44185b6562efe981b8b86e755 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 25 Jun 2024 13:18:10 +0200 Subject: [PATCH 05/15] bug: Add Flask-Limiter to 'install_requires' --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 55d8d13..e21bc4c 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ app.add_url_rule( site="maas.io/docs", template_path="docs/search.html", search_engine_id="xxxxxxxxxx", # Optional argument, required by some of our sites - request_limit="500/day", # Allows your to configure the limit at which the user will be forbidden to query more. Defaults to 500 per day + request_limit="500/day", # Allows your to configure the limit at which the user will be forbidden to query more. Defaults to 2000 per day ) ) ``` diff --git a/setup.py b/setup.py index 83c897b..1f77be6 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(), long_description=open("README.md").read(), long_description_content_type="text/markdown", - install_requires=["Flask>=1.0.2", "user-agents>=2.0.0", "limits>=3.2.0"], + install_requires=["Flask>=1.0.2", "user-agents>=2.0.0", "Flask-Limiter>=1.4"], tests_require=["httpretty"], ) From e6aca34d5aaa31cb3e6b0bdc856f1f748d163953 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 25 Jun 2024 13:57:50 +0200 Subject: [PATCH 06/15] chore: Fix failing test where key_func was being passed twice --- canonicalwebteam/search/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 62d1890..cc53b15 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -46,7 +46,7 @@ def build_search_view( app = flask.Flask(__name__) limiter = Limiter( - app, key_func=get_remote_address, default_limits=[request_limit] + get_remote_address, app=app, default_limits=[request_limit] ) @limiter.limit(request_limit) From d89722be9c91f1ad24e77e58f1c16fb60d719ef6 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 25 Jun 2024 13:57:50 +0200 Subject: [PATCH 07/15] chore: Fix failing test where key_func was being passed twice --- canonicalwebteam/search/views.py | 42 ++++++++++++++++---------------- setup.py | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 62d1890..f44674b 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -46,10 +46,9 @@ def build_search_view( app = flask.Flask(__name__) limiter = Limiter( - app, key_func=get_remote_address, default_limits=[request_limit] + get_remote_address, app=app, default_limits=[request_limit] ) - @limiter.limit(request_limit) def search_view(): """ Get search results from Google Custom Search @@ -68,28 +67,29 @@ def search_view(): results = None if query: - results = get_search_results( - session=session, - api_key=search_api_key, - search_engine_id=search_engine_id, - siteSearch=site_search, - site_restricted_search=site_restricted_search, - query=query, - start=start, - num=num, - ) - - return ( - flask.render_template( - template_path, + with limiter.limit(request_limit): + results = get_search_results( + session=session, + api_key=search_api_key, + search_engine_id=search_engine_id, + siteSearch=site_search, + site_restricted_search=site_restricted_search, query=query, start=start, num=num, - results=results, - siteSearch=site_search, - ), - {"X-Robots-Tag": "noindex"}, - ) + ) + + return ( + flask.render_template( + template_path, + query=query, + start=start, + num=num, + results=results, + siteSearch=site_search, + ), + {"X-Robots-Tag": "noindex"}, + ) else: return flask.render_template( diff --git a/setup.py b/setup.py index 1f77be6..dfc3642 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(), long_description=open("README.md").read(), long_description_content_type="text/markdown", - install_requires=["Flask>=1.0.2", "user-agents>=2.0.0", "Flask-Limiter>=1.4"], + install_requires=["canonicalwebteam.flask-base>=1.1.0", "user-agents>=2.0.0", "Flask-Limiter>=2.9.0"], tests_require=["httpretty"], ) From 5c3388af45d40b9b579f728efdd80fd6b68916c3 Mon Sep 17 00:00:00 2001 From: Pete Date: Wed, 24 Jul 2024 15:01:09 +0200 Subject: [PATCH 08/15] chore: Update to use flask_base 2.0.0 which itseld uess Flask 2.3.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dfc3642..7a7260a 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(), long_description=open("README.md").read(), long_description_content_type="text/markdown", - install_requires=["canonicalwebteam.flask-base>=1.1.0", "user-agents>=2.0.0", "Flask-Limiter>=2.9.0"], + install_requires=["canonicalwebteam.flask-base>=2.0.0", "user-agents>=2.0.0", "Flask-Limiter>=3.8.0"], tests_require=["httpretty"], ) From cbcd7fefc4b5b55bee8b892208f3666d9207d44e Mon Sep 17 00:00:00 2001 From: Pete Date: Wed, 24 Jul 2024 16:43:29 +0200 Subject: [PATCH 09/15] style: Format python --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a7260a..5042028 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,10 @@ packages=find_packages(), long_description=open("README.md").read(), long_description_content_type="text/markdown", - install_requires=["canonicalwebteam.flask-base>=2.0.0", "user-agents>=2.0.0", "Flask-Limiter>=3.8.0"], + install_requires=[ + "canonicalwebteam.flask-base>=2.0.0", + "user-agents>=2.0.0", + "Flask-Limiter>=3.8.0", + ], tests_require=["httpretty"], ) From ce0bf06f26405cfd0aaae8c4a3ec56d03419ecf6 Mon Sep 17 00:00:00 2001 From: Olwe Samuel Date: Thu, 25 Jul 2024 16:21:53 +0300 Subject: [PATCH 10/15] use pip to install package in tests --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2dcf00d..86fdc33 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -41,7 +41,9 @@ jobs: - name: Test Python run: | - python3 setup.py install --user test + pip install -e . + pip install httpretty + python -m unittest discover tests check-inclusive-naming: From 692ff98562fcd9ecb8136951b76e2b691cef2d05 Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 25 Jul 2024 19:20:29 +0200 Subject: [PATCH 11/15] feat: Update to use current app declaration --- CHANGES.txt | 2 +- canonicalwebteam/search/views.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 73938d7..f6297a0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,4 +11,4 @@ v1.2.5, 2022-07-12 -- Block some more bot useragents v1.2.6, 2022-07-13 -- Block one more useragent - "gh" v1.2.7, 2022-07-15 -- Block more user agents - "Petalbot" v1.3.0, 2023-02-20 -- Add rate limits -v1.3.1, 2-24-06-21 -- Migrate to Flask.Limiter for rate limits +v1.4.0, 2-24-06-21 -- Migrate to Flask.Limiter for rate limits diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index f44674b..90288c2 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -3,6 +3,7 @@ # Packages import flask +from flask import current_app as app from flask_limiter import Limiter from flask_limiter.util import get_remote_address @@ -44,7 +45,6 @@ def build_search_view( ) """ - app = flask.Flask(__name__) limiter = Limiter( get_remote_address, app=app, default_limits=[request_limit] ) diff --git a/setup.py b/setup.py index 5042028..4f9854f 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="canonicalwebteam.search", - version="1.3.1", + version="1.4.0", author="Canonical webteam", author_email="webteam@canonical.com", url="https://github.com/canonical/canonicalwebteam.search", From a3b0c8d59d70c2caef32aef0297f57ae03627d84 Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 25 Jul 2024 20:19:36 +0200 Subject: [PATCH 12/15] bug: Fix issue where app was not being correctly initialized --- canonicalwebteam/search/views.py | 40 ++++++++++++++++++-------------- tests/test_app.py | 5 +++- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 90288c2..8c9d276 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -3,7 +3,6 @@ # Packages import flask -from flask import current_app as app from flask_limiter import Limiter from flask_limiter.util import get_remote_address @@ -15,7 +14,13 @@ class NoAPIKeyError(Exception): pass +limiter = Limiter( + get_remote_address +) + + def build_search_view( + app, session, site=None, template_path="search.html", @@ -23,7 +28,7 @@ def build_search_view( site_restricted_search=False, request_limit="2000/day", ): - """ + """ Build and return a view function that will query the Google Custom Search API and then render search results using the provided template. @@ -32,12 +37,13 @@ def build_search_view( from canonicalwebteam.search import build_search_view - app = Flask() + app = Flask(__name__) session = talisker.requests.get_session() app.add_url_rule( "/search", "search", build_search_view( + app, session=session, site="snapcraft.io", template_path="search.html" @@ -45,9 +51,7 @@ def build_search_view( ) """ - limiter = Limiter( - get_remote_address, app=app, default_limits=[request_limit] - ) + limiter.init_app(app) def search_view(): """ @@ -65,7 +69,7 @@ def search_view(): num = params.get("num") site_search = site or params.get("siteSearch") or params.get("domain") results = None - + if query: with limiter.limit(request_limit): results = get_search_results( @@ -79,17 +83,17 @@ def search_view(): num=num, ) - return ( - flask.render_template( - template_path, - query=query, - start=start, - num=num, - results=results, - siteSearch=site_search, - ), - {"X-Robots-Tag": "noindex"}, - ) + return ( + flask.render_template( + template_path, + query=query, + start=start, + num=num, + results=results, + siteSearch=site_search, + ), + {"X-Robots-Tag": "noindex"}, + ) else: return flask.render_template( diff --git a/tests/test_app.py b/tests/test_app.py index 2920e9f..49cbabf 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -48,7 +48,7 @@ def setUp(self): # Default use-case self.app.add_url_rule( - "/search", "search", build_search_view(session=session) + "/search", "search", build_search_view(self.app, session=session) ) # Custom use-case @@ -56,6 +56,7 @@ def setUp(self): "/docs/search", "docs-search", build_search_view( + self.app, session=session, site="maas.io/docs", template_path="docs/search.html", @@ -67,6 +68,7 @@ def setUp(self): "/server/docs/search", "server-docs-search", build_search_view( + self.app, session=session, template_path="docs/search.html", site_restricted_search=True, @@ -78,6 +80,7 @@ def setUp(self): "/server/docs/limited/search", "server-docs-search-limited", build_search_view( + self.app, session=session, template_path="docs/search.html", site_restricted_search=True, From c806c573e335cd8567596c6273462b4101839602 Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 25 Jul 2024 20:21:39 +0200 Subject: [PATCH 13/15] style: Format python --- canonicalwebteam/search/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index 8c9d276..f585e88 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -28,7 +28,7 @@ def build_search_view( site_restricted_search=False, request_limit="2000/day", ): - """ + """ Build and return a view function that will query the Google Custom Search API and then render search results using the provided template. @@ -69,7 +69,7 @@ def search_view(): num = params.get("num") site_search = site or params.get("siteSearch") or params.get("domain") results = None - + if query: with limiter.limit(request_limit): results = get_search_results( From 3d4922ac5039e4aa2115645d42bda585c33934d1 Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 25 Jul 2024 20:23:29 +0200 Subject: [PATCH 14/15] chore: Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e21bc4c..559d63a 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ app.add_url_rule( "/docs/search", "docs-search", build_search_view( + app=app session=session, site="maas.io/docs", template_path="docs/search.html", From c622fded1387e4cd11a67f75d4b8d6fc95dc9e7a Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 25 Jul 2024 20:26:28 +0200 Subject: [PATCH 15/15] style: Format Python with Black --- canonicalwebteam/search/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/canonicalwebteam/search/views.py b/canonicalwebteam/search/views.py index f585e88..93162fb 100644 --- a/canonicalwebteam/search/views.py +++ b/canonicalwebteam/search/views.py @@ -14,9 +14,7 @@ class NoAPIKeyError(Exception): pass -limiter = Limiter( - get_remote_address -) +limiter = Limiter(get_remote_address) def build_search_view(