Cache results of common database queries

KerbalStuff/app.py

@@ -11,9 +11,11 @@
 
 import requests
 import werkzeug.wrappers
+from cachetools import TTLCache, cached
 from flask import Flask, render_template, g, url_for, Response, request
 from flask_login import LoginManager, current_user
 from flaskext.markdown import Markdown
+from sqlalchemy import desc
 from werkzeug.exceptions import HTTPException, InternalServerError
 
 from .blueprints.accounts import accounts
@@ -279,7 +281,7 @@ def inject() -> Dict[str, Any]:
     if request.cookies.get('dismissed_donation') is not None:
         dismissed_donation = True
     return {
-        'announcements': BlogPost.query.filter(BlogPost.announcement == True).order_by(BlogPost.created.desc()).all(),
+        'announcements': get_announcement_posts(),
         'many_paragraphs': many_paragraphs,
         'analytics_id': _cfg("google_analytics_id"),
         'analytics_domain': _cfg("google_analytics_domain"),
@@ -306,3 +308,8 @@ def inject() -> Dict[str, Any]:
         'donation_header_link': _cfgb('donation-header-link') if not dismissed_donation else False,
         'registration': _cfgb('registration')
     }
+
+
+@cached(cache=TTLCache(maxsize=1, ttl=1800))
+def get_announcement_posts() -> List[BlogPost]:
+    return BlogPost.query.filter(BlogPost.announcement == True).order_by(desc(BlogPost.created)).all()

KerbalStuff/blueprints/accounts.py

@@ -48,7 +48,6 @@ def register() -> Union[str, werkzeug.wrappers.Response]:
             if username is not None:
                 kwargs['username'] = username
             kwargs['registration'] = _cfgb('registration')
-            print("test")
             return render_template("register.html", **kwargs)
         # All valid, let's make them an account
         user = User(username=username, email=email)

KerbalStuff/blueprints/anonymous.py

@@ -1,43 +1,35 @@
 import os.path
 
+import werkzeug.wrappers
 from flask import Blueprint, render_template, send_from_directory, abort, request, Response
 from flask_login import current_user
 from sqlalchemy import desc
-import werkzeug.wrappers
 
-from ..common import dumb_object, paginate_mods, get_mods, get_game_info
+from ..common import dumb_object, paginate_query, get_paginated_mods, get_game_info, get_games, \
+    get_featured_mods, get_top_mods, get_new_mods, get_updated_mods
 from ..config import _cfg
 from ..database import db
-from ..objects import Featured, Mod, ModVersion, Game, User
-from ..search import search_mods
+from ..objects import Featured, Mod, ModVersion, User
 
 anonymous = Blueprint('anonymous', __name__, template_folder='../../templates/anonymous')
 
 
 @anonymous.route("/")
 def index() -> str:
-    games = Game.query.filter(Game.active == True).order_by(desc(Game.created))
-    return render_template("index.html",
-                           games=games)
+    return render_template("index.html", games=get_games())
 
 
 @anonymous.route("/<gameshort>")
 def game(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
-    featured = Featured.query.outerjoin(Mod).filter(
-        Mod.published, Mod.game_id == ga.id).order_by(desc(Featured.created)).limit(6)[:6]
-    # top = search_mods("", 1, 3)[0]
-    top = Mod.query.filter(Mod.published, Mod.game_id == ga.id).order_by(
-        desc(Mod.download_count)).limit(6)[:6]
-    new = Mod.query.filter(Mod.published, Mod.game_id == ga.id).order_by(
-        desc(Mod.created)).limit(6)[:6]
-    recent = Mod.query.filter(Mod.published, Mod.game_id == ga.id, Mod.versions.any(ModVersion.id != Mod.default_version_id)).order_by(desc(Mod.updated)).limit(6)[:6]
+    featured = get_featured_mods(ga.id, 6)
+    top = get_top_mods(ga.id, 6)
+    new = get_new_mods(ga.id, 6)
+    recent = get_updated_mods(ga.id, 6)
     user_count = User.query.count()
     mod_count = Mod.query.filter(Mod.game_id == ga.id, Mod.published == True).count()
-    yours = list()
-    if current_user:
-        yours = sorted(filter(lambda m: m.game_id == ga.id, current_user.following),
-            key=lambda m: m.updated, reverse=True)[:6]
+    following = sorted(filter(lambda m: m.game_id == ga.id, current_user.following),
+                       key=lambda m: m.updated, reverse=True)[:6] if current_user else list()
     return render_template("game.html",
                            ga=ga,
                            featured=featured,
@@ -46,7 +38,7 @@ def game(gameshort: str) -> str:
                            recent=recent,
                            user_count=user_count,
                            mod_count=mod_count,
-                           yours=yours)
+                           yours=following)
 
 
 @anonymous.route("/content/<path:path>")
@@ -59,27 +51,26 @@ def content(path: str) -> werkzeug.wrappers.Response:
 
 @anonymous.route("/browse")
 def browse() -> str:
-    featured = Featured.query.order_by(desc(Featured.created)).limit(6).all()
-    top = search_mods(None, '', 1, 6)[0]
-    new = Mod.query.filter(Mod.published).order_by(desc(Mod.created)).limit(6).all()
+    featured = get_featured_mods(None, 6)
+    top = get_top_mods(None, 6)
+    new = get_new_mods(None, 6)
     return render_template("browse.html", featured=featured, top=top, new=new)
 
 
 @anonymous.route("/browse/new")
 def browse_new() -> str:
     mods = Mod.query.filter(Mod.published).order_by(desc(Mod.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages,
                            url="/browse/new", name="Newest Mods", rss="/browse/new.rss")
 
 
 @anonymous.route("/browse/new.rss")
 def browse_new_rss() -> Response:
-    mods = Mod.query.filter(Mod.published).order_by(desc(Mod.created))
-    mods = mods.limit(30)
     site_name = _cfg('site-name')
     if not site_name:
         abort(404)
+    mods = get_new_mods(None, 30)
     return Response(render_template("rss.xml", mods=mods, title="New mods on " + site_name,
                                     description="The newest mods on " + site_name,
                                     url="/browse/new"), mimetype="text/xml")
@@ -88,18 +79,17 @@ def browse_new_rss() -> Response:
 @anonymous.route("/browse/updated")
 def browse_updated() -> str:
     mods = Mod.query.filter(Mod.published, Mod.versions.any(ModVersion.id != Mod.default_version_id)).order_by(desc(Mod.updated))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages,
-                           url="/browse/updated", name="Recently Updated Mods", rss="/browse/updated.rss", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/updated", name="Recently Updated Mods", rss="/browse/updated.rss")
 
 
 @anonymous.route("/browse/updated.rss")
 def browse_updated_rss() -> Response:
-    mods = Mod.query.filter(Mod.published, Mod.versions.any(ModVersion.id != Mod.default_version_id)).order_by(desc(Mod.updated))
-    mods = mods.limit(30)
     site_name = _cfg('site-name')
     if not site_name:
         abort(404)
+    mods = get_updated_mods(None, 30)
     return Response(render_template("rss.xml", mods=mods, title="Recently updated on " + site_name,
                                     description="Mods on " +
                                     site_name + " updated recently",
@@ -108,24 +98,23 @@ def browse_updated_rss() -> Response:
 
 @anonymous.route("/browse/top")
 def browse_top() -> str:
-    mods, page, total_pages = get_mods()
+    mods, page, total_pages = get_paginated_mods()
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages,
-                           url="/browse/top", name="Popular Mods", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/top", name="Popular Mods")
 
 
 @anonymous.route("/browse/featured")
 def browse_featured() -> str:
     mods = Featured.query.order_by(desc(Featured.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     mods = [f.mod for f in mods]
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages,
                            url="/browse/featured", name="Featured Mods", rss="/browse/featured.rss")
 
 
 @anonymous.route("/browse/featured.rss")
 def browse_featured_rss() -> Response:
-    mods = Featured.query.order_by(desc(Featured.created))
-    mods = mods.limit(30)
+    mods = get_featured_mods(None, 30)
     # Fix dates
     for f in mods:
         f.mod.created = f.created
@@ -141,27 +130,25 @@ def browse_featured_rss() -> Response:
 
 @anonymous.route("/browse/all")
 def browse_all() -> str:
-    mods, page, total_pages = get_mods()
+    mods, page, total_pages = get_paginated_mods()
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages,
-                           url="/browse/all", name="All Mods", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/all", name="All Mods")
 
 
 @anonymous.route("/<gameshort>/browse")
 def singlegame_browse(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
-    featured = Featured.query.outerjoin(Mod).filter(
-        Mod.game_id == ga.id).order_by(desc(Featured.created)).limit(6).all()
-    top = search_mods(ga, "", 1, 6)[0]
-    new = Mod.query.filter(Mod.published, Mod.game_id == ga.id).order_by(
-        desc(Mod.created)).limit(6).all()
+    featured = get_featured_mods(ga.id, 6)
+    top = get_top_mods(ga.id, 6)
+    new = get_new_mods(ga.id, 6)
     return render_template("browse.html", featured=featured, top=top, ga=ga, new=new)
 
 
 @anonymous.route("/<gameshort>/browse/new")
 def singlegame_browse_new(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
     mods = Mod.query.filter(Mod.published, Mod.game_id == ga.id).order_by(desc(Mod.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, ga=ga,
                            url="/browse/new", name="Newest Mods", rss="/browse/new.rss")
 
@@ -172,8 +159,7 @@ def singlegame_browse_new_rss(gameshort: str) -> Response:
     if not site_name:
         abort(404)
     ga = get_game_info(short=gameshort)
-    mods = Mod.query.filter(Mod.published, Mod.game_id == ga.id).order_by(desc(Mod.created))
-    mods = mods.limit(30)
+    mods = get_new_mods(ga.id, 30)
     return Response(render_template("rss.xml", mods=mods, title="New mods on " + site_name, ga=ga,
                                     description="The newest mods on " + site_name,
                                     url="/browse/new"), mimetype="text/xml")
@@ -183,9 +169,9 @@ def singlegame_browse_new_rss(gameshort: str) -> Response:
 def singlegame_browse_updated(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
     mods = Mod.query.filter(Mod.published, Mod.game_id == ga.id, Mod.versions.any(ModVersion.id != Mod.default_version_id)).order_by(desc(Mod.updated))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, ga=ga,
-                           url="/browse/updated", name="Recently Updated Mods", rss="/browse/updated.rss", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/updated", name="Recently Updated Mods", rss="/browse/updated.rss")
 
 
 @anonymous.route("/<gameshort>/browse/updated.rss")
@@ -194,8 +180,7 @@ def singlegame_browse_updated_rss(gameshort: str) -> Response:
     if not site_name:
         abort(404)
     ga = get_game_info(short=gameshort)
-    mods = Mod.query.filter(Mod.published, Mod.game_id == ga.id, Mod.versions.any(ModVersion.id != Mod.default_version_id)).order_by(desc(Mod.updated))
-    mods = mods.limit(30)
+    mods = get_updated_mods(ga.id, 30)
     return Response(render_template("rss.xml", mods=mods, title="Recently updated on " + site_name, ga=ga,
                                     description="Mods on " +
                                     site_name + " updated recently",
@@ -205,17 +190,17 @@ def singlegame_browse_updated_rss(gameshort: str) -> Response:
 @anonymous.route("/<gameshort>/browse/top")
 def singlegame_browse_top(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
-    mods, page, total_pages = get_mods(ga)
+    mods, page, total_pages = get_paginated_mods(ga)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, ga=ga,
-                           url="/browse/top", name="Popular Mods", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/top", name="Popular Mods")
 
 
 @anonymous.route("/<gameshort>/browse/featured")
 def singlegame_browse_featured(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
     mods = Featured.query.outerjoin(Mod).filter(
         Mod.game_id == ga.id).order_by(desc(Featured.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     mods = [f.mod for f in mods]
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, ga=ga,
                            url="/browse/featured", name="Featured Mods", rss="/browse/featured.rss")
@@ -227,9 +212,7 @@ def singlegame_browse_featured_rss(gameshort: str) -> Response:
     if not site_name:
         abort(404)
     ga = get_game_info(short=gameshort)
-    mods = Featured.query.outerjoin(Mod).filter(
-        Mod.game_id == ga.id).order_by(desc(Featured.created))
-    mods = mods.limit(30)
+    mods = get_featured_mods(ga.id, 30)
     # Fix dates
     for f in mods:
         f.mod.created = f.created
@@ -243,9 +226,9 @@ def singlegame_browse_featured_rss(gameshort: str) -> Response:
 @anonymous.route("/<gameshort>/browse/all")
 def singlegame_browse_all(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
-    mods, page, total_pages = get_mods(ga)
+    mods, page, total_pages = get_paginated_mods(ga)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, ga=ga,
-                           url="/browse/all", name="All Mods", site_name=_cfg('site-name'), support_mail=_cfg('support-mail'))
+                           url="/browse/all", name="All Mods")
 
 
 @anonymous.route("/about")
@@ -266,13 +249,13 @@ def privacy() -> str:
 @anonymous.route("/search")
 def search() -> str:
     query = request.args.get('query') or ''
-    mods, page, total_pages = get_mods(query=query)
+    mods, page, total_pages = get_paginated_mods(query=query)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, search=True, query=query)
 
 
 @anonymous.route("/<gameshort>/search")
 def singlegame_search(gameshort: str) -> str:
     ga = get_game_info(short=gameshort)
     query = request.args.get('query') or ''
-    mods, page, total_pages = get_mods(ga, query)
+    mods, page, total_pages = get_paginated_mods(ga, query)
     return render_template("browse-list.html", mods=mods, page=page, total_pages=total_pages, search=True, query=query, ga=ga)

KerbalStuff/blueprints/api.py

@@ -14,7 +14,7 @@
 
 from .accounts import check_password_criteria
 from ..ckan import send_to_ckan, notify_ckan
-from ..common import json_output, paginate_mods, with_session, get_mods, json_response, \
+from ..common import json_output, paginate_query, with_session, get_paginated_mods, json_response, \
     check_mod_editable, set_game_info, TRUE_STR, get_page
 from ..config import _cfg, _cfgi
 from ..database import db
@@ -334,22 +334,22 @@ def browse() -> Dict[str, Any]:
 @json_output
 def browse_new() -> Iterable[Dict[str, Any]]:
     mods = Mod.query.filter(Mod.published).order_by(desc(Mod.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return serialize_mod_list(mods)
 
 
 @api.route("/api/browse/top")
 @json_output
 def browse_top() -> Iterable[Dict[str, Any]]:
-    mods, *_ = get_mods()
+    mods, *_ = get_paginated_mods()
     return serialize_mod_list(mods)
 
 
 @api.route("/api/browse/featured")
 @json_output
 def browse_featured() -> Iterable[Dict[str, Any]]:
     mods = Featured.query.order_by(desc(Featured.created))
-    mods, page, total_pages = paginate_mods(mods)
+    mods, page, total_pages = paginate_query(mods)
     return serialize_mod_list((f.mod for f in mods))
 
 

KerbalStuff/blueprints/lists.py

@@ -6,7 +6,7 @@
 from sqlalchemy import desc, or_
 import werkzeug.wrappers
 
-from ..common import loginrequired, with_session, get_game_info, paginate_mods
+from ..common import loginrequired, with_session, get_game_info, paginate_query
 from ..database import db
 from ..objects import Mod, ModList, ModListItem, Game
 
@@ -36,7 +36,7 @@ def packs(gameshort: Optional[str]) -> str:
         .order_by(desc(ModList.created))
     if game:
         query = query.filter(ModList.game_id == game.id)
-    packs, page, total_pages = paginate_mods(query, 15)
+    packs, page, total_pages = paginate_query(query, 15)
     return render_template("packs.html", ga=game, game=game, packs=packs, page=page, total_pages=total_pages)