ref: switch from mywsgi/uwsgi to pyuwsgi

pyuwsgi provides wheels making it much faster to install whereas uwsgi isn't wheelable

Dockerfile

@@ -26,6 +26,7 @@ RUN set -ex; \
         liblz4-dev \
         libpcre3-dev \
         wget \
+        zlib1g-dev \
     '; \
     apt-get update; \
     apt-get install -y $buildDeps --no-install-recommends; \
@@ -41,7 +42,8 @@ RUN set -ex; \
     rm -rf /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder; \
     mkdir -p /var/lib/uwsgi; \
     mv dogstatsd_plugin.so /var/lib/uwsgi/; \
-    uwsgi --need-plugin=/var/lib/uwsgi/dogstatsd --help > /dev/null; \
+    # TODO: https://github.com/lincolnloop/pyuwsgi-wheels/pull/17
+    python -c 'import os, sys; sys.setdlopenflags(sys.getdlopenflags() | os.RTLD_GLOBAL); import pyuwsgi; pyuwsgi.run()' --need-plugin=/var/lib/uwsgi/dogstatsd --help > /dev/null; \
     \
     apt-get purge -y --auto-remove $buildDeps; \
     rm -rf /var/lib/apt/lists/*;

Makefile

@@ -22,6 +22,7 @@ backend-typing:
 	mypy snuba tests --strict --config-file mypy.ini --exclude 'tests/datasets|tests/query|tests/state|tests/snapshots|tests/clickhouse|tests/test_split.py|tests/test_consumer.py'
 
 install-python-dependencies:
+	pip uninstall -qqy uwsgi  # pip doesn't do well with swapping drop-ins
 	pip install -e .
 	pip install -r requirements-test.txt
 

mypy.ini

@@ -34,9 +34,6 @@ ignore_missing_imports = True
 [mypy-markdown]
 ignore_missing_imports = True
 
-[mypy-mywsgi]
-ignore_missing_imports = True
-
 [mypy-packaging]
 ignore_missing_imports = True
 

requirements.txt

@@ -9,7 +9,6 @@ Flask==2.2.2
 honcho==1.1.0
 jsonschema==4.9.1
 Markdown==3.4.1
-mywsgi==1.0.3
 mypy==0.812
 packaging==21.3
 # Do not update parsimonious, there is a performance issue in newer versions
@@ -27,6 +26,6 @@ sentry-relay==0.8.13
 sentry-sdk==1.9.4
 simplejson==3.17.6
 urllib3==1.26.11
-uWSGI==2.0.20
+pyuwsgi==2.0.20
 Werkzeug==2.2.2
 PyYAML==6.0

snuba/cli/admin.py

@@ -4,6 +4,7 @@
 import click
 
 from snuba.environment import setup_logging
+from snuba.utils import uwsgi
 
 
 @click.command()
@@ -34,11 +35,9 @@ def admin(
         WSGIRequestHandler.protocol_version = "HTTP/1.1"
         application.run(host=host, port=port, threaded=True, debug=debug)
     else:
-        import mywsgi
-
         if log_level:
             os.environ["LOG_LEVEL"] = log_level
-        mywsgi.run(
+        uwsgi.run(
             "snuba.admin.wsgi:application",
             f"{host}:{port}",
             processes=processes,

snuba/cli/api.py

@@ -5,6 +5,7 @@
 
 from snuba.attribution.log import flush_attribution_producer
 from snuba.environment import setup_logging
+from snuba.utils import uwsgi
 
 
 @click.command()
@@ -46,13 +47,11 @@ def api(
         WSGIRequestHandler.protocol_version = "HTTP/1.1"
         application.run(host=host, port=port, threaded=True, debug=debug)
     else:
-        import mywsgi
-
         if log_level:
             os.environ["LOG_LEVEL"] = log_level
 
         with flush_attribution_producer():
-            mywsgi.run(
+            uwsgi.run(
                 "snuba.web.wsgi:application",
                 f"{host}:{port}",
                 processes=processes,

snuba/utils/uwsgi.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import MutableMapping, NoReturn
+
+PYUWSGI_PROG = """\
+import os
+import sys
+
+orig = sys.getdlopenflags()
+sys.setdlopenflags(orig | os.RTLD_GLOBAL)
+try:
+    import pyuwsgi
+finally:
+    sys.setdlopenflags(orig)
+
+pyuwsgi.run()
+"""
+
+
+def _prepare_environ(
+    options: dict[str, bool | int | str | None], env: MutableMapping[str, str]
+) -> None:
+    for k, v in options.items():
+        if v is None:
+            continue
+        key = f"UWSGI_{k.upper().replace('-', '_')}"
+        if isinstance(v, str):
+            value = v
+        elif v is True:
+            value = "true"
+        elif v is False:
+            value = "false"
+        elif isinstance(v, int):
+            value = str(v)
+        else:
+            raise TypeError("Unknown option type: %r (%s)" % (k, type(v)))
+
+        env.setdefault(key, value)
+
+
+def run(module: str, bind: str, **kwargs: bool | int | str | None) -> NoReturn:
+    protocol = os.environ.pop("UWSGI_PROTOCOL", "http")
+    options: dict[str, bool | int | str | None] = {
+        "auto_procname": True,
+        "chmod_socket": 777,
+        "die_on_term": True,
+        "disable_write_exception": True,
+        "enable_threads": True,
+        "ignore_sigpipe": True,
+        "ignore_write_errors": True,
+        "lazy_apps": True,
+        "log_format": '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"',
+        "log_x_forwarded_for": True,
+        "master": True,
+        "module": module,
+        "need_app": True,
+        "processes": 1,
+        "protocol": protocol,
+        "single_interpreter": True,
+        "threads": 1,
+        "thunder_lock": True,
+        "vacuum": True,
+        "virtualenv": sys.prefix,
+        "wsgi_env_behavior": "holy",
+        f"{protocol}_socket": bind,
+        **kwargs,
+    }
+
+    _prepare_environ(options, os.environ)
+
+    # TODO: https://github.com/lincolnloop/pyuwsgi-wheels/pull/17
+    cmd = (sys.executable, "-c", PYUWSGI_PROG)
+    os.execvp(cmd[0], cmd)