pywb: Init at 2.7.4

maintainers/maintainer-list.nix

@@ -1284,6 +1284,12 @@
     githubId = 29845794;
     name = "Duncan Russell";
   };
+  anpandey = {
+    email = "anpandey@protonmail.com";
+    github = "anpandey";
+    githubId = 25336974;
+    name = "Ankit Pandey";
+  };
   anpin = {
     email = "pavel@anpin.fyi";
     github = "anpin";

pkgs/development/python-modules/py3amf/default.nix

@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, isPy3k
+, defusedxml
+}:
+
+buildPythonPackage rec {
+  pname = "Py3AMF";
+  version = "0.8.10";
+  format = "setuptools";
+
+  disabled = !isPy3k;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-bawtNKCdr1NR5lTozcMCazVgpttJjBfNzIQyKzFJlSw=";
+  };
+
+  propagatedBuildInputs = [ defusedxml ];
+
+  meta = with lib; {
+    description = "AMF (Action Message Format) support for Python 3";
+    homepage = "https://github.com/StdCarrot/Py3AMF";
+    license = licenses.mit;
+    maintainers = with maintainers; [ anpandey ];
+  };
+}

pkgs/tools/misc/pywb/default.nix

@@ -0,0 +1,91 @@
+{ lib, fetchFromGitHub, fetchPypi, python3, stdenv }:
+
+python3.pkgs.buildPythonApplication rec {
+  name = "pywb";
+  version = "2.7.4";
+  format = "setuptools";
+
+  src = fetchFromGitHub {
+    owner = "webrecorder";
+    repo = "pywb";
+    rev = "v-${version}";
+    sha256 = "sha256-M44TwBzXuF33HNLB2OjmBKNDG9kt/qhGEOu589YGvsQ=";
+    # TODO: We clone the wombat submodule, but we still need to implement
+    # building wombat.
+    fetchSubmodules = true;
+  };
+
+  patches = [ ./pywb.patch ];
+
+  nativeBuildInputs = with python3.pkgs; [
+    pythonRelaxDepsHook
+  ];
+
+  pythonRelaxDeps = [
+    "jinja2"
+    "redis"
+    "markupsafe"
+    "fakeredis"
+    "gevent"
+  ];
+
+  propagatedBuildInputs = with python3.pkgs; [
+    brotlipy
+    chardet
+    fakeredis
+    gevent
+    jinja2
+    markupsafe
+    portalocker
+    py3amf
+    pytest
+    python-dateutil
+    pyyaml
+    redis
+    requests
+    six
+    surt
+    ua-parser
+    warcio
+    webassets
+    webencodings
+    werkzeug
+    wsgiprox
+  ];
+
+  nativeCheckInputs = with python3.pkgs; [
+    pytestCheckHook
+    mock
+    webtest
+    urllib3
+    httpbin
+    flask
+    ujson
+    lxml
+    fakeredis
+  ];
+
+  doCheck = true;
+
+  disabledTests = [
+    # 400 Bad Request.
+    "test_integration"
+    "test_live_rewriter"
+    "test_redirect_classic"
+    "test_socks"
+    "test_cert_req"
+    "test_force_https"
+    # Disabled because of fakeredis patch.
+    "test_single_redis_entry"
+    "test_single_warc_record"
+    "test_redis_pending_count"
+  ];
+
+  meta = {
+    description = "Python web archiving toolkit for creating and replaying web archives";
+    homepage = "https://github.com/webrecorder/pywb";
+    license = with lib.licenses; [ gpl3Plus agpl3Plus ];
+    maintainers = with lib.maintainers; [ anpandey ];
+    platforms = lib.platforms.all;
+  };
+}

pkgs/tools/misc/pywb/pywb.patch

@@ -0,0 +1,100 @@
+diff --git a/babel.ini b/babel.ini
+index 3396a587..bd3ea595 100644
+--- a/babel.ini
++++ b/babel.ini
+@@ -1,2 +1,4 @@
+ [jinja2: pywb/templates/**.html]
+-extensions=jinja2.ext.i18n,jinja2.ext.autoescape,jinja2.ext.with_
++# https://jinja.palletsprojects.com/en/3.0.x/extensions/#with-statement with is
++# now built-in.
++extensions=jinja2.ext.i18n,jinja2.ext.autoescape
+diff --git a/pywb/apps/rewriterapp.py b/pywb/apps/rewriterapp.py
+index a0292732..601bb98f 100644
+--- a/pywb/apps/rewriterapp.py
++++ b/pywb/apps/rewriterapp.py
+@@ -63,8 +63,10 @@ class RewriterApp(object):
+         self.js_proxy_rw = RewriterWithJSProxy(replay_mod=self.replay_mod)
+
+         if not jinja_env:
++            # https://jinja.palletsprojects.com/en/3.0.x/extensions/#with-statement
++            # with is now built in.
+             jinja_env = JinjaEnv(globals={'static_path': 'static'},
+-                                 extensions=['jinja2.ext.i18n', 'jinja2.ext.with_'])
++                                 extensions=['jinja2.ext.i18n'])
+             jinja_env.jinja_env.install_null_translations()
+
+         self.jinja_env = jinja_env
+diff --git a/pywb/rewrite/templateview.py b/pywb/rewrite/templateview.py
+index 7f0cbc88..39c76ee7 100644
+--- a/pywb/rewrite/templateview.py
++++ b/pywb/rewrite/templateview.py
+@@ -5,7 +5,7 @@ from pywb.utils.loaders import load
+
+ from six.moves.urllib.parse import urlsplit, quote
+
+-from jinja2 import Environment, TemplateNotFound, contextfunction, select_autoescape
++from jinja2 import Environment, TemplateNotFound, pass_context, select_autoescape
+ from jinja2 import FileSystemLoader, PackageLoader, ChoiceLoader
+
+ from webassets.ext.jinja2 import AssetsExtension
+@@ -139,7 +139,7 @@ class JinjaEnv(object):
+             return loc_map.get(loc)
+
+         def override_func(jinja_env, name):
+-            @contextfunction
++            @pass_context
+             def get_override(context, text):
+                 translate = get_translate(context)
+                 if not translate:
+@@ -158,7 +158,7 @@ class JinjaEnv(object):
+
+         # Special _Q() function to return %-encoded text, necessary for use
+         # with text in banner
+-        @contextfunction
++        @pass_context
+         def quote_gettext(context, text):
+             translate = get_translate(context)
+             if not translate:
+@@ -171,7 +171,7 @@ class JinjaEnv(object):
+         self.jinja_env.globals['_Q'] = quote_gettext
+         self.jinja_env.globals['default_locale'] = default_locale
+
+-        @contextfunction
++        @pass_context
+         def switch_locale(context, locale):
+             environ = context.get('env')
+             curr_loc = environ.get('pywb_lang', '')
+@@ -188,7 +188,7 @@ class JinjaEnv(object):
+
+             return app_prefix + '/' + locale + request_uri
+
+-        @contextfunction
++        @pass_context
+         def get_locale_prefixes(context):
+             environ = context.get('env')
+             locale_prefixes = {}
+diff --git a/pywb/warcserver/test/testutils.py b/pywb/warcserver/test/testutils.py
+index 2d6444bd..ccb96a03 100644
+--- a/pywb/warcserver/test/testutils.py
++++ b/pywb/warcserver/test/testutils.py
+@@ -5,7 +5,7 @@ import shutil
+ import yaml
+ import time
+
+-from fakeredis import FakeStrictRedis, DATABASES
++from fakeredis import FakeStrictRedis
+ from mock import patch
+
+ from pywb.warcserver.basewarcserver import BaseWarcServer
+@@ -66,7 +66,10 @@ class FakeRedisTests(object):
+         super(FakeRedisTests, cls).setup_class()
+
+         del PUBSUBS[:]
+-        DATABASES.clear()
++        # DATABASES is no longer used in fakeredis. In fakeredis 0.16 this was
++        # how the list of all databases was cleared. We just comment this line
++        # out and disable the tests that this causes to fail.
++        # DATABASES.clear()
+
+         cls.redismock = patch('redis.StrictRedis', FakeStrictRedisSharedPubSub)
+         cls.redismock.start()

pkgs/top-level/all-packages.nix

@@ -12421,6 +12421,8 @@ with pkgs;
 
   pywal = with python3Packages; toPythonApplication pywal;
 
+  pywb = callPackage ../tools/misc/pywb {};
+
   pystring = callPackage ../development/libraries/pystring { };
 
   raysession = python3Packages.callPackage ../applications/audio/raysession {};

pkgs/top-level/python-packages.nix

@@ -9998,6 +9998,8 @@ self: super: with self; {
 
   pyalgotrade = callPackage ../development/python-modules/pyalgotrade { };
 
+  py3amf = callPackage ../development/python-modules/py3amf { };
+
   pyamg = callPackage ../development/python-modules/pyamg { };
 
   pyaml = callPackage ../development/python-modules/pyaml { };