pywb: init at 2.1.0; init pythonPackages.{warcio, surt, portalocker, certauth, wsgiprox, py3amf, redis_2, lupa, fakeredis}

pkgs/development/python-modules/certauth/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pyopenssl
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "certauth";
+  version = "1.2.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1lgsl053bbf1xjvwv7q8rhccg1jxzkya222pgkxdcjaa471s9l27";
+  };
+
+  propagatedBuildInputs = [ pyopenssl ];
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = with lib; {
+    description = "Certificate authority library for dynamically generating host certs";
+    homepage = https://github.com/ikreymer/certauth;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/fakeredis/default.nix

@@ -0,0 +1,43 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, lupa
+, pytest
+, redis
+, nose
+}:
+
+buildPythonPackage rec {
+  pname = "fakeredis";
+  version = "0.16.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "005gnzj8lffz6hv5ii980gv8ffsiilqijdifyrz7lnms0c1852ms";
+  };
+
+  postPatch = ''
+    substituteInPlace setup.py --replace "redis<3" "redis"
+  '';
+
+  propagatedBuildInputs = [ lupa redis ];
+
+  checkInputs = [ pytest nose ];
+
+  # test_pubsub_run_in_thread sometimes fails with
+  # `RuntimeError: dictionary changed size during iteration`
+  # https://github.com/jamesls/fakeredis/issues/230
+  #
+  # test_pipeline_as_context_manager fails with redis-py 3
+  # `AttributeError: module 'redis.client' has no attribute 'BasePipeline'`
+  checkPhase = ''
+    py.test -v -k "not test_pubsub_run_in_thread and not test_pipeline_as_context_manager"
+  '';
+
+  meta = with lib; {
+    description = "Fake implementation of redis API (redis-py) for testing purposes";
+    homepage = https://github.com/jamesls/fakeredis;
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/lupa/default.nix

@@ -0,0 +1,25 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, lua
+, pkgconfig
+}:
+
+buildPythonPackage rec {
+  pname = "lupa";
+  version = "1.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0xvfh27rv7ili9xc5nfg2nnnlj9iyxxcjinkcs71xz1spxp3zmyj";
+  };
+
+  buildInputs = [ lua pkgconfig ];
+
+  meta = with lib; {
+    description = "Integrates the runtime of Lua or LuaJIT2 into CPython";
+    homepage = https://github.com/scoder/lupa;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/portalocker/default.nix

@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "portalocker";
+  version = "1.3.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1832pnpb2c5basg47ks7ipfvvwflcm7s4r8whkir607mwjn75fmm";
+  };
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    # Based on https://github.com/WoLpH/portalocker/blob/master/pytest.ini
+    py.test tests/*.py --doctest-modules
+  '';
+
+  meta = with lib; {
+    description = "Easy API to file locking";
+    homepage = https://github.com/WoLpH/portalocker;
+    license = licenses.psfl;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/py3amf/default.nix

@@ -0,0 +1,35 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, isPy3k
+, defusedxml
+}:
+
+buildPythonPackage rec {
+  pname = "Py3AMF";
+  version = "0.8.9";
+
+  disabled = !isPy3k;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1pmh6p7mqdjmy2yj4mv6s1vnf7l9sn326dr6465mxhd6qz6mmgzi";
+  };
+
+  propagatedBuildInputs = [ defusedxml ];
+
+  preCheck = ''
+    # RuntimeError: generator raised StopIteration
+    rm pyamf/tests/test_remoting.py
+
+    # AssertionError: '500 Internal Server Error' != '200 OK'
+    rm pyamf/tests/gateway/test_wsgi.py
+  '';
+
+  meta = with lib; {
+    description = "AMF (Action Message Format) support for Python 3";
+    homepage = https://github.com/StdCarrot/Py3AMF;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/surt/default.nix

@@ -0,0 +1,34 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, six
+, tldextract
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "surt";
+  version = "0.3.0";
+
+  # Fetch from GitHub because the PyPI tarball does not include tests/
+  src = fetchFromGitHub {
+    owner = "internetarchive";
+    repo = "surt";
+    rev = version;
+    sha256 = "10l8zz9ps31rqhawdbzrcnfq9rkzmaaqrkq6i5c35386h0i22c3g";
+  };
+
+  propagatedBuildInputs = [ six tldextract ];
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = with lib; {
+    description = "Sort-friendly URI Reordering Transform (SURT)";
+    homepage = https://github.com/internetarchive/surt;
+    license = licenses.agpl3;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/warcio/default.nix

@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, six
+}:
+
+buildPythonPackage rec {
+  pname = "warcio";
+  version = "1.6.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1nyhghbag1chh5fml848x799mwgkgmz3l3ipv7lr6p0lj1jq8i1r";
+  };
+
+  propagatedBuildInputs = [ six ];
+
+  # Test suite makes DNS lookups and HTTP requests to example.com
+  # Most tests fail with OSError: [Errno 9] Bad file descriptor
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Streaming WARC/ARC library for fast web archive IO";
+    homepage = https://github.com/webrecorder/warcio;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/wsgiprox/default.nix

@@ -0,0 +1,40 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, certauth
+, gevent
+, mock
+, pytest
+, requests
+, six
+, websocket_client
+, waitress
+}:
+
+buildPythonPackage rec {
+  pname = "wsgiprox";
+  version = "1.5.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1d0x0ihms5sv9v8nrb65wlih6zcdh0644q6nawkzylmvfpwp2k2c";
+  };
+
+  propagatedBuildInputs = [ certauth six ];
+
+  # Don't provide uwsgi to skip tests that would hang on network access.
+  checkInputs = [ gevent mock pytest requests websocket_client waitress ];
+  checkPhase = ''
+    # Based on https://github.com/webrecorder/wsgiprox/blob/master/setup.py
+    py.test --doctest-module ./wsgiprox wsgiprox -v test/
+  '';
+
+  __darwinAllowLocalNetworking = true;
+
+  meta = with lib; {
+    description = "Middleware for adding HTTP/S proxy support to WSGI applications";
+    homepage = https://github.com/webrecorder/wsgiprox;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ivan ];
+  };
+}

pkgs/tools/backup/pywb/default.nix

@@ -0,0 +1,91 @@
+{ lib, python3Packages, fetchFromGitHub }:
+
+with lib;
+
+let
+  disabledTests = [
+    # `AssertionError` due to trailing comma in tuple representation
+    "test_another_remote_not_found"
+    "test_file_not_found"
+    "test_mem_agg_index_1"
+    "test_mem_agg_index_2"
+    "test_mem_agg_index_5"
+    "test_mem_agg_index_5_inverse_preset"
+    "test_mem_agg_not_found"
+    "test_agg_select_local"
+    "test_agg_select_local_postreq"
+
+    # `RuntimeError('generator raised StopIteration')`
+    "test_zipnum"
+
+    # Failures in sandbox due to lack of network access
+    "test_replay"
+    "test_record_skip_http_only_cookies_header"
+    "test_loaders"
+    "test_remote_loader"
+    "test_remote_closest_loader"
+    "test_remote_closest_wb_memnto_loader"
+    "test_all_not_found"
+    "test_agg_select_mem_1"
+    "test_agg_select_mem_2"
+    "test_agg_select_mem_unrewrite_headers"
+    "test_agg_select_live"
+    "test_force_https_root_replay_1"
+    "test_live_fallback"
+    "test_live_live_1"
+    "test_proxy_record_keep_percent"
+    "test_live_no_redir"
+    "test_root_replay_ts"
+    "test_root_replay_no_ts"
+    "test_root_replay_redir"
+
+    # `AssertionError: assert 3 == 4` due to another test being skipped
+    "test_cdx_all_coll"
+
+    # `AppError: Bad response: 404 Not Found` when using Python 2 on Linux
+    # or Python 3 on macOS, may fail in other cases
+    "test_timemap_all_coll"
+  ];
+
+  # Compose the final string expression, including the "-k" and the single quotes.
+  testExpression = optionalString (disabledTests != [])
+    "-k 'not ${concatStringsSep " and not " disabledTests}'";
+
+in
+
+python3Packages.buildPythonApplication rec {
+  name = "pywb-${version}";
+  version = "2.1.0";
+
+  # Fetch from GitHub because the PyPi tarball is missing requirements.txt
+  src = fetchFromGitHub {
+    owner = "webrecorder";
+    repo = "pywb";
+    # 2.1.0 was not tagged but pywb-2.1.0.tar.gz in PyPi was based on this commit
+    rev = "1b151b74bfee5bd465ccc70cc04a8d2b2e090db4";
+    sha256 = "0ahsk8fhrz0gdiq4i6b3m4jvhxbdsaz1mkdgvy26vb3r4ph24dfa";
+  };
+
+  propagatedBuildInputs = with python3Packages; [
+    brotlipy chardet gevent jinja2 portalocker py3amf pytest pyyaml redis
+    requests six surt warcio webassets webencodings werkzeug wsgiprox
+  ];
+
+  checkInputs = with python3Packages; [
+    fakeredis httpbin mock pytest ujson urllib3 webtest
+  ];
+  checkPhase = ''
+    # Based on https://github.com/webrecorder/pywb/blob/master/setup.py
+    py.test -v --doctest-module ./pywb/ tests/ ${testExpression}
+  '';
+
+  __darwinAllowLocalNetworking = true;
+
+  meta = {
+    description = "Web archiving toolkit for creating or replaying web archives";
+    homepage = https://github.com/webrecorder/pywb;
+    license = licenses.gpl3;
+    maintainers = with maintainers; [ ivan ];
+    platforms = platforms.all;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -5016,6 +5016,8 @@ in
 
   pywal = with python3Packages; toPythonApplication pywal;
 
+  pywb = callPackage ../tools/backup/pywb { };
+
   remarshal = callPackage ../development/tools/remarshal { };
 
   rig = callPackage ../tools/misc/rig { };