From f99bf9ba1fc8e4272fc724dc37c4efcef0d96ac5 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Mon, 24 Dec 2018 10:13:59 +0000
Subject: [PATCH 1/9] pythonPackages.warcio: init at 1.6.3

---
 .../python-modules/warcio/default.nix         | 28 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 30 insertions(+)
 create mode 100644 pkgs/development/python-modules/warcio/default.nix

diff --git a/pkgs/development/python-modules/warcio/default.nix b/pkgs/development/python-modules/warcio/default.nix
new file mode 100644
index 0000000000000..df16f514869be
--- /dev/null
+++ b/pkgs/development/python-modules/warcio/default.nix
@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, six
+}:
+
+buildPythonPackage rec {
+  pname = "warcio";
+  version = "1.6.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1nyhghbag1chh5fml848x799mwgkgmz3l3ipv7lr6p0lj1jq8i1r";
+  };
+
+  propagatedBuildInputs = [ six ];
+
+  # Test suite makes DNS lookups and HTTP requests to example.com
+  # Most tests fail with OSError: [Errno 9] Bad file descriptor
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Streaming WARC/ARC library for fast web archive IO";
+    homepage = https://github.com/webrecorder/warcio;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 33b9645a8a126..26ce4a4377be1 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -4230,6 +4230,8 @@ in {
 
   virtualenv = callPackage ../development/python-modules/virtualenv { };
 
+  warcio = callPackage ../development/python-modules/warcio { };
+
   webassets = callPackage ../development/python-modules/webassets { };
 
   webcolors = callPackage ../development/python-modules/webcolors { };

From 6688b6344e0bd3d24848bbf0c833f56d3558e928 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 02:35:36 +0000
Subject: [PATCH 2/9] pythonPackages.surt: init at 0.3.0

---
 .../python-modules/surt/default.nix           | 34 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 36 insertions(+)
 create mode 100644 pkgs/development/python-modules/surt/default.nix

diff --git a/pkgs/development/python-modules/surt/default.nix b/pkgs/development/python-modules/surt/default.nix
new file mode 100644
index 0000000000000..0223b108578ae
--- /dev/null
+++ b/pkgs/development/python-modules/surt/default.nix
@@ -0,0 +1,34 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, six
+, tldextract
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "surt";
+  version = "0.3.0";
+
+  # Fetch from GitHub because the PyPI tarball does not include tests/
+  src = fetchFromGitHub {
+    owner = "internetarchive";
+    repo = "surt";
+    rev = version;
+    sha256 = "10l8zz9ps31rqhawdbzrcnfq9rkzmaaqrkq6i5c35386h0i22c3g";
+  };
+
+  propagatedBuildInputs = [ six tldextract ];
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = with lib; {
+    description = "Sort-friendly URI Reordering Transform (SURT)";
+    homepage = https://github.com/internetarchive/surt;
+    license = licenses.agpl3;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 26ce4a4377be1..2d37e487022df 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -4481,6 +4481,8 @@ in {
 
   sure = callPackage ../development/python-modules/sure { };
 
+  surt = callPackage ../development/python-modules/surt { };
+
   svgwrite = callPackage ../development/python-modules/svgwrite { };
 
   freezegun = callPackage ../development/python-modules/freezegun { };

From 333a58e7f6edda997d9fe0b4a462f15d09311a12 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 02:45:28 +0000
Subject: [PATCH 3/9] pythonPackages.portalocker: init at 1.3.0

---
 .../python-modules/portalocker/default.nix    | 28 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 30 insertions(+)
 create mode 100644 pkgs/development/python-modules/portalocker/default.nix

diff --git a/pkgs/development/python-modules/portalocker/default.nix b/pkgs/development/python-modules/portalocker/default.nix
new file mode 100644
index 0000000000000..1d5fd45a5583e
--- /dev/null
+++ b/pkgs/development/python-modules/portalocker/default.nix
@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "portalocker";
+  version = "1.3.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1832pnpb2c5basg47ks7ipfvvwflcm7s4r8whkir607mwjn75fmm";
+  };
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    # Based on https://github.com/WoLpH/portalocker/blob/master/pytest.ini
+    py.test tests/*.py --doctest-modules
+  '';
+
+  meta = with lib; {
+    description = "Easy API to file locking";
+    homepage = https://github.com/WoLpH/portalocker;
+    license = licenses.psfl;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 2d37e487022df..d416a7591046f 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1457,6 +1457,8 @@ in {
 
   parsy = callPackage ../development/python-modules/parsy { };
 
+  portalocker = callPackage ../development/python-modules/portalocker { };
+
   portpicker = callPackage ../development/python-modules/portpicker { };
 
   pkginfo = callPackage ../development/python-modules/pkginfo { };

From 8efde845ae241ac181121132c115d2aa2fdde536 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 02:59:15 +0000
Subject: [PATCH 4/9] pythonPackages.certauth: init at 1.2.1

---
 .../python-modules/certauth/default.nix       | 30 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 32 insertions(+)
 create mode 100644 pkgs/development/python-modules/certauth/default.nix

diff --git a/pkgs/development/python-modules/certauth/default.nix b/pkgs/development/python-modules/certauth/default.nix
new file mode 100644
index 0000000000000..c2ec67ea5e00e
--- /dev/null
+++ b/pkgs/development/python-modules/certauth/default.nix
@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pyopenssl
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "certauth";
+  version = "1.2.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1lgsl053bbf1xjvwv7q8rhccg1jxzkya222pgkxdcjaa471s9l27";
+  };
+
+  propagatedBuildInputs = [ pyopenssl ];
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = with lib; {
+    description = "Certificate authority library for dynamically generating host certs";
+    homepage = https://github.com/ikreymer/certauth;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index d416a7591046f..f79303479e31e 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1293,6 +1293,8 @@ in {
 
   cerberus = callPackage ../development/python-modules/cerberus { };
 
+  certauth = callPackage ../development/python-modules/certauth { };
+
   certifi = callPackage ../development/python-modules/certifi { };
 
   characteristic = callPackage ../development/python-modules/characteristic { };

From 967e8cdde5a2e03d7de3edc236032179d4222781 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 03:13:51 +0000
Subject: [PATCH 5/9] pythonPackages.wsgiprox: init at 1.5.1

---
 .../python-modules/wsgiprox/default.nix       | 40 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 +
 2 files changed, 42 insertions(+)
 create mode 100644 pkgs/development/python-modules/wsgiprox/default.nix

diff --git a/pkgs/development/python-modules/wsgiprox/default.nix b/pkgs/development/python-modules/wsgiprox/default.nix
new file mode 100644
index 0000000000000..295fd1456a81d
--- /dev/null
+++ b/pkgs/development/python-modules/wsgiprox/default.nix
@@ -0,0 +1,40 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, certauth
+, gevent
+, mock
+, pytest
+, requests
+, six
+, websocket_client
+, waitress
+}:
+
+buildPythonPackage rec {
+  pname = "wsgiprox";
+  version = "1.5.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1d0x0ihms5sv9v8nrb65wlih6zcdh0644q6nawkzylmvfpwp2k2c";
+  };
+
+  propagatedBuildInputs = [ certauth six ];
+
+  # Don't provide uwsgi to skip tests that would hang on network access.
+  checkInputs = [ gevent mock pytest requests websocket_client waitress ];
+  checkPhase = ''
+    # Based on https://github.com/webrecorder/wsgiprox/blob/master/setup.py
+    py.test --doctest-module ./wsgiprox wsgiprox -v test/
+  '';
+
+  __darwinAllowLocalNetworking = true;
+
+  meta = with lib; {
+    description = "Middleware for adding HTTP/S proxy support to WSGI applications";
+    homepage = https://github.com/webrecorder/wsgiprox;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index f79303479e31e..bdf29c3a8eb51 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -4583,6 +4583,8 @@ in {
 
   webtest = callPackage ../development/python-modules/webtest { };
 
+  wsgiprox = callPackage ../development/python-modules/wsgiprox { };
+
   wsgiproxy2 = callPackage ../development/python-modules/wsgiproxy2 { };
 
   xcaplib = callPackage ../development/python-modules/xcaplib { };

From 652c549adfd5bac1565e5540072cd6a178686125 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 03:48:47 +0000
Subject: [PATCH 6/9] pythonPackages.py3amf: init at 0.8.9

---
 .../python-modules/py3amf/default.nix         | 35 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 37 insertions(+)
 create mode 100644 pkgs/development/python-modules/py3amf/default.nix

diff --git a/pkgs/development/python-modules/py3amf/default.nix b/pkgs/development/python-modules/py3amf/default.nix
new file mode 100644
index 0000000000000..cd68254df40b4
--- /dev/null
+++ b/pkgs/development/python-modules/py3amf/default.nix
@@ -0,0 +1,35 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, isPy3k
+, defusedxml
+}:
+
+buildPythonPackage rec {
+  pname = "Py3AMF";
+  version = "0.8.9";
+
+  disabled = !isPy3k;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1pmh6p7mqdjmy2yj4mv6s1vnf7l9sn326dr6465mxhd6qz6mmgzi";
+  };
+
+  propagatedBuildInputs = [ defusedxml ];
+
+  preCheck = ''
+    # RuntimeError: generator raised StopIteration
+    rm pyamf/tests/test_remoting.py
+
+    # AssertionError: '500 Internal Server Error' != '200 OK'
+    rm pyamf/tests/gateway/test_wsgi.py
+  '';
+
+  meta = with lib; {
+    description = "AMF (Action Message Format) support for Python 3";
+    homepage = https://github.com/StdCarrot/Py3AMF;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index bdf29c3a8eb51..d148d85f6fc77 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -562,6 +562,8 @@ in {
 
   pyamf = callPackage ../development/python-modules/pyamf { };
 
+  py3amf = callPackage ../development/python-modules/py3amf { };
+
   pyarrow = callPackage ../development/python-modules/pyarrow {
     inherit (pkgs) arrow-cpp cmake pkgconfig;
   };

From 9d17e0e08e2c8ee793812a5da04aebdbc80745d8 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 04:33:48 +0000
Subject: [PATCH 7/9] pythonPackages.lupa: init at 1.7

Use lua instead of luajit because of problems on Darwin:

  File "/private/var/folders/ym/xnfgyq_n2fzgzzx9gtccxdjm0000gn/T/nix-build-python3.7-lupa-1.7.drv-0/lupa-1.7/lupa/tests/test.py", line 29, in setUp
    self.lua = lupa.LuaRuntime(**self.lua_runtime_kwargs)
  File "lupa/_lupa.pyx", line 208, in lupa._lupa.LuaRuntime.__cinit__
    raise LuaError("Failed to initialise Lua runtime")
lupa._lupa.LuaError: Failed to initialise Lua runtime
---
 .../python-modules/lupa/default.nix           | 25 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 ++
 2 files changed, 27 insertions(+)
 create mode 100644 pkgs/development/python-modules/lupa/default.nix

diff --git a/pkgs/development/python-modules/lupa/default.nix b/pkgs/development/python-modules/lupa/default.nix
new file mode 100644
index 0000000000000..999f338c929c3
--- /dev/null
+++ b/pkgs/development/python-modules/lupa/default.nix
@@ -0,0 +1,25 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, lua
+, pkgconfig
+}:
+
+buildPythonPackage rec {
+  pname = "lupa";
+  version = "1.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0xvfh27rv7ili9xc5nfg2nnnlj9iyxxcjinkcs71xz1spxp3zmyj";
+  };
+
+  buildInputs = [ lua pkgconfig ];
+
+  meta = with lib; {
+    description = "Integrates the runtime of Lua or LuaJIT2 into CPython";
+    homepage = https://github.com/scoder/lupa;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index d148d85f6fc77..eeb5886a634a1 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1999,6 +1999,8 @@ in {
 
   luftdaten = callPackage ../development/python-modules/luftdaten { };
 
+  lupa = callPackage ../development/python-modules/lupa { };
+
   m2r = callPackage ../development/python-modules/m2r { };
 
   mailchimp = callPackage ../development/python-modules/mailchimp { };

From ca1a762605564e18362d88ec816e08413137f70e Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 04:38:23 +0000
Subject: [PATCH 8/9] pythonPackages.fakeredis: init at 0.16.0

---
 .../python-modules/fakeredis/default.nix      | 43 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  2 +
 2 files changed, 45 insertions(+)
 create mode 100644 pkgs/development/python-modules/fakeredis/default.nix

diff --git a/pkgs/development/python-modules/fakeredis/default.nix b/pkgs/development/python-modules/fakeredis/default.nix
new file mode 100644
index 0000000000000..5be787f2d304c
--- /dev/null
+++ b/pkgs/development/python-modules/fakeredis/default.nix
@@ -0,0 +1,43 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, lupa
+, pytest
+, redis
+, nose
+}:
+
+buildPythonPackage rec {
+  pname = "fakeredis";
+  version = "0.16.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "005gnzj8lffz6hv5ii980gv8ffsiilqijdifyrz7lnms0c1852ms";
+  };
+
+  postPatch = ''
+    substituteInPlace setup.py --replace "redis<3" "redis"
+  '';
+
+  propagatedBuildInputs = [ lupa redis ];
+
+  checkInputs = [ pytest nose ];
+
+  # test_pubsub_run_in_thread sometimes fails with
+  # `RuntimeError: dictionary changed size during iteration`
+  # https://github.com/jamesls/fakeredis/issues/230
+  #
+  # test_pipeline_as_context_manager fails with redis-py 3
+  # `AttributeError: module 'redis.client' has no attribute 'BasePipeline'`
+  checkPhase = ''
+    py.test -v -k "not test_pubsub_run_in_thread and not test_pipeline_as_context_manager"
+  '';
+
+  meta = with lib; {
+    description = "Fake implementation of redis API (redis-py) for testing purposes";
+    homepage = https://github.com/jamesls/fakeredis;
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ ivan ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index eeb5886a634a1..136e1f93f724b 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1771,6 +1771,8 @@ in {
 
   faker = callPackage ../development/python-modules/faker { };
 
+  fakeredis = callPackage ../development/python-modules/fakeredis { };
+
   fake_factory = callPackage ../development/python-modules/fake_factory { };
 
   factory_boy = callPackage ../development/python-modules/factory_boy { };

From acde7403eb744b2374c132145ce6b1950e059946 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 25 Dec 2018 05:22:25 +0000
Subject: [PATCH 9/9] pywb: init at 2.1.0

---
 pkgs/tools/backup/pywb/default.nix | 91 ++++++++++++++++++++++++++++++
 pkgs/top-level/all-packages.nix    |  2 +
 2 files changed, 93 insertions(+)
 create mode 100644 pkgs/tools/backup/pywb/default.nix

diff --git a/pkgs/tools/backup/pywb/default.nix b/pkgs/tools/backup/pywb/default.nix
new file mode 100644
index 0000000000000..8cf21b68aa5fc
--- /dev/null
+++ b/pkgs/tools/backup/pywb/default.nix
@@ -0,0 +1,91 @@
+{ lib, python3Packages, fetchFromGitHub }:
+
+with lib;
+
+let
+  disabledTests = [
+    # `AssertionError` due to trailing comma in tuple representation
+    "test_another_remote_not_found"
+    "test_file_not_found"
+    "test_mem_agg_index_1"
+    "test_mem_agg_index_2"
+    "test_mem_agg_index_5"
+    "test_mem_agg_index_5_inverse_preset"
+    "test_mem_agg_not_found"
+    "test_agg_select_local"
+    "test_agg_select_local_postreq"
+
+    # `RuntimeError('generator raised StopIteration')`
+    "test_zipnum"
+
+    # Failures in sandbox due to lack of network access
+    "test_replay"
+    "test_record_skip_http_only_cookies_header"
+    "test_loaders"
+    "test_remote_loader"
+    "test_remote_closest_loader"
+    "test_remote_closest_wb_memnto_loader"
+    "test_all_not_found"
+    "test_agg_select_mem_1"
+    "test_agg_select_mem_2"
+    "test_agg_select_mem_unrewrite_headers"
+    "test_agg_select_live"
+    "test_force_https_root_replay_1"
+    "test_live_fallback"
+    "test_live_live_1"
+    "test_proxy_record_keep_percent"
+    "test_live_no_redir"
+    "test_root_replay_ts"
+    "test_root_replay_no_ts"
+    "test_root_replay_redir"
+
+    # `AssertionError: assert 3 == 4` due to another test being skipped
+    "test_cdx_all_coll"
+
+    # `AppError: Bad response: 404 Not Found` when using Python 2 on Linux
+    # or Python 3 on macOS, may fail in other cases
+    "test_timemap_all_coll"
+  ];
+
+  # Compose the final string expression, including the "-k" and the single quotes.
+  testExpression = optionalString (disabledTests != [])
+    "-k 'not ${concatStringsSep " and not " disabledTests}'";
+
+in
+
+python3Packages.buildPythonApplication rec {
+  name = "pywb-${version}";
+  version = "2.1.0";
+
+  # Fetch from GitHub because the PyPi tarball is missing requirements.txt
+  src = fetchFromGitHub {
+    owner = "webrecorder";
+    repo = "pywb";
+    # 2.1.0 was not tagged but pywb-2.1.0.tar.gz in PyPi was based on this commit
+    rev = "1b151b74bfee5bd465ccc70cc04a8d2b2e090db4";
+    sha256 = "0ahsk8fhrz0gdiq4i6b3m4jvhxbdsaz1mkdgvy26vb3r4ph24dfa";
+  };
+
+  propagatedBuildInputs = with python3Packages; [
+    brotlipy chardet gevent jinja2 portalocker py3amf pytest pyyaml redis
+    requests six surt warcio webassets webencodings werkzeug wsgiprox
+  ];
+
+  checkInputs = with python3Packages; [
+    fakeredis httpbin mock pytest ujson urllib3 webtest
+  ];
+  checkPhase = ''
+    # Based on https://github.com/webrecorder/pywb/blob/master/setup.py
+    py.test -v --doctest-module ./pywb/ tests/ ${testExpression}
+  '';
+
+  __darwinAllowLocalNetworking = true;
+
+  meta = {
+    description = "Web archiving toolkit for creating or replaying web archives";
+    homepage = https://github.com/webrecorder/pywb;
+    license = licenses.gpl3;
+    maintainers = with maintainers; [ ivan ];
+    platforms = platforms.all;
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 821f9c7cb139a..ea526f44c4858 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -5016,6 +5016,8 @@ in
 
   pywal = with python3Packages; toPythonApplication pywal;
 
+  pywb = callPackage ../tools/backup/pywb { };
+
   remarshal = callPackage ../development/tools/remarshal { };
 
   rig = callPackage ../tools/misc/rig { };