diff --git a/pkgs/development/python-modules/certauth/default.nix b/pkgs/development/python-modules/certauth/default.nix new file mode 100644 index 0000000000000..c2ec67ea5e00e --- /dev/null +++ b/pkgs/development/python-modules/certauth/default.nix @@ -0,0 +1,30 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pyopenssl +, pytest +}: + +buildPythonPackage rec { + pname = "certauth"; + version = "1.2.1"; + + src = fetchPypi { + inherit pname version; + sha256 = "1lgsl053bbf1xjvwv7q8rhccg1jxzkya222pgkxdcjaa471s9l27"; + }; + + propagatedBuildInputs = [ pyopenssl ]; + + checkInputs = [ pytest ]; + checkPhase = '' + py.test + ''; + + meta = with lib; { + description = "Certificate authority library for dynamically generating host certs"; + homepage = https://github.com/ikreymer/certauth; + license = licenses.mit; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/fakeredis/default.nix b/pkgs/development/python-modules/fakeredis/default.nix new file mode 100644 index 0000000000000..5be787f2d304c --- /dev/null +++ b/pkgs/development/python-modules/fakeredis/default.nix @@ -0,0 +1,43 @@ +{ lib +, buildPythonPackage +, fetchPypi +, lupa +, pytest +, redis +, nose +}: + +buildPythonPackage rec { + pname = "fakeredis"; + version = "0.16.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "005gnzj8lffz6hv5ii980gv8ffsiilqijdifyrz7lnms0c1852ms"; + }; + + postPatch = '' + substituteInPlace setup.py --replace "redis<3" "redis" + ''; + + propagatedBuildInputs = [ lupa redis ]; + + checkInputs = [ pytest nose ]; + + # test_pubsub_run_in_thread sometimes fails with + # `RuntimeError: dictionary changed size during iteration` + # https://github.com/jamesls/fakeredis/issues/230 + # + # test_pipeline_as_context_manager fails with redis-py 3 + # `AttributeError: module 'redis.client' has no attribute 'BasePipeline'` + checkPhase = '' + py.test -v -k "not test_pubsub_run_in_thread and not test_pipeline_as_context_manager" + ''; + + meta = with lib; { + description = "Fake implementation of redis API (redis-py) for testing purposes"; + homepage = https://github.com/jamesls/fakeredis; + license = licenses.bsd3; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/lupa/default.nix b/pkgs/development/python-modules/lupa/default.nix new file mode 100644 index 0000000000000..999f338c929c3 --- /dev/null +++ b/pkgs/development/python-modules/lupa/default.nix @@ -0,0 +1,25 @@ +{ lib +, buildPythonPackage +, fetchPypi +, lua +, pkgconfig +}: + +buildPythonPackage rec { + pname = "lupa"; + version = "1.7"; + + src = fetchPypi { + inherit pname version; + sha256 = "0xvfh27rv7ili9xc5nfg2nnnlj9iyxxcjinkcs71xz1spxp3zmyj"; + }; + + buildInputs = [ lua pkgconfig ]; + + meta = with lib; { + description = "Integrates the runtime of Lua or LuaJIT2 into CPython"; + homepage = https://github.com/scoder/lupa; + license = licenses.mit; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/portalocker/default.nix b/pkgs/development/python-modules/portalocker/default.nix new file mode 100644 index 0000000000000..1d5fd45a5583e --- /dev/null +++ b/pkgs/development/python-modules/portalocker/default.nix @@ -0,0 +1,28 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pytest +}: + +buildPythonPackage rec { + pname = "portalocker"; + version = "1.3.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "1832pnpb2c5basg47ks7ipfvvwflcm7s4r8whkir607mwjn75fmm"; + }; + + checkInputs = [ pytest ]; + checkPhase = '' + # Based on https://github.com/WoLpH/portalocker/blob/master/pytest.ini + py.test tests/*.py --doctest-modules + ''; + + meta = with lib; { + description = "Easy API to file locking"; + homepage = https://github.com/WoLpH/portalocker; + license = licenses.psfl; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/py3amf/default.nix b/pkgs/development/python-modules/py3amf/default.nix new file mode 100644 index 0000000000000..cd68254df40b4 --- /dev/null +++ b/pkgs/development/python-modules/py3amf/default.nix @@ -0,0 +1,35 @@ +{ lib +, buildPythonPackage +, fetchPypi +, isPy3k +, defusedxml +}: + +buildPythonPackage rec { + pname = "Py3AMF"; + version = "0.8.9"; + + disabled = !isPy3k; + + src = fetchPypi { + inherit pname version; + sha256 = "1pmh6p7mqdjmy2yj4mv6s1vnf7l9sn326dr6465mxhd6qz6mmgzi"; + }; + + propagatedBuildInputs = [ defusedxml ]; + + preCheck = '' + # RuntimeError: generator raised StopIteration + rm pyamf/tests/test_remoting.py + + # AssertionError: '500 Internal Server Error' != '200 OK' + rm pyamf/tests/gateway/test_wsgi.py + ''; + + meta = with lib; { + description = "AMF (Action Message Format) support for Python 3"; + homepage = https://github.com/StdCarrot/Py3AMF; + license = licenses.mit; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/surt/default.nix b/pkgs/development/python-modules/surt/default.nix new file mode 100644 index 0000000000000..0223b108578ae --- /dev/null +++ b/pkgs/development/python-modules/surt/default.nix @@ -0,0 +1,34 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, six +, tldextract +, pytest +}: + +buildPythonPackage rec { + pname = "surt"; + version = "0.3.0"; + + # Fetch from GitHub because the PyPI tarball does not include tests/ + src = fetchFromGitHub { + owner = "internetarchive"; + repo = "surt"; + rev = version; + sha256 = "10l8zz9ps31rqhawdbzrcnfq9rkzmaaqrkq6i5c35386h0i22c3g"; + }; + + propagatedBuildInputs = [ six tldextract ]; + + checkInputs = [ pytest ]; + checkPhase = '' + py.test + ''; + + meta = with lib; { + description = "Sort-friendly URI Reordering Transform (SURT)"; + homepage = https://github.com/internetarchive/surt; + license = licenses.agpl3; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/warcio/default.nix b/pkgs/development/python-modules/warcio/default.nix new file mode 100644 index 0000000000000..df16f514869be --- /dev/null +++ b/pkgs/development/python-modules/warcio/default.nix @@ -0,0 +1,28 @@ +{ lib +, buildPythonPackage +, fetchPypi +, six +}: + +buildPythonPackage rec { + pname = "warcio"; + version = "1.6.3"; + + src = fetchPypi { + inherit pname version; + sha256 = "1nyhghbag1chh5fml848x799mwgkgmz3l3ipv7lr6p0lj1jq8i1r"; + }; + + propagatedBuildInputs = [ six ]; + + # Test suite makes DNS lookups and HTTP requests to example.com + # Most tests fail with OSError: [Errno 9] Bad file descriptor + doCheck = false; + + meta = with lib; { + description = "Streaming WARC/ARC library for fast web archive IO"; + homepage = https://github.com/webrecorder/warcio; + license = licenses.asl20; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/wsgiprox/default.nix b/pkgs/development/python-modules/wsgiprox/default.nix new file mode 100644 index 0000000000000..295fd1456a81d --- /dev/null +++ b/pkgs/development/python-modules/wsgiprox/default.nix @@ -0,0 +1,40 @@ +{ lib +, buildPythonPackage +, fetchPypi +, certauth +, gevent +, mock +, pytest +, requests +, six +, websocket_client +, waitress +}: + +buildPythonPackage rec { + pname = "wsgiprox"; + version = "1.5.1"; + + src = fetchPypi { + inherit pname version; + sha256 = "1d0x0ihms5sv9v8nrb65wlih6zcdh0644q6nawkzylmvfpwp2k2c"; + }; + + propagatedBuildInputs = [ certauth six ]; + + # Don't provide uwsgi to skip tests that would hang on network access. + checkInputs = [ gevent mock pytest requests websocket_client waitress ]; + checkPhase = '' + # Based on https://github.com/webrecorder/wsgiprox/blob/master/setup.py + py.test --doctest-module ./wsgiprox wsgiprox -v test/ + ''; + + __darwinAllowLocalNetworking = true; + + meta = with lib; { + description = "Middleware for adding HTTP/S proxy support to WSGI applications"; + homepage = https://github.com/webrecorder/wsgiprox; + license = licenses.asl20; + maintainers = with maintainers; [ ivan ]; + }; +} diff --git a/pkgs/tools/backup/pywb/default.nix b/pkgs/tools/backup/pywb/default.nix new file mode 100644 index 0000000000000..8cf21b68aa5fc --- /dev/null +++ b/pkgs/tools/backup/pywb/default.nix @@ -0,0 +1,91 @@ +{ lib, python3Packages, fetchFromGitHub }: + +with lib; + +let + disabledTests = [ + # `AssertionError` due to trailing comma in tuple representation + "test_another_remote_not_found" + "test_file_not_found" + "test_mem_agg_index_1" + "test_mem_agg_index_2" + "test_mem_agg_index_5" + "test_mem_agg_index_5_inverse_preset" + "test_mem_agg_not_found" + "test_agg_select_local" + "test_agg_select_local_postreq" + + # `RuntimeError('generator raised StopIteration')` + "test_zipnum" + + # Failures in sandbox due to lack of network access + "test_replay" + "test_record_skip_http_only_cookies_header" + "test_loaders" + "test_remote_loader" + "test_remote_closest_loader" + "test_remote_closest_wb_memnto_loader" + "test_all_not_found" + "test_agg_select_mem_1" + "test_agg_select_mem_2" + "test_agg_select_mem_unrewrite_headers" + "test_agg_select_live" + "test_force_https_root_replay_1" + "test_live_fallback" + "test_live_live_1" + "test_proxy_record_keep_percent" + "test_live_no_redir" + "test_root_replay_ts" + "test_root_replay_no_ts" + "test_root_replay_redir" + + # `AssertionError: assert 3 == 4` due to another test being skipped + "test_cdx_all_coll" + + # `AppError: Bad response: 404 Not Found` when using Python 2 on Linux + # or Python 3 on macOS, may fail in other cases + "test_timemap_all_coll" + ]; + + # Compose the final string expression, including the "-k" and the single quotes. + testExpression = optionalString (disabledTests != []) + "-k 'not ${concatStringsSep " and not " disabledTests}'"; + +in + +python3Packages.buildPythonApplication rec { + name = "pywb-${version}"; + version = "2.1.0"; + + # Fetch from GitHub because the PyPi tarball is missing requirements.txt + src = fetchFromGitHub { + owner = "webrecorder"; + repo = "pywb"; + # 2.1.0 was not tagged but pywb-2.1.0.tar.gz in PyPi was based on this commit + rev = "1b151b74bfee5bd465ccc70cc04a8d2b2e090db4"; + sha256 = "0ahsk8fhrz0gdiq4i6b3m4jvhxbdsaz1mkdgvy26vb3r4ph24dfa"; + }; + + propagatedBuildInputs = with python3Packages; [ + brotlipy chardet gevent jinja2 portalocker py3amf pytest pyyaml redis + requests six surt warcio webassets webencodings werkzeug wsgiprox + ]; + + checkInputs = with python3Packages; [ + fakeredis httpbin mock pytest ujson urllib3 webtest + ]; + checkPhase = '' + # Based on https://github.com/webrecorder/pywb/blob/master/setup.py + py.test -v --doctest-module ./pywb/ tests/ ${testExpression} + ''; + + __darwinAllowLocalNetworking = true; + + meta = { + description = "Web archiving toolkit for creating or replaying web archives"; + homepage = https://github.com/webrecorder/pywb; + license = licenses.gpl3; + maintainers = with maintainers; [ ivan ]; + platforms = platforms.all; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 821f9c7cb139a..ea526f44c4858 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5016,6 +5016,8 @@ in pywal = with python3Packages; toPythonApplication pywal; + pywb = callPackage ../tools/backup/pywb { }; + remarshal = callPackage ../development/tools/remarshal { }; rig = callPackage ../tools/misc/rig { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 33b9645a8a126..136e1f93f724b 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -562,6 +562,8 @@ in { pyamf = callPackage ../development/python-modules/pyamf { }; + py3amf = callPackage ../development/python-modules/py3amf { }; + pyarrow = callPackage ../development/python-modules/pyarrow { inherit (pkgs) arrow-cpp cmake pkgconfig; }; @@ -1293,6 +1295,8 @@ in { cerberus = callPackage ../development/python-modules/cerberus { }; + certauth = callPackage ../development/python-modules/certauth { }; + certifi = callPackage ../development/python-modules/certifi { }; characteristic = callPackage ../development/python-modules/characteristic { }; @@ -1457,6 +1461,8 @@ in { parsy = callPackage ../development/python-modules/parsy { }; + portalocker = callPackage ../development/python-modules/portalocker { }; + portpicker = callPackage ../development/python-modules/portpicker { }; pkginfo = callPackage ../development/python-modules/pkginfo { }; @@ -1765,6 +1771,8 @@ in { faker = callPackage ../development/python-modules/faker { }; + fakeredis = callPackage ../development/python-modules/fakeredis { }; + fake_factory = callPackage ../development/python-modules/fake_factory { }; factory_boy = callPackage ../development/python-modules/factory_boy { }; @@ -1993,6 +2001,8 @@ in { luftdaten = callPackage ../development/python-modules/luftdaten { }; + lupa = callPackage ../development/python-modules/lupa { }; + m2r = callPackage ../development/python-modules/m2r { }; mailchimp = callPackage ../development/python-modules/mailchimp { }; @@ -4230,6 +4240,8 @@ in { virtualenv = callPackage ../development/python-modules/virtualenv { }; + warcio = callPackage ../development/python-modules/warcio { }; + webassets = callPackage ../development/python-modules/webassets { }; webcolors = callPackage ../development/python-modules/webcolors { }; @@ -4479,6 +4491,8 @@ in { sure = callPackage ../development/python-modules/sure { }; + surt = callPackage ../development/python-modules/surt { }; + svgwrite = callPackage ../development/python-modules/svgwrite { }; freezegun = callPackage ../development/python-modules/freezegun { }; @@ -4575,6 +4589,8 @@ in { webtest = callPackage ../development/python-modules/webtest { }; + wsgiprox = callPackage ../development/python-modules/wsgiprox { }; + wsgiproxy2 = callPackage ../development/python-modules/wsgiproxy2 { }; xcaplib = callPackage ../development/python-modules/xcaplib { };