From f90b3bee3c74f089eb3807b688bcd2a4e87f54b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Aug 2023 12:01:03 -0400 Subject: [PATCH] Remove everything except the state needed to trigger the failure. Ref jaraco/jaraco.mongodb#36. --- .coveragerc | 9 - .editorconfig | 19 - .github/dependabot.yml | 8 - .github/workflows/main.yml | 133 ----- .pre-commit-config.yaml | 5 - .readthedocs.yaml | 12 - LICENSE | 17 - NEWS.rst | 630 ----------------------- README.rst | 185 ------- conftest.py | 35 -- docs/conf.py | 42 -- docs/history.rst | 8 - docs/index.rst | 77 --- jaraco/mongodb/__init__.py | 0 jaraco/mongodb/check-gridfs.py | 69 --- jaraco/mongodb/cli.py | 19 - jaraco/mongodb/codec.py | 44 -- jaraco/mongodb/compat.py | 47 -- jaraco/mongodb/fields.py | 31 -- jaraco/mongodb/fixtures.py | 59 --- jaraco/mongodb/helper.py | 83 --- jaraco/mongodb/insert-doc.py | 32 -- jaraco/mongodb/install.py | 89 ---- jaraco/mongodb/manage.py | 50 -- jaraco/mongodb/migration.py | 143 ----- jaraco/mongodb/monitor-index-creation.py | 37 -- jaraco/mongodb/move-gridfs.py | 152 ------ jaraco/mongodb/oplog.py | 622 +--------------------- jaraco/mongodb/pmxbot.py | 29 -- jaraco/mongodb/query.py | 62 --- jaraco/mongodb/repair-gridfs.py | 76 --- jaraco/mongodb/sampling.py | 43 -- jaraco/mongodb/service.py | 262 ---------- jaraco/mongodb/sessions.py | 208 -------- jaraco/mongodb/sharding.py | 41 -- jaraco/mongodb/testing.py | 69 --- jaraco/mongodb/tests/test_compat.py | 42 -- jaraco/mongodb/tests/test_fields.py | 10 - jaraco/mongodb/tests/test_insert_doc.py | 20 - jaraco/mongodb/tests/test_manage.py | 11 - jaraco/mongodb/tests/test_oplog.py | 100 ---- jaraco/mongodb/tests/test_service.py | 36 -- jaraco/mongodb/tests/test_sessions.py | 73 --- jaraco/mongodb/tests/test_testing.py | 65 --- jaraco/mongodb/timers.py | 25 - jaraco/mongodb/uri.py | 27 - mypy.ini | 5 - pyproject.toml | 5 - pytest.ini | 27 - setup.cfg | 56 +- tests/oplog.js | 269 ---------- towncrier.toml | 2 - tox.ini | 29 -- 53 files changed, 2 insertions(+), 4247 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .editorconfig delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .pre-commit-config.yaml delete mode 100644 .readthedocs.yaml delete mode 100644 LICENSE delete mode 100644 NEWS.rst delete mode 100644 README.rst delete mode 100644 conftest.py delete mode 100644 docs/history.rst delete mode 100644 jaraco/mongodb/__init__.py delete mode 100644 jaraco/mongodb/check-gridfs.py delete mode 100644 jaraco/mongodb/cli.py delete mode 100644 jaraco/mongodb/codec.py delete mode 100644 jaraco/mongodb/compat.py delete mode 100644 jaraco/mongodb/fields.py delete mode 100644 jaraco/mongodb/fixtures.py delete mode 100644 jaraco/mongodb/helper.py delete mode 100644 jaraco/mongodb/insert-doc.py delete mode 100644 jaraco/mongodb/install.py delete mode 100644 jaraco/mongodb/manage.py delete mode 100644 jaraco/mongodb/migration.py delete mode 100644 jaraco/mongodb/monitor-index-creation.py delete mode 100644 jaraco/mongodb/move-gridfs.py delete mode 100644 jaraco/mongodb/pmxbot.py delete mode 100644 jaraco/mongodb/query.py delete mode 100644 jaraco/mongodb/repair-gridfs.py delete mode 100644 jaraco/mongodb/sampling.py delete mode 100644 jaraco/mongodb/service.py delete mode 100644 jaraco/mongodb/sessions.py delete mode 100644 jaraco/mongodb/sharding.py delete mode 100644 jaraco/mongodb/testing.py delete mode 100644 jaraco/mongodb/tests/test_compat.py delete mode 100644 jaraco/mongodb/tests/test_fields.py delete mode 100644 jaraco/mongodb/tests/test_insert_doc.py delete mode 100644 jaraco/mongodb/tests/test_manage.py delete mode 100644 jaraco/mongodb/tests/test_oplog.py delete mode 100644 jaraco/mongodb/tests/test_service.py delete mode 100644 jaraco/mongodb/tests/test_sessions.py delete mode 100644 jaraco/mongodb/tests/test_testing.py delete mode 100644 jaraco/mongodb/timers.py delete mode 100644 jaraco/mongodb/uri.py delete mode 100644 mypy.ini delete mode 100644 pytest.ini delete mode 100644 tests/oplog.js delete mode 100644 towncrier.toml diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 0287948..0000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[run] -omit = - # leading `*/` for pytest-dev/pytest-cov#456 - */.tox/* -disable_warnings = - couldnt-parse - -[report] -show_missing = True diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 304196f..0000000 --- a/.editorconfig +++ /dev/null @@ -1,19 +0,0 @@ -root = true - -[*] -charset = utf-8 -indent_style = tab -indent_size = 4 -insert_final_newline = true -end_of_line = lf - -[*.py] -indent_style = space -max_line_length = 88 - -[*.{yml,yaml}] -indent_style = space -indent_size = 2 - -[*.rst] -indent_style = space diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 89ff339..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - allow: - - dependency-type: "all" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 600c00d..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: tests - -on: [push, pull_request] - -permissions: - contents: read - -env: - # Environment variables to support color support (jaraco/skeleton#66): - # Request colored output from CLI tools supporting it. Different tools - # interpret the value differently. For some, just being set is sufficient. - # For others, it must be a non-zero integer. For yet others, being set - # to a non-empty value is sufficient. For tox, it must be one of - # , 0, 1, false, no, off, on, true, yes. The only enabling value - # in common is "1". - FORCE_COLOR: 1 - # MyPy's color enforcement (must be a non-zero number) - MYPY_FORCE_COLOR: -42 - # Recognized by the `py` package, dependency of `pytest` (must be "1") - PY_COLORS: 1 - # Make tox-wrapped tools see color requests - TOX_TESTENV_PASSENV: >- - FORCE_COLOR - MYPY_FORCE_COLOR - NO_COLOR - PY_COLORS - PYTEST_THEME - PYTEST_THEME_MODE - - # Suppress noisy pip warnings - PIP_DISABLE_PIP_VERSION_CHECK: 'true' - PIP_NO_PYTHON_VERSION_WARNING: 'true' - PIP_NO_WARN_SCRIPT_LOCATION: 'true' - - # Disable the spinner, noise in GHA; TODO(webknjaz): Fix this upstream - # Must be "1". - TOX_PARALLEL_NO_SPINNER: 1 - - -jobs: - test: - strategy: - matrix: - python: - - "3.8" - - "3.11" - - "3.12" - platform: - - ubuntu-latest - - macos-latest - - windows-latest - include: - - python: "3.9" - platform: ubuntu-latest - - python: "3.10" - platform: ubuntu-latest - - python: pypy3.9 - platform: ubuntu-latest - exclude: - # Windows tests disabled due to #32 - - platform: windows-latest - runs-on: ${{ matrix.platform }} - continue-on-error: ${{ matrix.python == '3.12' }} - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - allow-prereleases: true - - name: Install tox - run: | - python -m pip install tox - # needed to run mongodb - # https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu-tarball/ - - uses: awalsh128/cache-apt-pkgs-action@latest - if: ${{ matrix.platform == 'ubuntu-latest' }} - with: - packages: libcurl4 libgssapi-krb5-2 libldap-2.5-0 libwrap0 libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit snmp openssl liblzma5 - - name: Run - run: tox - - docs: - runs-on: ubuntu-latest - env: - TOXENV: docs - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - - name: Install tox - run: | - python -m pip install tox - - name: Run - run: tox - - check: # This job does nothing and is only used for the branch protection - if: always() - - needs: - - test - - docs - - runs-on: ubuntu-latest - - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} - - release: - permissions: - contents: write - needs: - - check - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install tox - run: | - python -m pip install tox - - name: Run - run: tox -e release - env: - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index af50201..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -repos: -- repo: https://github.com/psf/black - rev: 22.6.0 - hooks: - - id: black diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 053c728..0000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -python: - install: - - path: . - extra_requirements: - - docs - -# required boilerplate readthedocs/readthedocs.org#10401 -build: - os: ubuntu-22.04 - tools: - python: "3" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 1bb5a44..0000000 --- a/LICENSE +++ /dev/null @@ -1,17 +0,0 @@ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. diff --git a/NEWS.rst b/NEWS.rst deleted file mode 100644 index e0725a4..0000000 --- a/NEWS.rst +++ /dev/null @@ -1,630 +0,0 @@ -v11.5.0 -======= - -Features --------- - -- Added special handling for MongoDB 7 query plan structure (#35) - - -Bugfixes --------- - -- Removed MongoDBInstance.mongod_args, removing default to ephemeralForTest storage engine, which no longer exists (#34) - - -v11.4.1 -======= - -Bugfixes --------- - -- Pin to Mongodb 6.0.9 to prevent installation of breaking 7.0 release (#34) - - -v11.4.0 -======= - -Features --------- - -- Added ``helpers.server_version``. (#28) -- ``oplog`` ``createIndexes`` support is now only applied on MongoDB 4.4 and later. (#28) - - -v11.3.0 -======= - -Features --------- - -- Require Python 3.8 or later. - - -v11.2.1 -======= - -#27: In oplog module, once again support createIndex operations -even on MongoDB 4.4 and later. - -``mongodb_instance`` now uses preferred simple ``fixture`` -instead of deprecated ``yield_fixture``. - -v11.2.0 -======= - -Rely on native f-strings and remove dependency on future-fstrings. - -v11.1.0 -======= - -#22: The pytest fixture now honors ``--mongodb-uri`` or -the environment variable ``MONGODB_URL`` to run tests -against an existing instance of MongoDB rather than starting -up a new one. - -v11.0.1 -======= - -Rely on PEP 420 for namespace package. - -v11.0.0 -======= - -Require Python 3.6 or later. - -#26: Removed ``--noprealloc`` and ``--smallfiles`` from -MongoDBReplicaSet class, restoring compatibility on -later MongoDB releases. - -10.3.0 -====== - -Added ``jaraco.mongodb.sampling`` with the new -``estimate`` function for estimating the count of -objects matching a query. - -10.2.0 -====== - -Remove dependency on ``namespace_format`` from -(otherwise pinned) ``jaraco.text`` and instead rely -on ``future-fstrings`` to provide for f-strings on -supported Python versions. - -10.1.3 -====== - -#25: Pin dependency on jaraco.text 2.x to avoid error. - -10.1.2 -====== - -Fixed DeprecationWarning in assert_distinct_covered. - -10.1.1 -====== - -Fix a couple of deprecation warnings, including an emergent -one on recent pytest versions. - -10.1 -==== - -Add ``codec`` module with support for parsing dates from -JSON input, suitable for making queries. - -10.0 -==== - -Switch to `pkgutil namespace technique -`_ -for the ``jaraco`` namespace. - -9.4 -=== - -``create_database_in_shard`` now also reports the 'nodes' -on which the database was created. - -9.3 -=== - -Added ``testing.assert_index_used`` function. - -9.2.1 -===== - -Removed deprecation of ``helper.connect_db``, as the -upstream implementation still doesn't provide for a -nice 'default'. - -9.2 -=== - -Disabled and deprecated ``helper.filter_warnings``. - -Deprecated ``helper.connect``. - -Deprecated ``helper.connect_db`` in favor of functions -now available in pymongo 3.5. - -Added ``helper.get_collection``. - -9.1 -=== - -#21: In ``mongodb_instance`` fixture, allow ``--port`` to be -passed as mongod args, overriding default behavior of starting -on an ephemeral port. - -9.0 -=== - -Refreshed project metadata, including conversion to declarative -config. Requires Setuptools 34.4 to install from sdist. - -8.1 -=== - -In ``query.upsert_and_fetch``, allow keyword arguments to pass -to the underlying call. - -Fix return value in ``query.upsert_and_fetch``. - -8.0 -=== - -MongoDB Instances are now started with -``--storageEngine ephemeralForTest`` instead of deferring to -the default storage engine. As a result, these options have -also been removed from the mongod invocation: - - - noprealloc - - nojournal - - syncdelay - - noauth - -This change also means that the ``soft_stop`` method has no -benefit and so has been removed. - -7.10 -==== - -MongoDBInstances will no longer attempt to store their data in -the root of the virtualenv (if present). Instead, they -unconditionally use a temp directory. - -7.9 -=== - -#12: Ensure indexes when moving files using ``move-gridfs`` script. - -7.8 -=== - -#19: Added Python 2 compatibility to the ``monitor-index-creation`` -script. - -7.7 -=== - -Added ``compat.Collection`` with ``save`` method added in 6.2. - -7.6 -=== - -No longer pass ``--ipv6`` to mongod in MongoDBInstance. IPv6 -is supported since MongoDB 3.0 without this option, and in -some environments, supplying this parameter causes the daemon -to fail to bind to any interfaces. - -7.5 -=== - -Added ``jaraco.mongodb.insert-doc`` to take a JSON document -from the command-line and insert it into the indicated collection. - -7.4 -=== - -#18: Allow pmxbot command to connect to the MongoDB database -other than localhost. - -7.3 -=== - -Add ``jaraco.mongodb.fields`` for escaping values for document -fields. - -7.2.3 -===== - -#17: Remove ``--nohttpinterface`` when constructing MongoDB -instances, following the `same approach taken by MongoDB -`_. - -7.2.2 -===== - -#16: Fixed monitor-index-creation script for MongoDB 3.2+. - -7.2.1 -===== - -Corrected oplog replication issues for MongoDB 3.6 (#13, -#14). - -7.2 -=== - -Moved ``Extend`` action in oplog module to -`jaraco.ui `_ 1.6. - -7.1 -=== - -In ``move-gridfs``, explicitly handle interrupt to allow a -move to complete and only stop between moves. - -7.0.2 -===== - -Fix AttributeError in ``move-gridfs`` get_args. - -7.0.1 -===== - -Miscellaneous packaging fixes. - -7.0 -=== - -Removed support for ``seconds`` argument to ``oplog`` -command. - -6.4 -=== - -``move-gridfs`` now accepts a limit-date option, allowing -for the archival of files older than a certain date. - -6.3.1 -===== - -#11: With save, only use replace when an _id is specified. - -6.3 -=== - -#10: MongoDBInstance now passes the subprocess output -through to stderr and stdout. Callers should either -capture this output separately (such as pytest already -does) or set a ``.process_kwargs`` property on the -instance to customize the ``stdout`` and/or ``stderr`` -parameters to Popen. - -6.2.1 -===== - -Use portend for finding available local port, eliminating -remaining DeprecationWarnings. - -6.2 -=== - -Add compat module and ``compat.save`` method for -supplying the ``Collection.save`` behavior, deprecated -in PyMongo. - -Updated PyMongo 3.0 API usage to eliminate -DeprecationWarnings. - -6.1.1 -===== - -#9: Fix issue with MongoDBInstance by using -``subprocess.PIPE`` for stdout. Users may read from -this pipe by reading ``instance.process.stdout``. - -6.1 -=== - -Now, suppress creation of MongoDBInstance log file in -fixture and MongoDBInstance service. - -6.0 -=== - -Removed workarounds module. - -5.6 -=== - -Added workarounds module with ``safe_upsert_27707``. - -5.5 -=== - -No longer startup MongoDBInstance with -``textSearchEnabled=true``, fixing startup on MongoDB 3.4 -and dropping implicit support for text search on MongoDB 2.4. - -#7: Oplog tool now supports MongoDB 3.4 for the tested -use cases. - -5.4 -=== - -``assert_covered`` now will fail when the candidate cursor -returns no results, as that's almost certainly not an effective -assertion. - -5.3 -=== - -Nicer rendering of operations in the oplog tool. - -In ``testing`` module, assertions now return the objects -on which they've asserted (for troubleshooting or additional -assertions). - -5.2.1 -===== - -#6: Oplog tool will now include, exclude, and apply namespace -renames on 'renameCollection' commands. - -5.2 -=== - -Oplog tool no longer has a default window of 86400 seconds, -but instead requires that a window or valid resume file -be specified. Additionally, there is no longer a default -resume file (avoiding potential issues with multiple -processes writing to the same file). - -Oplog tool now accepts a ``--window`` argument, preferred -to the now deprecated ``--seconds`` argument. Window -accepts simple time spans, like "3 days" or "04:20" (four -hours, twenty minutes). See the docs for `pytimeparse -`_ for specifics -on which formats are supported. - -5.1.1 -===== - -Fix version reporting when invoked with ``-m``. - -5.1 -=== - -Oplog tool no longer defaults to ``localhost`` for the dest, -but instead allows the value to be None. When combined with -``--dry-run``, dest is not needed and a connection is only -attempted if ``--dest`` is indicated. - -Oplog tool now logs the name and version on startup. - -5.0 -=== - -Removed ``oplog.increment_ts`` and ``Timestamp.next`` operation -(no longer needed). - -Ensure that ts is a oplog.Timestamp during ``save_ts``. - -4.4 -=== - -#3: ``create_db_in_shard`` no longer raises an exception when -the database happens to be created in the target shard. - -#5: Better MongoDB 3.2 support for oplog replication. - -Tests in continuous integration are now run against MongoDB -2.6, 3.0, and 3.2. - -4.3 -=== - -Oplog replay now warns if there are no operations preceding -the cutoff. - -4.2.2 -===== - -#2: Retain key order when loading Oplog events for replay. - -4.2.1 -===== - -Avoid race condition if an operation was being applied -when sync was cancelled. - -4.2 -=== - -``oplog`` now reports the failed operation when an oplog -entry fails to apply. - -4.1 -=== - -``oplog`` command now accepts multiple indications of the -following arguments:: - - - --ns - - --exclude - - --rename - -See the docstring for the implications of this change. - -4.0 -=== - -Drop support for Python 3.2. - -3.18.1 -====== - -Add helper module to docs. - -3.18 -==== - -Added ``sharding`` module with ``create_db_in_shard`` -function and pmxbot command. - -3.17 -==== - -Add Trove classifier for Pytest Framework. - -3.16 -==== - -Extract migration manager functionality from YouGov's -cases migration. - -3.15.2 -====== - -Correct syntax error. - -3.15.1 -====== - -Set a small batch size on fs query for move-gridfs to -prevent the cursor timing out while chunks are moved. - -3.15 -==== - -Add ``jaraco.mongodb.move-gridfs`` command. - -3.14 -==== - -Exposed ``mongod_args`` on ``MongoDBInstance`` -and ``MongoDBReplicaSet``. - -Allow arbitrary arguments to be included as mongodb -args with pytest plugin. For example:: - - pytest --mongod-args=--storageEngine=wiredTiger - -3.13 -==== - -Added ``manage`` module with support for purging all databases. -Added ``.purge_all_databases`` to MongoDBInstance. - -3.12 -==== - -Minor usability improvements in monitor-index-creation script. - -3.11 -==== - -Better error reporting in mongodb_instance fixture. - -3.10 -==== - -MongoDBInstance now allows for a ``.soft_stop`` and subsequent ``.start`` -to restart the instance against the same data_dir. - -3.8 -=== - -``repair-gridfs`` command now saves documents before removing -files. - -3.7 -=== - -Add ``helper.connect_gridfs`` function. - -Add script for removing corrupt GridFS files: -``jaraco.mongodb.repair-gridfs``. - -3.6 -=== - -Add ``helper`` and ``uri`` modules with functions to facilitate common -operations in PyMongo. - -3.5 -=== - -Add script for checking GridFS. Invoke with -``python -m jaraco.mongodb.check-gridfs``. - -3.4 -=== - -#1: Rename a namespace in index operations. - -3.3 -=== - -Add a ``dry-run`` option to suppress application of operations. - -3.0 -=== - -Oplog command no longer accepts '-h', '--host', '--to', '--port', '-p', -or '--from', but -instead accepts '--source' and '--dest' options for specifying source -and destination hosts/ports. - -2.8 -=== - -Adopt abandoned ``mongooplog_alt`` as ``jaraco.mongodb.oplog``. - -2.7 -=== - -Support PyMongo 2.x and 3.x. - -2.6 -=== - -Adopted ``service`` module from jaraco.test.services. - -2.4 -=== - -Add ``testing.assert_distinct_covered``. - -2.3 -=== - -Add ``query.compat_explain``, providing forward compatibility -for MongoDB 3.0 `explain changes -`_. - -``testing.assert_covered`` uses compat_explain for MongoDB 3.0 -compatibility. - -2.2 -=== - -Add query module with ``project`` function. - -2.0 -=== - -Removed references to ``jaraco.modb``. Instead, allow the Sessions object to -accept a ``codec`` parameter. Applications that currently depend on the -``use_modb`` functionality must instead use the following in the config:: - - "sessions.codec": jaraco.modb - -1.0 -=== - -Initial release, introducing ``sessions`` module based on ``yg.mongodb`` 2.9. diff --git a/README.rst b/README.rst deleted file mode 100644 index cd2c31e..0000000 --- a/README.rst +++ /dev/null @@ -1,185 +0,0 @@ -.. image:: https://img.shields.io/pypi/v/jaraco.mongodb.svg - :target: https://pypi.org/project/jaraco.mongodb - -.. image:: https://img.shields.io/pypi/pyversions/jaraco.mongodb.svg - -.. image:: https://github.com/PROJECT_PATH/workflows/tests/badge.svg - :target: https://github.com/PROJECT_PATH/actions?query=workflow%3A%22tests%22 - :alt: tests - -.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json - :target: https://github.com/astral-sh/ruff - :alt: Ruff - -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Code style: Black - -.. image:: https://readthedocs.org/projects/jaracomongodb/badge/?version=latest - :target: https://jaracomongodb.readthedocs.io/en/latest/?badge=latest - -.. image:: https://img.shields.io/badge/skeleton-2023-informational - :target: https://blog.jaraco.com/skeleton - -Migration Manager -================= - -``jaraco.mongodb.migration`` implements the Migration Manager as featured -at the `MongoWorld 2016 `_ presentation -`From the Polls to the Trolls -`_. -Use it to load documents of various schema versions into a target version that -your application expects. - -sessions -======== - -``jaraco.mongodb.sessions`` implements a CherryPy Sessions store backed by -MongoDB. - -By default, the session store will handle sessions with any objects that can -be inserted into a MongoDB collection naturally. - -To support richer objects, one may configure the codec to use ``jaraco.modb``. - -fields -====== - -``jaraco.mongodb.fields`` provides two functions, encode and decode, which -take arbitrary unicode text and transform it into values suitable as keys -on older versions of MongoDB by backslash-escaping the values. - -monitor-index-creation -====================== - -To monitor an ongoing index operation in a server, simply invoke: - - python -m jaraco.mongodb.monitor-index-creation mongodb://host/db - -move-gridfs -=========== - -To move files from one gridfs collection to another, invoke: - - python -m jaraco.mongodb.move-gridfs --help - -And follow the usage for moving all or some gridfs files and -optionally deleting the files after. - -oplog -===== - -This package provides an ``oplog`` module, which is based on the -`mongooplog-alt `_ project, -which itself is a Python remake of `official mongooplog utility -`_, -shipped with MongoDB starting from version 2.2 and deprecated in 3.2. -It reads oplog of a remote -server, and applies operations to the local server. This can be used to keep -independed replica set loosly synced in much the same way as Replica Sets -are synced, and may -be useful in various backup and migration scenarios. - -``oplog`` implements basic functionality of the official utility and -adds following features: - -* tailable oplog reader: runs forever polling new oplog event which is extremly - useful for keeping two independent replica sets in almost real-time sync. - -* option to sync only selected databases/collections. - -* option to exclude one or more namespaces (i.e. dbs or collections) from - being synced. - -* ability to "rename" dbs/collections on fly, i.e. destination namespaces can - differ from the original ones. This feature works on mongodb 1.8 and later. - Official utility only supports version 2.2.x and higher. - -* save last processed timestamp to file, resume from saved point later. - - -Invoke the command as a module script: ``python -m jaraco.mongodb.oplog``. - -Command-line options --------------------- - -Usage is as follows:: - - $ python -m jaraco.mongodb.oplog --help - usage: oplog.py [--help] [--source host[:port]] [--oplogns OPLOGNS] - [--dest host[:port]] [-w WINDOW] [-f] [--ns [NS [NS ...]]] - [-x [EXCLUDE [EXCLUDE ...]]] - [--rename [ns_old=ns_new [ns_old=ns_new ...]]] [--dry-run] - [--resume-file FILENAME] [-s SECONDS] [-l LOG_LEVEL] - - optional arguments: - --help show usage information - --source host[:port] Hostname of the mongod server from which oplog - operations are going to be pulled. Called "--from" in - mongooplog. - --oplogns OPLOGNS Source namespace for oplog - --dest host[:port] Hostname of the mongod server (or replica set as /s1,s2) to which oplog operations are going to be - applied. Default is "localhost". Called "--host" in - mongooplog. - -w WINDOW, --window WINDOW - Time window to query, like "3 days" or "24:00" (24 - hours, 0 minutes). - -f, --follow Wait for new data in oplog. Makes the utility polling - oplog forever (until interrupted). New data is going - to be applied immediately with at most one second - delay. - --ns [NS [NS ...]] Process only these namespaces, ignoring all others. - Space separated list of strings in form of ``dname`` - or ``dbname.collection``. May be specified multiple - times. - -x [EXCLUDE [EXCLUDE ...]], --exclude [EXCLUDE [EXCLUDE ...]] - List of space separated namespaces which should be - ignored. Can be in form of ``dname`` or - ``dbname.collection``. May be specified multiple - times. - --rename [ns_old=ns_new [ns_old=ns_new ...]] - Rename database(s) and/or collection(s). Operations on - namespace ``ns_old`` from the source server will be - applied to namespace ``ns_new`` on the destination - server. May be specified multiple times. - --dry-run Suppress application of ops. - --resume-file FILENAME - Read from and write to this file the last processed - timestamp. - -l LOG_LEVEL, --log-level LOG_LEVEL - Set log level (DEBUG, INFO, WARNING, ERROR) - -Example usages --------------- - -Consider the following sample usage:: - - python -m jaraco.mongodb.oplog --source prod.example.com:28000 --dest dev.example.com:28500 -f --exclude logdb data.transactions --seconds 600 - -This command is going to take operations from the last 10 minutes from prod, -and apply them to dev. Database ``logdb`` and collection ``transactions`` of -``data`` database will be omitted. After operations for the last minutes will -be applied, command will wait for new changes to come, keep running until -Ctrl+C or other termination signal recieved. - -The tool provides a ``--dry-run`` option and when logging at the DEBUG level will -emit the oplog entries. Combine these to use the tool as an oplog cat tool:: - - $ python -m jaraco.mongodb.oplog --dry-run -s 0 -f --source prod.example.com --ns survey_tabs -l DEBUG - - -Testing -------- - -Tests for ``oplog`` are written in javascript using test harness -which is used for testing MongoDB iteself. You can run the oplog suite with:: - - mongo tests/oplog.js - -Tests produce alot of output. Succesful execution ends with line like this:: - - ReplSetTest stopSet *** Shut down repl set - test worked **** - -These tests are run as part of the continuous integration and release acceptance -tests in Travis. diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 38a0ab3..0000000 --- a/conftest.py +++ /dev/null @@ -1,35 +0,0 @@ -import random - -import pytest - - -collect_ignore = [ - 'jaraco/mongodb/pmxbot.py', - # disable move-gridfs check, as it causes output capturing - # to be disabled. See pytest-dev/pytest#3752. - 'jaraco/mongodb/move-gridfs.py', -] - - -@pytest.fixture(scope='function') -def database(request, mongodb_instance): - """ - Return a clean MongoDB database suitable for testing. - """ - db_name = request.node.name.replace('.', '_') - database = mongodb_instance.get_connection()[db_name] - yield database - database.client.drop_database(db_name) - - -@pytest.fixture() -def bulky_collection(database): - """ - Generate a semi-bulky collection with a few dozen random - documents. - """ - coll = database.bulky - for _id in range(100): - doc = dict(_id=_id, val=random.randint(1, 100)) - coll.insert_one(doc) - return coll diff --git a/docs/conf.py b/docs/conf.py index 3e403c8..d3413f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,47 +1,5 @@ extensions = [ 'sphinx.ext.autodoc', - 'jaraco.packaging.sphinx', ] master_doc = "index" -html_theme = "furo" - -# Link dates and other references in the changelog -extensions += ['rst.linker'] -link_files = { - '../NEWS.rst': dict( - using=dict(GH='https://github.com'), - replace=[ - dict( - pattern=r'(Issue #|\B#)(?P\d+)', - url='{package_url}/issues/{issue}', - ), - dict( - pattern=r'(?m:^((?Pv?\d+(\.\d+){1,2}))\n[-=]+\n)', - with_scm='{text}\n{rev[timestamp]:%d %b %Y}\n', - ), - dict( - pattern=r'PEP[- ](?P\d+)', - url='https://peps.python.org/pep-{pep_number:0>4}/', - ), - ], - ) -} - -# Be strict about any broken references -nitpicky = True - -# Include Python intersphinx mapping to prevent failures -# jaraco/skeleton#51 -extensions += ['sphinx.ext.intersphinx'] -intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), -} - -# Preserve authored syntax for defaults -autodoc_preserve_defaults = True - -intersphinx_mapping.update( - cherrypy=('https://docs.cherrypy.dev/en/latest/', None), - pymongo=('https://pymongo.readthedocs.io/en/stable/', None), -) diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 5bdc232..0000000 --- a/docs/history.rst +++ /dev/null @@ -1,8 +0,0 @@ -:tocdepth: 2 - -.. _changes: - -History -******* - -.. include:: ../NEWS (links).rst diff --git a/docs/index.rst b/docs/index.rst index a108187..d06c625 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,81 +1,4 @@ -Welcome to |project| documentation! -=================================== - -.. sidebar-links:: - :home: - :pypi: - -.. toctree:: - :maxdepth: 1 - - history - - -.. automodule:: jaraco.mongodb - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.codec - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.fields - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.fixtures - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.helper - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.migration - :members: - :undoc-members: - :show-inheritance: - .. automodule:: jaraco.mongodb.oplog :members: :undoc-members: :show-inheritance: - -.. automodule:: jaraco.mongodb.query - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.sessions - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.sharding - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.timers - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: jaraco.mongodb.uri - :members: - :undoc-members: - :show-inheritance: - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/jaraco/mongodb/__init__.py b/jaraco/mongodb/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jaraco/mongodb/check-gridfs.py b/jaraco/mongodb/check-gridfs.py deleted file mode 100644 index da4e207..0000000 --- a/jaraco/mongodb/check-gridfs.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Script to check a GridFS instance for corrupted records. -""" - -import sys -import logging -import argparse - -import pymongo -from jaraco.ui import progress -from more_itertools.recipes import consume -from jaraco.itertools import Counter - -from jaraco.mongodb import helper -from jaraco.context import ExceptionTrap - - -log = logging.getLogger() - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--depth', - default=1024, - help="Bytes to read into each file during check", - ) - parser.add_argument('db', type=helper.connect_gridfs) - return parser.parse_args() - - -class FileChecker: - def __init__(self, gfs, depth): - self.gfs = gfs - self.depth = depth - - def run(self): - files = self.gfs.list() - bar = progress.TargetProgressBar(len(files)) - processed_files = map(self.process, bar.iterate(files)) - errors = filter(None, processed_files) - counter = Counter(errors) - consume(map(self.handle_trap, counter)) - return counter - - def process(self, filename): - file = self.gfs.get_last_version(filename) - with ExceptionTrap(pymongo.errors.PyMongoError) as trap: - file.read(self.depth) - trap.filename = filename - return trap - - def handle_trap(self, trap): - cls, exc, tb = trap.exc_info - log.error("Failed to read %s (%s)", trap.filename, exc) - - -def run(): - logging.basicConfig(stream=sys.stderr) - args = get_args() - - checker = FileChecker(args.db, args.depth) - counter = checker.run() - - print("Encountered", counter.count, "errors") - - -if __name__ == '__main__': - run() diff --git a/jaraco/mongodb/cli.py b/jaraco/mongodb/cli.py deleted file mode 100644 index 2ccecda..0000000 --- a/jaraco/mongodb/cli.py +++ /dev/null @@ -1,19 +0,0 @@ -import argparse - - -def extract_param(param, args, type=None): - """ - From a list of args, extract the one param if supplied, - returning the value and unused args. - - >>> extract_param('port', ['foo', '--port=999', 'bar'], type=int) - (999, ['foo', 'bar']) - >>> extract_param('port', ['foo', '--port', '999', 'bar'], type=int) - (999, ['foo', 'bar']) - >>> extract_param('port', ['foo', 'bar']) - (None, ['foo', 'bar']) - """ - parser = argparse.ArgumentParser() - parser.add_argument('--' + param, type=type) - res, unused = parser.parse_known_args(args) - return getattr(res, param), unused diff --git a/jaraco/mongodb/codec.py b/jaraco/mongodb/codec.py deleted file mode 100644 index 9432807..0000000 --- a/jaraco/mongodb/codec.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Decode support for JSON strings using ordered dictionaries and parsing -MongoDB-specific objects like dates. - -For example, if you have a JSON object representing a sort, you -want to retain the order of keys: - ->>> ob = decode('{"key1": 1, "key2": 2}') ->>> list(ob.keys()) -['key1', 'key2'] - -Or if you want to query by a date, PyMongo needs a Python datetime -object, which has no JSON representation, so this codec converts -``$date`` keys to date objects. - ->>> ob = decode('{"$gte": {"$date": "2019-01-01"}}') ->>> ob['$gte'] -datetime.datetime(2019, 1, 1, 0, 0) - -This function is useful in particular if you're accepting JSON queries -over an HTTP connection and you don't have the luxury of Javascript -expressions like you see in the Mongo shell or Compass. -""" - -import json -import functools -import collections - -import dateutil.parser -from jaraco.functools import compose - - -def maybe_date(obj): - """ - >>> maybe_date({"$date": "2019-01-01"}) - datetime.datetime(2019, 1, 1, 0, 0) - """ - return dateutil.parser.parse(obj['$date']) if list(obj) == ['$date'] else obj - - -smart_hook = compose(maybe_date, collections.OrderedDict) - - -decode = functools.partial(json.loads, object_pairs_hook=smart_hook) diff --git a/jaraco/mongodb/compat.py b/jaraco/mongodb/compat.py deleted file mode 100644 index e10418b..0000000 --- a/jaraco/mongodb/compat.py +++ /dev/null @@ -1,47 +0,0 @@ -import functools - -import pymongo.collection -from jaraco.collections import Projection - - -def save(coll, to_save): - """ - Pymongo has deprecated the save logic, even though - MongoDB still advertizes that logic in the core API: - https://docs.mongodb.com/manual/reference/method/db.collection.save/ - - This function provides a compatible interface. - """ - filter = Projection(['_id'], to_save) - upsert_replace = functools.partial(coll.replace_one, filter, upsert=True) - op = upsert_replace if filter else coll.insert_one - return op(to_save) - - -class Collection(pymongo.collection.Collection): - """ - Subclass of default Collection that provides a non-deprecated - save method. Don't use without first reading the cautions at - https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#q--a - - >> db = getfixture('database') - >> coll = Collection(db, 'mycoll') - >> coll.save({'foo': 'bar'}) - - >> ob = coll.find_one() - >> ob['foo'] = 'baz' - >> coll.save(ob) - - """ - - save = save - - -def query_or_command(op): - """ - Given an operation from currentOp, return the query or command - field as it changed in MongoDB 3.2 for indexing operations: - - https://docs.mongodb.com/manual/reference/method/db.currentOp/#active-indexing-operations - """ - return op.get('command') or op.get('query') diff --git a/jaraco/mongodb/fields.py b/jaraco/mongodb/fields.py deleted file mode 100644 index cac753a..0000000 --- a/jaraco/mongodb/fields.py +++ /dev/null @@ -1,31 +0,0 @@ -r""" -Backslash-escape text such that it is safe for MongoDB fields, -honoring `Restrictions on Field Names -`_. - ->>> decode(encode('my text with dots...')) -'my text with dots...' - ->>> decode(encode(r'my text with both \. and literal .')) -'my text with both \\. and literal .' - ->>> decode(encode('$leading dollar')) -'$leading dollar' -""" - -import re - - -def encode(text): - text = text.replace('\\', '\\\\') - text = re.sub(r'^\$', '\\$', text) - return text.replace('.', '\\D') - - -def unescape(match): - char = match.group(1) - return '.' if char == 'D' else char - - -def decode(encoded): - return re.sub(r'\\(.)', unescape, encoded) diff --git a/jaraco/mongodb/fixtures.py b/jaraco/mongodb/fixtures.py deleted file mode 100644 index af2eb36..0000000 --- a/jaraco/mongodb/fixtures.py +++ /dev/null @@ -1,59 +0,0 @@ -import shlex -import os - -import pytest - -try: - import pymongo -except ImportError: - pass - -from . import service - - -def pytest_addoption(parser): - parser.addoption( - '--mongod-args', - help="Arbitrary arguments to mongod", - ) - parser.addoption( - '--mongodb-uri', - help="URI to an extant MongoDB instance (supersedes ephemeral)", - ) - - -@pytest.fixture(scope='session') -def mongodb_instance(request): - if 'pymongo' not in globals(): - pytest.skip("pymongo not available") - - yield from _extant_instance(request.config) - yield from _ephemeral_instance(request.config) - - -def _extant_instance(config): - uri = config.getoption('mongodb_uri') or os.environ.get('MONGODB_URL') - if not uri: - return - yield service.ExtantInstance(uri) - - -def _ephemeral_instance(config): - params_raw = config.getoption('mongod_args') or '' - params = shlex.split(params_raw) - try: - instance = service.MongoDBInstance() - with instance.ensure(): - instance.merge_mongod_args(params) - instance.start() - pymongo.MongoClient(instance.get_connect_hosts()) - yield instance - except Exception as err: - raise - pytest.skip(f"MongoDB not available ({err})") - instance.stop() - - -@pytest.fixture(scope='session') -def mongodb_uri(mongodb_instance): - return mongodb_instance.get_uri() diff --git a/jaraco/mongodb/helper.py b/jaraco/mongodb/helper.py deleted file mode 100644 index 7a13b57..0000000 --- a/jaraco/mongodb/helper.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Helper functions to augment PyMongo -""" - -import warnings - -import pymongo -import gridfs - - -def filter_warnings(): - warnings.warn( - "filter_warnings is deprecated and has no effect; do not call", - DeprecationWarning, - ) - - -def connect(uri, factory=pymongo.MongoClient): - """ - Use the factory to establish a connection to uri. - """ - warnings.warn("do not use. Just call MongoClient directly.", DeprecationWarning) - return factory(uri) - - -def connect_db(uri, default_db_name=None, factory=pymongo.MongoClient): - """ - Use pymongo to parse a uri (possibly including database name) into - a connected database object. - - This serves as a convenience function for the common use case where one - wishes to get the Database object and is less concerned about the - intermediate MongoClient object that pymongo creates (though the - connection is always available as db.client). - - >>> db = connect_db( - ... 'mongodb://mongodb.localhost/mydb?readPreference=secondary') - >>> db.name - 'mydb' - >>> db.client.read_preference - Secondary(...) - - If no database is indicated in the uri, fall back to default. - - >>> db = connect_db('mongodb://mgo/', 'defaultdb') - >>> db.name - 'defaultdb' - - The default should only apply if no db was present in the URI. - - >>> db = connect_db('mongodb://mgo/mydb', 'defaultdb') - >>> db.name - 'mydb' - """ - uri_p = pymongo.uri_parser.parse_uri(uri) - client = factory(uri) - return client.get_database(uri_p['database'] or default_db_name) - - -def get_collection(uri): - return pymongo.uri_parser.parse_uri(uri)['collection'] - - -def connect_gridfs(uri, db=None): - """ - Construct a GridFS instance for a MongoDB URI. - """ - return gridfs.GridFS( - db or connect_db(uri), - collection=get_collection(uri) or 'fs', - ) - - -def server_version(conn): - """ - >>> conn = getfixture('mongodb_instance').get_connection() - >>> ver = server_version(conn) - >>> len(ver) - 3 - >>> set(map(type, ver)) - {} - """ - return tuple(map(int, conn.server_info()['version'].split('.'))) diff --git a/jaraco/mongodb/insert-doc.py b/jaraco/mongodb/insert-doc.py deleted file mode 100644 index c40ce06..0000000 --- a/jaraco/mongodb/insert-doc.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys -import json - -import argparse - -import pymongo.uri_parser - - -def get_collection(uri): - parsed = pymongo.uri_parser.parse_uri(uri) - client = pymongo.MongoClient(uri) - return client[parsed['database']][parsed['collection']] - - -def parse_args(): - parser = argparse.ArgumentParser( - "Insert a document from stdin into the specied collection" - ) - parser.add_argument( - 'collection', - metavar='collection_uri', - type=get_collection, - ) - return parser.parse_args() - - -def main(): - args = parse_args() - args.collection.insert_one(json.load(sys.stdin)) - - -__name__ == '__main__' and main() diff --git a/jaraco/mongodb/install.py b/jaraco/mongodb/install.py deleted file mode 100644 index 4317a8c..0000000 --- a/jaraco/mongodb/install.py +++ /dev/null @@ -1,89 +0,0 @@ -import json -import pathlib -import platform -import re -import tarfile -import urllib.request -import posixpath -import zipfile -import io - -import autocommand -from more_itertools import one - - -def get_download_url(): - source = 'https://www.mongodb.com/try/download/community' - with urllib.request.urlopen(source) as resp: - html = resp.read().decode('utf-8') - server_data = re.search( - r'', - html, - flags=re.DOTALL | re.MULTILINE, - ).group(1) - data = json.loads(server_data) - versions = data['components'][2]['props']['embeddedComponents'][0]['props'][ - 'items' - ][2]['embeddedComponents'][0]['props']['data'][0]['data'][0] - best_version = next(ver for ver in versions if versions[ver]['meta']['current']) - platforms = versions[best_version]['platforms'] - lookup = { - ('Darwin', 'arm64'): 'macOS ARM 64', - ('Darwin', 'x86_64'): 'macOS x64', - ('Linux', 'x86_64'): 'Ubuntu 22.04 x64', - ('Linux', 'aarch64'): 'Ubuntu 22.04 ARM 64', - ('Windows', 'x86_64'): 'Windows x64', - ('Windows', 'ARM64'): 'Windows x64', - } - plat_name = lookup[(platform.system(), platform.machine())] - format = 'zip' if 'Windows' in plat_name else 'tgz' - return platforms[plat_name][format] - - -class RootFinder(set): - def __call__(self, info, path): - self.add(self.root(info.name)) - return info - - @staticmethod - def root(name): - root, _, _ = name.partition(posixpath.sep) - return root - - @classmethod - def from_names(cls, names): - return cls(map(cls.root, names)) - - -def _extract_all(resp, target): - desig = resp.headers['Content-Type'].lower().replace('/', '_').replace('+', '_') - func_name = f'_extract_{desig}' - return globals()[func_name](resp, target) - - -def _extract_application_zip(resp, target): - data = io.BytesIO(resp.read()) - with zipfile.ZipFile(data) as obj: - roots = RootFinder.from_names(obj.namelist()) - obj.extractall( - target, - ) - return roots - - -def _extract_application_gzip(resp, target): - with tarfile.open(fileobj=resp, mode='r|*') as obj: - roots = RootFinder() - # python/typeshed#10514 - obj.extractall(target, filter=roots) # type: ignore - return roots - - -def install(target: pathlib.Path = pathlib.Path()): - url = get_download_url() - with urllib.request.urlopen(url) as resp: - roots = _extract_all(resp, target.expanduser()) - return target.joinpath(one(roots)) - - -autocommand.autocommand(__name__)(install) diff --git a/jaraco/mongodb/manage.py b/jaraco/mongodb/manage.py deleted file mode 100644 index b1ea3ad..0000000 --- a/jaraco/mongodb/manage.py +++ /dev/null @@ -1,50 +0,0 @@ -import re - - -def all_databases(client, exclude=['local']): - """ - Yield all databases except excluded (default - excludes 'local'). - """ - return ( - client[db_name] - for db_name in client.list_database_names() - if db_name not in exclude - ) - - -def all_collections(db): - """ - Yield all non-sytem collections in db. - """ - include_pattern = r'(?!system\.)' - return ( - db[name] - for name in db.list_collection_names() - if re.match(include_pattern, name) - ) - - -def purge_collection(coll): - coll.delete_all({}) - - -def safe_purge_collection(coll): - """ - Cannot remove documents from capped collections - in later versions of MongoDB, so drop the - collection instead. - """ - op = drop_collection if coll.options().get('capped', False) else purge_collection - return op(coll) - - -def drop_collection(coll): - coll.database.drop_collection(coll.name) - - -def purge_all_databases(client, op=drop_collection): - collections = (coll for db in all_databases(client) for coll in all_collections(db)) - - for coll in collections: - op(coll) diff --git a/jaraco/mongodb/migration.py b/jaraco/mongodb/migration.py deleted file mode 100644 index 14da850..0000000 --- a/jaraco/mongodb/migration.py +++ /dev/null @@ -1,143 +0,0 @@ -''' -Migration support as features in MongoWorld 2016 -From the Polls to the Trolls. - -The Manager class provides a general purpose support -for migrating documents to a target version through -a series of migration functions. -''' - -import re -import itertools - -from typing import Set, Callable - -from more_itertools import recipes - - -class Manager(object): - """ - A manager for facilitating the registration of migration functions - and applying those migrations to documents. - - To use, implement migration functions to and from each adjacent - version of your schema and decorate each with the register - classmethod. For example: - - >>> @Manager.register - ... def v1_to_2(manager, doc): - ... doc['foo'] = 'bar' - >>> @Manager.register - ... def v2_to_3(manager, doc): - ... doc['foo'] = doc['foo'] + ' baz' - - Note that in addition to the document, the migration manager is also - passed to the migration function, allowing for other context to be - made available during the migration. - - To create a manager for migrating documents to version 3: - - >>> mgr = Manager(3) - - Then, use the manager to migrate a document to a target version. - - >>> v1_doc = dict(version=1, data='stub') - >>> v3_doc = mgr.migrate_doc(v1_doc) - - >>> v3_doc['version'] - 3 - >>> v3_doc['foo'] - 'bar baz' - - Note that the document is modified in place: - - >>> v1_doc is v3_doc - True - - >>> Manager._upgrade_funcs.clear() - """ - - version_attribute_name = 'version' - _upgrade_funcs: Set[Callable] = set() - - def __init__(self, target_version): - self.target_version = target_version - - @classmethod - def register(cls, func): - """ - Decorate a migration function with this method - to make it available for migrating cases. - """ - cls._add_version_info(func) - cls._upgrade_funcs.add(func) - return func - - @staticmethod - def _add_version_info(func): - """ - Add .source and .target attributes to the registered function. - """ - pattern = r'v(?P\d+)_to_(?P\d+)$' - match = re.match(pattern, func.__name__) - if not match: - raise ValueError("migration function name must match " + pattern) - func.source, func.target = map(int, match.groups()) - - def migrate_doc(self, doc): - """ - Migrate the doc from its current version to the target version - and return it. - """ - orig_ver = doc.get(self.version_attribute_name, 0) - funcs = self._get_migrate_funcs(orig_ver, self.target_version) - for func in funcs: - func(self, doc) - doc[self.version_attribute_name] = func.target - return doc - - @classmethod - def _get_migrate_funcs(cls, orig_version, target_version): - """ - >>> @Manager.register - ... def v1_to_2(manager, doc): - ... doc['foo'] = 'bar' - >>> @Manager.register - ... def v2_to_1(manager, doc): - ... del doc['foo'] - >>> @Manager.register - ... def v2_to_3(manager, doc): - ... doc['foo'] = doc['foo'] + ' baz' - >>> funcs = list(Manager._get_migrate_funcs(1, 3)) - >>> len(funcs) - 2 - >>> funcs == [v1_to_2, v2_to_3] - True - >>> funcs = list(Manager._get_migrate_funcs(2, 1)) - >>> len(funcs) - 1 - >>> funcs == [v2_to_1] - True - - >>> Manager._upgrade_funcs.clear() - """ - direction = 1 if target_version > orig_version else -1 - versions = range(orig_version, target_version + direction, direction) - transitions = recipes.pairwise(versions) - return itertools.starmap(cls._get_func, transitions) - - @classmethod - def _get_func(cls, source_ver, target_ver): - """ - Return exactly one function to convert from source to target - """ - matches = ( - func - for func in cls._upgrade_funcs - if func.source == source_ver and func.target == target_ver - ) - try: - (match,) = matches - except ValueError: - raise ValueError(f"No migration from {source_ver} to {target_ver}") - return match diff --git a/jaraco/mongodb/monitor-index-creation.py b/jaraco/mongodb/monitor-index-creation.py deleted file mode 100644 index 7ed8829..0000000 --- a/jaraco/mongodb/monitor-index-creation.py +++ /dev/null @@ -1,37 +0,0 @@ -import time -import re -import argparse - -from jaraco.mongodb import helper -from .compat import query_or_command - - -def is_index_op(op): - cmd = query_or_command(op) or {} - return 'createIndexes' in cmd - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('db') - return parser.parse_args() - - -def run(): - db = helper.connect_db(get_args().db) - - while True: - ops = db.current_op()['inprog'] - index_op = next(filter(is_index_op, ops), None) - if not index_op: - print("No index operations in progress") - break - msg = index_op['msg'] - name = query_or_command(index_op)['indexes'][0]['name'] - pat = re.compile(r'Index Build( \(background\))?') - msg = pat.sub(name, msg, count=1) - print(msg, end='\r') - time.sleep(5) - - -__name__ == '__main__' and run() diff --git a/jaraco/mongodb/move-gridfs.py b/jaraco/mongodb/move-gridfs.py deleted file mode 100644 index bf1ee47..0000000 --- a/jaraco/mongodb/move-gridfs.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Script to move a subset of GridFS files from one db -(or collection) to another. - ->>> import io ->>> db_uri = getfixture('mongodb_uri') + '/move_gridfs_test' ->>> source = helper.connect_gridfs(db_uri + '.source') ->>> dest = helper.connect_gridfs(db_uri + '.dest') ->>> id = source.put(io.BytesIO(b'test'), filename='test.txt') ->>> mover = FileMove(source_gfs=source, dest_gfs=dest, delete=True) ->>> mover.ensure_indexes() ->>> mover.run(bar=None) ->>> source.list() -[] ->>> dest.exists(id) -True ->>> dest.list() -['test.txt'] -""" - -import sys -import logging -import argparse -import itertools -import signal - -import bson -import dateutil.parser -from jaraco.ui import progress -from more_itertools.recipes import consume - -from jaraco.mongodb import helper - - -log = logging.getLogger() - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('source_gfs', type=helper.connect_gridfs) - parser.add_argument('dest_gfs', type=helper.connect_gridfs) - parser.add_argument( - '--include', - help="a filter of files (regex) to include", - ) - parser.add_argument( - '--delete', - default=False, - action="store_true", - help="delete files after moving", - ) - parser.add_argument('--limit', type=int) - parser.add_argument( - '--limit-date', - type=dateutil.parser.parse, - help="only move files older than this date", - ) - return parser.parse_args() - - -class FileMove: - include = None - delete = False - limit = None - limit_date = None - - def __init__(self, **params): - vars(self).update(**params) - - def ensure_indexes(self): - """ - Create the same indexes that the GridFS API would have - """ - self.dest_gfs.new_file()._GridIn__ensure_indexes() - - @property - def filter(self): - filter = {} - if self.include: - filter.update(filename={"$regex": self.include}) - if self.limit_date: - id_max = bson.objectid.ObjectId.from_datetime(self.limit_date) - filter.update(_id={"$lt": id_max}) - return filter - - @property - def source_coll(self): - return self.source_gfs._GridFS__collection - - @property - def dest_coll(self): - return self.dest_gfs._GridFS__collection - - def run(self, bar=progress.TargetProgressBar): - files = self.source_coll.files.find( - self.filter, - batch_size=1, - ) - limit_files = itertools.islice(files, self.limit) - count = min(files.count(), self.limit or float('inf')) - progress = bar(count).iterate if bar else iter - with SignalTrap(progress(limit_files)) as items: - consume(map(self.process, items)) - - def process(self, file): - chunks = self.source_coll.chunks.find(dict(files_id=file['_id'])) - for chunk in chunks: - self.dest_coll.chunks.insert(chunk) - self.dest_coll.files.insert(file) - self.delete and self.source_gfs.delete(file['_id']) - - -class SignalTrap: - """ - A context manager for wrapping an iterable such that it - is only interrupted between iterations. - """ - - def __init__(self, iterable): - self.iterable = iterable - - def __enter__(self): - self.prev = signal.signal(signal.SIGINT, self.stop) - return self - - def __exit__(self, *args): - signal.signal(signal.SIGINT, self.prev) - del self.prev - - def __iter__(self): - return self - - def __next__(self): - return next(self.iterable) - - next = __next__ - - def stop(self, signal, frame): - self.iterable = iter([]) - - -def run(): - logging.basicConfig(stream=sys.stderr, level=logging.INFO) - args = get_args() - - mover = FileMove(**vars(args)) - mover.ensure_indexes() - mover.run() - - -if __name__ == '__main__': - run() diff --git a/jaraco/mongodb/oplog.py b/jaraco/mongodb/oplog.py index 42e2703..051f3e5 100644 --- a/jaraco/mongodb/oplog.py +++ b/jaraco/mongodb/oplog.py @@ -1,621 +1 @@ -import argparse -import time -import json -import logging -import pymongo -import bson.json_util -import re -import collections -import datetime -import operator - -try: - from importlib import metadata # type: ignore -except ImportError: - import importlib_metadata as metadata # type: ignore - -from typing import Dict, Any - -import cachetools -import jaraco.logging -import pytimeparse -from jaraco.functools import compose -from pymongo.cursor import CursorType -from jaraco.itertools import always_iterable -from jaraco.ui.cmdline import Extend - -from . import helper - - -def delta_from_seconds(seconds): - return datetime.timedelta(seconds=int(seconds)) - - -def parse_args(*args, **kwargs): - """ - Parse the args for the command. - - It should be possible for one to specify '--ns', '-x', and '--rename' - multiple times: - - >>> args = parse_args(['--ns', 'foo', 'bar', '--ns', 'baz']) - >>> args.ns - ['foo', 'bar', 'baz'] - - >>> parse_args(['-x', '--exclude']).exclude - [] - - >>> renames = parse_args(['--rename', 'a=b', '--rename', 'b=c']).rename - >>> len(renames) - 2 - - "..." below should be "jaraco." but for pytest-dev/pytest#3396 - >>> type(renames) - - """ - parser = argparse.ArgumentParser(add_help=False) - - parser.add_argument( - "--help", - help="show usage information", - action="help", - ) - - parser.add_argument( - "--source", - metavar="host[:port]", - help="""Hostname of the mongod server from which oplog - operations are going to be pulled. Called "--from" - in mongooplog.""", - ) - - parser.add_argument( - '--oplogns', - default='local.oplog.rs', - help="Source namespace for oplog", - ) - - parser.add_argument( - "--dest", - metavar="host[:port]", - help=""" - Hostname of the mongod server (or replica set as - /s1,s2) to which oplog operations - are going to be applied. Default is "localhost". - Called "--host" in mongooplog. - """, - ) - - parser.add_argument( - "-w", - "--window", - dest="start_ts", - metavar="WINDOW", - type=compose( - Timestamp.for_window, - delta_from_seconds, - pytimeparse.parse, - ), - help="""Time window to query, like "3 days" or "24:00" - (24 hours, 0 minutes).""", - ) - - parser.add_argument( - "-f", - "--follow", - action="store_true", - help="""Wait for new data in oplog. Makes the utility - polling oplog forever (until interrupted). New data - is going to be applied immediately with at most one - second delay.""", - ) - - parser.add_argument( - "--ns", - nargs="*", - default=[], - action=Extend, - help="""Process only these namespaces, ignoring all others. - Space separated list of strings in form of ``dname`` - or ``dbname.collection``. May be specified multiple times. - """, - ) - - parser.add_argument( - "-x", - "--exclude", - nargs="*", - default=[], - action=Extend, - help="""List of space separated namespaces which should be - ignored. Can be in form of ``dname`` or ``dbname.collection``. - May be specified multiple times. - """, - ) - - parser.add_argument( - "--rename", - nargs="*", - default=[], - metavar="ns_old=ns_new", - type=RenameSpec.from_spec, - action=Extend, - help=""" - Rename database(s) and/or collection(s). Operations on - namespace ``ns_old`` from the source server will be - applied to namespace ``ns_new`` on the destination server. - May be specified multiple times. - """, - ) - - parser.add_argument( - "--dry-run", - default=False, - action="store_true", - help="Suppress application of ops.", - ) - - parser.add_argument( - "--resume-file", - metavar="FILENAME", - type=ResumeFile, - default=NullResumeFile(), - help="""Read from and write to this file the last processed - timestamp.""", - ) - - jaraco.logging.add_arguments(parser) - - args = parser.parse_args(*args, **kwargs) - args.rename = Renamer(args.rename) - - args.start_ts = args.start_ts or args.resume_file.read() - - return args - - -class RenameSpec(object): - @classmethod - def from_spec(cls, string_spec): - """ - Construct RenameSpec from a pair separated by equal sign ('='). - """ - old_ns, new_ns = string_spec.split('=') - return cls(old_ns, new_ns) - - def __init__(self, old_ns, new_ns): - self.old_ns = old_ns - self.new_ns = new_ns - self.old_db, sep, self.old_coll = self.old_ns.partition('.') - self.new_db, sep, self.new_coll = self.new_ns.partition('.') - self.regex = re.compile(r"^{0}(\.|$)".format(re.escape(self.old_ns))) - - # ugly hack: append a period so the regex can match dot - # or end of string; requires .rstrip operation in __call__ also. - self.new_ns += "." - - def __call__(self, op): - """ - Apply this rename to the op - """ - self._handle_renameCollection(op) - if self.regex.match(op['ns']): - ns = self.regex.sub(self.new_ns, op['ns']).rstrip(".") - logging.debug("renaming %s to %s", op['ns'], ns) - op['ns'] = ns - if op['ns'].endswith('.system.indexes'): - # index operation; update ns in the op also. - self(op['o']) - self._handle_create(op) - - def _handle_create(self, op): - # MongoDB 3.4 introduces idIndex - if 'idIndex' in op.get('o', {}): - self(op['o']['idIndex']) - if self._matching_create_command(op, self.old_ns): - op['ns'] = self.new_db + '.$cmd' - op['o']['create'] = self.new_coll - - @staticmethod - def _matching_create_command(op, ns): - db, sep, coll = ns.partition('.') - return ( - op.get('op') == 'c' - and op['ns'] == db + '.$cmd' - and op['o'].get('create', None) == coll - ) - - @staticmethod - def _matching_renameCollection_command(op, ns): - db, sep, coll = ns.partition('.') - return ( - op.get('op') == 'c' - and ( - # seems command can happen in admin or the db - op['ns'] == 'admin.$cmd' - or op['ns'] == db + '.$cmd' - ) - and 'renameCollection' in op['o'] - and ( - op['o']['renameCollection'].startswith(ns) - or op['o']['to'].startswith(ns) - ) - ) - - def _handle_renameCollection(self, op): - if self._matching_renameCollection_command(op, self.old_ns): - cmd = op['o'] - rename_keys = 'renameCollection', 'to' - for key in rename_keys: - # todo, this is a mirror of the code in __call__; refactor - if self.regex.match(cmd[key]): - ns = self.regex.sub(self.new_ns, cmd[key]).rstrip(".") - logging.debug("renaming %s to %s", cmd[key], ns) - cmd[key] = ns - - def affects(self, ns): - return bool(self.regex.match(ns)) - - -class Renamer(list): - """ - >>> specs = [ - ... 'a=b', - ... 'alpha=gamma', - ... ] - >>> renames = Renamer.from_specs(specs) - >>> op = dict(ns='a.a') - >>> renames(op) - >>> op['ns'] - 'b.a' - >>> renames.affects('alpha.foo') - True - >>> renames.affects('b.gamma') - False - """ - - def invoke(self, op): - """ - Replace namespaces in op based on RenameSpecs in self. - """ - for rename in self: - rename(op) - - __call__ = invoke - - @classmethod - def from_specs(cls, specs): - return cls(map(RenameSpec.from_spec, always_iterable(specs))) - - def affects(self, ns): - """ - Return True if this renamer affects the indicated namespace. - """ - return any(rn.affects(ns) for rn in self) - - -def string_none(value): - """ - Convert the string 'none' to None - """ - is_string_none = not value or value.lower() == 'none' - return None if is_string_none else value - - -def _same_instance(client1, client2): - """ - Return True if client1 and client2 appear to reference the same - MongoDB instance. - """ - return client1._topology_settings.seeds == client2._topology_settings.seeds - - -def _full_rename(args): - """ - Return True only if the arguments passed specify exact namespaces - and to conduct a rename of every namespace. - """ - return args.ns and all(map(args.rename.affects, args.ns)) - - -def _resolve_shard(client): - """ - The destination cannot be a mongoS instance, as applyOps is - not an allowable command for mongoS instances, so if the - client is a connection to a mongoS instance, raise an error - or resolve the replica set. - """ - status = client.admin.command('serverStatus') - if status['process'] == 'mongos': - raise RuntimeError("Destination cannot be mongos") - return client - - -def _load_dest(host): - if not host: - return - return _resolve_shard(pymongo.MongoClient(host)) - - -def main(): - args = parse_args() - log_format = '%(asctime)s - %(levelname)s - %(message)s' - jaraco.logging.setup(args, format=log_format) - - logging.info(f"jaraco.mongodb.oplog {metadata.version('jaraco.mongodb')}") - logging.info("going to connect") - - src = pymongo.MongoClient(args.source) - dest = _load_dest(args.dest) - - if dest and _same_instance(src, dest) and not _full_rename(args): - logging.error( - "source and destination hosts can be the same only " - "when both --ns and --rename arguments are given" - ) - raise SystemExit(1) - - logging.info("connected") - - start = args.start_ts - if not start: - logging.error("Resume file or window required") - raise SystemExit(2) - - logging.info("starting from %s (%s)", start, start.as_datetime()) - db_name, sep, coll_name = args.oplogns.partition('.') - oplog_coll = src[db_name][coll_name] - num = 0 - - class_ = TailingOplog if args.follow else Oplog - generator = class_(oplog_coll) - - if not generator.has_ops_before(start): - logging.warning("No ops before start time; oplog may be overrun") - - try: - for num, doc in enumerate(generator.since(start)): - _handle(dest, doc, args, num) - last_handled = doc - logging.info("all done") - except KeyboardInterrupt: - logging.info("Got Ctrl+C, exiting...") - finally: - if 'last_handled' in locals(): - last = last_handled['ts'] - args.resume_file.save(last) - logging.info("last ts was %s (%s)", last, last.as_datetime()) - - -def applies_to_ns(op, ns): - return ( - op['ns'].startswith(ns) - or RenameSpec._matching_create_command(op, ns) - or RenameSpec._matching_renameCollection_command(op, ns) - ) - - -class NiceRepr(object): - """ - Adapt a Python representation of a MongoDB object - to make it appear nicely when rendered as a - string. - - >>> messy_doc = collections.OrderedDict([ - ... ('ts', bson.Timestamp(1111111111, 30), - ... )]) - >>> print(NiceRepr(messy_doc)) - {"ts": {"$timestamp": {"t": 1111111111, "i": 30}}} - """ - - def __init__(self, orig): - self.__orig = orig - - def __str__(self): - return bson.json_util.dumps(self.__orig) - - def __getattr__(self, attr): - return getattr(self.__orig, attr) - - -def _handle(dest, op, args, num): - # Skip "no operation" items - if op['op'] == 'n': - return - - # Skip excluded namespaces or namespaces that does not match --ns - excluded = any(applies_to_ns(op, ns) for ns in args.exclude) - included = any(applies_to_ns(op, ns) for ns in args.ns) - - if excluded or (args.ns and not included): - logging.log(logging.DEBUG - 1, "skipping %s", op) - return - - args.rename(op) - - logging.debug("applying op %s", NiceRepr(op)) - try: - args.dry_run or apply(dest, op) - except pymongo.errors.OperationFailure as e: - nice_op = NiceRepr(op) - msg = f'{e!r} applying {nice_op}' - logging.warning(msg) - - # Update status - ts = op['ts'] - if not num % 1000: - args.resume_file.save(ts) - logging.info( - "%s\t%s\t%s -> %s", - num, - ts.as_datetime(), - op.get('op'), - op.get('ns'), - ) - - -def apply(db, op): - """ - Apply operation in db - """ - dbname = op['ns'].split('.')[0] or "admin" - _db = db[dbname] - return _get_index_handler(db)(_db, op) or _apply_regular(_db, op) - - -@cachetools.cached({}, key=operator.attrgetter('address')) -def _get_index_handler(conn): - def _bypass(db, op): - pass - - return _apply_index_op if helper.server_version(conn) >= (4, 4) else _bypass - - -def _apply_index_op(db, op): - """ - Starting with MongoDB 4.2, index operations can no longer - be applied. Intercept the application and transform it into - a normal create index operation. - """ - if 'createIndexes' not in op['o']: - return - o = op['o'] - coll_name = o['createIndexes'] - key = list(o['key'].items()) - name = o['name'] - return db[coll_name].create_index(key, name=name) - - -def _apply_regular(db, op): - opts = bson.CodecOptions(uuid_representation=bson.binary.STANDARD) - db.command("applyOps", [op], codec_options=opts) - - -class Oplog(object): - find_params: Dict[str, Any] = {} - - def __init__(self, coll): - self.coll = coll.with_options( - codec_options=bson.CodecOptions( - document_class=collections.OrderedDict, - ), - ) - - def get_latest_ts(self): - cur = self.coll.find().sort('$natural', pymongo.DESCENDING).limit(-1) - latest_doc = next(cur) - return Timestamp.wrap(latest_doc['ts']) - - def query(self, spec): - return self.coll.find(spec, **self.find_params) - - def since(self, ts): - """ - Query the oplog for items since ts and then return - """ - spec = {'ts': {'$gt': ts}} - cursor = self.query(spec) - while True: - # todo: trap InvalidDocument errors: - # except bson.errors.InvalidDocument as e: - # logging.info(repr(e)) - for doc in cursor: - yield doc - if not cursor.alive: - break - time.sleep(1) - - def has_ops_before(self, ts): - """ - Determine if there are any ops before ts - """ - spec = {'ts': {'$lt': ts}} - return bool(self.coll.find_one(spec)) - - -class TailingOplog(Oplog): - find_params = dict( - cursor_type=CursorType.TAILABLE_AWAIT, - oplog_replay=True, - ) - - def since(self, ts): - """ - Tail the oplog, starting from ts. - """ - while True: - items = super(TailingOplog, self).since(ts) - for doc in items: - yield doc - ts = doc['ts'] - - -class Timestamp(bson.timestamp.Timestamp): - @classmethod - def wrap(cls, orig): - """ - Wrap an original timestamp as returned by a pymongo query - with a version of this class. - """ - # hack to give the timestamp this class' specialized methods - orig.__class__ = cls - return orig - - def dump(self, stream): - """Serialize self to text stream. - - Matches convention of mongooplog. - """ - items = ( - ('time', self.time), - ('inc', self.inc), - ) - # use ordered dict to retain order - ts = collections.OrderedDict(items) - json.dump(dict(ts=ts), stream) - - @classmethod - def load(cls, stream): - """Load a serialized version of self from text stream. - - Expects the format used by mongooplog. - """ - data = json.load(stream)['ts'] - return cls(data['time'], data['inc']) - - @classmethod - def for_window(cls, window): - """ - Given a timedelta window, return a timestamp representing - that time. - """ - utcnow = datetime.datetime.utcnow() - return cls(utcnow - window, 0) - - -class ResumeFile(str): - def save(self, ts): - """ - Save timestamp to file. - """ - with open(self, 'w') as f: - Timestamp.wrap(ts).dump(f) - - def read(self): - """ - Read timestamp from file. - """ - with open(self) as f: - return Timestamp.load(f) - - -class NullResumeFile(object): - def save(self, ts): - pass - - def read(self): - pass - - -if __name__ == '__main__': - main() +import inflect diff --git a/jaraco/mongodb/pmxbot.py b/jaraco/mongodb/pmxbot.py deleted file mode 100644 index 1db943e..0000000 --- a/jaraco/mongodb/pmxbot.py +++ /dev/null @@ -1,29 +0,0 @@ -import shlex -import contextlib - -import pmxbot.storage - -from . import sharding - - -class Storage(pmxbot.storage.SelectableStorage, pmxbot.storage.MongoDBStorage): - collection_name = 'unused' - - -def get_client(): - """ - Use the same MongoDB client as pmxbot if available. - """ - with contextlib.suppress(Exception): - store = Storage.from_URI() - assert isinstance(store, pmxbot.storage.MongoDBStorage) - return store.db.database.client - - -@pmxbot.core.command("create-db-in-shard") -def cdbs(client, event, channel, nick, rest): - """ - Create a database in a shard. !create-db-in-shard {db name} {shard name} - """ - db_name, shard = shlex.split(rest) - return sharding.create_db_in_shard(db_name, shard, client=get_client()) diff --git a/jaraco/mongodb/query.py b/jaraco/mongodb/query.py deleted file mode 100644 index 19e43a5..0000000 --- a/jaraco/mongodb/query.py +++ /dev/null @@ -1,62 +0,0 @@ -import pymongo - - -def project(*args, **kwargs): - """ - Build a projection for MongoDB. - - Due to https://jira.mongodb.org/browse/SERVER-3156, until MongoDB 2.6, - the values must be integers and not boolean. - - >>> project(a=True) == {'a': 1} - True - - Once MongoDB 2.6 is released, replace use of this function with a simple - dict. - """ - projection = dict(*args, **kwargs) - return {key: int(value) for key, value in projection.items()} - - -def compat_explain(cur): - """ - Simulate MongoDB 3.0 explain result on prior versions. - http://docs.mongodb.org/v3.0/reference/explain-results/ - """ - res = cur.explain() - if 'nscannedObjects' in res: - res['executionStats'] = dict( - nReturned=res.pop('n'), - totalKeysExamined=res.pop('nscanned'), - totalDocsExamined=res.pop('nscannedObjects'), - executionTimeMillis=res.pop('millis'), - ) - return res - - -def upsert_and_fetch(coll, doc, **kwargs): - """ - Fetch exactly one matching document or upsert - the document if not found, returning the matching - or upserted document. - - See https://jira.mongodb.org/browse/SERVER-28434 - describing the condition where MongoDB is uninterested in - providing an upsert and fetch behavior. - - >>> instance = getfixture('mongodb_instance').get_connection() - >>> coll = instance.test_upsert_and_fetch.items - >>> doc = {'foo': 'bar'} - >>> inserted = upsert_and_fetch(coll, doc) - >>> inserted - {...'foo': 'bar'...} - >>> upsert_and_fetch(coll, doc) == inserted - True - """ - return coll.find_one_and_update( - doc, - {"$setOnInsert": doc}, - upsert=True, - return_document=pymongo.ReturnDocument.AFTER, - **kwargs - ) diff --git a/jaraco/mongodb/repair-gridfs.py b/jaraco/mongodb/repair-gridfs.py deleted file mode 100644 index 7fe5094..0000000 --- a/jaraco/mongodb/repair-gridfs.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Script to repair broken GridFS files. It handles - -- Removing files with missing chunks. -""" - -import sys -import logging -import argparse - -import gridfs -from jaraco.ui import progress -from more_itertools.recipes import consume -from jaraco.itertools import Counter - -from jaraco.mongodb import helper -from jaraco.context import ExceptionTrap - - -log = logging.getLogger() - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('db', type=helper.connect_gridfs) - return parser.parse_args() - - -class FileRepair: - def __init__(self, gfs): - self.gfs = gfs - db = gfs._GridFS__database - coll = gfs._GridFS__collection - bu_coll_name = coll.name + '-saved' - self.backup_coll = db[bu_coll_name] - - def run(self): - files = self.gfs.list() - bar = progress.TargetProgressBar(len(files)) - processed_files = map(self.process, bar.iterate(files)) - errors = filter(None, processed_files) - counter = Counter(errors) - consume(map(self.handle_trap, counter)) - return counter - - def process(self, filename): - file = self.gfs.get_last_version(filename) - with ExceptionTrap(gridfs.errors.CorruptGridFile) as trap: - file.read(1) - trap.filename = filename - return trap - - def handle_trap(self, trap): - cls, exc, tb = trap.exc_info - spec = dict(filename=trap.filename) - for file_doc in self.gfs._GridFS__files.find(spec): - self.backup_coll.files.insert(file_doc) - chunk_spec = dict(files_id=file_doc['_id']) - for chunk in self.gfs._GridFS__chunks.find(chunk_spec): - self.backup_coll.chunks.insert(chunk) - log.info("Removing %s (%s)", trap.filename, exc) - self.gfs.delete(spec) - - -def run(): - logging.basicConfig(stream=sys.stderr, level=logging.INFO) - args = get_args() - - repair = FileRepair(args.db) - counter = repair.run() - - log.info("Removed %s corrupt files.", counter.count) - - -if __name__ == '__main__': - run() diff --git a/jaraco/mongodb/sampling.py b/jaraco/mongodb/sampling.py deleted file mode 100644 index 54921f0..0000000 --- a/jaraco/mongodb/sampling.py +++ /dev/null @@ -1,43 +0,0 @@ -import builtins - - -def estimate(coll, filter={}, sample=1): - """ - Estimate the number of documents in the collection - matching the filter. - - Sample may be a fixed number of documents to sample - or a percentage of the total collection size. - - >>> coll = getfixture('bulky_collection') - >>> estimate(coll) - 100 - >>> query = {"val": {"$gte": 50}} - >>> val = estimate(coll, filter=query) - >>> val > 0 - True - >>> val = estimate(coll, filter=query, sample=10) - >>> val > 0 - True - >>> val = estimate(coll, filter=query, sample=.1) - >>> val > 0 - True - """ - total = coll.estimated_document_count() - if not filter and sample == 1: - return total - if sample <= 1: - sample *= total - pipeline = list( - builtins.filter( - None, - [ - {'$sample': {'size': sample}} if sample < total else {}, - {'$match': filter}, - {'$count': 'matched'}, - ], - ) - ) - docs = next(coll.aggregate(pipeline)) - ratio = docs['matched'] / sample - return int(total * ratio) diff --git a/jaraco/mongodb/service.py b/jaraco/mongodb/service.py deleted file mode 100644 index 8c14e6c..0000000 --- a/jaraco/mongodb/service.py +++ /dev/null @@ -1,262 +0,0 @@ -import os -import sys -import tempfile -import subprocess -import glob -import collections -import importlib -import shutil -import functools -import logging -import datetime -import pathlib -import contextlib - -from typing import Dict, Any - -import portend -from jaraco.services import paths -from jaraco import services -from tempora import timing -from . import manage -from . import cli -from . import install - - -log = logging.getLogger(__name__) - - -class MongoDBFinder(paths.PathFinder): - windows_installed = glob.glob('/Program Files/MongoDB/Server/???/bin') - windows_paths = [ - # symlink Server/current to Server/X.X - '/Program Files/MongoDB/Server/current/bin', - # symlink MongoDB to mongodb-win32-x86_64-2008plus-X.X.X-rcX - '/Program Files/MongoDB/bin', - ] + list(reversed(windows_installed)) - heuristic_paths = [ - # on the path - '', - # 10gen Debian package - '/usr/bin', - # custom install in /opt - '/opt/mongodb/bin', - ] + windows_paths - - # allow the environment to stipulate where mongodb must - # be found. - env_paths = [ - os.path.join(os.environ[key], 'bin') - for key in ['MONGODB_HOME'] - if key in os.environ - ] - candidate_paths = env_paths or heuristic_paths - exe = 'mongod' - args = ['--version'] - - @classmethod - def find_binary(cls): - return os.path.join(cls.find_root(), cls.exe) - - @classmethod - @contextlib.contextmanager - def ensure(cls): - try: - yield cls.find_root() - except RuntimeError: - with tempfile.TemporaryDirectory() as tmp_dir: - tmp = pathlib.Path(tmp_dir) - root = install.install(target=tmp).joinpath('bin') - cls.candidate_paths.append(root) - yield root - cls.candidate_paths.remove(root) - - -class MongoDBService(MongoDBFinder, services.Subprocess, services.Service): - port = 27017 - - process_kwargs: Dict[str, Any] = {} - """ - keyword arguments to Popen to control the process creation - """ - - @services.Subprocess.PortFree() - def start(self): - super(MongoDBService, self).start() - # start the daemon - mongodb_data = os.path.join(sys.prefix, 'var', 'lib', 'mongodb') - cmd = [ - self.find_binary(), - '--dbpath=' + mongodb_data, - ] - self.process = subprocess.Popen(cmd, **self.process_kwargs) - self.wait_for_pattern(r'waiting for connections on port (?P\d+)') - log.info('%s listening on %s', self, self.port) - - -class MongoDBInstance(MongoDBFinder, services.Subprocess, services.Service): - process_kwargs: Dict[str, Any] = {} - """ - keyword arguments to Popen to control the process creation - """ - - def merge_mongod_args(self, add_args): - self.port, add_args[:] = cli.extract_param('port', add_args, type=int) - self.mongod_args = add_args - - def start(self): - super(MongoDBInstance, self).start() - if not hasattr(self, 'port') or not self.port: - self.port = portend.find_available_local_port() - self.data_dir = tempfile.mkdtemp() - cmd = [ - self.find_binary(), - '--dbpath', - self.data_dir, - '--port', - str(self.port), - ] + list(self.mongod_args) - if hasattr(self, 'bind_ip') and '--bind_ip' not in cmd: - cmd.extend(['--bind_ip', self.bind_ip]) - self.process = subprocess.Popen(cmd, **self.process_kwargs) - portend.occupied('localhost', self.port, timeout=10) - log.info(f'{self} listening on {self.port}') - - def get_connection(self): - pymongo = importlib.import_module('pymongo') - return pymongo.MongoClient('localhost', self.port) - - def purge_all_databases(self): - manage.purge_all_databases(self.get_connection()) - - def get_connect_hosts(self): - return [f'localhost:{self.port}'] - - def get_uri(self): - return 'mongodb://' + ','.join(self.get_connect_hosts()) - - def stop(self): - super(MongoDBInstance, self).stop() - shutil.rmtree(self.data_dir) - del self.data_dir - - -class ExtantInstance: - def __init__(self, uri): - self.uri = uri - - def get_connection(self): - pymongo = importlib.import_module('pymongo') - return pymongo.MongoClient(self.uri) - - def get_uri(self): - return self.uri - - -class MongoDBReplicaSet(MongoDBFinder, services.Service): - replica_set_name = 'test' - - mongod_parameters = ( - '--oplogSize', - '10', - ) - - def start(self): - super(MongoDBReplicaSet, self).start() - self.data_root = tempfile.mkdtemp() - self.instances = list(map(self.start_instance, range(3))) - # initialize the replica set - self.instances[0].connect().admin.command( - 'replSetInitiate', self.build_config() - ) - # wait until the replica set is initialized - get_repl_set_status = functools.partial( - self.instances[0].connect().admin.command, 'replSetGetStatus', 1 - ) - errors = importlib.import_module('pymongo.errors') - log.info('Waiting for replica set to initialize') - - watch = timing.Stopwatch() - while watch.elapsed < datetime.timedelta(minutes=5): - try: - res = get_repl_set_status() - if res.get('myState') != 1: - continue - except errors.OperationFailure: - continue - break - else: - raise RuntimeError("timeout waiting for replica set to start") - - def start_instance(self, number): - port = portend.find_available_local_port() - data_dir = os.path.join(self.data_root, repr(number)) - os.mkdir(data_dir) - cmd = [ - self.find_binary(), - '--dbpath', - data_dir, - '--port', - str(port), - '--replSet', - self.replica_set_name, - ] + list(self.mongod_parameters) - log_file = self.get_log(number) - process = subprocess.Popen(cmd, stdout=log_file) - portend.occupied('localhost', port, timeout=50) - log.info(f'{self}:{number} listening on {port}') - return InstanceInfo(data_dir, port, process, log_file) - - def get_log(self, number): - log_filename = os.path.join(self.data_root, f'r{number}.log') - log_file = open(log_filename, 'a') - return log_file - - def is_running(self): - return hasattr(self, 'instances') and all( - instance.process.returncode is None for instance in self.instances - ) - - def stop(self): - super(MongoDBReplicaSet, self).stop() - for instance in self.instances: - if instance.process.returncode is None: - instance.process.terminate() - instance.process.wait() - instance.log_file.close() - del self.instances - shutil.rmtree(self.data_root) - - def build_config(self): - return dict( - _id=self.replica_set_name, - members=[ - dict( - _id=number, - host=f'localhost:{instance.port}', - ) - for number, instance in enumerate(self.instances) - ], - ) - - def get_connect_hosts(self): - return [f'localhost:{instance.port}' for instance in self.instances] - - def get_uri(self): - return 'mongodb://' + ','.join(self.get_connect_hosts()) - - def get_connection(self): - pymongo = importlib.import_module('pymongo') - return pymongo.MongoClient(self.get_uri()) - - -InstanceInfoBase = collections.namedtuple( - 'InstanceInfoBase', 'path port process log_file' -) - - -class InstanceInfo(InstanceInfoBase): - def connect(self): - pymongo = __import__('pymongo') - rp = pymongo.ReadPreference.PRIMARY_PREFERRED - return pymongo.MongoClient(f'localhost:{self.port}', read_preference=rp) diff --git a/jaraco/mongodb/sessions.py b/jaraco/mongodb/sessions.py deleted file mode 100644 index 75b2872..0000000 --- a/jaraco/mongodb/sessions.py +++ /dev/null @@ -1,208 +0,0 @@ -""" -A MongoDB-backed CherryPy session store. - -Although this module requires CherryPy, it does not impose the requirement -on the package, as any user of this module will already require CherryPy. - -To enable these sessions, your code must call :meth:`Session.install()` and -then configure the CherryPy endpoint to use MongoDB sessions. For example:: - - jaraco.mongodb.sessions.Session.install() - - session_config = { - 'sessions.on': True, - 'sessions.storage_type': 'MongoDB', - 'sessions.database': pymongo.MongoClient().database, - } - config = { - '/': session_config, - } - - cherrypy.quickstart(..., config=config) - -The ``jaraco.modb`` module implements the codec interface, so may be used -to encode more complex objects in the session:: - - session_config.update({ - 'sessions.codec': jaraco.modb, - }) - -""" - -import datetime -import time -import logging -import pprint - -import pymongo.errors -import cherrypy -import dateutil.tz - -from . import timers -from . import compat - -log = logging.getLogger(__name__) - - -class LockTimeout(RuntimeError): - pass - - -class NullCodec(object): - def decode(self, data): - return data - - def encode(self, data): - return data - - -class Session(cherrypy.lib.sessions.Session): - """ - A MongoDB-backed CherryPy session store. Takes the following params: - - database: the pymongo Database object. - - collection_name: The name of the collection to use in the db. - - codec: An object with 'encode' and 'decode' methods, used to encode - objects before saving them to MongoDB and decode them when loaded - from MongoDB. - - lock_timeout: A timedelta or numeric seconds indicating how long - to block acquiring a lock. If None (default), acquiring a lock - will block indefinitely. - """ - - codec = NullCodec() - "by default, objects are passed directly to MongoDB" - - def __init__(self, id, **kwargs): - kwargs.setdefault('collection_name', 'sessions') - kwargs.setdefault('lock_timeout', None) - super(Session, self).__init__(id, **kwargs) - self.setup_expiration() - if isinstance(self.lock_timeout, (int, float)): - self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) - if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): - msg = "Lock timeout must be numeric seconds or a timedelta instance." - raise ValueError(msg) - - @classmethod - def install(cls): - """ - Add this session to the cherrypy session handlers. CherryPy looks - for session classes in vars(cherrypy.lib.sessions) with a name - in title-case followed by "Session". - """ - cherrypy.lib.sessions.MongodbSession = cls - - @property - def collection(self): - return self.database[self.collection_name] - - def setup_expiration(self): - """ - Use pymongo TTL index to automatically expire sessions. - """ - self.collection.create_index( - '_expiration_datetime', - expireAfterSeconds=0, - ) - - def _exists(self): - return bool(self.collection.find_one(self.id)) - - def _load(self): - filter = dict( - _id=self.id, - _expiration_datetime={'$exists': True}, - ) - projection = dict(_id=False) - doc = self.collection.find_one(filter, projection) - if not doc: - return - expiration_time = doc.pop('_expiration_datetime') - doc = self.codec.decode(doc) - return (doc, self._make_local(expiration_time)) - - @staticmethod - def _make_aware(local_datetime): - return local_datetime.replace(tzinfo=dateutil.tz.tzlocal()) - - @staticmethod - def _make_utc(local_datetime): - return Session._make_aware(local_datetime).astimezone(dateutil.tz.tzutc()) - - @staticmethod - def _make_local(utc_datetime): - """ - For a naive utc_datetime, return the same time in the local timezone - (also naive). - """ - return ( - utc_datetime.replace(tzinfo=dateutil.tz.tzutc()) - .astimezone(dateutil.tz.tzlocal()) - .replace(tzinfo=None) - ) - - def _save(self, expiration_datetime): - data = dict(self._data) - data = self.codec.encode(data) - # CherryPy defines the expiration in local time, which may be - # different for some hosts. Convert it to UTC before sticking - # it in the database. - expiration_datetime = self._make_utc(expiration_datetime) - data.update( - _expiration_datetime=expiration_datetime, - _id=self.id, - ) - try: - compat.save(self.collection, data) - except pymongo.errors.InvalidDocument: - log.warning( - "Unable to save session:\n%s", - pprint.pformat(data), - ) - raise - - def _delete(self): - self.collection.delete_one(self.id) - - def acquire_lock(self): - """ - Acquire the lock. Blocks indefinitely until lock is available - unless `lock_timeout` was supplied. If the lock_timeout elapses, - raises LockTimeout. - """ - # first ensure that a record exists for this session id - try: - self.collection.insert_one(dict(_id=self.id)) - except pymongo.errors.DuplicateKeyError: - pass - unlocked_spec = dict(_id=self.id, locked=None) - lock_timer = ( - timers.Timer.after(self.lock_timeout) - if self.lock_timeout - else timers.NeverExpires() - ) - while not lock_timer.expired(): - locked_spec = {'$set': dict(locked=datetime.datetime.utcnow())} - res = self.collection.update_one(unlocked_spec, locked_spec) - if res.raw_result['updatedExisting']: - # we have the lock - break - time.sleep(0.1) - else: - raise LockTimeout(f"Timeout acquiring lock for {self.id}") - self.locked = True - - def release_lock(self): - record_spec = dict(_id=self.id) - self.collection.update_one(record_spec, {'$unset': {'locked': 1}}) - # if no data was saved (no expiry), remove the record - record_spec.update(_expiration_datetime={'$exists': False}) - self.collection.delete_one(record_spec) - self.locked = False - - def __len__(self): - return self.collection.count() diff --git a/jaraco/mongodb/sharding.py b/jaraco/mongodb/sharding.py deleted file mode 100644 index 258d81e..0000000 --- a/jaraco/mongodb/sharding.py +++ /dev/null @@ -1,41 +0,0 @@ -import socket -import operator - -import pymongo - -hostname = socket.gethostname() -by_id = operator.itemgetter('_id') - - -def get_ids(collection): - return map(by_id, collection.find(projection=['_id'])) - - -def create_db_in_shard(db_name, shard, client=None): - """ - In a sharded cluster, create a database in a particular shard. - """ - client = client or pymongo.MongoClient() - # flush the router config to ensure it's not stale - res = client.admin.command('flushRouterConfig') - if not res.get('ok'): - raise RuntimeError("unable to flush router config") - if shard not in get_ids(client.config.shards): - raise ValueError(f"Unknown shard {shard}") - if db_name in get_ids(client.config.databases): - raise ValueError("database already exists") - # MongoDB doesn't have a 'create database' command, so insert an - # item into a collection and then drop the collection. - client[db_name].foo.insert({'foo': 1}) - client[db_name].foo.drop() - if client[db_name].collection_names(): - raise ValueError("database has collections") - primary = client['config'].databases.find_one(db_name)['primary'] - if primary != shard: - res = client.admin.command('movePrimary', value=db_name, to=shard) - if not res.get('ok'): - raise RuntimeError(str(res)) - return ( - f"Successfully created {db_name} in {shard} via {client.nodes} " - f"from {hostname}" - ) diff --git a/jaraco/mongodb/testing.py b/jaraco/mongodb/testing.py deleted file mode 100644 index 104c4eb..0000000 --- a/jaraco/mongodb/testing.py +++ /dev/null @@ -1,69 +0,0 @@ -import pprint -import textwrap - -from .query import compat_explain - - -def _rep_index_info(coll): - index_info = coll.index_information() - return "Indexes are:\n" + pprint.pformat(index_info) - - -def _mongo7_query_plan(plan): - """ - On MongoDB 7, the query plan is found in a separate field. - """ - return plan.get('queryPlan', plan) - - -def assert_index_used(cur): - """ - Explain the cursor and ensure that the index was used. - """ - explanation = compat_explain(cur) - plan = _mongo7_query_plan(explanation['queryPlanner']['winningPlan']) - assert plan['stage'] != 'COLLSCAN' - assert plan['inputStage']['stage'] == 'IXSCAN' - - -def assert_covered(cur): - """ - Use the best knowledge about Cursor.explain() to ensure that the query - was covered by an index. - """ - explanation = compat_explain(cur) - tmpl = textwrap.dedent( - """ - Query was not covered: - {explanation} - """ - ).lstrip() - report = tmpl.format(explanation=pprint.pformat(explanation)) - report += _rep_index_info(cur.collection) - stats = explanation['executionStats'] - assert stats['totalDocsExamined'] == 0, report - assert stats['totalKeysExamined'], "No documents matched" - return explanation - - -def assert_distinct_covered(coll, field, query): - """ - Ensure a distinct query is covered by an index. - """ - est = coll.estimated_document_count() - assert est, "Unable to assert without a document" - db = coll.database - res = db.command('distinct', coll.name, key=field, query=query) - assert 'stats' in res, "Stats not supplied. Maybe SERVER-9126?" - stats = res['stats'] - tmpl = textwrap.dedent( - """ - Distinct query was not covered: - {explanation} - """ - ).lstrip() - report = tmpl.format(explanation=pprint.pformat(stats)) - report += _rep_index_info(coll) - assert stats['nscannedObjects'] == 0, report - assert stats['n'], "No documents matched" - return stats diff --git a/jaraco/mongodb/tests/test_compat.py b/jaraco/mongodb/tests/test_compat.py deleted file mode 100644 index 1ebd482..0000000 --- a/jaraco/mongodb/tests/test_compat.py +++ /dev/null @@ -1,42 +0,0 @@ -from jaraco.mongodb import compat - - -def test_save_no_id(database): - doc = dict(foo='bar') - compat.save(database.test_coll, doc) - assert database.test_coll.find_one()['foo'] == 'bar' - - -def test_save_new_with_id(database): - doc = dict(foo='bar', _id=1) - compat.save(database.test_coll, doc) - assert database.test_coll.find_one() == doc - - -def test_save_replace_by_id(database): - compat.save(database.test_coll, dict(foo='bar', _id=1)) - - doc = dict(foo='baz', _id=1) - compat.save(database.test_coll, doc) - assert database.test_coll.count_documents({}) == 1 - assert database.test_coll.find_one() == doc - - -def test_save_no_id_extant_docs(database): - """ - When no id is supplied, a new document should be created. - """ - doc = dict(foo='bar') - compat.save(database.test_coll, dict(doc)) - assert database.test_coll.count_documents({}) == 1 - compat.save(database.test_coll, doc) - assert database.test_coll.count_documents({}) == 2 - - -def test_save_adds_id(database): - """ - Ensure _id is added to an inserted document. - """ - doc = dict(foo='bar') - compat.save(database.test_coll, doc) - assert '_id' in doc diff --git a/jaraco/mongodb/tests/test_fields.py b/jaraco/mongodb/tests/test_fields.py deleted file mode 100644 index e36a5aa..0000000 --- a/jaraco/mongodb/tests/test_fields.py +++ /dev/null @@ -1,10 +0,0 @@ -from jaraco.mongodb import fields - - -def test_insert_with_dots(mongodb_instance): - db = mongodb_instance.get_connection().test_db - field = fields.encode("foo.bar") - db.things.insert_one({field: "value"}) - doc = db.things.find_one({field: "value"}) - doc = {fields.decode(key): value for key, value in doc.items()} - assert doc['foo.bar'] == 'value' diff --git a/jaraco/mongodb/tests/test_insert_doc.py b/jaraco/mongodb/tests/test_insert_doc.py deleted file mode 100644 index 1a25e63..0000000 --- a/jaraco/mongodb/tests/test_insert_doc.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys -import subprocess -import json - - -def test_insert_doc_command(mongodb_instance): - uri = mongodb_instance.get_uri() + '/testdb.test_coll' - cmd = [ - sys.executable, - '-m', - 'jaraco.mongodb.insert-doc', - uri, - ] - proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) - doc = dict(test='value', test2=2) - proc.communicate(json.dumps(doc).encode('utf-8')) - assert not proc.wait() - (saved,) = mongodb_instance.get_connection().testdb.test_coll.find() - saved.pop('_id') - assert saved == doc diff --git a/jaraco/mongodb/tests/test_manage.py b/jaraco/mongodb/tests/test_manage.py deleted file mode 100644 index af35c7d..0000000 --- a/jaraco/mongodb/tests/test_manage.py +++ /dev/null @@ -1,11 +0,0 @@ -from jaraco.mongodb import manage - - -def test_purge_all_databases(mongodb_instance): - client = mongodb_instance.get_connection() - client.test_db.test_coll.insert_one({'a': 1}) - client.test_db2.test_coll.insert_one({'b': 2}) - manage.purge_all_databases(client) - indexes = {'system.indexes'} - assert set(client.test_db.list_collection_names()) <= indexes - assert set(client.test_db2.list_collection_names()) <= indexes diff --git a/jaraco/mongodb/tests/test_oplog.py b/jaraco/mongodb/tests/test_oplog.py deleted file mode 100644 index 0aa724b..0000000 --- a/jaraco/mongodb/tests/test_oplog.py +++ /dev/null @@ -1,100 +0,0 @@ -import functools - -import bson -import pytest -import jaraco.itertools - -from jaraco.mongodb import oplog -from jaraco.mongodb import service - - -class TestReplacer: - def test_rename_index_op_ns(self): - """ - As an index operation references namespaces, - when performing a rename operation, it's also important - to rename the ns in the op itself. - """ - op = { - 'ts': bson.Timestamp(1446495808, 3), - 'ns': 'airportlocker.system.indexes', - 'op': 'i', - 'o': { - 'ns': 'airportlocker.luggage.chunks', - 'key': {'files_id': 1, 'n': 1}, - 'name': 'files_id_1_n_1', - 'unique': True, - }, - } - - ren = oplog.Renamer.from_specs("airportlocker=airportlocker-us") - ren(op) - - assert op['ns'] == 'airportlocker-us.system.indexes' - assert op['o']['ns'] == 'airportlocker-us.luggage.chunks' - - def test_collection_rename_on_create_cmd(self): - """ - Starting in MongoDB 3.2, a create collection is required - before insert operations (apparently). Ensure that a renamed - command in a create collection is renamed. - """ - op = { - 'h': -4317026186822365585, - 't': 1, - 'ns': 'newdb.$cmd', - 'v': 2, - 'o': {'create': 'coll_1'}, - 'ts': bson.Timestamp(1470940276, 1), - 'op': 'c', - } - - ren = oplog.Renamer.from_specs("newdb.coll_1=newdb.coll_2") - ren(op) - - assert op['o']['create'] == 'coll_2' - - -def make_replicaset(request): - try: - r_set = service.MongoDBReplicaSet() - r_set.log_root = '' - r_set.start() - r_set.get_connection() - request.addfinalizer(r_set.stop) - except Exception as err: - pytest.skip(f"MongoDB not available ({err})") - return r_set - - -@pytest.fixture -def replicaset_factory(request): - """ - Return a factory that can generate MongoDB replica sets - """ - maker = functools.partial(make_replicaset, request) - return jaraco.itertools.infinite_call(maker) - - -class TestOplogReplication: - def test_index_deletion(self, replicaset_factory): - """ - A delete index operation should be able to be applied to a replica - """ - source = next(replicaset_factory).get_connection() - dest = next(replicaset_factory).get_connection() - source_oplog = oplog.Oplog(source.local.oplog.rs) - before_ts = source_oplog.get_latest_ts() - source.index_deletion_test.stuff.create_index("foo") - for op in source_oplog.since(before_ts): - oplog.apply(dest, op) - - id_index, foo_index = dest.index_deletion_test.stuff.list_indexes() - - after_ts = source_oplog.get_latest_ts() - source.index_deletion_test.stuff.drop_index("foo_1") - (delete_index_op,) = source_oplog.since(after_ts) - print("attempting", delete_index_op) - oplog.apply(dest, delete_index_op) - (only_index,) = dest.index_deletion_test.stuff.list_indexes() - assert only_index['name'] == '_id_' diff --git a/jaraco/mongodb/tests/test_service.py b/jaraco/mongodb/tests/test_service.py deleted file mode 100644 index ceae101..0000000 --- a/jaraco/mongodb/tests/test_service.py +++ /dev/null @@ -1,36 +0,0 @@ -import datetime - -import pymongo -import pytest -from tempora import timing - -from jaraco.mongodb import service - - -@pytest.mark.xfail(reason="#31") -def test_MongoDBReplicaSet_writable(): - rs = service.MongoDBReplicaSet() - rs.start() - try: - conn = pymongo.MongoClient(rs.get_connect_hosts()) - conn.database.collection.insert_one({'foo': 'bar'}) - finally: - rs.stop() - - -def test_MongoDBReplicaSet_starts_quickly(): - pytest.skip("Takes 20-30 seconds") - sw = timing.Stopwatch() - rs = service.MongoDBReplicaSet() - rs.start() - try: - elapsed = sw.split() - limit = datetime.timedelta(seconds=5) - assert elapsed < limit - finally: - rs.stop() - - -def test_fixture(mongodb_instance): - "Cause the fixture to be invoked" - pass diff --git a/jaraco/mongodb/tests/test_sessions.py b/jaraco/mongodb/tests/test_sessions.py deleted file mode 100644 index dc18c1a..0000000 --- a/jaraco/mongodb/tests/test_sessions.py +++ /dev/null @@ -1,73 +0,0 @@ -import functools -import datetime -import dateutil -import importlib - -import pytest - - -pytest.importorskip("cherrypy") -sessions = importlib.import_module('jaraco.mongodb.sessions') - - -@pytest.fixture(scope='function') -def database(request, mongodb_instance): - """ - Return a MongoDB database suitable for testing auth. Remove the - collection between every test. - """ - database = mongodb_instance.get_connection().sessions_test - request.addfinalizer(functools.partial(database.sessions.delete_many, {})) - return database - - -class TestSessions(object): - def test_time_conversion(self): - local_time = datetime.datetime.now().replace(microsecond=0) - local_time = sessions.Session._make_aware(local_time) - utc_aware = datetime.datetime.utcnow().replace( - tzinfo=dateutil.tz.tzutc(), microsecond=0 - ) - assert local_time == utc_aware - - def test_time_conversion2(self): - local_time = datetime.datetime.now().replace(microsecond=0) - round_local = sessions.Session._make_local( - sessions.Session._make_utc(local_time) - ) - assert round_local == local_time - assert round_local.tzinfo is None - - def test_session_persists(self, database): - session = sessions.Session(None, database=database) - session['x'] = 3 - session['y'] = "foo" - session.save() - session_id = session.id - del session - session = sessions.Session(session_id, database=database) - assert session['x'] == 3 - assert session['y'] == 'foo' - - def test_locked_session(self, database): - session = sessions.Session(None, database=database) - session.acquire_lock() - session['x'] = 3 - session['y'] = "foo" - session.save() - session_id = session.id - del session - session = sessions.Session(session_id, database=database) - assert session['x'] == 3 - - @pytest.mark.xfail - def test_numeric_keys(self, database): - session = sessions.Session(None, database=database, use_modb=True) - session.acquire_lock() - session[3] = 9 - session.save() - session_id = session.id - del session - session = sessions.Session(session_id, database=database, use_modb=True) - assert 3 in session - assert session[3] == 9 diff --git a/jaraco/mongodb/tests/test_testing.py b/jaraco/mongodb/tests/test_testing.py deleted file mode 100644 index 98a94e4..0000000 --- a/jaraco/mongodb/tests/test_testing.py +++ /dev/null @@ -1,65 +0,0 @@ -import functools - -import pytest - -from jaraco.mongodb import testing - - -@pytest.fixture -def indexed_collection(request, mongodb_instance): - _name = request.function.__name__ - db = mongodb_instance.get_connection()[_name] - dropper = functools.partial(db.client.drop_database, _name) - request.addfinalizer(dropper) - coll = db[_name] - coll.create_index('foo') - return coll - - -def test_assert_covered_passes(indexed_collection): - indexed_collection.insert_one({'foo': 'bar'}) - proj = {'_id': False, 'foo': True} - cur = indexed_collection.find({'foo': 'bar'}, proj) - testing.assert_covered(cur) - - -def test_assert_covered_empty(indexed_collection): - """ - assert_covered should raise an error it's trivially - covered (returns no results) - """ - cur = indexed_collection.find() - with pytest.raises(AssertionError): - testing.assert_covered(cur) - - -def test_assert_covered_null(indexed_collection): - """ - assert_covered should raise an error it's trivially - covered (returns no results) - """ - indexed_collection.insert_one({"foo": "bar"}) - proj = {'_id': False, 'foo': True} - cur = indexed_collection.find({"foo": "baz"}, proj) - with pytest.raises(AssertionError): - testing.assert_covered(cur) - - -def test_assert_index_used_passes(indexed_collection): - """ - assert_index_used should pass when the index is used, - even if the documents had to be hit. - """ - indexed_collection.insert_one({'foo': 'bar', 'bing': 'baz'}) - cur = indexed_collection.find({'foo': 'bar'}) - testing.assert_index_used(cur) - - -def test_assert_index_used_fails(indexed_collection): - """ - assert_index_used should fail when no index is used. - """ - indexed_collection.insert_one({'foo': 'bar', 'bing': 'baz'}) - cur = indexed_collection.find({'bing': 'baz'}) - with pytest.raises(AssertionError): - testing.assert_index_used(cur) diff --git a/jaraco/mongodb/timers.py b/jaraco/mongodb/timers.py deleted file mode 100644 index 6501923..0000000 --- a/jaraco/mongodb/timers.py +++ /dev/null @@ -1,25 +0,0 @@ -import datetime - - -class NeverExpires(object): - def expired(self): - return False - - -class Timer(object): - """ - A simple timer that will indicate when an expiration time has passed. - """ - - def __init__(self, expiration): - self.expiration = expiration - - @classmethod - def after(cls, elapsed): - """ - Return a timer that will expire after `elapsed` passes. - """ - return cls(datetime.datetime.utcnow() + elapsed) - - def expired(self): - return datetime.datetime.utcnow() >= self.expiration diff --git a/jaraco/mongodb/uri.py b/jaraco/mongodb/uri.py deleted file mode 100644 index 8eb4757..0000000 --- a/jaraco/mongodb/uri.py +++ /dev/null @@ -1,27 +0,0 @@ -import urllib.parse - -import jaraco.functools - - -@jaraco.functools.once -def _add_scheme(): - """ - urllib.parse doesn't support the mongodb scheme, but it's easy - to make it so. - """ - lists = [ - urllib.parse.uses_relative, - urllib.parse.uses_netloc, - urllib.parse.uses_query, - ] - for each in lists: - each.append('mongodb') - - -def join(base, new): - """ - Use urllib.parse to join the MongoDB URIs. - Registers the MongoDB scheme first. - """ - _add_scheme() - return urllib.parse.urljoin(base, new) diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index b6f9727..0000000 --- a/mypy.ini +++ /dev/null @@ -1,5 +0,0 @@ -[mypy] -ignore_missing_imports = True -# required to support namespace packages -# https://github.com/python/mypy/issues/14057 -explicit_package_bases = True diff --git a/pyproject.toml b/pyproject.toml index dce944d..5cbd2ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,3 @@ [build-system] requires = ["setuptools>=56", "setuptools_scm[toml]>=3.4.1"] build-backend = "setuptools.build_meta" - -[tool.black] -skip-string-normalization = true - -[tool.setuptools_scm] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index d9a15ed..0000000 --- a/pytest.ini +++ /dev/null @@ -1,27 +0,0 @@ -[pytest] -norecursedirs=dist build .tox .eggs -addopts=--doctest-modules -filterwarnings= - ## upstream - - # Ensure ResourceWarnings are emitted - default::ResourceWarning - - # shopkeep/pytest-black#55 - ignore: is not using a cooperative constructor:pytest.PytestDeprecationWarning - ignore:The \(fspath. py.path.local\) argument to BlackItem is deprecated.:pytest.PytestDeprecationWarning - ignore:BlackItem is an Item subclass and should not be a collector:pytest.PytestWarning - - # shopkeep/pytest-black#67 - ignore:'encoding' argument not specified::pytest_black - - # realpython/pytest-mypy#152 - ignore:'encoding' argument not specified::pytest_mypy - - # python/cpython#100750 - ignore:'encoding' argument not specified::platform - - # pypa/build#615 - ignore:'encoding' argument not specified::build.env - - ## end upstream diff --git a/setup.cfg b/setup.cfg index dcfad25..aa0a2e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,63 +19,9 @@ packages = find_namespace: include_package_data = true python_requires = >=3.8 install_requires = - pymongo>=3.0 - python-dateutil - jaraco.services>=2 - portend - jaraco.itertools>=2 - jaraco.functools>=2 - jaraco.ui>=2 - jaraco.context>=2 - more_itertools - jaraco.logging>=2 - tempora - pytimeparse - jaraco.collections>=2 - importlib_metadata; python_version < "3.8" - autocommand - cachetools - -[options.packages.find] -exclude = - build* - dist* - docs* - tests* + inflect [options.extras_require] -testing = - # upstream - pytest >= 6 - pytest-checkdocs >= 2.4 - pytest-black >= 0.3.7; \ - # workaround for jaraco/skeleton#22 - python_implementation != "PyPy" - pytest-cov - pytest-mypy >= 0.9.1; \ - # workaround for jaraco/skeleton#22 - python_implementation != "PyPy" - pytest-enabler >= 2.2 - pytest-ruff - - # local - cherrypy - types-python-dateutil - types-cachetools - docs = # upstream sphinx >= 3.5 - jaraco.packaging >= 9.3 - rst.linker >= 1.9 - furo - sphinx-lint - - # local - cherrypy - -[options.entry_points] -pytest11 = - MongoDB = jaraco.mongodb.fixtures -pmxbot_handlers = - create in MongoDB shard = jaraco.mongodb.pmxbot diff --git a/tests/oplog.js b/tests/oplog.js deleted file mode 100644 index 5e9708d..0000000 --- a/tests/oplog.js +++ /dev/null @@ -1,269 +0,0 @@ -/** Test suite runner. */ -function runTests() { - var cases = [ - test_basicOperations, - test_excludeNamespaces, - test_includeMatchingNamespaces, - test_renameNamespaces, - test_renameNamespacesIndexes, - test_renameNamespacesRenameOps, - test_resumeFromSavedTimestamp, - ]; - - cases.forEach(function(test) { - var env = setUp(); - print("============================"); - print(" " + test.name); - print("============================"); - test(env.rs1, env.rs2); - tearDown(env); - }); -} - -/** Initialize test environment. */ -function setUp() { - MongoRunner.dataPath = '/tmp/' - opts = {}; - var rs1 = new ReplSetTest({ - name: 'rs1', - nodes: [opts], - startPort: 31001, - }); - var rs2 = new ReplSetTest({ - name: 'rs2', - nodes: [opts], - startPort: 31002, - }); - - rs1.startSet({oplogSize: 1}) - rs1.initiate(); - rs1.waitForMaster(); - - rs2.startSet({oplogSize: 1}) - rs2.initiate(); - rs2.waitForMaster(); - - return {rs1: rs1, rs2: rs2}; -} - -/** Clean up after the tests. */ -function tearDown(env) { - env.rs1.stopSet(); - env.rs2.stopSet(); - removeFile('mongooplog.ts'); -} - - -/* - * Check that oplog records can be applied from one replica set to another - */ -function test_basicOperations(rs1, rs2) { - var src = rs1.getPrimary(); - var dst = rs2.getPrimary(); - var srcColl = src.getDB("test").coll; - var dstColl = dst.getDB("test").coll; - - // Insert some data in source db - srcColl.insert({"answer": "unknown"}); - srcColl.update({"answer": "unknown"}, {"$set": {"answer": 42}}); - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', src.host, - '--dest', dst.host - ); - - // Check that all operations got applied - assert(dstColl.findOne()); - assert.eq(dstColl.findOne().answer, 42); -} - -function test_excludeNamespaces(rs1, rs2) { - // Given operations on several different namespaces - var srcDb1 = rs1.getPrimary().getDB('testdb'); - var srcDb2 = rs1.getPrimary().getDB('test_ignored_db'); - - srcDb1.include_coll.insert({msg: "This namespace should be transfered"}); - srcDb1.exclude_coll.insert({msg: "This namespace should be ignored"}); - srcDb2.coll.insert({msg: "This whole db should be ignored"}); - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - // Ignore two namespaces: a collection and a whole database - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host, - '--exclude', 'testdb.exclude_coll', 'test_ignored_db' - ); - - // Changes in namespaces that are not in --exclude list should be delivered - var destDb1 = rs2.getPrimary().getDB('testdb'); - var destDb2 = rs2.getPrimary().getDB('test_ignored_db'); - - assert.eq(destDb1.include_coll.count(), 1); - - // Changes in excluded namespaces should not be on dest server - assert.eq(destDb1.exclude_coll.count(), 0); - assert.eq(destDb2.coll.count(), 0); -} - -function test_includeMatchingNamespaces(rs1, rs2) { - // Given operations on several different namespaces - var srcDb1 = rs1.getPrimary().getDB('testdb'); - var srcDb2 = rs1.getPrimary().getDB('test_ignored_db'); - - srcDb1.include_coll.insert({msg: "This namespace should be transfered"}); - srcDb1.other_coll.insert({msg: "This namespace should be ignored"}); - srcDb2.coll.insert({msg: "This whole db should be ignored"}); - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - // Process only one namespace (a collection) - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host, - '--ns', 'testdb.include_coll' - ); - - // Only changes in namespaces specified in --ns should be delivered - var destDb1 = rs2.getPrimary().getDB('testdb'); - var destDb2 = rs2.getPrimary().getDB('test_ignored_db'); - - assert.eq(destDb1.include_coll.count(), 1); - - // All other namespaces should be ignored - assert.eq(destDb1.exclude_coll.count(), 0); - assert.eq(destDb2.coll.count(), 0); -} - -function test_renameNamespaces(rs1, rs2) { - - // Given operations on different namespaces - var srcDb1 = rs1.getPrimary().getDB('renamedb'); - var srcDb2 = rs1.getPrimary().getDB('testdb'); - - srcDb1.coll_1.insert({msg: "All collections in this db "}); - srcDb1.coll_2.insert({msg: " should be moved to other db"}); - - srcDb2.renameMe.insert({msg: "Only this collection must be renamed"}); - srcDb2.notMe.insert({msg: "...but not this"}); - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - // Rename one db and one collection during transfer - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host, - '--rename', 'renamedb=newdb', 'testdb.renameMe=testdb.newMe' - ) - - // Namespaces (databases and collections) given in --rename argument - // should be actually renamed on destination server - var dest = rs2.getPrimary(); - assert(dest.getDB('newdb').coll_1.findOne()); - assert(dest.getDB('newdb').coll_2.findOne()); - assert(dest.getDB('testdb').newMe.findOne()); - - // Old namespaces should not appear on destination server - assert(dest.getDB('renamedb').coll_1.findOne() == null); - assert(dest.getDB('renamedb').coll_2.findOne() == null); - assert(dest.getDB('testdb').renameMe.findOne() == null); -} - -function test_renameNamespacesIndexes(rs1, rs2) { - - // Given operations on different namespaces - var srcDb1 = rs1.getPrimary().getDB('testdb'); - - srcDb1.coll_1.ensureIndex({'msg': 1}) - srcDb1.coll_1.insert({msg: "This message is indexed"}); - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - // Rename one db and one collection during transfer - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host, - '--rename', 'testdb.coll_1=testdb.coll_new' - ) - - // Namespaces in index operation - // should be actually renamed on destination server - var dest = rs2.getPrimary(); - assert(dest.getDB('testdb').coll_new.findOne()); - - // The index should have been created on the new collection - assert(dest.getDB('testdb').coll_new.getIndexes()[1]['name'] == 'msg_1'); - - // Old namespaces should not appear on destination server - assert(dest.getDB('testdb').coll_1.findOne() == null); -} - -function test_renameNamespacesRenameOps(rs1, rs2) { - - // When renaming a db, rename operations on - // collections in that db should be honored. - - // Given operations on different namespaces - var srcDb1 = rs1.getPrimary().getDB('testdb'); - - srcDb1.coll_oops.insert({msg: "This collection gets renamed"}); - srcDb1.coll_oops.renameCollection('coll_1') - - // Invoke mongooplog-alt to transfer changes from rs1 to rs2 - // Rename one db and one collection during transfer - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host, - '--rename', 'testdb=newdb' - ) - - // Only coll_1 should exist in the dest - var dest = rs2.getPrimary(); - assert(dest.getDB('newdb').coll_oops.findOne() == null); - assert(dest.getDB('newdb').coll_1.findOne()); - -} - -function test_resumeFromSavedTimestamp(rs1, rs2) { - var srcDb = rs1.getPrimary().getDB('testdb'); - var destDb = rs2.getPrimary().getDB('testdb'); - var destLocal = rs2.getPrimary().getDB('local'); - - // 1. Do some operation on source db and replicate it to the dest db - srcDb.test_coll.insert({msg: "Hello world!"}); - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--window', '1 d', - '--resume-file', 'oplog-resume-test.ts', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host - ); - - // 2. Notice oplog size on dest server - var oplogSizeAfterStep1 = destLocal.oplog.rs.count(); - - // 3. Do one more operation on source and replicate it one more time - srcDb.test_coll.remove({msg: "Hello world!"}); - runMongoProgram( - 'python', '-m', 'jaraco.mongodb.oplog', '-l', '9', - '--resume-file', 'oplog-resume-test.ts', - '--source', rs1.getPrimary().host, - '--dest', rs2.getPrimary().host - ); - - // 4. mongooplog-alt should process only the last one operation. - // Thus, oplog size must increase by 1 - var oplogSizeAfterStep2 = destLocal.oplog.rs.count(); - assert.eq(oplogSizeAfterStep2 - oplogSizeAfterStep1, 1); -} - -runTests(); diff --git a/towncrier.toml b/towncrier.toml deleted file mode 100644 index 6fa480e..0000000 --- a/towncrier.toml +++ /dev/null @@ -1,2 +0,0 @@ -[tool.towncrier] -title_format = "{version}" diff --git a/tox.ini b/tox.ini index e51d652..b3aa864 100644 --- a/tox.ini +++ b/tox.ini @@ -11,35 +11,6 @@ extras = [testenv:docs] extras = docs - testing changedir = docs commands = python -m sphinx -W --keep-going . {toxinidir}/build/html - python -m sphinxlint - -[testenv:finalize] -skip_install = True -deps = - towncrier - jaraco.develop >= 7.23 -passenv = * -commands = - python -m jaraco.develop.finalize - - -[testenv:release] -skip_install = True -deps = - build - twine>=3 - jaraco.develop>=7.1 -passenv = - TWINE_PASSWORD - GITHUB_TOKEN -setenv = - TWINE_USERNAME = {env:TWINE_USERNAME:__token__} -commands = - python -c "import shutil; shutil.rmtree('dist', ignore_errors=True)" - python -m build - python -m twine upload dist/* - python -m jaraco.develop.create-github-release