diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c01c84bcd..e2bcc4ab2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,7 @@ - - ## Description + -## Changelog description - - ## Status - [ ] Code base linted - [ ] Ready to go diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 3047d3f50..f27950201 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -19,7 +19,7 @@ jobs: with: fetch-depth: 0 # needed for tag/version - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index ceda1fe55..ef2634dc7 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -28,7 +28,7 @@ jobs: run: sudo apt update && sudo apt install postgresql postgresql-contrib - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/full_tests.yml b/.github/workflows/full_tests.yml index 3183474f6..51fe91b5b 100644 --- a/.github/workflows/full_tests.yml +++ b/.github/workflows/full_tests.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -39,11 +39,9 @@ jobs: # QCFractal Server ################################################# - name: Configure conda for the QCFractal server - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-server environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_server.yaml auto-activate-base: false @@ -72,11 +70,9 @@ jobs: # QCFractalCompute worker ################################################# - name: Configure conda for QCFractalCompute worker - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-worker environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_worker.yaml auto-activate-base: false @@ -104,11 +100,9 @@ jobs: # QCPortal client ################################################# - name: Configure conda for the QCPortal client - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-qcportal environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_qcportal.yaml auto-activate-base: false diff --git a/.github/workflows/full_tests_snowflake.yml b/.github/workflows/full_tests_snowflake.yml index 44e5a47e5..9b6a0138b 100644 --- a/.github/workflows/full_tests_snowflake.yml +++ b/.github/workflows/full_tests_snowflake.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -34,11 +34,9 @@ jobs: fetch-depth: 0 - name: Configure conda for the snowflake - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-snowflake environment-file: qcarchivetesting/conda-envs/fulltest_snowflake.yaml auto-activate-base: false diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 493710905..26e7ff3f2 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -1,6 +1,35 @@ Release Notes ============= +0.55 / 2024-05-23 +------------------- + +Some moderate improvements - mainly, the use of ASPW for SQLite, and the removal of the never-used manager log and +serverinfo log tables. Also adds maintenance jobs for removing old access log and completed internal jobs. + +This release also adds the ability to more-strictly handle queue tags (to prevent managers with `*` as a tag from pulling +everything). + +In addition, lots of smaller bug fixes and improvements. + +Notable PRs: + +- (:pr:`819`) Fix native_files fields return from server & ORM +- (:pr:`821`) Enable dumping a database from a snowflake +- (:pr:`822`) Fix missing client on cached dataset records +- (:pr:`825`) Fix database constraint violation when program version contains uppercase characters +- (:pr:`826`) Fix type of results in ServiceSubtaskRecord +- (:pr:`829`) Replace sqlite3 (python stdlib module) with APSW +- (:pr:`830`) Remove server stats & compute manager logs +- (:pr:`831`) Add internal job to delete old access log entries +- (:pr:`832`) Add internal job to delete old, finished internal jobs +- (:pr:`834`) Add property to get errored child records +- (:pr:`835`) Better handling of queue tags (strict queue tags and case insensitivity) +- (:pr:`836`) Replace gunicorn with waitress and fix logging issues +- (:pr:`837`) Commit to database after every returned task is processed (to help prevent deadlocks) +- (:pr:`838`) Better handling of passwords with `init-db` + + 0.54.1 / 2024-04-12 ------------------- diff --git a/qcarchivetesting/conda-envs/fulltest_qcportal.yaml b/qcarchivetesting/conda-envs/fulltest_qcportal.yaml index 5f480abd6..5feedf2d6 100644 --- a/qcarchivetesting/conda-envs/fulltest_qcportal.yaml +++ b/qcarchivetesting/conda-envs/fulltest_qcportal.yaml @@ -12,7 +12,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas diff --git a/qcarchivetesting/conda-envs/fulltest_server.yaml b/qcarchivetesting/conda-envs/fulltest_server.yaml index 4d77382bd..dceb74bdf 100644 --- a/qcarchivetesting/conda-envs/fulltest_server.yaml +++ b/qcarchivetesting/conda-envs/fulltest_server.yaml @@ -16,7 +16,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas diff --git a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml index 3dc8ca25d..aa0fe8c48 100644 --- a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml +++ b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml @@ -1,7 +1,6 @@ name: qcarchive-snowflake channels: - conda-forge - - conda-forge/label/libint_dev - defaults dependencies: @@ -17,7 +16,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas @@ -44,8 +43,8 @@ dependencies: - torsiondrive # Worker codes below - - qcengine - - psi4>=1.7 + - qcengine<0.70a0 + - psi4>=1.9.1 - rdkit # Testing packages diff --git a/qcarchivetesting/conda-envs/fulltest_worker.yaml b/qcarchivetesting/conda-envs/fulltest_worker.yaml index 26afa2c62..cd0ea2e0b 100644 --- a/qcarchivetesting/conda-envs/fulltest_worker.yaml +++ b/qcarchivetesting/conda-envs/fulltest_worker.yaml @@ -1,7 +1,6 @@ name: qcarchive-worker channels: - conda-forge - - conda-forge/label/libint_dev - defaults dependencies: @@ -16,7 +15,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas @@ -30,8 +29,8 @@ dependencies: - parsl # Worker codes below - - qcengine - - psi4>=1.7 + - qcengine<0.70a0 + - psi4>=1.9.1 - rdkit # Geometric service diff --git a/qcarchivetesting/pyproject.toml b/qcarchivetesting/pyproject.toml index a8cc47de4..eae4055c2 100644 --- a/qcarchivetesting/pyproject.toml +++ b/qcarchivetesting/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] dependencies = [ "qcfractal", - "qcengine", + "qcengine<0.70a0", "pytest", ] diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml deleted file mode 100644 index 97b5e57a6..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml +++ /dev/null @@ -1,30 +0,0 @@ -common: - adapter: dask - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 2 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false - -cluster: - node_exclusivity: True - scheduler: local diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml deleted file mode 100644 index 4e01e649e..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml +++ /dev/null @@ -1,30 +0,0 @@ -common: - adapter: parsl - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 1 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false - -cluster: - node_exclusivity: True - scheduler: local diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml deleted file mode 100644 index 4bfbcaf89..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml +++ /dev/null @@ -1,26 +0,0 @@ -common: - adapter: pool - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 1 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false diff --git a/qcfractal/qcfractal/components/internal_jobs/test_client.py b/qcfractal/qcfractal/components/internal_jobs/test_client.py index ded340ddc..f23aef068 100644 --- a/qcfractal/qcfractal/components/internal_jobs/test_client.py +++ b/qcfractal/qcfractal/components/internal_jobs/test_client.py @@ -111,7 +111,7 @@ def test_internal_jobs_client_cancel_running(snowflake: QCATestingSnowflake): job_1 = snowflake_client.get_internal_job(id_1) assert job_1.status == InternalJobStatusEnum.cancelled - assert job_1.progress < 50 + assert job_1.progress < 70 assert job_1.result == "Internal job cancelled" finally: diff --git a/qcfractal/qcfractal/components/internal_jobs/test_socket.py b/qcfractal/qcfractal/components/internal_jobs/test_socket.py index d1fd26c0e..3b8aa066b 100644 --- a/qcfractal/qcfractal/components/internal_jobs/test_socket.py +++ b/qcfractal/qcfractal/components/internal_jobs/test_socket.py @@ -127,7 +127,7 @@ def test_internal_jobs_socket_recover(storage_socket: SQLAlchemySocket, session: end_event = threading.Event() th = threading.Thread(target=storage_socket.internal_jobs.run_loop, args=(end_event,)) th.start() - time.sleep(15) + time.sleep(30) try: session.expire(job_1) diff --git a/qcfractal/qcfractal/qcfractal_server_cli.py b/qcfractal/qcfractal/qcfractal_server_cli.py index 11bde9442..944a5bb22 100644 --- a/qcfractal/qcfractal/qcfractal_server_cli.py +++ b/qcfractal/qcfractal/qcfractal_server_cli.py @@ -547,6 +547,12 @@ def server_start_api(config): def server_upgrade_db(config): + + # Always set logging level to INFO, otherwise things are a bit quiet + root_logger = logging.getLogger() + if root_logger.level > logging.INFO: + root_logger.setLevel(logging.INFO) + logger = logging.getLogger(__name__) # Don't use start_database - we don't want to create the socket (which diff --git a/qcfractalcompute/qcfractalcompute/test_compute_manager.py b/qcfractalcompute/qcfractalcompute/test_compute_manager.py index 6bad2c813..2b5cdd02e 100644 --- a/qcfractalcompute/qcfractalcompute/test_compute_manager.py +++ b/qcfractalcompute/qcfractalcompute/test_compute_manager.py @@ -226,13 +226,13 @@ def test_manager_missed_heartbeats_shutdown(snowflake: QCATestingSnowflake): snowflake.stop_api() - for i in range(60): + for i in range(90): time.sleep(1) if not compute_thread.is_alive(): break else: - raise RuntimeError("Compute thread did not stop in 60 seconds") + raise RuntimeError("Compute thread did not stop in 90 seconds") compute_thread._compute_thread.join(5) assert compute_thread.is_alive() is False diff --git a/qcportal/pyproject.toml b/qcportal/pyproject.toml index 3d379e724..5b3db3028 100644 --- a/qcportal/pyproject.toml +++ b/qcportal/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "pydantic", "zstandard", "apsw", - "qcelemental", + "qcelemental<0.70a0", "tabulate", "tqdm", "pandas",