cleanup

setup.py

@@ -57,6 +57,10 @@
  classifiers=[
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
  "Development Status :: 2 - Pre-Alpha",
  "Intended Audience :: Science/Research",
  "Intended Audience :: System Administrators",

src/maggma/api/resource/__init__.py

@@ -8,8 +8,8 @@
 from maggma.api.resource.aggregation import AggregationResource
 from maggma.api.resource.post_resource import PostOnlyResource
 from maggma.api.resource.read_resource import ReadOnlyResource, attach_query_ops
-from maggma.api.resource.submission import SubmissionResource
 from maggma.api.resource.s3_url import S3URLResource
+from maggma.api.resource.submission import SubmissionResource
 
 __all__ = [
  "Resource",

src/maggma/api/resource/aggregation.py

@@ -1,14 +1,14 @@
 from typing import Any, Dict, List, Optional, Type
 
-from fastapi import HTTPException, Response, Request
+from fastapi import HTTPException, Request, Response
 from pydantic import BaseModel
 from pymongo import timeout as query_timeout
 from pymongo.errors import NetworkTimeout, PyMongoError
 
 from maggma.api.models import Meta
 from maggma.api.models import Response as ResponseModel
 from maggma.api.query_operator import QueryOperator
-from maggma.api.resource import Resource, HeaderProcessor
+from maggma.api.resource import HeaderProcessor, Resource
 from maggma.api.resource.utils import attach_query_ops
 from maggma.api.utils import STORE_PARAMS, merge_queries
 from maggma.core import Store

src/maggma/api/resource/core.py

@@ -2,7 +2,7 @@
 from abc import ABCMeta, abstractmethod
 from typing import Dict, Type
 
-from fastapi import APIRouter, FastAPI, Response, Request
+from fastapi import APIRouter, FastAPI, Request, Response
 from monty.json import MontyDecoder, MSONable
 from pydantic import BaseModel
 from starlette.responses import RedirectResponse

src/maggma/api/resource/read_resource.py

@@ -1,23 +1,21 @@
 from inspect import signature
 from typing import Any, Dict, List, Optional, Type, Union
 
-from fastapi import Depends, HTTPException, Path, Request
-from fastapi import Response
+import orjson
+from fastapi import Depends, HTTPException, Path, Request, Response
 from pydantic import BaseModel
 from pymongo import timeout as query_timeout
 from pymongo.errors import NetworkTimeout, PyMongoError
 
 from maggma.api.models import Meta
 from maggma.api.models import Response as ResponseModel
 from maggma.api.query_operator import PaginationQuery, QueryOperator, SparseFieldsQuery
-from maggma.api.resource import Resource, HintScheme, HeaderProcessor
+from maggma.api.resource import HeaderProcessor, HintScheme, Resource
 from maggma.api.resource.utils import attach_query_ops, generate_query_pipeline
 from maggma.api.utils import STORE_PARAMS, merge_queries, serialization_helper
 from maggma.core import Store
 from maggma.stores import MongoStore, S3Store
 
-import orjson
-
 
 class ReadOnlyResource(Resource):
  """

src/maggma/api/resource/s3_url.py

@@ -1,18 +1,16 @@
 from datetime import datetime, timedelta
 from typing import List, Optional
-from botocore.exceptions import ClientError
 
-from fastapi import HTTPException, Path, Request
-from fastapi import Response
+import orjson
+from botocore.exceptions import ClientError
+from fastapi import HTTPException, Path, Request, Response
 
-from maggma.api.models import S3URLDoc
 from maggma.api.models import Response as ResponseModel
-from maggma.api.resource import Resource, HeaderProcessor
+from maggma.api.models import S3URLDoc
+from maggma.api.resource import HeaderProcessor, Resource
 from maggma.api.utils import serialization_helper
 from maggma.stores.aws import S3Store
 
-import orjson
-
 
 class S3URLResource(Resource):
  """

src/maggma/api/utils.py

@@ -1,9 +1,9 @@
+import base64
 import inspect
 import sys
 from typing import Any, Callable, Dict, List, Optional, Type
-from bson.objectid import ObjectId
-import base64
 
+from bson.objectid import ObjectId
 from monty.json import MSONable
 from pydantic import BaseModel
 from pydantic.schema import get_flat_models_from_model

src/maggma/cli/__init__.py

@@ -5,20 +5,19 @@
 import asyncio
 import logging
 import sys
-from itertools import chain
 from datetime import datetime
+from itertools import chain
 
 import click
 from monty.serialization import loadfn
 
 from maggma.cli.distributed import find_port
 from maggma.cli.multiprocessing import multi
 from maggma.cli.serial import serial
-from maggma.cli.source_loader import ScriptFinder, load_builder_from_source
 from maggma.cli.settings import CLISettings
+from maggma.cli.source_loader import ScriptFinder, load_builder_from_source
 from maggma.utils import ReportingHandler, TqdmLoggingHandler
 
-
 sys.meta_path.append(ScriptFinder())
 
 settings = CLISettings()
@@ -130,7 +129,7 @@ def run(
 ):
  # Import profiler and setup directories to dump profiler output
  if memray:
- from memray import Tracker, FileDestination
+ from memray import FileDestination, Tracker
 
  if memray_dir:
  import os

src/maggma/cli/distributed.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 # coding utf-8
 
+import asyncio
 import json
-from logging import getLogger
 import socket as pysocket
-from typing import List
-import numpy as np
-from time import perf_counter
-import asyncio
+from logging import getLogger
 from random import randint
+from time import perf_counter
+from typing import List
 
+import numpy as np
+import zmq
 from monty.json import jsanitize
 from monty.serialization import MontyDecoder
 
@@ -18,8 +19,6 @@
 from maggma.core import Builder
 from maggma.utils import tqdm
 
-import zmq
-
 settings = CLISettings()
 
 

src/maggma/cli/multiprocessing.py

@@ -1,12 +1,7 @@
 #!/usr/bin/env python
 # coding utf-8
 
-from asyncio import (
- BoundedSemaphore,
- Queue,
- gather,
- get_event_loop,
-)
+from asyncio import BoundedSemaphore, Queue, gather, get_event_loop
 from concurrent.futures import ProcessPoolExecutor
 from logging import getLogger
 from types import GeneratorType

src/maggma/cli/rabbitmq.py

@@ -3,20 +3,20 @@
 
 import asyncio
 import json
-from logging import getLogger
 import socket as pysocket
-from typing import List, Literal
-import numpy as np
-from time import perf_counter
+from logging import getLogger
 from random import randint
+from time import perf_counter
+from typing import List, Literal
 
+import numpy as np
 from monty.json import jsanitize
 from monty.serialization import MontyDecoder
 
 from maggma.cli.multiprocessing import multi
 from maggma.cli.settings import CLISettings
 from maggma.core import Builder
-from maggma.utils import tqdm, Timeout
+from maggma.utils import Timeout, tqdm
 
 try:
  import pika

src/maggma/stores/aws.py

@@ -8,8 +8,8 @@
 from concurrent.futures import wait
 from concurrent.futures.thread import ThreadPoolExecutor
 from hashlib import sha1
-from typing import Dict, Iterator, List, Optional, Tuple, Union
 from json import dumps
+from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 import msgpack # type: ignore
 from monty.msgpack import default as monty_default

src/maggma/stores/azure.py

@@ -2,15 +2,15 @@
 """
 Advanced Stores for connecting to Microsoft Azure data
 """
+import os
 import threading
 import warnings
 import zlib
 from concurrent.futures import wait
 from concurrent.futures.thread import ThreadPoolExecutor
 from hashlib import sha1
-from typing import Dict, Iterator, List, Optional, Tuple, Union
 from json import dumps
-import os
+from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 import msgpack # type: ignore
 from monty.msgpack import default as monty_default
@@ -19,11 +19,11 @@
 from maggma.utils import grouper, to_isoformat_ceil_ms
 
 try:
+ import azure
  import azure.storage.blob as azure_blob
- from azure.storage.blob import BlobServiceClient, ContainerClient
- from azure.identity import DefaultAzureCredential
  from azure.core.exceptions import ResourceExistsError
- import azure
+ from azure.identity import DefaultAzureCredential
+ from azure.storage.blob import BlobServiceClient, ContainerClient
 except (ImportError, ModuleNotFoundError):
  azure_blob = None # type: ignore
  ContainerClient = None

src/maggma/stores/file_store.py

@@ -4,20 +4,20 @@
 using typical maggma access patterns.
 """
 
+import fnmatch
 import hashlib
 import os
-import fnmatch
 import re
-
 import warnings
-from pathlib import Path
 from datetime import datetime, timezone
-from typing import Dict, List, Optional, Union, Iterator, Callable
+from pathlib import Path
+from typing import Callable, Dict, Iterator, List, Optional, Union
 
-from pymongo import UpdateOne
 from monty.io import zopen
-from maggma.core import StoreError, Sort
-from maggma.stores.mongolike import MemoryStore, JSONStore
+from pymongo import UpdateOne
+
+from maggma.core import Sort, StoreError
+from maggma.stores.mongolike import JSONStore, MemoryStore
 
 # These keys are automatically populated by the FileStore.read() method and
 # hence are not allowed to be manually overwritten
@@ -264,7 +264,7 @@ def _create_record_from_file(self, f: Path) -> Dict:
  self.key: file_id,
  }
 
- def connect(self, force_reset: bool = False):
+ def connect(self):
  """
  Connect to the source data
 
@@ -273,9 +273,6 @@ def connect(self, force_reset: bool = False):
 
  If there is a metadata .json file in the directory, read its
  contents into the MemoryStore
-
- Args:
- force_reset: whether to reset the connection or not
  """
  # read all files and place them in the MemoryStore
  # use super.update to bypass the read_only guard statement

src/maggma/stores/gridfs.py

@@ -8,15 +8,15 @@
 import copy
 import json
 import zlib
-from ruamel import yaml
 from datetime import datetime
-from pymongo.errors import ConfigurationError
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import gridfs
 from monty.json import jsanitize
 from pydash import get, has
 from pymongo import MongoClient, uri_parser
+from pymongo.errors import ConfigurationError
+from ruamel import yaml
 
 from maggma.core import Sort, Store, StoreError
 from maggma.stores.mongolike import MongoStore, SSHTunnel

src/maggma/stores/mongolike.py

@@ -5,23 +5,24 @@
 various utilities
 """
 
-from pathlib import Path
-from ruamel import yaml
+import warnings
 from itertools import chain, groupby
+from pathlib import Path
 from socket import socket
-import warnings
+
+from ruamel import yaml
 
 try:
  from typing import (
+ Any,
+ Callable,
  Dict,
  Iterator,
  List,
+ Literal,
  Optional,
  Tuple,
  Union,
- Any,
- Callable,
- Literal,
  )
 except ImportError:
  from typing import Dict, Iterator, List, Optional, Tuple, Union, Any, Callable

src/maggma/stores/shared_stores.py

@@ -1,9 +1,11 @@
-from monty.json import MontyDecoder
-from threading import Lock
-from maggma.core.store import Store, Sort
-from typing import Dict, Iterator, List, Optional, Tuple, Union, Callable, Any
 from functools import partial
 from multiprocessing.managers import BaseManager
+from threading import Lock
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
+
+from monty.json import MontyDecoder
+
+from maggma.core.store import Sort, Store
 
 
 class StoreFacade(Store):

src/maggma/utils.py

@@ -7,11 +7,11 @@
 import signal
 import uuid
 from datetime import datetime, timedelta
-from dateutil import parser
 from importlib import import_module
 from typing import Dict, Iterable, Optional, Union
 
 from bson.json_util import ObjectId
+from dateutil import parser
 from pydash.objects import get, has, set_
 from pydash.objects import unset as _unset
 from pydash.utilities import to_path

tests/api/test_aggregation_resource.py

@@ -1,15 +1,13 @@
+from datetime import datetime
 from random import randint
 
 import pytest
 from fastapi import FastAPI
 from pydantic import BaseModel, Field
 from starlette.testclient import TestClient
 
-from datetime import datetime
 from maggma.api.query_operator.core import QueryOperator
-
 from maggma.api.resource import AggregationResource
-
 from maggma.stores import MemoryStore