Merge 'dev' into 'main': add the lint-code workflow and pypots-cli

.github/workflows/linting.yml

@@ -0,0 +1,31 @@
+name: Linting
+
+on:
+ push:
+ branches:
+ - main
+ - dev
+ pull_request:
+ branches:
+ - main
+ - dev
+
+jobs:
+ flake8:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: "3.10"
+
+ - name: Install Flake8
+ run: |
+ pip install flake8
+
+ - name: Run linting
+ run: |
+ flake8 .

docs/conf.py

@@ -10,13 +10,8 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
-import sys
-from os.path import abspath
-
-sys.path.insert(0, abspath(".."))
 from pypots.__version__ import version
 
-sys.path.insert(0, abspath("../pypots"))
 # -- Project information -----------------------------------------------------
 project = "PyPOTS"
 copyright = "2023, Wenjie Du"

environment-dev.yml

@@ -0,0 +1,26 @@
+name: pypots-dev
+channels:
+ - conda-forge
+ - pytorch
+ - pyg
+ - nodefaults
+dependencies:
+ - conda-forge::python
+ - conda-forge::pip
+ - conda-forge::scipy
+ - conda-forge::numpy # numpy should >= 1.23.3, otherwise may encounter "number not available" when torch>1.11
+ - conda-forge::scikit-learn # sklearn should >= 0.24.1
+ - conda-forge::pandas
+ - conda-forge::h5py
+ - conda-forge::tensorboard
+ - conda-forge::pytest-cov
+ - conda-forge::pytest-xdist
+ - conda-forge::coverage
+ - conda-forge::pycorruptor
+ - conda-forge::tsdb
+ - conda-forge::black
+ - conda-forge::flake8
+ - pytorch::pytorch
+ - pyg::pyg
+ - pyg::pytorch-scatter
+ - pyg::pytorch-sparse

pypots/__init__.py

@@ -14,4 +14,5 @@
  "clustering",
  "forecasting",
  "utils",
+ "__version__",
 ]

pypots/classification/brits.py

@@ -5,7 +5,7 @@
 # Created by Wenjie Du <wenjay.du@gmail.com>
 # License: GPL-v3
 
-from typing import Tuple, Optional, Union
+from typing import Optional, Union
 
 import torch
 import torch.nn as nn

pypots/data/__init__.py

@@ -9,20 +9,41 @@
 from pypots.data.dataset_for_brits import DatasetForBRITS
 from pypots.data.dataset_for_grud import DatasetForGRUD
 from pypots.data.dataset_for_mit import DatasetForMIT
-
 from pypots.data.generating import (
  gene_complete_random_walk,
  gene_random_walk_for_classification,
+ gene_incomplete_random_walk_dataset,
+ gene_physionet2012,
+)
+from pypots.data.load_specific_datasets import (
+ list_supported_datasets,
+ load_specific_dataset,
 )
-
 from pypots.data.utils import (
  masked_fill,
  mcar,
  pickle_load,
  pickle_dump,
 )
 
-from pypots.data.load_specific_datasets import (
- list_supported_datasets,
- load_specific_dataset,
-)
+__all__ = [
+ # datasets
+ "BaseDataset",
+ "DatasetForMIT",
+ "DatasetForBRITS",
+ "DatasetForGRUD",
+ "DatasetForGRUD",
+ # data generation
+ "gene_complete_random_walk",
+ "gene_random_walk_for_classification",
+ "gene_incomplete_random_walk_dataset",
+ "gene_physionet2012",
+ # list and load datasets
+ "list_supported_datasets",
+ "load_specific_dataset",
+ # utils
+ "masked_fill",
+ "mcar",
+ "pickle_load",
+ "pickle_dump",
+]

pypots/imputation/brits.py

@@ -10,7 +10,7 @@
 # License: GPL-v3
 
 import math
-from typing import Tuple, Any, Union, Optional
+from typing import Tuple, Union, Optional
 
 import h5py
 import numpy as np

pypots/tests/test_utils.py

@@ -46,5 +46,11 @@ def test_saving_log_into_file(self):
  shutil.rmtree("test_log", ignore_errors=True)
 
 
+class TestPyPOTSCLI(unittest.TestCase):
+ def test_pypots_cli(self):
+ # TODO: need more test cases here
+ os.system("python pypots/utils/commands/pypots_cli.py")
+
+
 if __name__ == "__main__":
  unittest.main()

pypots/utils/commands/__init__.py

@@ -0,0 +1,21 @@
+"""
+
+"""
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: GLP-v3
+
+
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseCommand(ABC):
+ @staticmethod
+ @abstractmethod
+ def register_subcommand(parser: ArgumentParser):
+ raise NotImplementedError()
+
+ @abstractmethod
+ def run(self):
+ raise NotImplementedError()

pypots/utils/commands/dev.py

@@ -0,0 +1,115 @@
+"""
+CLI tools to help the development team build PyPOTS.
+"""
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: GLP-v3
+
+import os
+from argparse import ArgumentParser, Namespace
+
+from pypots.utils.commands import BaseCommand
+from pypots.utils.logging import logger
+
+IMPORT_ERROR_MESSAGE = (
+ " `pypots-cli dev` command is for PyPOTS developers to run tests easily. "
+ "Therefore, you need a complete PyPOTS development environment. However, you are missing some dependencies. "
+ "Please refer to https://github.com/WenjieDu/PyPOTS/blob/main/environment-dev.yml for dependency details. "
+)
+
+
+def dev_command_factory(args: Namespace):
+ return DevCommand(
+ args.run_tests,
+ args.k,
+ args.show_coverage,
+ args.lint_code,
+ )
+
+
+class DevCommand(BaseCommand):
+ @staticmethod
+ def register_subcommand(parser: ArgumentParser):
+ train_parser = parser.add_parser("dev", help="CLI tool to help development ")
+ train_parser.add_argument(
+ "--run_tests",
+ dest="run_tests",
+ action="store_true",
+ help="run all test cases",
+ )
+ train_parser.add_argument(
+ "--show_coverage",
+ dest="show_coverage",
+ action="store_true",
+ help="show the code coverage report after running tests",
+ )
+ train_parser.add_argument(
+ "-k",
+ type=str,
+ default="",
+ required=False,
+ help="the -k option of pytest. Description of -k option in pytest: "
+ "only run tests which match the given substring expression. An expression is a python evaluatable "
+ "expression where all names are substring-matched against test names and their parent classes. "
+ "Example: -k 'test_method or test_other' matches all test functions and classes whose name contains "
+ "'test_method' or 'test_other', while -k 'not test_method' matches those that don't contain "
+ "'test_method' in their names. -k 'not test_method and not test_other' will eliminate the matches. "
+ "Additionally keywords are matched to classes and functions containing extra names in their "
+ "'extra_keyword_matches' set, as well as functions which have names assigned directly to them. The "
+ "matching is case-insensitive.",
+ )
+ train_parser.add_argument(
+ "--lint_code",
+ dest="lint_code",
+ action="store_true",
+ help="run Black and Flake8 to lint code",
+ )
+ train_parser.set_defaults(func=dev_command_factory)
+
+ def __init__(
+ self,
+ run_tests: bool,
+ k: str,
+ show_coverage: bool,
+ lint_code: bool,
+ ):
+ self._run_tests = run_tests
+ self._k = k
+ self._show_coverage = show_coverage
+ self._lint_code = lint_code
+
+ def run(self):
+ if self._run_tests:
+ try:
+ pytest_command = f"pytest -k {self._k}" if self._k else "pytest"
+ command_to_run_test = (
+ f"coverage run -m {pytest_command}"
+ if self._show_coverage
+ else pytest_command
+ )
+ logger.info(f"Executing '{command_to_run_test}'...")
+
+ os.system(command_to_run_test)
+ if self._show_coverage:
+ os.system("coverage report -m")
+ os.system("rm -rf .coverage")
+ else:
+ logger.info(
+ "Omit the code coverage report. Enable it by using --show_coverage if in need."
+ )
+ os.system("rm -rf .pytest_cache")
+
+ except ImportError:
+ raise ImportError(IMPORT_ERROR_MESSAGE)
+ except Exception as e:
+ raise RuntimeError(e)
+ elif self._lint_code:
+ try:
+ logger.info("Reformatting with Black...")
+ os.system("black .")
+ logger.info("Linting with Flake8...")
+ os.system("flake8 .")
+ except ImportError:
+ raise ImportError(IMPORT_ERROR_MESSAGE)
+ except Exception as e:
+ raise RuntimeError(e)

pypots/utils/commands/pypots_cli.py

@@ -0,0 +1,36 @@
+"""
+PyPOTS CLI (Command Line Interface) tool
+"""
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: GLP-v3
+
+
+from argparse import ArgumentParser
+
+from pypots.utils.commands.dev import DevCommand
+
+
+def main():
+ parser = ArgumentParser(
+ "PyPOTS Command-Line-Interface tool", usage="pypots-cli <command> [<args>]"
+ )
+ commands_parser = parser.add_subparsers(help="pypots-cli command helpers")
+
+ # Register commands here
+ DevCommand.register_subcommand(commands_parser)
+
+ # parse all arguments
+ args = parser.parse_args()
+
+ if not hasattr(args, "func"):
+ parser.print_help()
+ exit(1)
+
+ # then run
+ service = args.func(args)
+ service.run()
+
+
+if __name__ == "__main__":
+ main()

setup.cfg

@@ -0,0 +1,18 @@
+# This file stores some meta configurations for project PyPOTS.
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: GLP-v3
+
+[flake8]
+# People may argue that coding style is personal. This may be true if the project is personal and one works like a
+# hermit, but to PyPOTS and its community, the answer is NO.
+# We use Black and Flake8 to lint code style and keep the style consistent across all commits and pull requests.
+# Black only reformats the code, and Flake8 is necessary for checking for some other issues not covered by Black.
+
+# The Black line length is default as 88, while the default of Flake8 is 79. However, considering our monitors are
+# much more advanced nowadays, I extend the maximum line length to 120, like other project e.g. transformers. People
+# who prefer the default setting can keep using 88 or 79 while coding. Please ensure your code lines not exceeding 120.
+max-line-length = 120
+extend-ignore =
+ # why ignore E203? Refer to https://github.com/PyCQA/pycodestyle/issues/373
+ E203,

setup.py

@@ -44,5 +44,22 @@
  "tsdb",
  "h5py",
  ],
+ python_requires=">=3.7.0",
  setup_requires=["setuptools>=38.6.0"],
+ entry_points={
+ "console_scripts": ["pypots-cli=pypots.utils.commands.pypots_cli:main"]
+ },
+ classifiers=[
+ "Intended Audience :: Developers",
+ "Intended Audience :: Education",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ ],
 )