Merge pull request #79 from dunnkers/fix-deprecations

Fix deprecations & improve docs

.github/workflows/python-app.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macOS-latest, windows-latest]
     name: Python ${{ matrix.python-version }} - ${{ matrix.os }}
 
@@ -43,6 +43,8 @@ jobs:
           WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
         run: |
           pytest tests --cov=./ --cov-report=xml
+      # The following should only run once per build
+      # → e.g. we configure it only to trigger for python==3.9 and os==ubuntu
       - name: Codecov
-        uses: codecov/codecov-action@v2.1.0
         if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v2.1.0

.gitignore

@@ -150,4 +150,7 @@ artifacts
 .DS_Store
 
 # fseval
-output
+output
+
+# Eggs (deps installed in requirements.txt using `-e`)
+src

examples/quick-start-structured-configs/benchmark.py

@@ -32,7 +32,7 @@ def fit(self, X, y):
 cs.store(name="my_config", node=my_config)
 
 
-@hydra.main(config_path="conf", config_name="my_config")
+@hydra.main(config_path="conf", config_name="my_config", version_base="1.1")
 def main(cfg: PipelineConfig) -> None:
     run_pipeline(cfg)
 

examples/quick-start-yaml/benchmark.py

@@ -18,7 +18,7 @@ def fit(self, X, y):
         self.feature_importances_ = scores
 
 
-@hydra.main(config_path="conf", config_name="my_config")
+@hydra.main(config_path="conf", config_name="my_config", version_base="1.1")
 def main(cfg: PipelineConfig) -> None:
     run_pipeline(cfg)
 

fseval/metrics/feature_importances.py

@@ -37,7 +37,7 @@ def _build_table(self, feature_vector: np.ndarray):
                         "class": class_index,
                     }
                 )
-                df = df.append(df_class)
+                df = pd.concat([df, df_class])
 
             return df
 
@@ -88,14 +88,14 @@ def score_ranking(
             estimated_df = self._build_table(estimated)
             estimated_df["group"] = "estimated"
             estimated_df["bootstrap_state"] = bootstrap_state
-            table = table.append(estimated_df)
+            table = pd.concat([table, estimated_df])
 
         if feature_importances is not None:
             ground_truth = self._normalize_feature_importances(feature_importances)
             ground_truth_df = self._build_table(ground_truth)
             ground_truth_df["group"] = "ground_truth"
             ground_truth_df["bootstrap_state"] = bootstrap_state
-            table = table.append(ground_truth_df)
+            table = pd.concat([table, ground_truth_df])
 
         if not table.empty:
             callbacks.on_table(table, "feature_importances")

fseval/pipelines/_experiment.py

@@ -176,7 +176,7 @@ def postfit(self):
                 estimator.postfit()
 
     def _aggregate_dataframe_scores(self, a: pd.DataFrame, b: pd.DataFrame):
-        return a.append(b)
+        return pd.concat([a, b])
 
     def _aggregate_dict_scores(self, a: Dict, b: Dict):
         aggregated = {}

fseval/utils/hydra_utils.py

@@ -20,7 +20,9 @@ def _ensure_hydra_initialized(
     """Initializes Hydra only if it is not already initialized."""
     gh = GlobalHydra()
     if not gh.is_initialized():
-        initialize_config_module(config_module=config_module, job_name=job_name)
+        initialize_config_module(
+            config_module=config_module, job_name=job_name, version_base="1.1"
+        )
 
 
 def get_config(

requirements.txt

@@ -1,5 +1,5 @@
-hydra-core==1.1.2
-hydra-colorlog==1.1.0
+hydra-core>=1.1.2
+hydra-colorlog>=1.1.0
 numpy>=1.19
 pandas>=1.1
 scikit-learn>=0.24

setup.py

@@ -6,7 +6,7 @@
     LONG_DESC = fh.read()
     setup(
         name="fseval",
-        version="3.0.2",
+        version="3.0.3",
         packages=find_namespace_packages(include=["hydra_plugins.*"])
         + find_packages(include=["fseval", "fseval.*"]),
         entry_points={"console_scripts": ["fseval = fseval.main:main"]},
@@ -28,8 +28,8 @@
         },
         include_package_data=True,
         install_requires=[
-            "hydra-core==1.1.2",
-            "hydra-colorlog==1.1.0",
+            "hydra-core>=1.1.2",
+            "hydra-colorlog>=1.1.0",
             "numpy>=1.19",
             "pandas>=1.1",
             "scikit-learn>=0.24",
@@ -47,6 +47,7 @@
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",
+            "Programming Language :: Python :: 3.10",
             "Operating System :: POSIX :: Linux",
             "Operating System :: MacOS",
             "Operating System :: Microsoft :: Windows",

tests/unit/callbacks/test_to_wandb.py

@@ -1,3 +1,4 @@
+from time import sleep
 import pandas as pd
 import pytest
 import wandb
@@ -112,6 +113,18 @@ def test_on_summary(wandb_callback: WandbCallback):
     # → assertions in `test_on_end`
 
 
+@pytest.mark.xfail(
+    reason="""
+    The Weights-and-Biases platform has proven to have too large delays to be able to
+    run integration tests on it consistently. Alhough the WandB platform does
+    eventually upload all the data that is being sent to it, all logging is queued,
+    making it hard to rely on it in integration-/ unit test settings.
+    Therefore this test is marked as optional to prevent unnecessary build failures.
+    The test is kept in the codebase, however, for convenience in local debugging.
+
+    See https://github.com/dunnkers/fseval/issues/78
+"""
+)
 @pytest.mark.dependency(
     depends=[
         "test_on_begin",

tests/unit/storage/test_wandb.py

@@ -44,7 +44,18 @@ def test_save(request: FixtureRequest):
     cache.set("filename", filename)
 
 
-@pytest.mark.xfail(reason="Weights and Biases might not have saved the file correctly.")
+@pytest.mark.xfail(
+    reason="""
+    The Weights-and-Biases platform has proven to have too large delays to be able to
+    run integration tests on it consistently. Alhough the WandB platform does
+    eventually upload all the data that is being sent to it, all logging is queued,
+    making it hard to rely on it in integration-/ unit test settings.
+    Therefore this test is marked as optional to prevent unnecessary build failures.
+    The test is kept in the codebase, however, for convenience in local debugging.
+
+    See https://github.com/dunnkers/fseval/issues/78
+"""
+)
 @pytest.mark.dependency(depends=["test_save"])
 def test_load(request: FixtureRequest):
     # retrieve previous run id from cache

website/docs/motivation.md

@@ -0,0 +1,22 @@
+---
+sidebar_position: 0
+---
+
+# Motivation
+
+fseval helps you benchmark **Feature Selection** and **Feature Ranking** algorithms. Any algorithm that ranks features in importance.
+
+It comes useful if you are one of the following types of users:
+1. **Feature Selection / Feature Ranker algorithm authors**. You are the author of a novel Feature Selection algorithm. Now, you have to prove the performance of your algorithm against other competitors. Therefore, you are going to run a large-scale benchmark. Many authors, however, spend much time rewriting similar pipelines to benchmark their algorithms. fseval helps you run benchmarks in a structured manner, on supercomputer clusters or on the cloud.
+1. **Interpretable AI method authors**. You wrote a new Interpretable AI method that aims to find out which features are most meaningful by ranking them. Now, the challenge is to find out how well your method ranked those features. fseval can help with this.
+1. **Machine Learning practitioners**. You have a dataset and want to find out with exactly what features your models will perform best. You can use fseval to try multiple Feature Selection or Feature Ranking algorithms.
+
+
+
+Key features 🚀:
+- Easily benchmark Feature Ranking algorithms
+- Built on [Hydra](https://hydra.cc/)
+- Support for distributed systems (SLURM through the [Submitit launcher](https://hydra.cc/docs/plugins/submitit_launcher), AWS support through the [Ray launcher](https://hydra.cc/docs/plugins/ray_launcher/))
+- Reproducible experiments (your entire experiment can be described and reproduced by 1 YAML file)
+- Send experiment results directly to a dashboard (integration with [Weights and Biases](https://wandb.ai/) is built-in)
+- Export your data to any SQL database (integration with [SQLAlchemy](https://www.sqlalchemy.org/))

website/docs/quick-start.mdx

@@ -2,7 +2,7 @@
 sidebar_position: 1
 ---
 
-# Quick start
+# Getting started
 
 <!-- Docusaurus -->
 import Tabs from '@theme/Tabs';
@@ -31,35 +31,30 @@ import BenchmarkPy from '!!raw-loader!../../examples/quick-start-structured-conf
 import AsciinemaPlayer from '../src/components/AsciinemaPlayer';
 import 'asciinema-player/dist/bundle/asciinema-player.css';
 
-fseval helps you benchmark **Feature Selection** and **Feature Ranking** algorithms. Any algorithm that ranks features in importance.
+To get started, there's two main resources.
 
-It comes useful if you are one of the following types of users:
-1. **Feature Selection / Feature Ranker algorithm authors**. You are the author of a novel Feature Selection algorithm. Now, you have to prove the performance of your algorithm against other competitors. Therefore, you are going to run a large-scale benchmark. Many authors, however, spend much time rewriting similar pipelines to benchmark their algorithms. fseval helps you run benchmarks in a structured manner, on supercomputer clusters or on the cloud.
-1. **Interpretable AI method authors**. You wrote a new Interpretable AI method that aims to find out which features are most meaningful by ranking them. Now, the challenge is to find out how well your method ranked those features. fseval can help with this.
-1. **Machine Learning practitioners**. You have a dataset and want to find out with exactly what features your models will perform best. You can use fseval to try multiple Feature Selection or Feature Ranking algorithms.
+1. A [Google Colab](https://colab.research.google.com/drive/1Bsuxxuw0-mEsYRSnNbmvD_wNUAkOPiQa?usp=sharing)
+  
+  [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1Bsuxxuw0-mEsYRSnNbmvD_wNUAkOPiQa?usp=sharing)
+2. The [⚡️ Quick start](#%EF%B8%8F-quick-start) guide below 👇🏻
 
 
+## ⚡️ Quick start
 
-Key features 🚀:
-- Easily benchmark Feature Ranking algorithms
-- Built on [Hydra](https://hydra.cc/)
-- Support for distributed systems (SLURM through the [Submitit launcher](https://hydra.cc/docs/plugins/submitit_launcher), AWS support through the [Ray launcher](https://hydra.cc/docs/plugins/ray_launcher/))
-- Reproducible experiments (your entire experiment can be described and reproduced by 1 YAML file)
-- Send experiment results directly to a dashboard (integration with [Weights and Biases](https://wandb.ai/) is built-in)
-- Export your data to any SQL database (integration with [SQLAlchemy](https://www.sqlalchemy.org/))
-
-## Getting started
+Let's run our first experiment. The goal will be to compare two feature selectors _ANOVA F-Value_ and _Mutual Info_.
 
 Install fseval:
-```shell
+
+```
 pip install fseval
 ```
 
-Given the following [configuration](https://github.com/dunnkers/fseval/tree/master/examples/quick-start):
 
 <Tabs groupId="config-representation">
 <TabItem value="yaml" label="YAML" default>
 
+Given the following [configuration](https://github.com/dunnkers/fseval/tree/master/examples/quick-start-yaml):
+
 <FileTreeCodeViewer treeId="tree-1" template={{
   root: {
     "conf": {
@@ -87,6 +82,7 @@ Given the following [configuration](https://github.com/dunnkers/fseval/tree/mast
 </TabItem>
 <TabItem value="structured" label="Structured Config">
 
+Given the following [configuration](https://github.com/dunnkers/fseval/tree/master/examples/quick-start-structured-configs):
 
 <FileTreeCodeViewer treeId="tree-2" template={{
   root: {

website/zip-examples.sh

@@ -0,0 +1,3 @@
+cd ../examples
+zip -r ../website/static/quick-start-structured-configs quick-start-structured-configs
+zip -r ../website/static/quick-start-yaml quick-start-yaml