Merge main

microsoft · Mar 29, 2023 · c31d3af · c31d3af
1 parent 651347b
commit c31d3af
Show file tree

Hide file tree

Showing 63 changed files with 1,508 additions and 7,195 deletions.
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -9,6 +9,3 @@ jobs:
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
         sync-labels: true
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -58,7 +58,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -72,12 +72,10 @@ jobs:
     - name: Install pip dependencies
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
-        pip install .[datasets,docs,tests]
+        pip install .[docs,tests] planetary_computer pystac pytest-rerunfailures
         pip list
     - name: Run notebook checks
-      env:
-        MLHUB_API_KEY: ${{ secrets.MLHUB_API_KEY }}
-      run: pytest --nbmake docs/tutorials --durations=10
+      run: pytest --nbmake --durations=10 --reruns=10 docs/tutorials
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -83,7 +83,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -106,7 +106,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -43,7 +43,7 @@ jobs:
         python-version: ['3.8', '3.9', '3.10']
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -95,7 +95,7 @@ jobs:
       MPLBACKEND: Agg
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:

diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repo
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.5.0
     - name: Set up python
       uses: actions/setup-python@v4.5.0
       with:
@@ -30,12 +30,10 @@ jobs:
     - name: Install pip dependencies
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
-        pip install .[datasets,docs,tests]
+        pip install .[docs,tests] planetary_computer pystac pytest-rerunfailures
         pip list
     - name: Run notebook checks
-      env:
-        MLHUB_API_KEY: ${{ secrets.MLHUB_API_KEY }}
-      run: pytest --nbmake --nbmake-timeout=3000 docs/tutorials --durations=10
+      run: pytest --nbmake --durations=10 --reruns=10 docs/tutorials
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
diff --git a/benchmark.py b/benchmark.py
@@ -7,7 +7,7 @@
 import os
 import time
 
-import lightning as pl
+import lightning.pytorch as pl
 import torch
 import torch.nn as nn
 import torch.optim as optim

diff --git a/docs/api/datamodules.rst b/docs/api/datamodules.rst
@@ -43,6 +43,7 @@ EuroSAT
 ^^^^^^^
 
 .. autoclass:: EuroSATDataModule
+.. autoclass:: EuroSAT100DataModule
 
 FAIR1M
 ^^^^^^

diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst
@@ -195,6 +195,7 @@ EuroSAT
 ^^^^^^^
 
 .. autoclass:: EuroSAT
+.. autoclass:: EuroSAT100
 
 FAIR1M
 ^^^^^^

diff --git a/docs/conf.py b/docs/conf.py
@@ -62,6 +62,7 @@
     ("py:class", "segmentation_models_pytorch.base.model.SegmentationModel"),
     ("py:class", "timm.models.resnet.ResNet"),
     ("py:class", "timm.models.vision_transformer.VisionTransformer"),
+    ("py:class", "torch.optim.lr_scheduler.LRScheduler"),
     ("py:class", "torchvision.models._api.WeightsEnum"),
     ("py:class", "torchvision.models.resnet.ResNet"),
 ]

diff --git a/docs/tutorials/benchmarking.ipynb b/docs/tutorials/benchmarking.ipynb
@@ -33,7 +33,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -58,7 +58,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {
     "gather": {
      "logged": 1629238744113
@@ -90,12 +90,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_root = tempfile.gettempdir()\n",
-    "naip_root = os.path.join(data_root, \"naip\")\n",
+    "naip_root = os.path.join(tempfile.gettempdir(), \"naip\")\n",
     "naip_url = (\n",
     "    \"https://naipeuwest.blob.core.windows.net/naip/v002/de/2018/de_060cm_2018/38075/\"\n",
     ")\n",
@@ -118,12 +117,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "chesapeake_root = os.path.join(data_root, \"chesapeake\")\n",
-    "\n",
+    "chesapeake_root = os.path.join(tempfile.gettempdir(), \"chesapeake\")\n",
     "chesapeake = ChesapeakeDE(chesapeake_root, download=True)"
    ]
   },
@@ -143,7 +141,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {
     "gather": {
      "logged": 1629238744228
@@ -167,6 +165,34 @@
     "    return toc - tic, i"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The following variables can be modified to control the number of samples drawn per epoch."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "nbmake": {
+     "mock": {
+      "batch_size": 1,
+      "length": 1,
+      "size": 1,
+      "stride": 1000000
+     }
+    }
+   },
+   "outputs": [],
+   "source": [
+    "size = 1000\n",
+    "length = 888\n",
+    "batch_size = 12\n",
+    "stride = 500"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -183,7 +209,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {
     "gather": {
      "logged": 1629248963725
@@ -197,24 +223,15 @@
     "outputId": "edcc8199-bd09-4832-e50c-7be8ac78995b",
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "296.582683801651 74\n",
-      "54.20210099220276 74\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "for cache in [False, True]:\n",
     "    chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
     "    naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
     "    dataset = chesapeake & naip\n",
-    "    sampler = RandomGeoSampler(dataset, size=1000, length=888)\n",
+    "    sampler = RandomGeoSampler(dataset, size=size, length=length)\n",
     "    dataloader = DataLoader(\n",
-    "        dataset, batch_size=12, sampler=sampler, collate_fn=stack_samples\n",
+    "        dataset, batch_size=batch_size, sampler=sampler, collate_fn=stack_samples\n",
     "    )\n",
     "    duration, count = time_epoch(dataloader)\n",
     "    print(duration, count)"
@@ -236,7 +253,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {
     "gather": {
      "logged": 1629239313388
@@ -250,24 +267,15 @@
     "outputId": "159ce99f-a438-4ecc-d218-9b9e28d02055",
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "391.90197944641113 74\n",
-      "118.0611424446106 74\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "for cache in [False, True]:\n",
     "    chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
     "    naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
     "    dataset = chesapeake & naip\n",
-    "    sampler = GridGeoSampler(dataset, size=1000, stride=500)\n",
+    "    sampler = GridGeoSampler(dataset, size=size, stride=stride)\n",
     "    dataloader = DataLoader(\n",
-    "        dataset, batch_size=12, sampler=sampler, collate_fn=stack_samples\n",
+    "        dataset, batch_size=batch_size, sampler=sampler, collate_fn=stack_samples\n",
     "    )\n",
     "    duration, count = time_epoch(dataloader)\n",
     "    print(duration, count)"
@@ -289,7 +297,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {
     "gather": {
      "logged": 1629249843438
@@ -303,22 +311,15 @@
     "outputId": "497f6869-1ab7-4db7-bbce-e943b493ca41",
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "230.51380324363708 74\n",
-      "53.99923872947693 74\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "for cache in [False, True]:\n",
     "    chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
     "    naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
     "    dataset = chesapeake & naip\n",
-    "    sampler = RandomBatchGeoSampler(dataset, size=1000, batch_size=12, length=888)\n",
+    "    sampler = RandomBatchGeoSampler(\n",
+    "        dataset, size=size, batch_size=batch_size, length=length\n",
+    "    )\n",
     "    dataloader = DataLoader(dataset, batch_sampler=sampler, collate_fn=stack_samples)\n",
     "    duration, count = time_epoch(dataloader)\n",
     "    print(duration, count)"
@@ -349,10 +350,10 @@
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
-   "name": "ipython",
+   "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.7"
+   "version": "3.10.8"
   },
   "nteract": {
    "version": "nteract-front-end@1.0.0"
-Original file line number
+Diff line change
@@ Expand Up / @@ -43,6 +43,7 @@ EuroSAT @@
     ^^^^^^^
     .. autoclass:: EuroSATDataModule
+    .. autoclass:: EuroSAT100DataModule
     FAIR1M
     ^^^^^^
@@ Expand Down @@