From 511ec78014a8347c3e6e9ad4bee92577ec8016b2 Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Tue, 10 Jan 2023 18:45:25 +0000
Subject: [PATCH 1/7] Add python integration test using GitHub workflow of
 partitioning_nt

---
 .github/workflows/python-app.yaml             |  38 ++++++
 oneflux_steps/Makefile                        |   2 +-
 pytest.ini                                    |   5 +
 tests/context.py                              |  19 ---
 tests/{ => python}/__init__.py                |   0
 tests/python/integration/test_partitioning.py | 116 ++++++++++++++++++
 tests/python/test_context.py                  |  14 +++
 tests/test_context.py                         |  21 ----
 8 files changed, 174 insertions(+), 41 deletions(-)
 create mode 100644 .github/workflows/python-app.yaml
 create mode 100644 pytest.ini
 delete mode 100644 tests/context.py
 rename tests/{ => python}/__init__.py (100%)
 create mode 100644 tests/python/integration/test_partitioning.py
 create mode 100644 tests/python/test_context.py
 delete mode 100644 tests/test_context.py

diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml
new file mode 100644
index 00000000..9bf13cce
--- /dev/null
+++ b/.github/workflows/python-app.yaml
@@ -0,0 +1,38 @@
+name: ONEFlux CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 2.7.18
+      uses: actions/setup-python@v3
+      with:
+        python-version: "2.7.18"
+    - name: Install OneFLUX
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel pytest
+        make
+    - name: Download data
+      run: |
+        # get US-ARc_sample data for tests
+        mkdir -p ./tests/data/test_input
+        mkdir -p ./tests/data/test_output
+        wget ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip
+        wget ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip
+        unzip US-ARc_sample_input.zip -d ./tests/data/test_input
+        unzip US-ARc_sample_output.zip -d ./tests/data/test_output
+    - name: Run pytest
+      run: |
+        export PYTHONPATH=/home/runner/work/ONEFlux/ONEFlux:$PYTHONPATH
+        pytest 
diff --git a/oneflux_steps/Makefile b/oneflux_steps/Makefile
index d56115bf..a5e22cc9 100644
--- a/oneflux_steps/Makefile
+++ b/oneflux_steps/Makefile
@@ -31,7 +31,7 @@ CC := gcc -O3
 MKDIR = mkdir -p
 
 # copy file command (verbose, keep file metadata)
-COPY = cp -av
+COPY = cp -v
 
 SRCDIR := $(shell pwd)/
 TGTDIR := ${HOME}/bin/oneflux/
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000..f5276a23
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+log_cli = 1
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+log_cli_date_format=%Y-%m-%d %H:%M:%S
diff --git a/tests/context.py b/tests/context.py
deleted file mode 100644
index a619dd4a..00000000
--- a/tests/context.py
+++ /dev/null
@@ -1,19 +0,0 @@
-'''
-For license information:
-see LICENSE file or headers in oneflux.__init__.py
-
-Context file for tests; import resolution when path not set
-
-@author: Gilberto Pastorello
-@contact: gzpastorello@lbl.gov
-@date: 2017-01-31
-'''
-import sys
-
-# support execution without package set (e.g., call unittest from command line)
-if __package__ is None:
-    import os
-    path = os.path.dirname(os.path.dirname(__file__))
-    sys.path.insert(0, path)
-
-import oneflux
diff --git a/tests/__init__.py b/tests/python/__init__.py
similarity index 100%
rename from tests/__init__.py
rename to tests/python/__init__.py
diff --git a/tests/python/integration/test_partitioning.py b/tests/python/integration/test_partitioning.py
new file mode 100644
index 00000000..1e313782
--- /dev/null
+++ b/tests/python/integration/test_partitioning.py
@@ -0,0 +1,116 @@
+import pytest
+import os, glob
+import errno
+import shutil
+import urllib
+from distutils.dir_util import copy_tree
+import logging
+import time
+
+_log = logging.getLogger(__name__)
+
+@pytest.fixture(scope="module")
+def get_data():
+    '''
+    Utilising python to obtain sample test data. Function currently unused. 
+    as a fixture in this class. 
+    '''
+    from zipfile import ZipFile
+    urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip') 
+    urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip') 
+  
+    input_zip = "US-ARc_sample_input.zip"
+    output_zip = "US-ARc_sample_output.zip"
+    
+    with ZipFile(input_zip) as zi, ZipFile(output_zip) as zo:
+        zi.extractall(path='tests/data/test_input')
+        zo.extractall(path='tests/data/test_output')
+
+def equal_csv(csv_1, csv_2):
+    '''
+    Check equality of two csv files.
+    '''
+    _log.info("Check csv equality")
+    start = time.time()
+    with open(csv_1, 'r') as t1, open(csv_2, 'r') as t2:
+        fileone = t1.readlines()
+        filetwo = t2.readlines()
+        for line in filetwo:
+            if line not in fileone:
+                return False
+
+        _log.info("total time", start - time.time())
+
+        return True
+
+
+@pytest.fixture
+def setup_data():
+    '''
+    Set up input data for run_partition_nt test. 
+    
+    Create data directory for tests './tests/integration/step10' and copy 
+    data from expected output ('./datadir/test_output/US-ARc_sample_output')
+    to this directory.
+    '''
+    try:
+        os.mkdir('tests/integration/data/step_10')
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            print("directory exists")
+            
+    testdata = 'tests/python/integration/input/step_10/US-ARc_sample_input'
+    
+    copy_tree('tests/data/test_input/', testdata)
+
+    refoutdir = 'tests/data/test_output/US-ARc_sample_output'
+
+    copy_tree(os.path.join(refoutdir, '07_meteo_proc'), \
+        os.path.join(testdata, '07_meteo_proc'))
+    copy_tree(os.path.join(refoutdir, '08_nee_proc'), \
+        os.path.join(testdata, '08_nee_proc/'))
+    copy_tree(os.path.join(refoutdir, '02_qc_auto'), \
+        os.path.join(testdata, '02_qc_auto/'))
+    
+    
+def test_run_partition_nt(setup_data):
+    '''
+    Run partition_nt on single percentile.
+    '''
+    datadir = "./tests/python/integration/input/step_10/"
+    refoutdir = "./tests/data/test_output/"
+    siteid = "US-ARc"
+    sitedir = "US-ARc_sample_input"
+    years = [2005] # years = [2005, 2006]
+    # PROD_TO_COMPARE = ['c', 'y']
+    PROD_TO_COMPARE = ['y',]
+    # PERC_TO_COMPARE = ['1.25', '3.75',]
+    PERC_TO_COMPARE = ['1.25',]
+    
+    from oneflux.tools.partition_nt import remove_previous_run, run_python
+    remove_previous_run(datadir=datadir, siteid=siteid, sitedir=sitedir, python=True, 
+                        prod_to_compare=PROD_TO_COMPARE, perc_to_compare=PERC_TO_COMPARE,
+                        years_to_compare=years)
+
+    run_python(datadir=datadir, siteid=siteid, sitedir=sitedir, prod_to_compare=PROD_TO_COMPARE,
+               perc_to_compare=PERC_TO_COMPARE, years_to_compare=years)
+    
+    # check whether csv of "output" is same as csv of reference
+
+    # the generated output is actually in the "input" directory.
+    rootdir = os.path.join(datadir, sitedir, "10_nee_partition_nt")
+    nee_y_files = glob.glob(os.path.join(rootdir, "nee_y_1.25_US-ARc_2005*"))
+    nee_y_files = filter(lambda x: not x.endswith('_orig.csv'), nee_y_files)
+    
+    # paths to the "reference" output data
+    refoutdir = os.path.join(refoutdir, "US-ARc_sample_output", "10_nee_partition_nt")
+    ref_nee_y_files = glob.glob(os.path.join(refoutdir, "nee_y_1.25_US-ARc_2005*"))
+   
+    assert len(nee_y_files) == len(ref_nee_y_files)
+    retval = True 
+    for f, b in zip(nee_y_files, ref_nee_y_files):
+        print(f, b)
+        assert equal_csv(f, b) == True
+
+    # clean up data. 
+    # shutil.rmtree(datadir)
diff --git a/tests/python/test_context.py b/tests/python/test_context.py
new file mode 100644
index 00000000..c8b3c3f7
--- /dev/null
+++ b/tests/python/test_context.py
@@ -0,0 +1,14 @@
+'''
+For license information:
+see LICENSE file or headers in oneflux.__init__.py
+
+Simple context/import setup test
+
+@author: Gilberto Pastorello
+@contact: gzpastorello@lbl.gov
+@date: 2017-01-31
+'''
+
+def test_import_oneflux():
+    import oneflux
+    assert hasattr(oneflux, '__version__') == True
diff --git a/tests/test_context.py b/tests/test_context.py
deleted file mode 100644
index 9ec95d17..00000000
--- a/tests/test_context.py
+++ /dev/null
@@ -1,21 +0,0 @@
-'''
-For license information:
-see LICENSE file or headers in oneflux.__init__.py
-
-Simple context/import setup test
-
-@author: Gilberto Pastorello
-@contact: gzpastorello@lbl.gov
-@date: 2017-01-31
-'''
-import unittest
-
-from context import oneflux
-
-class BasicTest(unittest.TestCase):
-    def test_context(self):
-        """Test import by checking imported 'oneflux' module has '__version__' attribute"""
-        self.assertTrue(hasattr(oneflux, '__version__'))
-
-if __name__ == '__main__':
-    unittest.main()

From e0ea70e506475a954d5f903be705f163afbbd500 Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Fri, 18 Oct 2024 10:34:03 +0000
Subject: [PATCH 2/7] Add upper bound to statsmodels

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 44066824..268aa467 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 numpy>=1.11.0,<1.16.0
 scipy>=0.17.0
 matplotlib>=1.5.1
-statsmodels>=0.8.0
+statsmodels>=0.8.0,<0.11.0

From f3d77e476315ae1e912f3584b82f268cd533b236 Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Thu, 17 Oct 2024 15:45:55 +0000
Subject: [PATCH 3/7] Change name of test reference data and change assert
 statement

---
 .github/workflows/python-app.yaml             |  2 +-
 tests/python/integration/test_partitioning.py | 16 ++++++++--------
 tests/python/test_context.py                  |  3 ++-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml
index 9bf13cce..af538cad 100644
--- a/.github/workflows/python-app.yaml
+++ b/.github/workflows/python-app.yaml
@@ -31,7 +31,7 @@ jobs:
         wget ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip
         wget ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip
         unzip US-ARc_sample_input.zip -d ./tests/data/test_input
-        unzip US-ARc_sample_output.zip -d ./tests/data/test_output
+        unzip US-ARc_sample_output.zip -d ./tests/data/test_reference
     - name: Run pytest
       run: |
         export PYTHONPATH=/home/runner/work/ONEFlux/ONEFlux:$PYTHONPATH
diff --git a/tests/python/integration/test_partitioning.py b/tests/python/integration/test_partitioning.py
index 1e313782..0c176697 100644
--- a/tests/python/integration/test_partitioning.py
+++ b/tests/python/integration/test_partitioning.py
@@ -12,8 +12,8 @@
 @pytest.fixture(scope="module")
 def get_data():
     '''
-    Utilising python to obtain sample test data. Function currently unused. 
-    as a fixture in this class. 
+    Obtain sample test data using a fixture
+    Function currently unused.
     '''
     from zipfile import ZipFile
     urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip') 
@@ -24,7 +24,7 @@ def get_data():
     
     with ZipFile(input_zip) as zi, ZipFile(output_zip) as zo:
         zi.extractall(path='tests/data/test_input')
-        zo.extractall(path='tests/data/test_output')
+        zo.extractall(path='tests/data/test_reference')
 
 def equal_csv(csv_1, csv_2):
     '''
@@ -55,6 +55,7 @@ def setup_data():
     '''
     try:
         os.mkdir('tests/integration/data/step_10')
+        os.mkdir('tests/data/test_reference')
     except OSError as e:
         if e.errno == errno.EEXIST:
             print("directory exists")
@@ -63,7 +64,7 @@ def setup_data():
     
     copy_tree('tests/data/test_input/', testdata)
 
-    refoutdir = 'tests/data/test_output/US-ARc_sample_output'
+    refoutdir = 'tests/data/test_reference/US-ARc_sample_output'
 
     copy_tree(os.path.join(refoutdir, '07_meteo_proc'), \
         os.path.join(testdata, '07_meteo_proc'))
@@ -78,7 +79,7 @@ def test_run_partition_nt(setup_data):
     Run partition_nt on single percentile.
     '''
     datadir = "./tests/python/integration/input/step_10/"
-    refoutdir = "./tests/data/test_output/"
+    refoutdir = "./tests/data/test_reference/"
     siteid = "US-ARc"
     sitedir = "US-ARc_sample_input"
     years = [2005] # years = [2005, 2006]
@@ -95,7 +96,7 @@ def test_run_partition_nt(setup_data):
     run_python(datadir=datadir, siteid=siteid, sitedir=sitedir, prod_to_compare=PROD_TO_COMPARE,
                perc_to_compare=PERC_TO_COMPARE, years_to_compare=years)
     
-    # check whether csv of "output" is same as csv of reference
+    # check whether csv of output in input directory is same as csv of reference
 
     # the generated output is actually in the "input" directory.
     rootdir = os.path.join(datadir, sitedir, "10_nee_partition_nt")
@@ -107,10 +108,9 @@ def test_run_partition_nt(setup_data):
     ref_nee_y_files = glob.glob(os.path.join(refoutdir, "nee_y_1.25_US-ARc_2005*"))
    
     assert len(nee_y_files) == len(ref_nee_y_files)
-    retval = True 
     for f, b in zip(nee_y_files, ref_nee_y_files):
         print(f, b)
-        assert equal_csv(f, b) == True
+        assert equal_csv(f, b) 
 
     # clean up data. 
     # shutil.rmtree(datadir)
diff --git a/tests/python/test_context.py b/tests/python/test_context.py
index c8b3c3f7..9104458a 100644
--- a/tests/python/test_context.py
+++ b/tests/python/test_context.py
@@ -11,4 +11,5 @@
 
 def test_import_oneflux():
     import oneflux
-    assert hasattr(oneflux, '__version__') == True
+
+    assert hasattr(oneflux, "__version__")

From 6017186c5556b68c9d80a3793e67e89a17ff3409 Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Fri, 18 Oct 2024 09:16:15 +0000
Subject: [PATCH 4/7] Update action for Python2 support

Github no longer provides support for python2.7 from their action/setup-python action
---
 .github/workflows/python-app.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml
index af538cad..c9288505 100644
--- a/.github/workflows/python-app.yaml
+++ b/.github/workflows/python-app.yaml
@@ -12,12 +12,10 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-20.04
+    container:
+      image: python:2.7.18-buster
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 2.7.18
-      uses: actions/setup-python@v3
-      with:
-        python-version: "2.7.18"
+    - uses: actions/checkout@v4
     - name: Install OneFLUX
       run: |
         python -m pip install --upgrade pip

From 272ddb9a7ac0618e1fecfe8e835648a17f3cf06e Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Fri, 18 Oct 2024 10:21:57 +0000
Subject: [PATCH 5/7] Add docstrings

---
 tests/python/integration/test_partitioning.py | 16 ++++++++--------
 tests/python/test_context.py                  |  3 +++
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/python/integration/test_partitioning.py b/tests/python/integration/test_partitioning.py
index 0c176697..3c12a62b 100644
--- a/tests/python/integration/test_partitioning.py
+++ b/tests/python/integration/test_partitioning.py
@@ -11,10 +11,10 @@
 
 @pytest.fixture(scope="module")
 def get_data():
-    '''
+    """
     Obtain sample test data using a fixture
     Function currently unused.
-    '''
+    """
     from zipfile import ZipFile
     urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip') 
     urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip') 
@@ -27,9 +27,9 @@ def get_data():
         zo.extractall(path='tests/data/test_reference')
 
 def equal_csv(csv_1, csv_2):
-    '''
+    """
     Check equality of two csv files.
-    '''
+    """
     _log.info("Check csv equality")
     start = time.time()
     with open(csv_1, 'r') as t1, open(csv_2, 'r') as t2:
@@ -46,13 +46,13 @@ def equal_csv(csv_1, csv_2):
 
 @pytest.fixture
 def setup_data():
-    '''
+    """
     Set up input data for run_partition_nt test. 
     
     Create data directory for tests './tests/integration/step10' and copy 
     data from expected output ('./datadir/test_output/US-ARc_sample_output')
     to this directory.
-    '''
+    """
     try:
         os.mkdir('tests/integration/data/step_10')
         os.mkdir('tests/data/test_reference')
@@ -75,9 +75,9 @@ def setup_data():
     
     
 def test_run_partition_nt(setup_data):
-    '''
+    """
     Run partition_nt on single percentile.
-    '''
+    """
     datadir = "./tests/python/integration/input/step_10/"
     refoutdir = "./tests/data/test_reference/"
     siteid = "US-ARc"
diff --git a/tests/python/test_context.py b/tests/python/test_context.py
index 9104458a..1fa60871 100644
--- a/tests/python/test_context.py
+++ b/tests/python/test_context.py
@@ -10,6 +10,9 @@
 '''
 
 def test_import_oneflux():
+    """
+    Test import by checking imported 'oneflux' module has '__version__' attribute
+    """
     import oneflux
 
     assert hasattr(oneflux, "__version__")

From f659e654a509996eab51a2bfc616ea8c2cc2f29a Mon Sep 17 00:00:00 2001
From: Matthew Archer <ma595@cam.ac.uk>
Date: Fri, 18 Oct 2024 10:28:20 +0000
Subject: [PATCH 6/7] Apply style changes using Ruff

---
 tests/python/integration/test_partitioning.py | 112 +++++++++++-------
 tests/python/test_context.py                  |   5 +-
 2 files changed, 73 insertions(+), 44 deletions(-)

diff --git a/tests/python/integration/test_partitioning.py b/tests/python/integration/test_partitioning.py
index 3c12a62b..4c3e08e7 100644
--- a/tests/python/integration/test_partitioning.py
+++ b/tests/python/integration/test_partitioning.py
@@ -9,6 +9,7 @@
 
 _log = logging.getLogger(__name__)
 
+
 @pytest.fixture(scope="module")
 def get_data():
     """
@@ -16,15 +17,21 @@ def get_data():
     Function currently unused.
     """
     from zipfile import ZipFile
-    urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip') 
-    urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip') 
-  
+
+    urllib.urlopen(
+        "ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip"
+    )
+    urllib.urlopen(
+        "ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip"
+    )
+
     input_zip = "US-ARc_sample_input.zip"
     output_zip = "US-ARc_sample_output.zip"
-    
+
     with ZipFile(input_zip) as zi, ZipFile(output_zip) as zo:
-        zi.extractall(path='tests/data/test_input')
-        zo.extractall(path='tests/data/test_reference')
+        zi.extractall(path="tests/data/test_input")
+        zo.extractall(path="tests/data/test_reference")
+
 
 def equal_csv(csv_1, csv_2):
     """
@@ -32,7 +39,7 @@ def equal_csv(csv_1, csv_2):
     """
     _log.info("Check csv equality")
     start = time.time()
-    with open(csv_1, 'r') as t1, open(csv_2, 'r') as t2:
+    with open(csv_1, "r") as t1, open(csv_2, "r") as t2:
         fileone = t1.readlines()
         filetwo = t2.readlines()
         for line in filetwo:
@@ -47,33 +54,37 @@ def equal_csv(csv_1, csv_2):
 @pytest.fixture
 def setup_data():
     """
-    Set up input data for run_partition_nt test. 
-    
-    Create data directory for tests './tests/integration/step10' and copy 
+    Set up input data for run_partition_nt test.
+
+    Create data directory for tests './tests/integration/step10' and copy
     data from expected output ('./datadir/test_output/US-ARc_sample_output')
     to this directory.
     """
     try:
-        os.mkdir('tests/integration/data/step_10')
-        os.mkdir('tests/data/test_reference')
+        os.mkdir("tests/integration/data/step_10")
+        os.mkdir("tests/data/test_reference")
     except OSError as e:
         if e.errno == errno.EEXIST:
             print("directory exists")
-            
-    testdata = 'tests/python/integration/input/step_10/US-ARc_sample_input'
-    
-    copy_tree('tests/data/test_input/', testdata)
-
-    refoutdir = 'tests/data/test_reference/US-ARc_sample_output'
-
-    copy_tree(os.path.join(refoutdir, '07_meteo_proc'), \
-        os.path.join(testdata, '07_meteo_proc'))
-    copy_tree(os.path.join(refoutdir, '08_nee_proc'), \
-        os.path.join(testdata, '08_nee_proc/'))
-    copy_tree(os.path.join(refoutdir, '02_qc_auto'), \
-        os.path.join(testdata, '02_qc_auto/'))
-    
-    
+
+    testdata = "tests/python/integration/input/step_10/US-ARc_sample_input"
+
+    copy_tree("tests/data/test_input/", testdata)
+
+    refoutdir = "tests/data/test_reference/US-ARc_sample_output"
+
+    copy_tree(
+        os.path.join(refoutdir, "07_meteo_proc"),
+        os.path.join(testdata, "07_meteo_proc"),
+    )
+    copy_tree(
+        os.path.join(refoutdir, "08_nee_proc"), os.path.join(testdata, "08_nee_proc/")
+    )
+    copy_tree(
+        os.path.join(refoutdir, "02_qc_auto"), os.path.join(testdata, "02_qc_auto/")
+    )
+
+
 def test_run_partition_nt(setup_data):
     """
     Run partition_nt on single percentile.
@@ -82,35 +93,52 @@ def test_run_partition_nt(setup_data):
     refoutdir = "./tests/data/test_reference/"
     siteid = "US-ARc"
     sitedir = "US-ARc_sample_input"
-    years = [2005] # years = [2005, 2006]
+    years = [2005]  # years = [2005, 2006]
     # PROD_TO_COMPARE = ['c', 'y']
-    PROD_TO_COMPARE = ['y',]
+    PROD_TO_COMPARE = [
+        "y",
+    ]
     # PERC_TO_COMPARE = ['1.25', '3.75',]
-    PERC_TO_COMPARE = ['1.25',]
-    
+    PERC_TO_COMPARE = [
+        "1.25",
+    ]
+
     from oneflux.tools.partition_nt import remove_previous_run, run_python
-    remove_previous_run(datadir=datadir, siteid=siteid, sitedir=sitedir, python=True, 
-                        prod_to_compare=PROD_TO_COMPARE, perc_to_compare=PERC_TO_COMPARE,
-                        years_to_compare=years)
 
-    run_python(datadir=datadir, siteid=siteid, sitedir=sitedir, prod_to_compare=PROD_TO_COMPARE,
-               perc_to_compare=PERC_TO_COMPARE, years_to_compare=years)
-    
+    remove_previous_run(
+        datadir=datadir,
+        siteid=siteid,
+        sitedir=sitedir,
+        python=True,
+        prod_to_compare=PROD_TO_COMPARE,
+        perc_to_compare=PERC_TO_COMPARE,
+        years_to_compare=years,
+    )
+
+    run_python(
+        datadir=datadir,
+        siteid=siteid,
+        sitedir=sitedir,
+        prod_to_compare=PROD_TO_COMPARE,
+        perc_to_compare=PERC_TO_COMPARE,
+        years_to_compare=years,
+    )
+
     # check whether csv of output in input directory is same as csv of reference
 
     # the generated output is actually in the "input" directory.
     rootdir = os.path.join(datadir, sitedir, "10_nee_partition_nt")
     nee_y_files = glob.glob(os.path.join(rootdir, "nee_y_1.25_US-ARc_2005*"))
-    nee_y_files = filter(lambda x: not x.endswith('_orig.csv'), nee_y_files)
-    
+    nee_y_files = filter(lambda x: not x.endswith("_orig.csv"), nee_y_files)
+
     # paths to the "reference" output data
     refoutdir = os.path.join(refoutdir, "US-ARc_sample_output", "10_nee_partition_nt")
     ref_nee_y_files = glob.glob(os.path.join(refoutdir, "nee_y_1.25_US-ARc_2005*"))
-   
+
     assert len(nee_y_files) == len(ref_nee_y_files)
     for f, b in zip(nee_y_files, ref_nee_y_files):
         print(f, b)
-        assert equal_csv(f, b) 
+        assert equal_csv(f, b)
 
-    # clean up data. 
+    # clean up data.
     # shutil.rmtree(datadir)
diff --git a/tests/python/test_context.py b/tests/python/test_context.py
index 1fa60871..d334642a 100644
--- a/tests/python/test_context.py
+++ b/tests/python/test_context.py
@@ -1,4 +1,4 @@
-'''
+"""
 For license information:
 see LICENSE file or headers in oneflux.__init__.py
 
@@ -7,7 +7,8 @@
 @author: Gilberto Pastorello
 @contact: gzpastorello@lbl.gov
 @date: 2017-01-31
-'''
+"""
+
 
 def test_import_oneflux():
     """

From 20a9d4a7362cfd652f91249e7210a278d9f1ff44 Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Mon, 25 Nov 2024 14:18:10 +0000
Subject: [PATCH 7/7] Update comment as -a flag has been removed from cp

The -a flag causes the build to break on HPC systems.
---
 oneflux_steps/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oneflux_steps/Makefile b/oneflux_steps/Makefile
index a5e22cc9..019db440 100644
--- a/oneflux_steps/Makefile
+++ b/oneflux_steps/Makefile
@@ -30,7 +30,7 @@ CC := gcc -O3
 # create directory command
 MKDIR = mkdir -p
 
-# copy file command (verbose, keep file metadata)
+# copy file command (verbose)
 COPY = cp -v
 
 SRCDIR := $(shell pwd)/