Adds YAPF to requirements and as a CI step

.github/workflows/run-tests.yml

@@ -25,6 +25,9 @@ jobs:
 
       - uses: ./.github/actions/build-autoreduction
 
+      - name: YAPF
+        run: python -m yapf --exclude '**/migrations' --parallel --diff --recursive .
+
       - name: Run unit tests
         run: >-
           pytest --ignore=systemtests --cov=build --cov=monitors --cov=queue_processors

.unibeautifyignore

@@ -0,0 +1 @@
+**/migrations

.unibeautifyrc.json

@@ -0,0 +1,8 @@
+{
+    "Python": {
+        "beautifiers": [
+            "YAPF"
+        ],
+        "wrap_line_length": 120
+    }
+}

WebApp/autoreduce_webapp/autoreduce_webapp/backends.py

@@ -47,8 +47,7 @@ def authenticate(request, token=None):
                     # This will be checked upon each login
                     # pylint: disable=no-member
                     user.is_superuser = icat.is_admin(int(person['usernumber']))
-                    user.is_staff = (icat.is_instrument_scientist(
-                        int(person['usernumber'])) or user.is_superuser)
+                    user.is_staff = (icat.is_instrument_scientist(int(person['usernumber'])) or user.is_superuser)
                 user.save()
                 return user
         return None

WebApp/autoreduce_webapp/autoreduce_webapp/icat_cache.py

@@ -59,8 +59,7 @@ def open_icat(self):
 
     def is_valid(self, cache_obj):
         """ Check whether a cache object is fresh and is not None. """
-        return cache_obj and (cache_obj.created +
-                              datetime.timedelta(seconds=self.cache_lifetime) > timezone.now())
+        return cache_obj and (cache_obj.created + datetime.timedelta(seconds=self.cache_lifetime) > timezone.now())
 
     @staticmethod
     def to_list(target_list):
@@ -87,17 +86,21 @@ def update_cache(self, obj_type, obj_id):
             # Check func_list for the attributes that each model should have,
             # and the corresponding ICATCommunication function to query for it;
             #  call it for each, building a dict, and then splice it into the constructor kwargs.
-            new_obj = obj_type(**{attr: (getattr(self.icat, func)(obj_id)
-                                         if typ is None
-                                         else self.to_list(getattr(self.icat, func)(obj_id)))
-                                  for (func, model, attr, typ) in FUNC_LIST if model == obj_type})
+            new_obj = obj_type(
+                **{
+                    attr: (getattr(self.icat, func)
+                           (obj_id) if typ is None else self.to_list(getattr(self.icat, func)(obj_id)))
+                    for (func, model, attr, typ) in FUNC_LIST if model == obj_type
+                })
         else:
             # In this case, ICATCommunication returns all the ExperimentCache
             # fields in one query, so we splice that into the constructor.
-            new_obj = obj_type(**{attr: str(val)
-                                  for attr, val in
-                                  list(self.icat.get_experiment_details(obj_id).items())
-                                  if attr != "reference_number"})
+            new_obj = obj_type(
+                **{
+                    attr: str(val)
+                    for attr, val in list(self.icat.get_experiment_details(obj_id).items())
+                    if attr != "reference_number"
+                })
         new_obj.id_name = obj_id
         new_obj.save()
         return new_obj
@@ -135,8 +138,7 @@ def get_valid_experiments_for_instruments(self, user_number, instruments):
         user_experiments = set(self.get_associated_experiments(user_number))
         for instrument_name in instruments:
             instrument_experiments = self.get_valid_experiments_for_instrument(instrument_name)
-            experiment_dict[instrument_name] = list(user_experiments.
-                                                    intersection(instrument_experiments))
+            experiment_dict[instrument_name] = list(user_experiments.intersection(instrument_experiments))
         return experiment_dict
 
     def get_experiment_details(self, experiment_number):
@@ -150,14 +152,11 @@ def get_experiment_details(self, experiment_number):
 # Here we define (ICATCommunication function to wrap, Cache object type,
 # field of object to get, type of list element if the field is a list)
 FUNC_LIST = [("get_owned_instruments", UserCache, "owned_instruments", str),
-             ("get_valid_instruments", UserCache, "valid_instruments", str),
-             ("is_admin", UserCache, "is_admin", None),
+             ("get_valid_instruments", UserCache, "valid_instruments", str), ("is_admin", UserCache, "is_admin", None),
              ("is_instrument_scientist", UserCache, "is_instrument_scientist", None),
              ("get_associated_experiments", UserCache, "associated_experiments", int),
-             ("get_upcoming_experiments_for_instrument",
-              InstrumentCache, "upcoming_experiments", int),
-             ("get_valid_experiments_for_instrument",
-              InstrumentCache, "valid_experiments", int)]
+             ("get_upcoming_experiments_for_instrument", InstrumentCache, "upcoming_experiments", int),
+             ("get_valid_experiments_for_instrument", InstrumentCache, "valid_experiments", int)]
 
 
 def make_member_func(obj_type, cache_attr, list_type):
@@ -190,6 +189,7 @@ def member_func(self, obj_id):
             attr = map(list_type, filter(isvalid, attr.split(",")))
 
         return attr
+
     return member_func
 
 

WebApp/autoreduce_webapp/autoreduce_webapp/icat_communication.py

@@ -24,7 +24,6 @@
 
 from reduction_viewer.models import Setting
 
-
 LOGGER = logging.getLogger(__name__)
 sys.path.insert(0, BASE_DIR)
 
@@ -75,9 +74,8 @@ def get_experiment_details(self, reference_number):
 
         if reference_number > 0:
             try:
-                investigation = self.client.search("SELECT i from Investigation i where i.name = '"
-                                                   + str(reference_number) +
-                                                   "' INCLUDE i.investigationInstruments."
+                investigation = self.client.search("SELECT i from Investigation i where i.name = '" +
+                                                   str(reference_number) + "' INCLUDE i.investigationInstruments."
                                                    "instrument, i.investigationUsers.user")
 
                 trimmed_investigation = {
@@ -87,7 +85,8 @@ def get_experiment_details(self, reference_number):
                     'title': smart_str(investigation[0].title),
                     'summary': smart_str(investigation[0].summary),
                     'instrument': investigation[0].investigationInstruments[0].instrument.fullName,
-                    'pi': ''}
+                    'pi': ''
+                }
 
                 for investigation_user in investigation[0].investigationUsers:
                     if investigation_user.role == 'principal_experimenter':
@@ -103,7 +102,8 @@ def get_experiment_details(self, reference_number):
             'end_date': 'N/A',
             'title': 'N/A',
             'summary': u'N/A',
-            'pi': ''}
+            'pi': ''
+        }
 
         return trimmed_investigation
 
@@ -118,17 +118,16 @@ def get_valid_instruments(self, user_number):
 
         instruments = set()
         if self.is_admin(user_number):
-            self._add_list_to_set(self.client.search("SELECT inst.fullName FROM Instrument inst"),
-                                  instruments)
+            self._add_list_to_set(self.client.search("SELECT inst.fullName FROM Instrument inst"), instruments)
         else:
             self._add_list_to_set(self.get_owned_instruments(user_number), instruments)
-            self._add_list_to_set(self.client.search("SELECT inst.fullName FROM Instrument inst"
-                                                     " JOIN inst.investigationInstruments ii"
-                                                     " WHERE ii.investigation.id IN"
-                                                     " (SELECT i.id from Investigation i JOIN"
-                                                     " i.investigationUsers iu WHERE"
-                                                     " iu.user.name = 'uows/"
-                                                     + str(user_number) + "')"), instruments)
+            self._add_list_to_set(
+                self.client.search("SELECT inst.fullName FROM Instrument inst"
+                                   " JOIN inst.investigationInstruments ii"
+                                   " WHERE ii.investigation.id IN"
+                                   " (SELECT i.id from Investigation i JOIN"
+                                   " i.investigationUsers iu WHERE"
+                                   " iu.user.name = 'uows/" + str(user_number) + "')"), instruments)
         return sorted(instruments)
 
     def get_owned_instruments(self, user_number):
@@ -140,10 +139,10 @@ def get_owned_instruments(self, user_number):
             raise TypeError("User number must be a number")
 
         instruments = set()
-        self._add_list_to_set(self.client.search("SELECT ins.instrument.fullName from"
-                                                 " InstrumentScientist ins WHERE"
-                                                 " ins.user.name = 'uows/"
-                                                 + str(user_number) + "'"), instruments)
+        self._add_list_to_set(
+            self.client.search("SELECT ins.instrument.fullName from"
+                               " InstrumentScientist ins WHERE"
+                               " ins.user.name = 'uows/" + str(user_number) + "'"), instruments)
         return sorted(instruments)
 
     def is_instrument_scientist(self, user_number):
@@ -167,9 +166,8 @@ def is_on_experiment_team(self, reference_number, user_number):
 
         is_on_team = self.client.search("SELECT i.name from Investigation i JOIN"
                                         " i.investigationUsers iu where"
-                                        " iu.user.name = 'uows/"
-                                        + str(user_number) + "' and i.name = '"
-                                        + str(reference_number) + "'")
+                                        " iu.user.name = 'uows/" + str(user_number) + "' and i.name = '" +
+                                        str(reference_number) + "'")
         if is_on_team:
             return True
         return False
@@ -184,10 +182,10 @@ def get_associated_experiments(self, user_number):
             raise TypeError("User number must be a number")
 
         experiments = set()
-        self._add_list_to_set(self.client.search("SELECT i.name from Investigation i JOIN"
-                                                 " i.investigationUsers iu where"
-                                                 " iu.user.name = 'uows/"
-                                                 + str(user_number) + "'"), experiments)
+        self._add_list_to_set(
+            self.client.search("SELECT i.name from Investigation i JOIN"
+                               " i.investigationUsers iu where"
+                               " iu.user.name = 'uows/" + str(user_number) + "'"), experiments)
         return sorted(experiments, reverse=True)
 
     # pylint: disable=invalid-name
@@ -209,18 +207,16 @@ def get_valid_experiments_for_instruments(self, user_number, instruments):
         # pylint: disable=bare-except
         except:
             number_of_years = 3
-        years_back = datetime.datetime.now() - datetime.timedelta(days=(number_of_years*365.24))
+        years_back = datetime.datetime.now() - datetime.timedelta(days=(number_of_years * 365.24))
 
         for instrument in instruments:
             experiments = set()
-            self._add_list_to_set(self.client.search("SELECT i.name FROM Investigation i"
-                                                     " JOIN i.investigationInstruments inst"
-                                                     " WHERE i.name NOT LIKE 'CAL%' and"
-                                                     " i.endDate > '" + str(years_back) +
-                                                     "' and (inst.instrument.name = '"
-                                                     + instrument +
-                                                     "' OR inst.instrument.fullName = '"
-                                                     + instrument + "')"), experiments)
+            self._add_list_to_set(
+                self.client.search("SELECT i.name FROM Investigation i"
+                                   " JOIN i.investigationInstruments inst"
+                                   " WHERE i.name NOT LIKE 'CAL%' and"
+                                   " i.endDate > '" + str(years_back) + "' and (inst.instrument.name = '" + instrument +
+                                   "' OR inst.instrument.fullName = '" + instrument + "')"), experiments)
 
             instruments_dict[instrument] = sorted(experiments, reverse=True)
 
@@ -239,17 +235,15 @@ def get_valid_experiments_for_instrument(self, instrument):
         # pylint: disable=bare-except
         except:
             number_of_years = 3
-        years_back = datetime.datetime.now() - datetime.timedelta(days=(number_of_years*365.24))
+        years_back = datetime.datetime.now() - datetime.timedelta(days=(number_of_years * 365.24))
 
         experiments = set()
-        self._add_list_to_set(self.client.search("SELECT i.name FROM Investigation i JOIN"
-                                                 " i.investigationInstruments inst WHERE"
-                                                 " i.name NOT LIKE 'CAL%' and i.endDate > '"
-                                                 + str(years_back) +
-                                                 "' and (inst.instrument.name = '"
-                                                 + instrument +
-                                                 "' OR inst.instrument.fullName = '"
-                                                 + instrument+"')"), experiments)
+        self._add_list_to_set(
+            self.client.search("SELECT i.name FROM Investigation i JOIN"
+                               " i.investigationInstruments inst WHERE"
+                               " i.name NOT LIKE 'CAL%' and i.endDate > '" + str(years_back) +
+                               "' and (inst.instrument.name = '" + instrument + "' OR inst.instrument.fullName = '" +
+                               instrument + "')"), experiments)
         return sorted(experiments, reverse=True)
 
     # pylint: disable=invalid-name
@@ -262,13 +256,13 @@ def get_upcoming_experiments_for_instrument(self, instrument):
             raise Exception("At least one instrument must be supplied")
 
         experiments = set()
-        self._add_list_to_set(self.client.search("SELECT i.name FROM Investigation i JOIN"
-                                                 " i.investigationInstruments inst WHERE"
-                                                 " i.name NOT LIKE 'CAL%' and"
-                                                 " i.endDate > CURRENT_TIMESTAMP and"
-                                                 " (inst.instrument.name = '" +
-                                                 instrument + "' OR inst.instrument.fullName = '"
-                                                 + instrument + "')"), experiments)
+        self._add_list_to_set(
+            self.client.search("SELECT i.name FROM Investigation i JOIN"
+                               " i.investigationInstruments inst WHERE"
+                               " i.name NOT LIKE 'CAL%' and"
+                               " i.endDate > CURRENT_TIMESTAMP and"
+                               " (inst.instrument.name = '" + instrument + "' OR inst.instrument.fullName = '" +
+                               instrument + "')"), experiments)
         return sorted(experiments, reverse=True)
 
     def is_admin(self, user_number):
@@ -277,8 +271,7 @@ def is_admin(self, user_number):
         """
         LOGGER.debug("Calling ICATCommunication.is_admin")
         admin_group = 'Autoreduce Admins'
-        if self.client.search("SELECT g FROM Grouping g JOIN g.userGroups ug WHERE g.name = '"
-                              + admin_group +
+        if self.client.search("SELECT g FROM Grouping g JOIN g.userGroups ug WHERE g.name = '" + admin_group +
                               "' and ug.user.name = 'uows/" + str(user_number) + "'"):
             return True
         return False
@@ -297,12 +290,12 @@ def get_run_details(self, instrument, start_run_number, end_run_number):
 
         return self.client.search("SELECT dfp FROM DatafileParameter dfp JOIN "
                                   "dfp.datafile.dataset.investigation.investigationInstruments "
-                                  "ii WHERE dfp.type.name='run_number' and dfp.numericValue >= "
-                                  + str(start_run_number)+" and dfp.numericValue <= " +
-                                  str(end_run_number)+" and ii.instrument.fullName = '" +
-                                  instrument + "' and dfp.datafile.dataset.investigation.name "
-                                               "not LIKE 'CAL%%' include "
-                                               "dfp.datafile.dataset.investigation")
+                                  "ii WHERE dfp.type.name='run_number' and dfp.numericValue >= " +
+                                  str(start_run_number) + " and dfp.numericValue <= " + str(end_run_number) +
+                                  " and ii.instrument.fullName = '" + instrument +
+                                  "' and dfp.datafile.dataset.investigation.name "
+                                  "not LIKE 'CAL%%' include "
+                                  "dfp.datafile.dataset.investigation")
 
     @staticmethod
     def post_process(_):

WebApp/autoreduce_webapp/autoreduce_webapp/templatetags/natural_time_difference.py

@@ -12,7 +12,6 @@
 # pylint: disable=relative-import
 from .common_helpers import get_var
 
-
 # pylint:disable=invalid-name
 register = Library()
 
@@ -62,8 +61,7 @@ def natural_time_difference(_, token):
     """
     args = token.split_contents()[1:]
     if len(args) != 2:
-        raise TemplateSyntaxError('%r tag requires two datetimes.'
-                                  % token.contents.split()[0])
+        raise TemplateSyntaxError('%r tag requires two datetimes.' % token.contents.split()[0])
     return NaturalTimeDifferenceNode(*args)
 
 

WebApp/autoreduce_webapp/autoreduce_webapp/templatetags/replace.py

@@ -11,7 +11,6 @@
 # pylint: disable=relative-import
 from .common_helpers import get_var
 
-
 # pylint:disable=invalid-name
 register = Library()
 
@@ -42,6 +41,6 @@ def replace(_, token):
     """
     args = token.split_contents()[1:]
     if len(args) != 3:
-        raise TemplateSyntaxError('%r tag requires a string, an old value, and a new value.'
-                                  % token.contents.split()[0])
+        raise TemplateSyntaxError('%r tag requires a string, an old value, and a new value.' %
+                                  token.contents.split()[0])
     return ReplaceNode(*args)

WebApp/autoreduce_webapp/autoreduce_webapp/templatetags/view.py

@@ -106,8 +106,7 @@ def do_view(_, token):
     kwargs = {}
     tokens = token.split_contents()
     if len(tokens) < 2:
-        raise TemplateSyntaxError("%r tag requires one or more arguments" %
-                                  token.contents.split()[0])
+        raise TemplateSyntaxError("%r tag requires one or more arguments" % token.contents.split()[0])
     _ = tokens.pop(0)
     url_or_view = tokens.pop(0)
     # pylint: disable=redefined-argument-from-local
@@ -116,7 +115,7 @@ def do_view(_, token):
         if equals == -1:
             args.append(token)
         else:
-            kwargs[str(token[:equals])] = token[equals+1:]
+            kwargs[str(token[:equals])] = token[equals + 1:]
     return ViewNode(url_or_view, args, kwargs)
 
 

WebApp/autoreduce_webapp/autoreduce_webapp/uows_client.py

@@ -16,7 +16,6 @@
 # pylint: disable=relative-import
 from .settings import UOWS_URL
 
-
 LOGGER = logging.getLogger(__name__)