#520 automate course updates across semesters

.github/workflows/ci.yaml

@@ -97,7 +97,7 @@ jobs:
 
       - name: run Lighthouse CI
         run: |
-          npm install -g @lhci/cli@0.7.x &&
+          npm install -g @lhci/cli &&
           lhci autorun
         env:
           LHCI_GITHUB_APP_TOKEN: ${{secrets.LHCI_GITHUB_APP_TOKEN}}

docker-compose.production.yml

@@ -4,6 +4,7 @@ version: "3.4"
 # web -> api -> db [dependency chain]
 
 services:
+
   yacs_web:
     ports:
       - 80:8080
@@ -37,7 +38,7 @@ services:
       - PROXY_CACHE_VALID=200=1s
       # ===== HTTPS config =====
       - AUTO_LETS_ENCRYPT=${AUTO_LE:-no}
-      - GENERATE_SELF_SIGNED_SSL=${SELF_CERT:-no}
+      - GENERATE_SELF_SIGNED_SSL=${SELF_CERT:-yes}
       - REDIRECT_HTTP_TO_HTTPS=yes
       # ===== ModSecurity =====
       - USE_MODSECURITY=no
@@ -75,3 +76,15 @@ services:
       - POSTGRES_DB=yacs
       - POSTGRES_USER=yacs
       - POSTGRES_PASSWORD=${DB_PASS:-easy_dev_pass}
+
+
+  yacs_updater:
+    restart: unless-stopped
+    container_name: yacs_updater
+    stdin_open: true
+    tty: true
+    build:
+      context: ./rpi_data
+      dockerfile: Dockerfile
+    depends_on:
+      - yacs_web

rpi_data/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.8.12-slim-bullseye
+
+RUN mkdir -p /usr/src
+WORKDIR /usr/src
+COPY modules ./modules/
+COPY requirements.txt ./
+COPY school-department-mapping.yaml ./
+COPY automate-semesters.sh ./
+COPY website_checker.py ./
+
+RUN apt update && apt install cron curl -y && \
+  pip install --no-cache-dir -r requirements.txt && \
+  chmod 0744 automate-semesters.sh && \
+  crontab -l | { cat; echo "0 * * * * /usr/src/automate-semesters.sh"; } | crontab -
+
+CMD ["cron", "-f"]

rpi_data/automate-semesters.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+post() {
+	test=${TESTING:-0}
+
+	if [ ${test} == 0 ]
+	then
+		curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@${1}" \
+			-A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" -k $API_ENDPOINT
+	else
+		curl --insecure -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@${1}" \
+			-A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" -k $API_ENDPOINT
+	fi
+}
+
+spring() {
+	local year=${1}
+	source_url=https://sis.rpi.edu/reg/zfs${year}01.htm
+
+	python3 ./website_checker.py ${source_url}
+	exitCode=$?
+	if [ ${exitCode} == 0 ]
+	then
+		export SEMESTER="SPRING ${year}"
+		SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py
+		cat p* > spring-${year}.csv
+		rm -rf p*.csv
+		post spring-${year}.csv
+	else
+		echo "The requested Spring SIS page returned error 404." >&2
+	fi
+}
+
+summer() {
+	local year=${1}
+	source_url_a=https://sis.rpi.edu/reg/zfs${year}05.htm
+	source_url_b=https://sis.rpi.edu/reg/zfs${year}0502.htm
+	source_url_c=https://sis.rpi.edu/reg/zfs${year}0503.htm
+
+	python3 ./website_checker.py ${source_url_a} ${source_url_b} ${source_url}
+	exitCode=$?
+	if [ ${exitCode} == 0 ]
+    then
+		export SEMESTER="SUMMER ${year}"
+		SOURCE_URL=${source_url_a} HEADERS=True DEST=p0.csv python3 ./modules/rpi-parse.py
+		SOURCE_URL=${source_url_b} HEADERS=False DEST=p1.csv python3 ./modules/rpi-parse.py
+		SOURCE_URL=${source_url_c} HEADERS=False DEST=p2.csv python3 ./modules/rpi-parse.py
+		cat p* > summer-${year}.csv
+		rm -rf p*.csv
+		post summer-${year}.csv
+	else
+		echo "A requested Summer SIS page returned error 404." >&2
+	fi
+}
+
+fall() {
+	local year=${1}
+	source_url=https://sis.rpi.edu/reg/zfs${year}09.htm
+
+	python3 ./website_checker.py ${source_url}
+	exitCode=$?
+	if [ ${exitCode} == 0 ]
+	then
+		export SEMESTER="FALL ${year}"
+		SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py
+		cat p* > fall-${year}.csv
+		rm -rf p*.csv
+		post fall-${year}.csv
+	else
+		echo "A requested Fall SIS page returned error 404." >&2
+	fi
+}
+
+
+MONTH=$(date +%m)
+YEAR=$(date +%Y)
+API_ENDPOINT=https://yacs_web:8443/api/bulkCourseUpload
+if [[ ${MONTH} -ge 1 && ${MONTH} -lt 5 ]]
+then
+	spring ${YEAR}
+	summer ${YEAR}
+fi
+
+if [[ ${MONTH} -ge 5 && ${MONTH} -lt 9 ]]
+then
+	summer ${YEAR}
+	fall ${YEAR}
+fi
+
+if [[ ${MONTH} -ge 9 && ${MONTH} -le 12 ]]
+then
+	fall ${YEAR}
+	spring ${YEAR}
+fi
+
+rm *.csv

rpi_data/modules/rpi-parse.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
 # %%
-import json
 import pandas as pd
 import os
 from fetch_catalog_course_info import acalog_client as AcalogClient

rpi_data/website_checker.py

@@ -0,0 +1,20 @@
+import requests
+import sys
+
+"""Script to determine if a website returns a 404 error."""
+
+
+def main():
+
+    for url in range(1, len(sys.argv)):
+        URL = sys.argv[url]
+        response = requests.get(URL)
+        if response.status_code == 404:
+            print(URL)
+            exit(1)
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()
+

src/api/requirements.txt

@@ -7,4 +7,4 @@ pytest==6.2.5
 uvicorn==0.14.0
 python-multipart==0.0.5
 fastapi-cache2==0.1.8
-itsdangerous==2.0.1
+itsdangerous==2.0.1

src/web/src/components/Header.vue

@@ -14,19 +14,16 @@
       YACS
     </b-navbar-brand>
     <div>
-      <b-dropdown
-        variant="outline-primary"
-        size="sm"
-        :text="selectedSemester"
-        class="m-md-2"
-      >
-        <b-dropdown-item
-          v-for="option in semesterOptions"
-          :key="option.value"
-          :value="option.value"
-          @click="selectSemester(option.value)"
-        >
-          {{ option.value }}
+      <b-dropdown variant="outline-primary" 
+                        size = "sm" 
+                        :text= selectedSemester
+                        class= "m-md-2">
+        <b-dropdown-item v-for="option in semesterOptions" 
+                        :key="option.value" 
+                        :value="option.value" 
+                        @click="selectSemester(option.value)">
+
+          {{option.value}}
         </b-dropdown-item>
       </b-dropdown>
     </div>
@@ -95,13 +92,8 @@
 </template>
 
 <script>
-import {
-  SELECT_SEMESTER,
-  COOKIE_DARK_MODE,
-  TOGGLE_DARK_MODE,
-  SAVE_DARK_MODE,
-  RESET_DARK_MODE,
-} from "@/store";
+import { SELECT_SEMESTER, COOKIE_DARK_MODE, TOGGLE_DARK_MODE, SAVE_DARK_MODE,
+  RESET_DARK_MODE } from "@/store";
 import { mapState, mapActions, mapGetters } from "vuex";
 import LoginComponent from "@/components/Login";
 import { userTypes } from "../store/modules/user";
@@ -177,14 +169,14 @@ export default {
     }),
     ...mapState({ sessionId: userTypes.state.SESSION_ID }),
     ...mapState(["semesters", "selectedSemester"]),
-
+    
     semesterOptions() {
       return this.semesters.map(({ semester }) => ({
         text: semester,
         value: semester,
       }));
     },
-  },
+  }
 };
 </script>