diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ddad772ff..ce1e4e60e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,7 +97,7 @@ jobs: - name: run Lighthouse CI run: | - npm install -g @lhci/cli@0.7.x && + npm install -g @lhci/cli && lhci autorun env: LHCI_GITHUB_APP_TOKEN: ${{secrets.LHCI_GITHUB_APP_TOKEN}} diff --git a/.github/workflows/prod-update.yaml b/.github/workflows/prod-update.yaml deleted file mode 100644 index 92a2f4676..000000000 --- a/.github/workflows/prod-update.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Semester Data Update - -on: - schedule: - - cron: "0 * * * *" - workflow_dispatch: - -jobs: - update: - runs-on: ubuntu-latest - - strategy: - matrix: - semester: [fall-2022] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-parser-${{ hashFiles('rpi_data/modules/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-parser - - - name: Install parser dependencies - run: pip install -r requirements.txt - working-directory: rpi_data/modules - - - name: Get course data - run: bash $SCRIPT_NAME - working-directory: rpi_data - env: - SCRIPT_NAME: get-${{ matrix.semester }}.sh - - - name: Update prod - run: | - curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@$FILE_NAME" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $AUTH_ARGS $API_ENDPOINT - env: - FILE_NAME: ${{ matrix.semester }}.csv - AUTH_ARGS: ${{ secrets.AUTH_ARGS }} - API_ENDPOINT: ${{ secrets.UPDATE_API_ENDPOINT }} - working-directory: rpi_data diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 2687eaeb7..ddf98c42a 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -4,6 +4,7 @@ version: "3.4" # web -> api -> db [dependency chain] services: + yacs_web: ports: - 80:8080 @@ -37,7 +38,7 @@ services: - PROXY_CACHE_VALID=200=1s # ===== HTTPS config ===== - AUTO_LETS_ENCRYPT=${AUTO_LE:-no} - - GENERATE_SELF_SIGNED_SSL=${SELF_CERT:-no} + - GENERATE_SELF_SIGNED_SSL=${SELF_CERT:-yes} - REDIRECT_HTTP_TO_HTTPS=yes # ===== ModSecurity ===== - USE_MODSECURITY=no @@ -75,3 +76,15 @@ services: - POSTGRES_DB=yacs - POSTGRES_USER=yacs - POSTGRES_PASSWORD=${DB_PASS:-easy_dev_pass} + + + yacs_updater: + restart: unless-stopped + container_name: yacs_updater + stdin_open: true + tty: true + build: + context: ./rpi_data + dockerfile: Dockerfile + depends_on: + - yacs_web diff --git a/rpi_data/fall-2020.csv b/rpi_data/CSVs/fall-2020.csv similarity index 100% rename from rpi_data/fall-2020.csv rename to rpi_data/CSVs/fall-2020.csv diff --git a/rpi_data/fall-2021.csv b/rpi_data/CSVs/fall-2021.csv similarity index 100% rename from rpi_data/fall-2021.csv rename to rpi_data/CSVs/fall-2021.csv diff --git a/rpi_data/spring-2020.csv b/rpi_data/CSVs/spring-2020.csv similarity index 100% rename from rpi_data/spring-2020.csv rename to rpi_data/CSVs/spring-2020.csv diff --git a/rpi_data/spring-2021.csv b/rpi_data/CSVs/spring-2021.csv similarity index 100% rename from rpi_data/spring-2021.csv rename to rpi_data/CSVs/spring-2021.csv diff --git a/rpi_data/spring-2022.csv b/rpi_data/CSVs/spring-2022.csv similarity index 100% rename from rpi_data/spring-2022.csv rename to rpi_data/CSVs/spring-2022.csv diff --git a/rpi_data/summer-2020.csv b/rpi_data/CSVs/summer-2020.csv similarity index 100% rename from rpi_data/summer-2020.csv rename to rpi_data/CSVs/summer-2020.csv diff --git a/rpi_data/summer-2021.csv b/rpi_data/CSVs/summer-2021.csv similarity index 100% rename from rpi_data/summer-2021.csv rename to rpi_data/CSVs/summer-2021.csv diff --git a/rpi_data/Dockerfile b/rpi_data/Dockerfile new file mode 100644 index 000000000..f0808f46f --- /dev/null +++ b/rpi_data/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.8.12-slim-bullseye + +RUN mkdir -p /usr/src +WORKDIR /usr/src +COPY modules ./modules/ +COPY requirements.txt ./ +COPY school-department-mapping.yaml ./ +COPY automate-semesters.sh ./ +COPY website_checker.py ./ + +RUN apt update && apt install cron curl -y && \ + pip install --no-cache-dir -r requirements.txt && \ + chmod 0744 automate-semesters.sh && \ + crontab -l | { cat; echo "0 * * * * /usr/src/automate-semesters.sh"; } | crontab - + +CMD ["cron", "-f"] \ No newline at end of file diff --git a/rpi_data/automate-semesters.sh b/rpi_data/automate-semesters.sh new file mode 100755 index 000000000..9cc9fbd84 --- /dev/null +++ b/rpi_data/automate-semesters.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +post() { + test=${TESTING:-0} + + if [ ${test} == 0 ] + then + curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@${1}" \ + -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" -k $API_ENDPOINT + else + curl --insecure -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@${1}" \ + -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" -k $API_ENDPOINT + fi +} + +spring() { + local year=${1} + source_url=https://sis.rpi.edu/reg/zfs${year}01.htm + + python3 ./website_checker.py ${source_url} + exitCode=$? + if [ ${exitCode} == 0 ] + then + export SEMESTER="SPRING ${year}" + SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py + cat p* > spring-${year}.csv + rm -rf p*.csv + post spring-${year}.csv + else + echo "The requested Spring SIS page returned error 404." >&2 + fi +} + +summer() { + local year=${1} + source_url_a=https://sis.rpi.edu/reg/zfs${year}05.htm + source_url_b=https://sis.rpi.edu/reg/zfs${year}0502.htm + source_url_c=https://sis.rpi.edu/reg/zfs${year}0503.htm + + python3 ./website_checker.py ${source_url_a} ${source_url_b} ${source_url} + exitCode=$? + if [ ${exitCode} == 0 ] + then + export SEMESTER="SUMMER ${year}" + SOURCE_URL=${source_url_a} HEADERS=True DEST=p0.csv python3 ./modules/rpi-parse.py + SOURCE_URL=${source_url_b} HEADERS=False DEST=p1.csv python3 ./modules/rpi-parse.py + SOURCE_URL=${source_url_c} HEADERS=False DEST=p2.csv python3 ./modules/rpi-parse.py + cat p* > summer-${year}.csv + rm -rf p*.csv + post summer-${year}.csv + else + echo "A requested Summer SIS page returned error 404." >&2 + fi +} + +fall() { + local year=${1} + source_url=https://sis.rpi.edu/reg/zfs${year}09.htm + + python3 ./website_checker.py ${source_url} + exitCode=$? + if [ ${exitCode} == 0 ] + then + export SEMESTER="FALL ${year}" + SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py + cat p* > fall-${year}.csv + rm -rf p*.csv + post fall-${year}.csv + else + echo "A requested Fall SIS page returned error 404." >&2 + fi +} + + +MONTH=$(date +%m) +YEAR=$(date +%Y) +API_ENDPOINT=https://yacs_web:8443/api/bulkCourseUpload +if [[ ${MONTH} -ge 1 && ${MONTH} -lt 5 ]] +then + spring ${YEAR} + summer ${YEAR} +fi + +if [[ ${MONTH} -ge 5 && ${MONTH} -lt 9 ]] +then + summer ${YEAR} + fall ${YEAR} +fi + +if [[ ${MONTH} -ge 9 && ${MONTH} -le 12 ]] +then + fall ${YEAR} + spring ${YEAR} +fi + +rm *.csv diff --git a/rpi_data/get-fall-2020.sh b/rpi_data/get-fall-2020.sh deleted file mode 100644 index b24ba62ca..000000000 --- a/rpi_data/get-fall-2020.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="FALL 2020" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202009.htm DEST=p0.csv HEADERS=True python modules/rpi-parse.py -cat p* > fall-2020.csv -rm -rf p*.csv diff --git a/rpi_data/get-fall-2021.sh b/rpi_data/get-fall-2021.sh deleted file mode 100644 index 8eeed32b0..000000000 --- a/rpi_data/get-fall-2021.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="FALL 2021" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202109.htm DEST=p0.csv HEADERS=True python modules/rpi-parse.py -cat p* > fall-2021.csv -rm -rf p*.csv diff --git a/rpi_data/get-fall-2022.sh b/rpi_data/get-fall-2022.sh deleted file mode 100644 index d0bfe9ee2..000000000 --- a/rpi_data/get-fall-2022.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="FALL 2022" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202209.htm DEST=p0.csv HEADERS=True python modules/rpi-parse.py -cat p* > fall-2022.csv -rm -rf p*.csv diff --git a/rpi_data/get-spring-2020.sh b/rpi_data/get-spring-2020.sh deleted file mode 100644 index 0ea93c99b..000000000 --- a/rpi_data/get-spring-2020.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SPRING 2020" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202001.htm DEST=p0.csv HEADERS=True python modules/rpi-parse.py -cat p* > spring-2020.csv -rm -rf p*.csv diff --git a/rpi_data/get-spring-2021.sh b/rpi_data/get-spring-2021.sh deleted file mode 100755 index f09889ca9..000000000 --- a/rpi_data/get-spring-2021.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SPRING 2021" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202101.htm DEST=p0.csv HEADERS=True python modules/rpi-parse.py -cat p* > spring-2021.csv -rm -rf p*.csv diff --git a/rpi_data/get-spring-2022.sh b/rpi_data/get-spring-2022.sh deleted file mode 100755 index fddfac86f..000000000 --- a/rpi_data/get-spring-2022.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SPRING 2022" -SOURCE_URL=https://sis.rpi.edu/reg/zfs202201.htm DEST=p0.csv HEADERS=True python3 modules/rpi-parse.py -cat p* > spring-2022.csv -rm -rf p*.csv \ No newline at end of file diff --git a/rpi_data/get-summer-2020.sh b/rpi_data/get-summer-2020.sh deleted file mode 100644 index 6a3260a01..000000000 --- a/rpi_data/get-summer-2020.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SUMMER 2020" -SOURCE_URL=https://sis.rpi.edu/reg/zs20200501.htm DEST=p0.csv HEADERS=True python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20200502.htm DEST=p1.csv HEADERS=False python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20200503.htm DEST=p2.csv HEADERS=False python ./modules/rpi-parse.py -cat p* > summer-2020.csv -rm -rf p*.csv diff --git a/rpi_data/get-summer-2021.sh b/rpi_data/get-summer-2021.sh deleted file mode 100755 index 3a11d993a..000000000 --- a/rpi_data/get-summer-2021.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SUMMER 2021" -SOURCE_URL=https://sis.rpi.edu/reg/zs202105.htm DEST=p0.csv HEADERS=True python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20210502.htm DEST=p1.csv HEADERS=False python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20210503.htm DEST=p2.csv HEADERS=False python ./modules/rpi-parse.py -cat p* > summer-2021.csv -rm -rf p*.csv diff --git a/rpi_data/get-summer-2022.sh b/rpi_data/get-summer-2022.sh deleted file mode 100644 index 56feea573..000000000 --- a/rpi_data/get-summer-2022.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -export SEMESTER="SUMMER 2022" -SOURCE_URL=https://sis.rpi.edu/reg/zs202205.htm DEST=p0.csv HEADERS=True python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20220502.htm DEST=p1.csv HEADERS=False python ./modules/rpi-parse.py -SOURCE_URL=https://sis.rpi.edu/reg/zs20220503.htm DEST=p2.csv HEADERS=False python ./modules/rpi-parse.py -cat p* > summer-2022.csv -rm -rf p*.csv diff --git a/rpi_data/modules/rpi-parse.py b/rpi_data/modules/rpi-parse.py index 26d6049f1..25827c3d3 100644 --- a/rpi_data/modules/rpi-parse.py +++ b/rpi_data/modules/rpi-parse.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # coding: utf-8 # %% -import json import pandas as pd import os from fetch_catalog_course_info import acalog_client as AcalogClient diff --git a/rpi_data/modules/requirements.txt b/rpi_data/requirements.txt similarity index 100% rename from rpi_data/modules/requirements.txt rename to rpi_data/requirements.txt diff --git a/rpi_data/website_checker.py b/rpi_data/website_checker.py new file mode 100644 index 000000000..f5e25da50 --- /dev/null +++ b/rpi_data/website_checker.py @@ -0,0 +1,20 @@ +import requests +import sys + +"""Script to determine if a website returns a 404 error.""" + + +def main(): + + for url in range(1, len(sys.argv)): + URL = sys.argv[url] + response = requests.get(URL) + if response.status_code == 404: + print(URL) + exit(1) + exit(0) + + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/src/api/requirements.txt b/src/api/requirements.txt index 29768f14e..94e638779 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -7,4 +7,4 @@ pytest==6.2.5 uvicorn==0.14.0 python-multipart==0.0.5 fastapi-cache2==0.1.8 -itsdangerous==2.0.1 +itsdangerous==2.0.1 \ No newline at end of file diff --git a/src/web/src/components/Header.vue b/src/web/src/components/Header.vue index 03e47cde5..11443d163 100644 --- a/src/web/src/components/Header.vue +++ b/src/web/src/components/Header.vue @@ -14,19 +14,16 @@ YACS
- - - {{ option.value }} + + + + {{option.value}}
@@ -95,13 +92,8 @@