diff --git a/.github/workflows/prod-update.yaml b/.github/workflows/prod-update.yaml deleted file mode 100644 index 756755de4..000000000 --- a/.github/workflows/prod-update.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Semester Data Update - -on: - schedule: - - cron: "0 * * * *" - workflow_dispatch: - -jobs: - update: - runs-on: ubuntu-latest - - strategy: - matrix: - semester: [spring-2022] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-parser-${{ hashFiles('rpi_data/modules/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-parser - - - name: Install parser dependencies - run: pip install -r requirements.txt - working-directory: rpi_data/modules - - - name: Get course data - run: bash $SCRIPT_NAME - working-directory: rpi_data - env: - SCRIPT_NAME: get-${{ matrix.semester }}.sh - - - name: Update prod - run: | - curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=@$FILE_NAME" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $AUTH_ARGS $API_ENDPOINT - env: - FILE_NAME: ${{ matrix.semester }}.csv - AUTH_ARGS: ${{ secrets.AUTH_ARGS }} - API_ENDPOINT: ${{ secrets.UPDATE_API_ENDPOINT }} - working-directory: rpi_data diff --git a/rpi_data/Dockerfile b/rpi_data/Dockerfile index 8a06d7256..3d078869c 100644 --- a/rpi_data/Dockerfile +++ b/rpi_data/Dockerfile @@ -14,6 +14,6 @@ RUN chmod +x /etc/cron.d/cronjob && \ chmod +x /usr/src/automate-semesters.sh && \ touch /var/log/cron.log -RUN apt update && apt install cron -y && \ - pip3 install --no-cache-dir -r requirements.txt +RUN apt update && apt install cron curl -y && \ + pip install --no-cache-dir -r requirements.txt CMD cron && tail -f /var/log/cron.log \ No newline at end of file diff --git a/rpi_data/automate-semesters.sh b/rpi_data/automate-semesters.sh index fa30ce7d1..0e571b5df 100755 --- a/rpi_data/automate-semesters.sh +++ b/rpi_data/automate-semesters.sh @@ -8,12 +8,13 @@ spring() { exitCode=$? if [ ${exitCode} == 0 ] then - export SEMESTER="SPRING ${year}" - SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py - cat p* > spring-${year}.csv - rm -rf p*.csv + export SEMESTER="SPRING ${year}" + SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py + cat p* > spring-${year}.csv + rm -rf p*.csv + curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=spring-${year}" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $API_ENDPOINT else - echo "The requested Spring SIS page returned error 404." >&2 + echo "The requested Spring SIS page returned error 404." >&2 fi } @@ -26,13 +27,14 @@ summer() { python3 ./website_checker.py ${source_url_a} ${source_url_b} ${source_url} exitCode=$? if [ ${exitCode} == 0 ] - then + then export SEMESTER="SUMMER ${year}" SOURCE_URL=${source_url_a} HEADERS=True DEST=p0.csv python3 ./modules/rpi-parse.py SOURCE_URL=${source_url_b} HEADERS=False DEST=p1.csv python3 ./modules/rpi-parse.py SOURCE_URL=${source_url_c} HEADERS=False DEST=p2.csv python3 ./modules/rpi-parse.py cat p* > summer-${year}.csv rm -rf p*.csv + curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=summer-${year}" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $API_ENDPOINT else echo "A requested Summer SIS page returned error 404." >&2 fi @@ -46,11 +48,11 @@ fall() { exitCode=$? if [ ${exitCode} == 0 ] then - echo "Fall" export SEMESTER="FALL ${year}" SOURCE_URL=${source_url} DEST=p0.csv HEADERS=True python3 ./modules/rpi-parse.py cat p* > fall-${year}.csv rm -rf p*.csv + curl -X POST -H "Content-Type: multipart/form-data" -F "isPubliclyVisible=on" -F "file=fall-${year}" -A "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0" $API_ENDPOINT else echo "A requested Fall SIS page returned error 404." >&2 fi @@ -59,6 +61,7 @@ fall() { MONTH=$(date +%m) YEAR=$(date +%Y) +API_ENDPOINT=https://yacs_web/api/bulkCourseUpload if [[ ${MONTH} -ge 1 && ${MONTH} -lt 5 ]] then spring ${YEAR}