diff --git a/.env b/.env index 4855d5b5..e8387199 100644 --- a/.env +++ b/.env @@ -1,10 +1,12 @@ +ENVIRONMENT=dev DB_NAME=campus_db DB_ROOT_PASSWORD=secret_root_password DB_PORT=3306 + APNS_KEY_ID= APNS_TEAM_ID= APNS_P8_FILE_PATH=/secrets/AuthKey_XXXX.p8 -ENVIRONMENT=dev - SENTRY_DSN= + +CAMPUS_API_TOKEN= diff --git a/README.md b/README.md index 9617731c..825ce41f 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ The following environment variables need to be set for the server to work proper * [REQUIRED] `APNS_KEY_ID`: The key ID of the APNs key => APNs Key needs to be downloaded from the Apple Developer Portal the name of the file also contains the key ID. * [REQUIRED] `APNS_TEAM_ID`: The team ID of the iOS app can be found in AppStoreConnect. * [REQUIRED] `APNS_P8_FILE_PATH`: The path to the APNs key file (e.g. `/secrets/AuthKey_XXXX.p8`) in the docker container. The file itself needs to exist in the same directory as the `docker-compose.yml` file and called `apns_auth_key.p8`. + * [REQUIRED] `CAMPUS_API_TOKEN`: A token used to authenticate with TUMonline (used for example for the grades) ## InfluxDB InfluxDB can be used to store metrics. diff --git a/deployment/charts/backend/templates/deployments/backend-v2.yaml b/deployment/charts/backend/templates/deployments/backend-v2.yaml index f037f8f9..99c36826 100644 --- a/deployment/charts/backend/templates/deployments/backend-v2.yaml +++ b/deployment/charts/backend/templates/deployments/backend-v2.yaml @@ -48,6 +48,11 @@ spec: secretKeyRef: name: backend-api-keys key: SENTRY_DSN + - name: CAMPUS_API_TOKEN + valueFrom: + secretKeyRef: + name: backend-api-keys + key: CAMPUS_API_TOKEN - name: DB_DSN value: "{{ $db.username }}:{{ $db.password }}@tcp(tca-backend-mariadb.{{ $.Values.namespace }}.svc.cluster.local:3306)/{{ $db.database }}?charset=utf8mb4&parseTime=True&loc=Local" - name: APNS_KEY_ID @@ -105,6 +110,7 @@ metadata: app.kubernetes.io/part-of: tum-campus-app app.kubernetes.io/name: backend-v2 data: + CAMPUS_API_TOKEN: {{ $.Values.backend.campusApiToken | b64enc }} SENTRY_DSN: {{ $.Values.backend.sentry.dsn | b64enc }} apns_auth_key.p8: {{ $.Values.backend.apns.auth_key }} APNS_KEY_ID: {{ $.Values.backend.apns.key_id | b64enc }} diff --git a/deployment/charts/backend/values.yaml b/deployment/charts/backend/values.yaml index ec5d6476..36d04422 100644 --- a/deployment/charts/backend/values.yaml +++ b/deployment/charts/backend/values.yaml @@ -41,6 +41,7 @@ mariadb: backend: + campusApiToken: changeme-changeme-changeme sentry: dsn: changeme-changeme-changeme apns: diff --git a/docker-compose.yaml b/docker-compose.yaml index a7d45018..6e1cf07a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,6 +17,7 @@ services: - APNS_TEAM_ID=${APNS_TEAM_ID} - APNS_P8_FILE_PATH=${APNS_P8_FILE_PATH} - MensaCronDisabled=false + - CAMPUS_API_TOKEN=${CAMPUS_API_TOKEN} volumes: - backend-storage:/Storage - ./apns_auth_key.p8:${APNS_P8_FILE_PATH} diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index b79ec94e..746455e4 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -4,6 +4,7 @@ package campus_api import ( "encoding/xml" "errors" + "fmt" "io" "net/http" @@ -11,49 +12,44 @@ import ( log "github.com/sirupsen/logrus" ) -const ( - CampusApiUrl = "https://campus.tum.de/tumonline" - CampusQueryToken = "pToken" - CampusGradesPath = "/wbservicesbasic.noten" -) +func FetchExamResultsPublished(token string) (*model.TUMAPIPublishedExamResults, error) { + var examResultsPublished model.TUMAPIPublishedExamResults + err := RequestCampusApi("/wbservicesbasic.pruefungenErgebnisse", token, &examResultsPublished) + if err != nil { + return nil, err + } -var ( - ErrCannotCreateRequest = errors.New("cannot create http request") - ErrWhileFetchingGrades = errors.New("error while fetching grades") - ErrorWhileUnmarshalling = errors.New("error while unmarshalling") -) + return &examResultsPublished, nil +} func FetchGrades(token string) (*model.IOSGrades, error) { - - requestUrl := CampusApiUrl + CampusGradesPath - req, err := http.NewRequest(http.MethodGet, requestUrl, nil) - + var grades model.IOSGrades + err := RequestCampusApi("/wbservicesbasic.noten", token, &grades) if err != nil { - log.WithError(err).Error("Failed to create api-request") - return nil, ErrCannotCreateRequest + return nil, err } - q := req.URL.Query() - q.Add(CampusQueryToken, token) - - req.URL.RawQuery = q.Encode() + return &grades, nil +} - resp, err := http.DefaultClient.Do(req) +func RequestCampusApi(path string, token string, response any) error { + requestUrl := fmt.Sprintf("https://campus.tum.de/tumonline%s?pToken=%s", path, token) + resp, err := http.Get(requestUrl) if err != nil { - log.WithError(err).Error("failed to fetch grades") - return nil, ErrWhileFetchingGrades + log.WithError(err).WithField("path", path).Error("Error while fetching url") + return errors.New("error while fetching " + path) } defer func(Body io.ReadCloser) { - if err := Body.Close(); err != nil { - log.WithError(err).Error("Could not close body") + err := Body.Close() + if err != nil { + log.WithError(err).Error("Error while closing body") } }(resp.Body) - var grades model.IOSGrades - if err = xml.NewDecoder(resp.Body).Decode(&grades); err != nil { - log.WithError(err).Error("could not unmarshall grades") - return nil, ErrorWhileUnmarshalling + if err = xml.NewDecoder(resp.Body).Decode(&response); err != nil { + log.WithError(err).WithField("path", path).Error("Error while unmarshalling") + return errors.New("error while unmarshalling") } - return &grades, nil + return nil } diff --git a/server/backend/cron/cronjobs.go b/server/backend/cron/cronjobs.go index 5d2bd422..29e90954 100644 --- a/server/backend/cron/cronjobs.go +++ b/server/backend/cron/cronjobs.go @@ -30,6 +30,7 @@ const ( CanteenHeadcount = "canteenHeadCount" IOSNotifications = "iosNotifications" IOSActivityReset = "iosActivityReset" + NewExamResultsHook = "newExamResultsHook" /* MensaType = "mensa" KinoType = "kino" @@ -59,7 +60,7 @@ func (c *CronService) Run() error { var res []model.Crontab c.db.Model(&model.Crontab{}). - Where("`interval` > 0 AND (lastRun+`interval`) < ? AND type IN (?, ?, ?, ?, ?, ?, ?)", + Where("`interval` > 0 AND (lastRun+`interval`) < ? AND type IN (?, ?, ?, ?, ?, ?, ?, ?)", time.Now().Unix(), NewsType, FileDownloadType, @@ -68,6 +69,7 @@ func (c *CronService) Run() error { CanteenHeadcount, IOSNotifications, IOSActivityReset, + NewExamResultsHook, ). Scan(&res) @@ -104,6 +106,8 @@ func (c *CronService) Run() error { if env.IsMensaCronActive() { g.Go(c.averageRatingComputation) } + case NewExamResultsHook: + g.Go(func() error { return c.newExamResultsHookCron() }) /* TODO: Implement handlers for other cronjobs case MensaType: diff --git a/server/backend/cron/newExamResultsHook.go b/server/backend/cron/newExamResultsHook.go new file mode 100644 index 00000000..122d636c --- /dev/null +++ b/server/backend/cron/newExamResultsHook.go @@ -0,0 +1,15 @@ +package cron + +import ( + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" + "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_scheduling" +) + +func (c *CronService) newExamResultsHookCron() error { + repo := new_exam_results_scheduling.NewRepository(c.db) + devicesRepo := ios_device.NewRepository(c.db) + + service := new_exam_results_scheduling.NewService(repo, devicesRepo, c.APNs) + + return service.HandleScheduledCron() +} diff --git a/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go b/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go index 3108b090..05c03edb 100644 --- a/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go +++ b/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go @@ -48,7 +48,6 @@ func (r *Repository) ApnsUrl() string { if env.IsProd() { return ApnsProductionURL } - return ApnsDevelopmentURL } @@ -87,7 +86,6 @@ func (r *Repository) SendNotification(notification *model.IOSNotificationPayload url := r.ApnsUrl() + "/3/device/" + notification.DeviceId body, _ := notification.MarshalJSON() - client := r.httpClient req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) // can be e.g. alert or background @@ -99,7 +97,7 @@ func (r *Repository) SendNotification(notification *model.IOSNotificationPayload bearer := r.Token.GenerateNewTokenIfExpired() req.Header.Set("authorization", "bearer "+bearer) - resp, err := client.Do(req) + resp, err := r.httpClient.Do(req) if err != nil { log.WithError(err).Error("Could not send notification") return nil, ErrCouldNotSendNotification diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go new file mode 100644 index 00000000..165113b7 --- /dev/null +++ b/server/backend/migration/20230530000000.go @@ -0,0 +1,67 @@ +package migration + +import ( + _ "embed" + "time" + + "github.com/TUM-Dev/Campus-Backend/server/model" + "github.com/go-gormigrate/gormigrate/v2" + "github.com/guregu/null" + "gorm.io/gorm" +) + +type PublishedExamResult struct { + Date time.Time + ExamID string `gorm:"primary_key"` + LectureTitle string + LectureType string + LectureSem string + Published bool +} + +type NewExamResultsSubscriber struct { + CallbackUrl string `gorm:"primary_key"` + ApiKey null.String + CreatedAt time.Time `gorm:"autoCreateTime"` + LastNotifiedAt null.Time +} + +func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20230530000000", + Migrate: func(tx *gorm.DB) error { + + if err := tx.AutoMigrate( + &PublishedExamResult{}, + &NewExamResultsSubscriber{}, + ); err != nil { + return err + } + + err := SafeEnumMigrate(tx, model.Crontab{}, "type", "newExamResultsHook") + if err != nil { + return err + } + + return tx.Create(&model.Crontab{ + Interval: 60 * 10, // Every 10 minutes + Type: null.StringFrom("newExamResultsHook"), + }).Error + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Migrator().DropTable(&PublishedExamResult{}); err != nil { + return err + } + if err := tx.Migrator().DropTable(&NewExamResultsSubscriber{}); err != nil { + return err + } + + err := SafeEnumRollback(tx, model.Crontab{}, "type", "newExamResultsHook") + if err != nil { + return err + } + + return tx.Delete(&model.Crontab{}, "type = ?", "newExamResultsHook").Error + }, + } +} diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 7cd9acf6..c572d806 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -50,6 +50,7 @@ func (m TumDBMigrator) Migrate() error { m.migrate20221210000000(), m.migrate20230825000000(), m.migrate20230904000000(), + m.migrate20230530000000(), }) err := mig.Migrate() return err diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go new file mode 100644 index 00000000..4daada15 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go @@ -0,0 +1,41 @@ +package new_exam_results_scheduling + +import ( + "github.com/TUM-Dev/Campus-Backend/server/model" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type Repository struct { + DB *gorm.DB +} + +func (repository *Repository) StoreExamResultsPublished(examResultsPublished []model.PublishedExamResult) error { + db := repository.DB + + return db.Transaction(func(tx *gorm.DB) error { + err := tx.Where("1 = 1").Delete(&model.PublishedExamResult{}).Error + + if err != nil { + return err + } + + // disabled logging because this query always prints a warning because it takes longer then normal + // to execute because we bulk insert a lot of data + return tx.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}). + Create(examResultsPublished).Error + }) +} + +func (repository *Repository) FindAllExamResultsPublished() (*[]model.PublishedExamResult, error) { + var results []model.PublishedExamResult + err := repository.DB.Find(&results).Error + + return &results, err +} + +func NewRepository(db *gorm.DB) *Repository { + return &Repository{ + DB: db, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go new file mode 100644 index 00000000..30c266f4 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go @@ -0,0 +1,99 @@ +package new_exam_results_scheduling + +import ( + "os" + + "github.com/TUM-Dev/Campus-Backend/server/backend/campus_api" + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns" + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" + "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_subscriber" + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" +) + +var ( + CampusApiToken = os.Getenv("CAMPUS_API_TOKEN") +) + +type Service struct { + Repository *Repository + DevicesRepository *ios_device.Repository + Priority *model.IOSSchedulingPriority + APNs *ios_apns.Service +} + +func (service *Service) HandleScheduledCron() error { + log.Info("Fetching published exam results") + + apiResult, err := campus_api.FetchExamResultsPublished(CampusApiToken) + if err != nil { + return err + } + + var apiExamResults []model.PublishedExamResult + for _, apiExamResult := range apiResult.ExamResults { + apiExamResults = append(apiExamResults, *apiExamResult.ToDBExamResult()) + } + + storedExamResults, err := service.Repository.FindAllExamResultsPublished() + if err != nil { + return err + } + + newPublishedExamResults := service.findNewPublishedExamResults(&apiExamResults, storedExamResults) + + if len(*newPublishedExamResults) > 0 { + service.notifySubscribers(newPublishedExamResults) + } else { + log.Info("No new published exam results") + } + + return service.Repository.StoreExamResultsPublished(apiExamResults) +} + +func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamResults *[]model.PublishedExamResult) *[]model.PublishedExamResult { + var apiExamResultsMap = make(map[string]model.PublishedExamResult) + for _, apiExamResult := range *apiExamResults { + apiExamResultsMap[apiExamResult.ExamID] = apiExamResult + } + + var storedExamResultsMap = make(map[string]model.PublishedExamResult) + for _, storedExamResult := range *storedExamResults { + storedExamResultsMap[storedExamResult.ExamID] = storedExamResult + } + + var newPublishedExamResults []model.PublishedExamResult + + for id, result := range apiExamResultsMap { + if storedResult, ok := storedExamResultsMap[id]; ok && !storedResult.Published && result.Published { + newPublishedExamResults = append(newPublishedExamResults, result) + } + } + + return &newPublishedExamResults +} + +func (service *Service) notifySubscribers(newPublishedExamResults *[]model.PublishedExamResult) { + log.Infof("Notifying subscribers about %d published exam results", len(*newPublishedExamResults)) + + subscribersRepo := new_exam_results_subscriber.NewRepository(service.Repository.DB) + subscribersService := new_exam_results_subscriber.NewService(subscribersRepo) + + err := subscribersService.NotifySubscribers(newPublishedExamResults) + if err != nil { + log.WithError(err).Error("Failed to notify subscribers") + return + } +} + +func NewService(repository *Repository, + devicesRepository *ios_device.Repository, + apnsService *ios_apns.Service, +) *Service { + return &Service{ + Repository: repository, + DevicesRepository: devicesRepository, + Priority: model.DefaultIOSSchedulingPriority(), + APNs: apnsService, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go new file mode 100644 index 00000000..5482daae --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -0,0 +1,59 @@ +package new_exam_results_subscriber + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +type Repository struct { + DB *gorm.DB +} + +func (repository *Repository) FindAllSubscribers() (*[]model.NewExamResultsSubscriber, error) { + db := repository.DB + + var subscribers []model.NewExamResultsSubscriber + + err := db.Find(&subscribers).Error + + return &subscribers, err +} + +func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsSubscriber, newGrades *[]model.PublishedExamResult) error { + url := subscriber.CallbackUrl + + body, err := json.Marshal(newGrades) + if err != nil { + log.WithError(err).Error("Error while marshalling newGrades") + return err + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) + if err != nil { + log.WithError(err).Error("Error while creating request") + return err + } + req.Header.Set("Content-Type", "application/json") + if subscriber.ApiKey.Valid { + req.Header.Set("Authorization", subscriber.ApiKey.String) + } + + _, err = http.DefaultClient.Do(req) + if err != nil { + log.WithField("url", url).WithError(err).Error("Error while fetching url") + return err + } + + return nil +} + +func NewRepository(db *gorm.DB) *Repository { + return &Repository{ + DB: db, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go new file mode 100644 index 00000000..a13bda3e --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go @@ -0,0 +1,34 @@ +package new_exam_results_subscriber + +import ( + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" +) + +type Service struct { + Repository *Repository +} + +func (service *Service) NotifySubscribers(newGrades *[]model.PublishedExamResult) error { + repository := service.Repository + + subscribers, err := repository.FindAllSubscribers() + if err != nil { + return err + } + + for _, subscriber := range *subscribers { + if err := repository.NotifySubscriber(&subscriber, newGrades); err != nil { + log.WithError(err).Error("Failed to notify subscriber") + continue + } + } + + return nil +} + +func NewService(repository *Repository) *Service { + return &Service{ + Repository: repository, + } +} diff --git a/server/model/crontab.go b/server/model/crontab.go index 183721e3..de013c86 100644 --- a/server/model/crontab.go +++ b/server/model/crontab.go @@ -14,6 +14,6 @@ type Crontab struct { Cron int64 `gorm:"primary_key;AUTO_INCREMENT;column:cron;type:int;" json:"cron"` Interval int32 `gorm:"column:interval;type:int;default:7200;" json:"interval"` LastRun int32 `gorm:"column:lastRun;type:int;default:0;" json:"last_run"` - Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'kino', 'roomfinder', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount');" json:"type"` + Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'kino', 'roomfinder', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` ID null.Int `gorm:"column:id;type:int;" json:"id"` } diff --git a/server/model/examResultPublished.go b/server/model/examResultPublished.go new file mode 100644 index 00000000..5cd24ffe --- /dev/null +++ b/server/model/examResultPublished.go @@ -0,0 +1,58 @@ +package model + +import ( + "encoding/xml" + "time" +) + +type campusApiBool bool + +func (p *campusApiBool) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var value string + if err := d.DecodeElement(&value, &start); err != nil { + return err + } + switch value { + case "J": + *p = true + default: + *p = false + } + return nil +} + +type TUMAPIPublishedExamResults struct { + XMLName xml.Name `xml:"pruefungen"` + ExamResults []TUMAPIPublishedExamResult `xml:"pruefung"` +} + +type TUMAPIPublishedExamResult struct { + XMLName xml.Name `xml:"pruefung"` + Date customDate `xml:"datum"` + ExamID string `xml:"pv_term_nr"` + LectureTitle string `xml:"lv_titel"` + LectureNumber string `xml:"lv_nummer"` + LectureSem string `xml:"lv_semester"` + LectureType string `xml:"lv_typ"` + Published campusApiBool `xml:"note_veroeffentlicht"` +} + +func (examResult *TUMAPIPublishedExamResult) ToDBExamResult() *PublishedExamResult { + return &PublishedExamResult{ + Date: examResult.Date.Time, + ExamID: examResult.ExamID, + LectureTitle: examResult.LectureTitle, + LectureType: examResult.LectureType, + LectureSem: examResult.LectureSem, + Published: bool(examResult.Published), + } +} + +type PublishedExamResult struct { + Date time.Time `json:"date"` + ExamID string `gorm:"primary_key" json:"examId"` + LectureTitle string `json:"lectureTitle"` + LectureType string `json:"lectureType"` + LectureSem string `json:"lectureSem"` + Published bool `json:"published"` +} diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go new file mode 100644 index 00000000..56bb5661 --- /dev/null +++ b/server/model/newExamResultsSubscriber.go @@ -0,0 +1,14 @@ +package model + +import ( + "time" + + "github.com/guregu/null" +) + +type NewExamResultsSubscriber struct { + CallbackUrl string `gorm:"primary_key" json:"callbackUrl"` + ApiKey null.String `json:"-"` + CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` + LastNotifiedAt null.Time `json:"lastNotifiedAt"` +}