From 0ae6ce1524b58d7655b7189f23912d1126f69dc0 Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 18 Jun 2023 21:53:30 +0200 Subject: [PATCH 01/14] feat: implemented base grade notification hook service --- server/backend/campus_api/campusApi.go | 49 +++++---- server/backend/cron/cronjobs.go | 6 +- server/backend/cron/newExamResultsHook.go | 15 +++ server/backend/migration/20230530000000.go | 51 +++++++++ server/backend/migration/migration.go | 1 + .../newExamResultsSchedulingRepository.go | 37 +++++++ .../newExamResultsSchedulingService.go | 100 ++++++++++++++++++ .../newExamResultsSubscriberRepository.go | 57 ++++++++++ .../newExamResultsSubscriberService.go | 35 ++++++ server/model/crontab.go | 2 +- server/model/examResultPublished.go | 62 +++++++++++ server/model/newExamResultsSubscriber.go | 13 +++ 12 files changed, 408 insertions(+), 20 deletions(-) create mode 100644 server/backend/cron/newExamResultsHook.go create mode 100644 server/backend/migration/20230530000000.go create mode 100644 server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go create mode 100644 server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go create mode 100644 server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go create mode 100644 server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go create mode 100644 server/model/examResultPublished.go create mode 100644 server/model/newExamResultsSubscriber.go diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index fc59c0da..8d2fc5b3 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -11,37 +11,51 @@ import ( ) const ( - CampusApiUrl = "https://campus.tum.de/tumonline" - CampusQueryToken = "pToken" - CampusGradesPath = "/wbservicesbasic.noten" + CampusApiUrl = "https://campus.tum.de/tumonline" + CampusQueryToken = "pToken" + CampusGradesPath = "/wbservicesbasic.noten" + CampusExamResultsPublished = "/wbservicesbasic.pruefungenErgebnisse" ) var ( ErrCannotCreateRequest = errors.New("cannot create http request") - ErrWhileFetchingGrades = errors.New("error while fetching grades") ErrorWhileUnmarshalling = errors.New("error while unmarshalling") ) +func FetchExamResultsPublished(token string) (*model.TUMAPIExamResultsPublished, error) { + var examResultsPublished model.TUMAPIExamResultsPublished + err := RequestCampusApi(CampusExamResultsPublished, token, &examResultsPublished) + if err != nil { + return nil, err + } + + return &examResultsPublished, nil +} + func FetchGrades(token string) (*model.IOSGrades, error) { + var grades model.IOSGrades + err := RequestCampusApi(CampusGradesPath, token, &grades) + if err != nil { + return nil, err + } - requestUrl := CampusApiUrl + CampusGradesPath + return &grades, nil +} + +func RequestCampusApi(path string, token string, response any) error { + requestUrl := "https://exams.free.beeceptor.com/" req, err := http.NewRequest(http.MethodGet, requestUrl, nil) if err != nil { log.Errorf("Error while creating request: %s", err) - return nil, ErrCannotCreateRequest + return ErrCannotCreateRequest } - q := req.URL.Query() - q.Add(CampusQueryToken, token) - - req.URL.RawQuery = q.Encode() - resp, err := http.DefaultClient.Do(req) if err != nil { - log.Errorf("Error while fetching grades: %s", err) - return nil, ErrWhileFetchingGrades + log.Errorf("Error while fetching %s: %s", path, err) + return errors.New("error while fetching " + path) } defer func(Body io.ReadCloser) { @@ -51,13 +65,12 @@ func FetchGrades(token string) (*model.IOSGrades, error) { } }(resp.Body) - var grades model.IOSGrades - err = xml.NewDecoder(resp.Body).Decode(&grades) + err = xml.NewDecoder(resp.Body).Decode(&response) if err != nil { - log.Errorf("Error while unmarshalling grades: %s", err) - return nil, ErrorWhileUnmarshalling + log.Errorf("Error while unmarshalling %s: %s", path, err) + return ErrorWhileUnmarshalling } - return &grades, nil + return nil } diff --git a/server/backend/cron/cronjobs.go b/server/backend/cron/cronjobs.go index df39184b..b084d8cb 100644 --- a/server/backend/cron/cronjobs.go +++ b/server/backend/cron/cronjobs.go @@ -28,6 +28,7 @@ const ( StorageDir = "/Storage/" // target location of files IOSNotifications = "iosNotifications" IOSActivityReset = "iosActivityReset" + NewExamResultsHook = "newExamResultsHook" /* MensaType = "mensa" ChatType = "chat" @@ -58,7 +59,7 @@ func (c *CronService) Run() error { var res []model.Crontab c.db.Model(&model.Crontab{}). - Where("`interval` > 0 AND (lastRun+`interval`) < ? AND type IN (?, ?, ?, ?, ?, ?, ?)", + Where("`interval` > 0 AND (lastRun+`interval`) < ? AND type IN (?, ?, ?, ?, ?, ?, ?, ?)", time.Now().Unix(), NewsType, FileDownloadType, @@ -67,6 +68,7 @@ func (c *CronService) Run() error { CanteenHeadcount, IOSNotifications, IOSActivityReset, + NewExamResultsHook, ). Scan(&res) @@ -98,6 +100,8 @@ func (c *CronService) Run() error { if c.useMensa { g.Go(c.averageRatingComputation) } + case NewExamResultsHook: + g.Go(func() error { return c.newExamResultsHookCron() }) /* TODO: Implement handlers for other cronjobs case MensaType: diff --git a/server/backend/cron/newExamResultsHook.go b/server/backend/cron/newExamResultsHook.go new file mode 100644 index 00000000..122d636c --- /dev/null +++ b/server/backend/cron/newExamResultsHook.go @@ -0,0 +1,15 @@ +package cron + +import ( + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" + "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_scheduling" +) + +func (c *CronService) newExamResultsHookCron() error { + repo := new_exam_results_scheduling.NewRepository(c.db) + devicesRepo := ios_device.NewRepository(c.db) + + service := new_exam_results_scheduling.NewService(repo, devicesRepo, c.APNs) + + return service.HandleScheduledCron() +} diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go new file mode 100644 index 00000000..3cd2e0e6 --- /dev/null +++ b/server/backend/migration/20230530000000.go @@ -0,0 +1,51 @@ +package migration + +import ( + "database/sql" + _ "embed" + "github.com/TUM-Dev/Campus-Backend/server/backend/cron" + "github.com/TUM-Dev/Campus-Backend/server/model" + "github.com/go-gormigrate/gormigrate/v2" + "github.com/guregu/null" + "gorm.io/gorm" +) + +func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20230530000000", + Migrate: func(tx *gorm.DB) error { + + if err := tx.AutoMigrate( + &model.ExamResultPublished{}, + &model.NewExamResultsSubscriber{}, + ); err != nil { + return err + } + + err := SafeEnumMigrate(tx, model.Crontab{}, "type", cron.NewExamResultsHook) + if err != nil { + return err + } + + return tx.Create(&model.Crontab{ + Interval: 60, // Every 5 minutes + Type: null.String{NullString: sql.NullString{String: cron.NewExamResultsHook, Valid: true}}, + }).Error + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Migrator().DropTable(&model.ExamResultPublished{}); err != nil { + return err + } + if err := tx.Migrator().DropTable(&model.NewExamResultsSubscriber{}); err != nil { + return err + } + + err := SafeEnumRollback(tx, model.Crontab{}, "type", cron.NewExamResultsHook) + if err != nil { + return err + } + + return tx.Delete(&model.Crontab{}, "type = ?", cron.NewExamResultsHook).Error + }, + } +} diff --git a/server/backend/migration/migration.go b/server/backend/migration/migration.go index 5302bf11..83050cf4 100644 --- a/server/backend/migration/migration.go +++ b/server/backend/migration/migration.go @@ -41,6 +41,7 @@ func (m TumDBMigrator) Migrate() error { m.migrate20220713000000(), m.migrate20221119131300(), m.migrate20221210000000(), + m.migrate20230530000000(), }) err := mig.Migrate() return err diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go new file mode 100644 index 00000000..8d241128 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go @@ -0,0 +1,37 @@ +package new_exam_results_scheduling + +import ( + "github.com/TUM-Dev/Campus-Backend/server/model" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type Repository struct { + DB *gorm.DB +} + +func (repository *Repository) StoreExamResultsPublished(examResultsPublished []model.ExamResultPublished) error { + db := repository.DB + + db.Where("1 = 1").Delete(&model.ExamResultPublished{}) + + return db. + Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}). + Create(examResultsPublished).Error +} + +func (repository *Repository) FindAllExamResultsPublished() (*[]model.ExamResultPublished, error) { + db := repository.DB + + var results []model.ExamResultPublished + + err := db.Find(&results).Error + + return &results, err +} + +func NewRepository(db *gorm.DB) *Repository { + return &Repository{ + DB: db, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go new file mode 100644 index 00000000..3cf343b3 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go @@ -0,0 +1,100 @@ +package new_exam_results_scheduling + +import ( + "github.com/TUM-Dev/Campus-Backend/server/backend/campus_api" + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns" + "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" + "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_subscriber" + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" +) + +const ( + MaxRoutineCount = 10 + MockAPIToken = "DDF9A212B2F80A01C6D0307B8455EEAA" +) + +type Service struct { + Repository *Repository + DevicesRepository *ios_device.Repository + Priority *model.IOSSchedulingPriority + APNs *ios_apns.Service +} + +func (service *Service) HandleScheduledCron() error { + log.Info("Fetching published exam results") + + apiResult, err := campus_api.FetchExamResultsPublished(MockAPIToken) + if err != nil { + return err + } + + var apiExamResults []model.ExamResultPublished + for _, apiExamResult := range apiResult.ExamResults { + apiExamResults = append(apiExamResults, *apiExamResult.ToDBExamResult()) + } + + storedExamResults, err := service.Repository.FindAllExamResultsPublished() + if err != nil { + return err + } + + newPublishedExamResults := service.findNewPublishedExamResults(&apiExamResults, storedExamResults) + + if len(*newPublishedExamResults) > 0 { + service.notifySubscribers(newPublishedExamResults) + } else { + log.Info("No new published exam results") + } + + service.Repository.StoreExamResultsPublished(apiExamResults) + + return nil +} + +func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamResults *[]model.ExamResultPublished) *[]model.ExamResultPublished { + var apiExamResultsMap = make(map[string]model.ExamResultPublished) + for _, apiExamResult := range *apiExamResults { + apiExamResultsMap[apiExamResult.ExamID] = apiExamResult + } + + var storedExamResultsMap = make(map[string]model.ExamResultPublished) + for _, storedExamResult := range *storedExamResults { + storedExamResultsMap[storedExamResult.ExamID] = storedExamResult + } + + var newPublishedExamResults []model.ExamResultPublished + + for id, result := range apiExamResultsMap { + if storedResult, ok := storedExamResultsMap[id]; ok && !storedResult.Published && result.Published { + newPublishedExamResults = append(newPublishedExamResults, result) + } + } + + return &newPublishedExamResults +} + +func (service *Service) notifySubscribers(newPublishedExamResults *[]model.ExamResultPublished) { + log.Infof("Notifying subscribers about %d published exam results", len(*newPublishedExamResults)) + + subscribersRepo := new_exam_results_subscriber.NewRepository(service.Repository.DB) + subscribersService := new_exam_results_subscriber.NewService(subscribersRepo) + + err := subscribersService.NotifySubscribers(newPublishedExamResults) + if err != nil { + log.WithError(err).Error("Failed to notify subscribers") + return + } +} + +func NewService(repository *Repository, + devicesRepository *ios_device.Repository, + apnsService *ios_apns.Service, +) *Service { + return &Service{ + Repository: repository, + DevicesRepository: devicesRepository, + Priority: model.DefaultIOSSchedulingPriority(), + APNs: apnsService, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go new file mode 100644 index 00000000..6344e205 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -0,0 +1,57 @@ +package new_exam_results_subscriber + +import ( + "bytes" + "encoding/json" + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" + "gorm.io/gorm" + "net/http" +) + +type Repository struct { + DB *gorm.DB +} + +func (repository *Repository) FindAllSubscribers() (*[]model.NewExamResultsSubscriber, error) { + db := repository.DB + + var subscribers []model.NewExamResultsSubscriber + + err := db.Find(&subscribers).Error + + return &subscribers, err +} + +func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsSubscriber, newGrades *[]model.ExamResultPublished) error { + url := subscriber.CallbackUrl + + body, err := json.Marshal(newGrades) + if err != nil { + log.WithError(err).Errorf("Error while marshalling newGrades") + return err + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) + + req.Header.Set("Content-Type", "application/json") + + if err != nil { + log.WithError(err).Errorf("Error while creating request") + return err + } + + _, err = http.DefaultClient.Do(req) + if err != nil { + log.WithError(err).Errorf("Error while fetching %s", url) + return err + } + + return nil +} + +func NewRepository(db *gorm.DB) *Repository { + return &Repository{ + DB: db, + } +} diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go new file mode 100644 index 00000000..e3d739c7 --- /dev/null +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go @@ -0,0 +1,35 @@ +package new_exam_results_subscriber + +import ( + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" +) + +type Service struct { + Repository *Repository +} + +func (service *Service) NotifySubscribers(newGrades *[]model.ExamResultPublished) error { + repository := service.Repository + + subscribers, err := repository.FindAllSubscribers() + if err != nil { + return err + } + + for _, subscriber := range *subscribers { + err := repository.NotifySubscriber(&subscriber, newGrades) + if err != nil { + log.WithError(err).Error("Failed to notify subscriber") + continue + } + } + + return nil +} + +func NewService(repository *Repository) *Service { + return &Service{ + Repository: repository, + } +} diff --git a/server/model/crontab.go b/server/model/crontab.go index 54f175ea..b81e35dc 100644 --- a/server/model/crontab.go +++ b/server/model/crontab.go @@ -18,7 +18,7 @@ type Crontab struct { //[ 2] lastRun int null: false primary: false isArray: false auto: false col: int len: -1 default: [0] LastRun int32 `gorm:"column:lastRun;type:int;default:0;" json:"last_run"` //[ 3] type char(10) null: true primary: false isArray: false auto: false col: char len: 10 default: [] - Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount');" json:"type"` + Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` //[ 4] id int null: true primary: false isArray: false auto: false col: int len: -1 default: [] ID null.Int `gorm:"column:id;type:int;" json:"id"` } diff --git a/server/model/examResultPublished.go b/server/model/examResultPublished.go new file mode 100644 index 00000000..ac394141 --- /dev/null +++ b/server/model/examResultPublished.go @@ -0,0 +1,62 @@ +package model + +import ( + "encoding/xml" + "time" +) + +type campusApiBool bool + +type TUMAPIExamResultsPublished struct { + XMLName xml.Name `xml:"pruefungen"` + ExamResults []TUMAPIExamResultPublished `xml:"pruefung"` +} + +type TUMAPIExamResultPublished struct { + XMLName xml.Name `xml:"pruefung"` + Date customDate `xml:"datum"` + ExamID string `xml:"pv_term_nr"` + LectureTitle string `xml:"lv_titel"` + LectureNumber string `xml:"lv_nummer"` + LectureSem string `xml:"lv_semester"` + LectureType string `xml:"lv_typ"` + Published campusApiBool `xml:"note_veroeffentlicht"` +} + +func (examResult *TUMAPIExamResultPublished) ToDBExamResult() *ExamResultPublished { + return &ExamResultPublished{ + Date: examResult.Date.Time, + ExamID: examResult.ExamID, + LectureTitle: examResult.LectureTitle, + LectureType: examResult.LectureType, + LectureSem: examResult.LectureSem, + Published: bool(examResult.Published), + } +} + +type ExamResultPublished struct { + Date time.Time `json:"date"` + ExamID string `gorm:"primary_key" json:"examId"` + LectureTitle string `json:"lectureTitle"` + LectureType string `json:"lectureType"` + LectureSem string `json:"lectureSem"` + Published bool `json:"published"` +} + +func (ExamResultPublished) TableName() string { + return "exam_results_published" +} + +func (p *campusApiBool) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var value string + if err := d.DecodeElement(&value, &start); err != nil { + return err + } + switch value { + case "J": + *p = true + default: + *p = false + } + return nil +} diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go new file mode 100644 index 00000000..4adb7715 --- /dev/null +++ b/server/model/newExamResultsSubscriber.go @@ -0,0 +1,13 @@ +package model + +import ( + "database/sql" + "time" +) + +type NewExamResultsSubscriber struct { + ID int32 `gorm:"primary_key;AUTO_INCREMENT;" json:"id"` + CallbackUrl string `json:"callbackUrl"` + CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` + LastNotifiedAt sql.NullTime `json:"lastNotifiedAt"` +} From dc5d93738421623d3a2b79ae27286fac8545256a Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 2 Jul 2023 16:39:23 +0200 Subject: [PATCH 02/14] feat: added api key option --- .../newExamResultsSubscriberRepository.go | 4 ++++ server/model/newExamResultsSubscriber.go | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go index 6344e205..76ef8863 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -36,6 +36,10 @@ func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsS req.Header.Set("Content-Type", "application/json") + if subscriber.ApiKey.Valid { + req.Header.Set("Authorization", subscriber.ApiKey.String) + } + if err != nil { log.WithError(err).Errorf("Error while creating request") return err diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go index 4adb7715..52975d72 100644 --- a/server/model/newExamResultsSubscriber.go +++ b/server/model/newExamResultsSubscriber.go @@ -6,8 +6,9 @@ import ( ) type NewExamResultsSubscriber struct { - ID int32 `gorm:"primary_key;AUTO_INCREMENT;" json:"id"` - CallbackUrl string `json:"callbackUrl"` - CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` - LastNotifiedAt sql.NullTime `json:"lastNotifiedAt"` + ID int32 `gorm:"primary_key;AUTO_INCREMENT;" json:"id"` + CallbackUrl string `json:"callbackUrl"` + ApiKey sql.NullString `json:"-"` + CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` + LastNotifiedAt sql.NullTime `json:"lastNotifiedAt"` } From 2f96449fadf6b6c3c434938274f57a8740739b41 Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 17 Sep 2023 18:27:35 +0200 Subject: [PATCH 03/14] fix: campusAPI Url change & new error calls --- server/backend/campus_api/campusApi.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index 8d2fc5b3..d20e8ec8 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -4,6 +4,7 @@ package campus_api import ( "encoding/xml" "errors" + "fmt" "github.com/TUM-Dev/Campus-Backend/server/model" log "github.com/sirupsen/logrus" "io" @@ -43,32 +44,32 @@ func FetchGrades(token string) (*model.IOSGrades, error) { } func RequestCampusApi(path string, token string, response any) error { - requestUrl := "https://exams.free.beeceptor.com/" + requestUrl := fmt.Sprintf("%s%s?%s=%s", CampusApiUrl, path, CampusQueryToken, token) req, err := http.NewRequest(http.MethodGet, requestUrl, nil) if err != nil { - log.Errorf("Error while creating request: %s", err) + log.WithError(err).Error("Error while creating request") return ErrCannotCreateRequest } resp, err := http.DefaultClient.Do(req) if err != nil { - log.Errorf("Error while fetching %s: %s", path, err) + log.WithError(err).WithField("path", path).Error("Error while fetching url") return errors.New("error while fetching " + path) } defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { - log.Errorf("Error while closing body: %s", err) + log.WithError(err).Error("Error while closing body") } }(resp.Body) err = xml.NewDecoder(resp.Body).Decode(&response) if err != nil { - log.Errorf("Error while unmarshalling %s: %s", path, err) + log.WithError(err).WithField("path", path).Error("Error while unmarshalling") return ErrorWhileUnmarshalling } From 2ada07c90ab6e9bedc94bee0fe1ade0485531c05 Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 17 Sep 2023 18:40:26 +0200 Subject: [PATCH 04/14] fix: exam scheduling comments => moved CAMPUS_API_TOKEN into .env --- .env | 4 +++- .../newExamResultsSchedulingRepository.go | 22 +++++++++++-------- .../newExamResultsSchedulingService.go | 12 +++++----- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.env b/.env index 33bc1ff7..7a48aca7 100644 --- a/.env +++ b/.env @@ -7,4 +7,6 @@ APNS_P8_FILE_PATH=/secrets/AuthKey_XXXX.p8 ENVIRONMENT=dev -SENTRY_DSN= \ No newline at end of file +SENTRY_DSN= + +CAMPUS_API_TOKEN= \ No newline at end of file diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go index 8d241128..47d5931b 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go @@ -13,19 +13,23 @@ type Repository struct { func (repository *Repository) StoreExamResultsPublished(examResultsPublished []model.ExamResultPublished) error { db := repository.DB - db.Where("1 = 1").Delete(&model.ExamResultPublished{}) - - return db. - Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}). - Create(examResultsPublished).Error + return db.Transaction(func(tx *gorm.DB) error { + err := tx.Where("1 = 1").Delete(&model.ExamResultPublished{}).Error + + if err != nil { + return err + } + + // disabled logging because this query always prints a warning because it takes longer then normal + // to execute because we bulk insert a lot of data + return tx.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}). + Create(examResultsPublished).Error + }) } func (repository *Repository) FindAllExamResultsPublished() (*[]model.ExamResultPublished, error) { - db := repository.DB - var results []model.ExamResultPublished - - err := db.Find(&results).Error + err := repository.DB.Find(&results).Error return &results, err } diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go index 3cf343b3..b1f6cf5c 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go @@ -7,11 +7,11 @@ import ( "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_subscriber" "github.com/TUM-Dev/Campus-Backend/server/model" log "github.com/sirupsen/logrus" + "os" ) -const ( - MaxRoutineCount = 10 - MockAPIToken = "DDF9A212B2F80A01C6D0307B8455EEAA" +var ( + CampusApiToken = os.Getenv("CAMPUS_API_TOKEN") ) type Service struct { @@ -24,7 +24,7 @@ type Service struct { func (service *Service) HandleScheduledCron() error { log.Info("Fetching published exam results") - apiResult, err := campus_api.FetchExamResultsPublished(MockAPIToken) + apiResult, err := campus_api.FetchExamResultsPublished(CampusApiToken) if err != nil { return err } @@ -47,9 +47,7 @@ func (service *Service) HandleScheduledCron() error { log.Info("No new published exam results") } - service.Repository.StoreExamResultsPublished(apiExamResults) - - return nil + return service.Repository.StoreExamResultsPublished(apiExamResults) } func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamResults *[]model.ExamResultPublished) *[]model.ExamResultPublished { From 346d799fb68e6d4cf6e7c1c680c9ddce6af1cff7 Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 17 Sep 2023 18:45:45 +0200 Subject: [PATCH 05/14] fix: directly added primary key to callback url --- server/model/newExamResultsSubscriber.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go index 52975d72..e68a4f60 100644 --- a/server/model/newExamResultsSubscriber.go +++ b/server/model/newExamResultsSubscriber.go @@ -6,8 +6,7 @@ import ( ) type NewExamResultsSubscriber struct { - ID int32 `gorm:"primary_key;AUTO_INCREMENT;" json:"id"` - CallbackUrl string `json:"callbackUrl"` + CallbackUrl string `gorm:"primary_key" json:"callbackUrl"` ApiKey sql.NullString `json:"-"` CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` LastNotifiedAt sql.NullTime `json:"lastNotifiedAt"` From f391cb5227ee10872382f4a3a2816ffe3488192c Mon Sep 17 00:00:00 2001 From: Anton Wyrowski Date: Sun, 17 Sep 2023 18:53:27 +0200 Subject: [PATCH 06/14] fix: formatting --- server/model/crontab.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/model/crontab.go b/server/model/crontab.go index c98c20c2..7ef742f9 100644 --- a/server/model/crontab.go +++ b/server/model/crontab.go @@ -11,9 +11,9 @@ func (Crontab) TableName() string { // Crontab struct is a row record of the crontab table in the tca database type Crontab struct { - Cron int32 `gorm:"primary_key;AUTO_INCREMENT;column:cron;type:int;" json:"cron"` - Interval int32 `gorm:"column:interval;type:int;default:7200;" json:"interval"` - LastRun int32 `gorm:"column:lastRun;type:int;default:0;" json:"last_run"` - Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` - ID null.Int `gorm:"column:id;type:int;" json:"id"` + Cron int32 `gorm:"primary_key;AUTO_INCREMENT;column:cron;type:int;" json:"cron"` + Interval int32 `gorm:"column:interval;type:int;default:7200;" json:"interval"` + LastRun int32 `gorm:"column:lastRun;type:int;default:0;" json:"last_run"` + Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` + ID null.Int `gorm:"column:id;type:int;" json:"id"` } From 45f9fac6e56d859527bd8e15771fb00d2815831c Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:14:16 +0200 Subject: [PATCH 07/14] Apply suggestions from code review --- server/backend/migration/20230530000000.go | 2 +- .../newExamResultsSubscriberRepository.go | 6 +++--- server/model/crontab.go | 2 +- server/model/newExamResultsSubscriber.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 3cd2e0e6..600778e3 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -29,7 +29,7 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { return tx.Create(&model.Crontab{ Interval: 60, // Every 5 minutes - Type: null.String{NullString: sql.NullString{String: cron.NewExamResultsHook, Valid: true}}, + Type: null.StringFrom(cron.NewExamResultsHook), }).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go index 76ef8863..31883672 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -28,7 +28,7 @@ func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsS body, err := json.Marshal(newGrades) if err != nil { - log.WithError(err).Errorf("Error while marshalling newGrades") + log.WithError(err).Error("Error while marshalling newGrades") return err } @@ -41,13 +41,13 @@ func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsS } if err != nil { - log.WithError(err).Errorf("Error while creating request") + log.WithError(err).Error("Error while creating request") return err } _, err = http.DefaultClient.Do(req) if err != nil { - log.WithError(err).Errorf("Error while fetching %s", url) + log.WithField("url", url).WithError(err).Error("Error while fetching url") return err } diff --git a/server/model/crontab.go b/server/model/crontab.go index 03ecf399..de013c86 100644 --- a/server/model/crontab.go +++ b/server/model/crontab.go @@ -14,6 +14,6 @@ type Crontab struct { Cron int64 `gorm:"primary_key;AUTO_INCREMENT;column:cron;type:int;" json:"cron"` Interval int32 `gorm:"column:interval;type:int;default:7200;" json:"interval"` LastRun int32 `gorm:"column:lastRun;type:int;default:0;" json:"last_run"` - Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'chat', 'kino', 'roomfinder', 'ticketsale', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` + Type null.String `gorm:"column:type;type:enum ('news', 'mensa', 'kino', 'roomfinder', 'alarm', 'fileDownload','dishNameDownload','averageRatingComputation', 'iosNotifications', 'iosActivityReset', 'canteenHeadCount', 'newExamResultsHook');" json:"type"` ID null.Int `gorm:"column:id;type:int;" json:"id"` } diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go index e68a4f60..16d362c6 100644 --- a/server/model/newExamResultsSubscriber.go +++ b/server/model/newExamResultsSubscriber.go @@ -7,7 +7,7 @@ import ( type NewExamResultsSubscriber struct { CallbackUrl string `gorm:"primary_key" json:"callbackUrl"` - ApiKey sql.NullString `json:"-"` + ApiKey null.String `json:"-"` CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` - LastNotifiedAt sql.NullTime `json:"lastNotifiedAt"` + LastNotifiedAt null.Time `json:"lastNotifiedAt"` } From 76e6ab2dbe392e4853c78753b7a0262a2762ceab Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:15:04 +0200 Subject: [PATCH 08/14] ran the pre-commit hook --- .env | 2 +- client/go.mod | 2 +- client/go.sum | 4 ++-- server/backend/campus_api/campusApi.go | 5 +++-- server/backend/migration/20230530000000.go | 2 +- .../newExamResultsSchedulingService.go | 3 ++- .../newExamResultsSubscriberRepository.go | 3 ++- server/model/newExamResultsSubscriber.go | 7 ++++--- 8 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 7a48aca7..b286ff71 100644 --- a/.env +++ b/.env @@ -9,4 +9,4 @@ ENVIRONMENT=dev SENTRY_DSN= -CAMPUS_API_TOKEN= \ No newline at end of file +CAMPUS_API_TOKEN= diff --git a/client/go.mod b/client/go.mod index 3d345397..7915adde 100644 --- a/client/go.mod +++ b/client/go.mod @@ -3,7 +3,7 @@ module github.com/TUM-Dev/Campus-Backend/client go 1.21 require ( - github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919155641-f895a75987e0 + github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919162132-71bec88330f7 github.com/sirupsen/logrus v1.9.3 google.golang.org/grpc v1.58.1 ) diff --git a/client/go.sum b/client/go.sum index ba3cbef0..a8611e9f 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,5 +1,5 @@ -github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919155641-f895a75987e0 h1:rZqUJmywWU9aV9Bk/IYKIYN76+nYcATD2q+pFVBYQN4= -github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919155641-f895a75987e0/go.mod h1:fjoLL3rbdY6wTRJIksekT2p3OUp5ocFfXjB/avV/TVI= +github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919162132-71bec88330f7 h1:TDgiN5Z1vi3V0Qo94MIXURiD9+U7TGiRtiUIqN/rulo= +github.com/TUM-Dev/Campus-Backend/server v0.0.0-20230919162132-71bec88330f7/go.mod h1:fjoLL3rbdY6wTRJIksekT2p3OUp5ocFfXjB/avV/TVI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index d20e8ec8..54778f70 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -5,10 +5,11 @@ import ( "encoding/xml" "errors" "fmt" - "github.com/TUM-Dev/Campus-Backend/server/model" - log "github.com/sirupsen/logrus" "io" "net/http" + + "github.com/TUM-Dev/Campus-Backend/server/model" + log "github.com/sirupsen/logrus" ) const ( diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 600778e3..0bcb1b2e 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -1,8 +1,8 @@ package migration import ( - "database/sql" _ "embed" + "github.com/TUM-Dev/Campus-Backend/server/backend/cron" "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go index b1f6cf5c..21537d3e 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go @@ -1,13 +1,14 @@ package new_exam_results_scheduling import ( + "os" + "github.com/TUM-Dev/Campus-Backend/server/backend/campus_api" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_apns" "github.com/TUM-Dev/Campus-Backend/server/backend/ios_notifications/ios_device" "github.com/TUM-Dev/Campus-Backend/server/backend/new_exam_results_hook/new_exam_results_subscriber" "github.com/TUM-Dev/Campus-Backend/server/model" log "github.com/sirupsen/logrus" - "os" ) var ( diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go index 31883672..31b38973 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -3,10 +3,11 @@ package new_exam_results_subscriber import ( "bytes" "encoding/json" + "net/http" + "github.com/TUM-Dev/Campus-Backend/server/model" log "github.com/sirupsen/logrus" "gorm.io/gorm" - "net/http" ) type Repository struct { diff --git a/server/model/newExamResultsSubscriber.go b/server/model/newExamResultsSubscriber.go index 16d362c6..56bb5661 100644 --- a/server/model/newExamResultsSubscriber.go +++ b/server/model/newExamResultsSubscriber.go @@ -1,13 +1,14 @@ package model import ( - "database/sql" "time" + + "github.com/guregu/null" ) type NewExamResultsSubscriber struct { - CallbackUrl string `gorm:"primary_key" json:"callbackUrl"` + CallbackUrl string `gorm:"primary_key" json:"callbackUrl"` ApiKey null.String `json:"-"` - CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` + CreatedAt time.Time `gorm:"autoCreateTime" json:"createdAt"` LastNotifiedAt null.Time `json:"lastNotifiedAt"` } From 8070d61dabe3a9cd1e92122136d1d34e8b129bef Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:17:01 +0200 Subject: [PATCH 09/14] renamed `ExamResultPublished` -> `PublishedExamResult` --- server/backend/campus_api/campusApi.go | 4 +- server/backend/migration/20230530000000.go | 4 +- .../newExamResultsSchedulingRepository.go | 8 ++-- .../newExamResultsSchedulingService.go | 12 ++--- .../newExamResultsSubscriberRepository.go | 2 +- .../newExamResultsSubscriberService.go | 2 +- server/model/examResultPublished.go | 44 +++++++++---------- 7 files changed, 36 insertions(+), 40 deletions(-) diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index 54778f70..29ec93ff 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -24,8 +24,8 @@ var ( ErrorWhileUnmarshalling = errors.New("error while unmarshalling") ) -func FetchExamResultsPublished(token string) (*model.TUMAPIExamResultsPublished, error) { - var examResultsPublished model.TUMAPIExamResultsPublished +func FetchExamResultsPublished(token string) (*model.TUMAPIPublishedExamResults, error) { + var examResultsPublished model.TUMAPIPublishedExamResults err := RequestCampusApi(CampusExamResultsPublished, token, &examResultsPublished) if err != nil { return nil, err diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 0bcb1b2e..38a7f014 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -16,7 +16,7 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { Migrate: func(tx *gorm.DB) error { if err := tx.AutoMigrate( - &model.ExamResultPublished{}, + &model.PublishedExamResult{}, &model.NewExamResultsSubscriber{}, ); err != nil { return err @@ -33,7 +33,7 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { }).Error }, Rollback: func(tx *gorm.DB) error { - if err := tx.Migrator().DropTable(&model.ExamResultPublished{}); err != nil { + if err := tx.Migrator().DropTable(&model.PublishedExamResult{}); err != nil { return err } if err := tx.Migrator().DropTable(&model.NewExamResultsSubscriber{}); err != nil { diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go index 47d5931b..4daada15 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingRepository.go @@ -10,11 +10,11 @@ type Repository struct { DB *gorm.DB } -func (repository *Repository) StoreExamResultsPublished(examResultsPublished []model.ExamResultPublished) error { +func (repository *Repository) StoreExamResultsPublished(examResultsPublished []model.PublishedExamResult) error { db := repository.DB return db.Transaction(func(tx *gorm.DB) error { - err := tx.Where("1 = 1").Delete(&model.ExamResultPublished{}).Error + err := tx.Where("1 = 1").Delete(&model.PublishedExamResult{}).Error if err != nil { return err @@ -27,8 +27,8 @@ func (repository *Repository) StoreExamResultsPublished(examResultsPublished []m }) } -func (repository *Repository) FindAllExamResultsPublished() (*[]model.ExamResultPublished, error) { - var results []model.ExamResultPublished +func (repository *Repository) FindAllExamResultsPublished() (*[]model.PublishedExamResult, error) { + var results []model.PublishedExamResult err := repository.DB.Find(&results).Error return &results, err diff --git a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go index 21537d3e..30c266f4 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go +++ b/server/backend/new_exam_results_hook/new_exam_results_scheduling/newExamResultsSchedulingService.go @@ -30,7 +30,7 @@ func (service *Service) HandleScheduledCron() error { return err } - var apiExamResults []model.ExamResultPublished + var apiExamResults []model.PublishedExamResult for _, apiExamResult := range apiResult.ExamResults { apiExamResults = append(apiExamResults, *apiExamResult.ToDBExamResult()) } @@ -51,18 +51,18 @@ func (service *Service) HandleScheduledCron() error { return service.Repository.StoreExamResultsPublished(apiExamResults) } -func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamResults *[]model.ExamResultPublished) *[]model.ExamResultPublished { - var apiExamResultsMap = make(map[string]model.ExamResultPublished) +func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamResults *[]model.PublishedExamResult) *[]model.PublishedExamResult { + var apiExamResultsMap = make(map[string]model.PublishedExamResult) for _, apiExamResult := range *apiExamResults { apiExamResultsMap[apiExamResult.ExamID] = apiExamResult } - var storedExamResultsMap = make(map[string]model.ExamResultPublished) + var storedExamResultsMap = make(map[string]model.PublishedExamResult) for _, storedExamResult := range *storedExamResults { storedExamResultsMap[storedExamResult.ExamID] = storedExamResult } - var newPublishedExamResults []model.ExamResultPublished + var newPublishedExamResults []model.PublishedExamResult for id, result := range apiExamResultsMap { if storedResult, ok := storedExamResultsMap[id]; ok && !storedResult.Published && result.Published { @@ -73,7 +73,7 @@ func (service *Service) findNewPublishedExamResults(apiExamResults, storedExamRe return &newPublishedExamResults } -func (service *Service) notifySubscribers(newPublishedExamResults *[]model.ExamResultPublished) { +func (service *Service) notifySubscribers(newPublishedExamResults *[]model.PublishedExamResult) { log.Infof("Notifying subscribers about %d published exam results", len(*newPublishedExamResults)) subscribersRepo := new_exam_results_subscriber.NewRepository(service.Repository.DB) diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go index 31b38973..dc3f047e 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -24,7 +24,7 @@ func (repository *Repository) FindAllSubscribers() (*[]model.NewExamResultsSubsc return &subscribers, err } -func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsSubscriber, newGrades *[]model.ExamResultPublished) error { +func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsSubscriber, newGrades *[]model.PublishedExamResult) error { url := subscriber.CallbackUrl body, err := json.Marshal(newGrades) diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go index e3d739c7..d5d373ae 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go @@ -9,7 +9,7 @@ type Service struct { Repository *Repository } -func (service *Service) NotifySubscribers(newGrades *[]model.ExamResultPublished) error { +func (service *Service) NotifySubscribers(newGrades *[]model.PublishedExamResult) error { repository := service.Repository subscribers, err := repository.FindAllSubscribers() diff --git a/server/model/examResultPublished.go b/server/model/examResultPublished.go index ac394141..5cd24ffe 100644 --- a/server/model/examResultPublished.go +++ b/server/model/examResultPublished.go @@ -7,12 +7,26 @@ import ( type campusApiBool bool -type TUMAPIExamResultsPublished struct { +func (p *campusApiBool) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var value string + if err := d.DecodeElement(&value, &start); err != nil { + return err + } + switch value { + case "J": + *p = true + default: + *p = false + } + return nil +} + +type TUMAPIPublishedExamResults struct { XMLName xml.Name `xml:"pruefungen"` - ExamResults []TUMAPIExamResultPublished `xml:"pruefung"` + ExamResults []TUMAPIPublishedExamResult `xml:"pruefung"` } -type TUMAPIExamResultPublished struct { +type TUMAPIPublishedExamResult struct { XMLName xml.Name `xml:"pruefung"` Date customDate `xml:"datum"` ExamID string `xml:"pv_term_nr"` @@ -23,8 +37,8 @@ type TUMAPIExamResultPublished struct { Published campusApiBool `xml:"note_veroeffentlicht"` } -func (examResult *TUMAPIExamResultPublished) ToDBExamResult() *ExamResultPublished { - return &ExamResultPublished{ +func (examResult *TUMAPIPublishedExamResult) ToDBExamResult() *PublishedExamResult { + return &PublishedExamResult{ Date: examResult.Date.Time, ExamID: examResult.ExamID, LectureTitle: examResult.LectureTitle, @@ -34,7 +48,7 @@ func (examResult *TUMAPIExamResultPublished) ToDBExamResult() *ExamResultPublish } } -type ExamResultPublished struct { +type PublishedExamResult struct { Date time.Time `json:"date"` ExamID string `gorm:"primary_key" json:"examId"` LectureTitle string `json:"lectureTitle"` @@ -42,21 +56,3 @@ type ExamResultPublished struct { LectureSem string `json:"lectureSem"` Published bool `json:"published"` } - -func (ExamResultPublished) TableName() string { - return "exam_results_published" -} - -func (p *campusApiBool) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var value string - if err := d.DecodeElement(&value, &start); err != nil { - return err - } - switch value { - case "J": - *p = true - default: - *p = false - } - return nil -} From fe3cfca37847c603331c651d41b943a55458eb0e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:19:38 +0200 Subject: [PATCH 10/14] inlined the models in the migration --- server/backend/migration/20230530000000.go | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 38a7f014..9e9bfb26 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -2,6 +2,7 @@ package migration import ( _ "embed" + "time" "github.com/TUM-Dev/Campus-Backend/server/backend/cron" "github.com/TUM-Dev/Campus-Backend/server/model" @@ -10,14 +11,30 @@ import ( "gorm.io/gorm" ) +type PublishedExamResult struct { + Date time.Time + ExamID string `gorm:"primary_key"` + LectureTitle string + LectureType string + LectureSem string + Published bool +} + +type NewExamResultsSubscriber struct { + CallbackUrl string `gorm:"primary_key"` + ApiKey null.String + CreatedAt time.Time `gorm:"autoCreateTime"` + LastNotifiedAt null.Time +} + func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { return &gormigrate.Migration{ ID: "20230530000000", Migrate: func(tx *gorm.DB) error { if err := tx.AutoMigrate( - &model.PublishedExamResult{}, - &model.NewExamResultsSubscriber{}, + &PublishedExamResult{}, + &NewExamResultsSubscriber{}, ); err != nil { return err } @@ -33,10 +50,10 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { }).Error }, Rollback: func(tx *gorm.DB) error { - if err := tx.Migrator().DropTable(&model.PublishedExamResult{}); err != nil { + if err := tx.Migrator().DropTable(&PublishedExamResult{}); err != nil { return err } - if err := tx.Migrator().DropTable(&model.NewExamResultsSubscriber{}); err != nil { + if err := tx.Migrator().DropTable(&NewExamResultsSubscriber{}); err != nil { return err } From 5bcb00516982bc5cef5ce05ef211a44bdde5111c Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:20:55 +0200 Subject: [PATCH 11/14] decoupled cron and the migration --- server/backend/migration/20230530000000.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 9e9bfb26..9278c595 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -4,7 +4,6 @@ import ( _ "embed" "time" - "github.com/TUM-Dev/Campus-Backend/server/backend/cron" "github.com/TUM-Dev/Campus-Backend/server/model" "github.com/go-gormigrate/gormigrate/v2" "github.com/guregu/null" @@ -39,14 +38,14 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { return err } - err := SafeEnumMigrate(tx, model.Crontab{}, "type", cron.NewExamResultsHook) + err := SafeEnumMigrate(tx, model.Crontab{}, "type", "newExamResultsHook") if err != nil { return err } return tx.Create(&model.Crontab{ Interval: 60, // Every 5 minutes - Type: null.StringFrom(cron.NewExamResultsHook), + Type: null.StringFrom("newExamResultsHook"), }).Error }, Rollback: func(tx *gorm.DB) error { @@ -57,12 +56,12 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { return err } - err := SafeEnumRollback(tx, model.Crontab{}, "type", cron.NewExamResultsHook) + err := SafeEnumRollback(tx, model.Crontab{}, "type", "newExamResultsHook") if err != nil { return err } - return tx.Delete(&model.Crontab{}, "type = ?", cron.NewExamResultsHook).Error + return tx.Delete(&model.Crontab{}, "type = ?", "newExamResultsHook").Error }, } } From 95c40ba5a30ab812f3722bd1a84f1c85a30ec02a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:27:26 +0200 Subject: [PATCH 12/14] made shure the CAMPUS_API_TOKEN is configured in the deployment and developemnt --- .env | 4 ++-- README.md | 1 + .../charts/backend/templates/deployments/backend-v2.yaml | 6 ++++++ deployment/charts/backend/values.yaml | 1 + docker-compose.yaml | 1 + 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.env b/.env index b286ff71..e8387199 100644 --- a/.env +++ b/.env @@ -1,12 +1,12 @@ +ENVIRONMENT=dev DB_NAME=campus_db DB_ROOT_PASSWORD=secret_root_password DB_PORT=3306 + APNS_KEY_ID= APNS_TEAM_ID= APNS_P8_FILE_PATH=/secrets/AuthKey_XXXX.p8 -ENVIRONMENT=dev - SENTRY_DSN= CAMPUS_API_TOKEN= diff --git a/README.md b/README.md index 9617731c..825ce41f 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ The following environment variables need to be set for the server to work proper * [REQUIRED] `APNS_KEY_ID`: The key ID of the APNs key => APNs Key needs to be downloaded from the Apple Developer Portal the name of the file also contains the key ID. * [REQUIRED] `APNS_TEAM_ID`: The team ID of the iOS app can be found in AppStoreConnect. * [REQUIRED] `APNS_P8_FILE_PATH`: The path to the APNs key file (e.g. `/secrets/AuthKey_XXXX.p8`) in the docker container. The file itself needs to exist in the same directory as the `docker-compose.yml` file and called `apns_auth_key.p8`. + * [REQUIRED] `CAMPUS_API_TOKEN`: A token used to authenticate with TUMonline (used for example for the grades) ## InfluxDB InfluxDB can be used to store metrics. diff --git a/deployment/charts/backend/templates/deployments/backend-v2.yaml b/deployment/charts/backend/templates/deployments/backend-v2.yaml index f037f8f9..99c36826 100644 --- a/deployment/charts/backend/templates/deployments/backend-v2.yaml +++ b/deployment/charts/backend/templates/deployments/backend-v2.yaml @@ -48,6 +48,11 @@ spec: secretKeyRef: name: backend-api-keys key: SENTRY_DSN + - name: CAMPUS_API_TOKEN + valueFrom: + secretKeyRef: + name: backend-api-keys + key: CAMPUS_API_TOKEN - name: DB_DSN value: "{{ $db.username }}:{{ $db.password }}@tcp(tca-backend-mariadb.{{ $.Values.namespace }}.svc.cluster.local:3306)/{{ $db.database }}?charset=utf8mb4&parseTime=True&loc=Local" - name: APNS_KEY_ID @@ -105,6 +110,7 @@ metadata: app.kubernetes.io/part-of: tum-campus-app app.kubernetes.io/name: backend-v2 data: + CAMPUS_API_TOKEN: {{ $.Values.backend.campusApiToken | b64enc }} SENTRY_DSN: {{ $.Values.backend.sentry.dsn | b64enc }} apns_auth_key.p8: {{ $.Values.backend.apns.auth_key }} APNS_KEY_ID: {{ $.Values.backend.apns.key_id | b64enc }} diff --git a/deployment/charts/backend/values.yaml b/deployment/charts/backend/values.yaml index ec5d6476..36d04422 100644 --- a/deployment/charts/backend/values.yaml +++ b/deployment/charts/backend/values.yaml @@ -41,6 +41,7 @@ mariadb: backend: + campusApiToken: changeme-changeme-changeme sentry: dsn: changeme-changeme-changeme apns: diff --git a/docker-compose.yaml b/docker-compose.yaml index a7d45018..6e1cf07a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,6 +17,7 @@ services: - APNS_TEAM_ID=${APNS_TEAM_ID} - APNS_P8_FILE_PATH=${APNS_P8_FILE_PATH} - MensaCronDisabled=false + - CAMPUS_API_TOKEN=${CAMPUS_API_TOKEN} volumes: - backend-storage:/Storage - ./apns_auth_key.p8:${APNS_P8_FILE_PATH} From d874fa04bf12df38c403f32b29283f08bf190f26 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:35:18 +0200 Subject: [PATCH 13/14] inlined a few variables --- server/backend/campus_api/campusApi.go | 35 ++++--------------- .../ios_apns/iosAPNsRepository.go | 4 +-- .../newExamResultsSubscriberRepository.go | 11 +++--- .../newExamResultsSubscriberService.go | 3 +- 4 files changed, 12 insertions(+), 41 deletions(-) diff --git a/server/backend/campus_api/campusApi.go b/server/backend/campus_api/campusApi.go index 29ec93ff..746455e4 100644 --- a/server/backend/campus_api/campusApi.go +++ b/server/backend/campus_api/campusApi.go @@ -12,21 +12,9 @@ import ( log "github.com/sirupsen/logrus" ) -const ( - CampusApiUrl = "https://campus.tum.de/tumonline" - CampusQueryToken = "pToken" - CampusGradesPath = "/wbservicesbasic.noten" - CampusExamResultsPublished = "/wbservicesbasic.pruefungenErgebnisse" -) - -var ( - ErrCannotCreateRequest = errors.New("cannot create http request") - ErrorWhileUnmarshalling = errors.New("error while unmarshalling") -) - func FetchExamResultsPublished(token string) (*model.TUMAPIPublishedExamResults, error) { var examResultsPublished model.TUMAPIPublishedExamResults - err := RequestCampusApi(CampusExamResultsPublished, token, &examResultsPublished) + err := RequestCampusApi("/wbservicesbasic.pruefungenErgebnisse", token, &examResultsPublished) if err != nil { return nil, err } @@ -36,7 +24,7 @@ func FetchExamResultsPublished(token string) (*model.TUMAPIPublishedExamResults, func FetchGrades(token string) (*model.IOSGrades, error) { var grades model.IOSGrades - err := RequestCampusApi(CampusGradesPath, token, &grades) + err := RequestCampusApi("/wbservicesbasic.noten", token, &grades) if err != nil { return nil, err } @@ -45,21 +33,12 @@ func FetchGrades(token string) (*model.IOSGrades, error) { } func RequestCampusApi(path string, token string, response any) error { - requestUrl := fmt.Sprintf("%s%s?%s=%s", CampusApiUrl, path, CampusQueryToken, token) - req, err := http.NewRequest(http.MethodGet, requestUrl, nil) - - if err != nil { - log.WithError(err).Error("Error while creating request") - return ErrCannotCreateRequest - } - - resp, err := http.DefaultClient.Do(req) - + requestUrl := fmt.Sprintf("https://campus.tum.de/tumonline%s?pToken=%s", path, token) + resp, err := http.Get(requestUrl) if err != nil { log.WithError(err).WithField("path", path).Error("Error while fetching url") return errors.New("error while fetching " + path) } - defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { @@ -67,11 +46,9 @@ func RequestCampusApi(path string, token string, response any) error { } }(resp.Body) - err = xml.NewDecoder(resp.Body).Decode(&response) - - if err != nil { + if err = xml.NewDecoder(resp.Body).Decode(&response); err != nil { log.WithError(err).WithField("path", path).Error("Error while unmarshalling") - return ErrorWhileUnmarshalling + return errors.New("error while unmarshalling") } return nil diff --git a/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go b/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go index 3108b090..05c03edb 100644 --- a/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go +++ b/server/backend/ios_notifications/ios_apns/iosAPNsRepository.go @@ -48,7 +48,6 @@ func (r *Repository) ApnsUrl() string { if env.IsProd() { return ApnsProductionURL } - return ApnsDevelopmentURL } @@ -87,7 +86,6 @@ func (r *Repository) SendNotification(notification *model.IOSNotificationPayload url := r.ApnsUrl() + "/3/device/" + notification.DeviceId body, _ := notification.MarshalJSON() - client := r.httpClient req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) // can be e.g. alert or background @@ -99,7 +97,7 @@ func (r *Repository) SendNotification(notification *model.IOSNotificationPayload bearer := r.Token.GenerateNewTokenIfExpired() req.Header.Set("authorization", "bearer "+bearer) - resp, err := client.Do(req) + resp, err := r.httpClient.Do(req) if err != nil { log.WithError(err).Error("Could not send notification") return nil, ErrCouldNotSendNotification diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go index dc3f047e..5482daae 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberRepository.go @@ -34,17 +34,14 @@ func (repository *Repository) NotifySubscriber(subscriber *model.NewExamResultsS } req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) - - req.Header.Set("Content-Type", "application/json") - - if subscriber.ApiKey.Valid { - req.Header.Set("Authorization", subscriber.ApiKey.String) - } - if err != nil { log.WithError(err).Error("Error while creating request") return err } + req.Header.Set("Content-Type", "application/json") + if subscriber.ApiKey.Valid { + req.Header.Set("Authorization", subscriber.ApiKey.String) + } _, err = http.DefaultClient.Do(req) if err != nil { diff --git a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go index d5d373ae..a13bda3e 100644 --- a/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go +++ b/server/backend/new_exam_results_hook/new_exam_results_subscriber/newExamResultsSubscriberService.go @@ -18,8 +18,7 @@ func (service *Service) NotifySubscribers(newGrades *[]model.PublishedExamResult } for _, subscriber := range *subscribers { - err := repository.NotifySubscriber(&subscriber, newGrades) - if err != nil { + if err := repository.NotifySubscriber(&subscriber, newGrades); err != nil { log.WithError(err).Error("Failed to notify subscriber") continue } From 21e0891ea700734b73673d476eb7ee73ec68e492 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 19 Sep 2023 20:39:25 +0200 Subject: [PATCH 14/14] increased the amount of time between newExamResultsHook's --- server/backend/migration/20230530000000.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/backend/migration/20230530000000.go b/server/backend/migration/20230530000000.go index 9278c595..165113b7 100644 --- a/server/backend/migration/20230530000000.go +++ b/server/backend/migration/20230530000000.go @@ -44,7 +44,7 @@ func (m TumDBMigrator) migrate20230530000000() *gormigrate.Migration { } return tx.Create(&model.Crontab{ - Interval: 60, // Every 5 minutes + Interval: 60 * 10, // Every 10 minutes Type: null.StringFrom("newExamResultsHook"), }).Error },