Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Quiz exercises: Enable manual quiz evaluation for course quizzes #8837

Merged
merged 9 commits into from
Jun 21, 2024
Merged

Quiz exercises: Enable manual quiz evaluation for course quizzes #8837

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b89da8b
Development: Fix two null pointer exceptions
Jun 21, 2024
ab424df
move quiz evaluation to new service
Jun 21, 2024
f4eeaac
add REST call to evaluate a course quiz
Jun 21, 2024
cd004e5
add button
Jun 21, 2024
8633fb6
Improve authorization
Hialus Jun 21, 2024
d33ce5a
UI improvements
Hialus Jun 21, 2024
4956f1e
Add tests for the evaluate endpoint
Hialus Jun 21, 2024
c61fd49
Fix server tests
Hialus Jun 21, 2024
f6af6d3
Implement feedback
Hialus Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions src/main/java/de/tum/in/www1/artemis/domain/Result.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import de.tum.in.www1.artemis.domain.enumeration.SubmissionType;
import de.tum.in.www1.artemis.domain.hestia.CoverageFileReport;
import de.tum.in.www1.artemis.domain.participation.Participation;
import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.domain.quiz.QuizExercise;
import de.tum.in.www1.artemis.domain.quiz.QuizSubmission;
import de.tum.in.www1.artemis.domain.view.QuizView;
Expand Down Expand Up @@ -513,17 +512,14 @@ public void setAssessmentNote(AssessmentNote assessmentNote) {
}
}

// jhipster-needle-entity-add-getters-setters - JHipster will add getters and setters here, do not remove

/**
* Updates the attributes "score" and "successful" by evaluating its submission.
* <b>Important</b>: the quizSubmission has to be loaded with eager submitted answers, otherwise this method will not work correctly
*
* @param quizExercise the quiz exercise for which the submission should be evaluated, must contain access to the course to calculate the score correctly
*/
public void evaluateQuizSubmission() {
public void evaluateQuizSubmission(@NotNull QuizExercise quizExercise) {
if (submission instanceof QuizSubmission quizSubmission) {
// get the exercise this result belongs to
StudentParticipation studentParticipation = (StudentParticipation) getParticipation();
QuizExercise quizExercise = (QuizExercise) studentParticipation.getExercise();
// update score
setScore(quizExercise.getScoreForSubmission(quizSubmission), quizExercise.getCourseViaExerciseGroupOrCourseMember());
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/de/tum/in/www1/artemis/service/dto/UserRoleDTO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ public record UserRoleDTO(Long userId, String username, UserRole role) {

@SuppressWarnings("unused")
UserRoleDTO(Long userId, String username, String role) {
this(userId, username, UserRole.valueOf(role));
this(userId, username, role != null ? UserRole.valueOf(role) : null);
}
}
140 changes: 2 additions & 138 deletions src/main/java/de/tum/in/www1/artemis/service/exam/ExamQuizService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,17 @@

import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE;

import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import jakarta.validation.constraints.NotNull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;

import de.tum.in.www1.artemis.domain.Result;
import de.tum.in.www1.artemis.domain.Submission;
import de.tum.in.www1.artemis.domain.enumeration.AssessmentType;
import de.tum.in.www1.artemis.domain.enumeration.InitializationState;
import de.tum.in.www1.artemis.domain.exam.StudentExam;
import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.domain.quiz.QuizExercise;
import de.tum.in.www1.artemis.domain.quiz.QuizSubmission;
import de.tum.in.www1.artemis.repository.QuizExerciseRepository;
Expand All @@ -33,7 +23,6 @@
import de.tum.in.www1.artemis.repository.SubmittedAnswerRepository;
import de.tum.in.www1.artemis.service.ResultService;
import de.tum.in.www1.artemis.service.quiz.QuizStatisticService;
import de.tum.in.www1.artemis.service.util.TimeLogUtil;
import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;

@Profile(PROFILE_CORE)
Expand All @@ -46,8 +35,6 @@ public class ExamQuizService {

private final QuizStatisticService quizStatisticService;

private final ResultService resultService;

private final StudentParticipationRepository studentParticipationRepository;

private final ResultRepository resultRepository;
Expand All @@ -66,29 +53,10 @@ public ExamQuizService(StudentParticipationRepository studentParticipationReposi
this.submissionRepository = submissionRepository;
this.quizExerciseRepository = quizExerciseRepository;
this.quizStatisticService = quizStatisticService;
this.resultService = resultService;
this.quizSubmissionRepository = quizSubmissionRepository;
this.submittedAnswerRepository = submittedAnswerRepository;
}

/**
* Evaluate the given quiz exercise by evaluate the submission for each participation (there is only one for each participation in exams)
* and update the statistics with the generated results.
*
* @param quizExerciseId the id of the QuizExercise that should be evaluated
*/
public void evaluateQuizAndUpdateStatistics(@NotNull Long quizExerciseId) {
long start = System.nanoTime();
log.info("Starting quiz evaluation for quiz {}", quizExerciseId);
// We have to load the questions and statistics so that we can evaluate and update and we also need the participations and submissions that exist for this exercise so that
// they can be evaluated
var quizExercise = quizExerciseRepository.findByIdWithQuestionsAndStatisticsElseThrow(quizExerciseId);
Set<Result> createdResults = evaluateSubmissions(quizExercise);
log.info("Quiz evaluation for quiz {} finished after {} with {} created results", quizExercise.getId(), TimeLogUtil.formatDurationFrom(start), createdResults.size());
quizStatisticService.updateStatistics(createdResults, quizExercise);
log.info("Statistic update for quiz {} finished after {}", quizExercise.getId(), TimeLogUtil.formatDurationFrom(start));
}

/**
* This method is intended to be called after a user submits a test run. We calculate the achieved score in the quiz exercises immediately and attach a result.
* Note: We do not insert the result of this test run quiz participation into the quiz statistics.
Expand Down Expand Up @@ -117,7 +85,7 @@ public void evaluateQuizParticipationsForTestRunAndTestExam(StudentExam studentE
result.setSubmission(quizSubmission);
// calculate scores and update result and submission accordingly
quizSubmission.calculateAndUpdateScores(quizExercise.getQuizQuestions());
result.evaluateQuizSubmission();
result.evaluateQuizSubmission(quizExercise);
// remove submission to follow save order for ordered collections
result.setSubmission(null);
if (studentExam.isTestExam()) {
Expand All @@ -137,7 +105,7 @@ public void evaluateQuizParticipationsForTestRunAndTestExam(StudentExam studentE
quizSubmission.calculateAndUpdateScores(quizExercise.getQuizQuestions());
// prevent a lazy exception in the evaluateQuizSubmission method
result.setParticipation(participation);
result.evaluateQuizSubmission();
result.evaluateQuizSubmission(quizExercise);
if (studentExam.isTestExam()) {
result.rated(true);
}
Expand All @@ -152,108 +120,4 @@ public void evaluateQuizParticipationsForTestRunAndTestExam(StudentExam studentE
}
}
}

/**
* // @formatter:off
* Evaluate the given quiz exercise by performing the following actions for each participation:
* 1. Get the submission for each participation (there should be only one as in exam mode, the submission gets created upfront and will be updated)
* - If no submission is found, print a warning and continue as we cannot evaluate that submission
* - If more than one submission is found, select one of them
* 2. mark submission and participation as evaluated
* 3. Create a new result for the selected submission and calculate scores
* 4. Save the updated submission & participation and the newly created result
* <p>
* After processing all participations, the created results will be returned for further processing
* Note: We ignore test run participations
* // @formatter:on
*
* @param quizExercise the id of the QuizExercise that should be evaluated
* @return the newly generated results
*/
private Set<Result> evaluateSubmissions(@NotNull QuizExercise quizExercise) {
Set<Result> createdResults = new HashSet<>();
List<StudentParticipation> studentParticipations = studentParticipationRepository.findAllWithEagerLegalSubmissionsAndEagerResultsByExerciseId(quizExercise.getId());
submittedAnswerRepository.loadQuizSubmissionsSubmittedAnswers(studentParticipations);

for (var participation : studentParticipations) {
if (!participation.isTestRun()) {
try {
// reconnect so that the quiz questions are available later on (otherwise there will be a org.hibernate.LazyInitializationException)
participation.setExercise(quizExercise);
Set<Submission> submissions = participation.getSubmissions();
QuizSubmission quizSubmission;
if (submissions.isEmpty()) {
log.warn("Found no submissions for participation {} (Participant {}) in quiz {}", participation.getId(), participation.getParticipant().getName(),
quizExercise.getId());
continue;
}
else if (submissions.size() > 1) {
log.warn("Found multiple ({}) submissions for participation {} (Participant {}) in quiz {}, taking the one with highest id", submissions.size(),
participation.getId(), participation.getParticipant().getName(), quizExercise.getId());
List<Submission> submissionsList = new ArrayList<>(submissions);

// Load submission with highest id
submissionsList.sort(Comparator.comparing(Submission::getId).reversed());
quizSubmission = (QuizSubmission) submissionsList.getFirst();
}
else {
quizSubmission = (QuizSubmission) submissions.iterator().next();
}

participation.setInitializationState(InitializationState.FINISHED);

boolean resultExisting = false;
// create new result if none is existing
Result result;
if (participation.getResults().isEmpty()) {
result = new Result().participation(participation);
}
else {
resultExisting = true;
result = participation.getResults().iterator().next();
}
// Only create Results once after the first evaluation
if (!resultExisting) {
// delete result from quizSubmission, to be able to set a new one
if (quizSubmission.getLatestResult() != null) {
resultService.deleteResult(quizSubmission.getLatestResult(), true);
}
result.setRated(true);
result.setAssessmentType(AssessmentType.AUTOMATIC);
result.setCompletionDate(ZonedDateTime.now());

// set submission to calculate scores
result.setSubmission(quizSubmission);
// calculate scores and update result and submission accordingly
quizSubmission.calculateAndUpdateScores(quizExercise.getQuizQuestions());
result.evaluateQuizSubmission();
// remove submission to follow save order for ordered collections
result.setSubmission(null);

// NOTE: we save participation, submission and result here individually so that one exception (e.g. duplicated key) cannot destroy multiple student answers
submissionRepository.save(quizSubmission);
result = resultRepository.save(result);

// add result to participation
participation.addResult(result);
studentParticipationRepository.save(participation);

// add result to submission
result.setSubmission(quizSubmission);
quizSubmission.addResult(result);
submissionRepository.save(quizSubmission);

// Add result so that it can be returned (and processed later)
createdResults.add(result);
}
}
catch (Exception e) {
log.error("Exception in evaluateExamQuizExercise() for user {} in quiz {}: {}", participation.getParticipantIdentifier(), quizExercise.getId(), e.getMessage(),
e);
}
}

}
return createdResults;
}
}
13 changes: 7 additions & 6 deletions src/main/java/de/tum/in/www1/artemis/service/exam/ExamService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
import de.tum.in.www1.artemis.service.notifications.GroupNotificationService;
import de.tum.in.www1.artemis.service.plagiarism.PlagiarismCaseService.PlagiarismMapping;
import de.tum.in.www1.artemis.service.quiz.QuizPoolService;
import de.tum.in.www1.artemis.service.quiz.QuizResultService;
import de.tum.in.www1.artemis.service.util.TimeLogUtil;
import de.tum.in.www1.artemis.web.rest.dto.BonusExampleDTO;
import de.tum.in.www1.artemis.web.rest.dto.BonusResultDTO;
Expand All @@ -124,6 +125,8 @@ public class ExamService {

private static final int EXAM_ACTIVE_DAYS = 7;

private final QuizResultService quizResultService;

@Value("${artemis.course-archives-path}")
private Path examArchivesDirPath;

Expand All @@ -137,8 +140,6 @@ public class ExamService {

private final QuizExerciseRepository quizExerciseRepository;

private final ExamQuizService examQuizService;

private final ExamDateService examDateService;

private final ExamLiveEventsService examLiveEventsService;
Expand Down Expand Up @@ -191,22 +192,21 @@ public class ExamService {

private static final String NOT_ALLOWED_TO_ACCESS_THE_GRADE_SUMMARY = "You are not allowed to access the grade summary of a student exam ";

public ExamService(ExamDateService examDateService, ExamRepository examRepository, StudentExamRepository studentExamRepository, ExamQuizService examQuizService,
public ExamService(ExamDateService examDateService, ExamRepository examRepository, StudentExamRepository studentExamRepository,
InstanceMessageSendService instanceMessageSendService, TutorLeaderboardService tutorLeaderboardService, StudentParticipationRepository studentParticipationRepository,
ComplaintRepository complaintRepository, ComplaintResponseRepository complaintResponseRepository, UserRepository userRepository,
ProgrammingExerciseRepository programmingExerciseRepository, QuizExerciseRepository quizExerciseRepository, ExamLiveEventsService examLiveEventsService,
ResultRepository resultRepository, SubmissionRepository submissionRepository, CourseExamExportService courseExamExportService, GitService gitService,
GroupNotificationService groupNotificationService, GradingScaleRepository gradingScaleRepository, PlagiarismCaseRepository plagiarismCaseRepository,
AuthorizationCheckService authorizationCheckService, BonusService bonusService, ExerciseDeletionService exerciseDeletionService,
SubmittedAnswerRepository submittedAnswerRepository, AuditEventRepository auditEventRepository, CourseScoreCalculationService courseScoreCalculationService,
CourseRepository courseRepository, QuizPoolService quizPoolService) {
CourseRepository courseRepository, QuizPoolService quizPoolService, QuizResultService quizResultService) {
this.examDateService = examDateService;
this.examRepository = examRepository;
this.studentExamRepository = studentExamRepository;
this.userRepository = userRepository;
this.studentParticipationRepository = studentParticipationRepository;
this.programmingExerciseRepository = programmingExerciseRepository;
this.examQuizService = examQuizService;
this.instanceMessageSendService = instanceMessageSendService;
this.complaintRepository = complaintRepository;
this.complaintResponseRepository = complaintResponseRepository;
Expand All @@ -229,6 +229,7 @@ public ExamService(ExamDateService examDateService, ExamRepository examRepositor
this.courseRepository = courseRepository;
this.quizPoolService = quizPoolService;
this.defaultObjectMapper = new ObjectMapper();
this.quizResultService = quizResultService;
}

/**
Expand Down Expand Up @@ -1138,7 +1139,7 @@ public Integer evaluateQuizExercises(Exam exam) {
long start = System.nanoTime();
log.debug("Evaluating {} quiz exercises in exam {}", quizExercises.size(), exam.getId());
// Evaluate all quizzes for that exercise
quizExercises.stream().map(Exercise::getId).forEach(examQuizService::evaluateQuizAndUpdateStatistics);
quizExercises.stream().map(Exercise::getId).forEach(quizResultService::evaluateQuizAndUpdateStatistics);
if (log.isDebugEnabled()) {
log.debug("Evaluated {} quiz exercises in exam {} in {}", quizExercises.size(), exam.getId(), TimeLogUtil.formatDurationFrom(start));
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/de/tum/in/www1/artemis/service/quiz/QuizExerciseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ private void updateResultsOnQuizChanges(QuizExercise quizExercise) {
// update Successful-Flag in Result
StudentParticipation studentParticipation = (StudentParticipation) result.getParticipation();
studentParticipation.setExercise(quizExercise);
result.evaluateQuizSubmission();
result.evaluateQuizSubmission(quizExercise);

submissions.add(quizSubmission);
}
Expand Down
Loading
Loading