From 4f55dbbb43465b03072b340ca2037f45b66722c2 Mon Sep 17 00:00:00 2001 From: Wilfried BARADAT Date: Tue, 28 May 2024 17:25:47 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=94=8A(xi)=20add=20possibility=20to?= =?UTF-8?q?=20ignore=20error=20when=20indexing=20courses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validation errors that happens on course indexing were not ignored by the `ignore_errors` flag. Fixing it. --- src/api/core/warren/xi/indexers/moodle/etl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/api/core/warren/xi/indexers/moodle/etl.py b/src/api/core/warren/xi/indexers/moodle/etl.py index c4a7c31f..00c93f8c 100644 --- a/src/api/core/warren/xi/indexers/moodle/etl.py +++ b/src/api/core/warren/xi/indexers/moodle/etl.py @@ -49,7 +49,14 @@ async def _extract(self) -> List[Course]: def _transform(self, raw: List[Course]) -> Iterator[ExperienceCreate]: """Transform courses into experiences.""" - return (course.to_experience(base_url=self._lms.url) for course in raw) + for course in raw: + try: + yield course.to_experience(base_url=self._lms.url) + except ValidationError as err: + if not self._ignore_errors: + raise err + logger.exception("Skipping invalid course %s", course.id) + pass async def _load(self, data: Iterator[ExperienceCreate]) -> None: """Load experiences into the Experience Index (XI).""" @@ -121,6 +128,7 @@ def _transform(self, raw: List[Section]) -> Iterator[ExperienceCreate]: if not self._ignore_errors: raise err logger.exception("Skipping invalid module %s", module.id) + pass async def _load(self, data: Iterator[ExperienceCreate]) -> None: """Load experiences into the Experience Index (XI) and create relations.""" From f16437d15f67cb913da81a22b64d67a62a7a0f8b Mon Sep 17 00:00:00 2001 From: Wilfried BARADAT Date: Tue, 28 May 2024 17:35:05 +0200 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8(xi)=20add=20a=20xi=20command=20to?= =?UTF-8?q?=20index=20all=20courses=20and=20their=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a useful `warren xi index all` command to index all courses of a LMS instance, and their content. --- CHANGELOG.md | 1 + src/api/core/warren/cli.py | 46 +++++++++++++++++++- src/api/core/warren/tests/test_cli.py | 60 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c15eb84a..7ab11004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to ### Added - Add dedicated views depending on LTI roles +- Add a warren xi index all CLI command ### Changed diff --git a/src/api/core/warren/cli.py b/src/api/core/warren/cli.py index 03350f0b..ac3d9098 100644 --- a/src/api/core/warren/cli.py +++ b/src/api/core/warren/cli.py @@ -314,9 +314,53 @@ def xi_index_course_content( moodle_ws_token: str, ignore_errors: bool, ): - """Index LMS course content.""" + """Index LMS content of a course.""" asyncio.run( _xi_index_course_content( course_id, xi_url, moodle_url, moodle_ws_token, ignore_errors ) ) + + +async def _xi_index_all( + xi_url: str, + moodle_url: str, + moodle_ws_token: str, + ignore_errors: bool, +): + """Index LMS courses and their content. + + Nota bene: as we are calling multiple asynchronous functions, we need + to wrap calls in a single async function called in a synchronous Click + command using the asyncio.run method. Calling asyncio.run multiple times + can close the execution loop unexpectedly. + """ + lms = Moodle(url=moodle_url, token=moodle_ws_token) + xi = ExperienceIndex(url=xi_url) + + indexer_courses = Courses(lms=lms, xi=xi, ignore_errors=ignore_errors) + await indexer_courses.execute() + + experiences = await xi.experience.read(aggregation_level=AggregationLevel.THREE) + for experience in experiences: + course = await xi.experience.get(object_id=experience.id) + if course is None: + raise click.BadParameter( + f"Unknown course {experience.id}. Course indexation has failed!" + ) + indexer_content = CourseContent( + course=course, lms=lms, xi=xi, ignore_errors=ignore_errors + ) + await indexer_content.execute() + + +@xi_index.command("all") +@click.option("--xi-url", "-x", default="") +@click.option("--moodle-url", "-u", default="") +@click.option("--moodle-ws-token", "-t", default="") +@click.option("--ignore-errors/--no-ignore-errors", "-I/-F", default=False) +def xi_index_all( + xi_url: str, moodle_url: str, moodle_ws_token: str, ignore_errors: bool +): + """Index all LMS courses and their content.""" + asyncio.run(_xi_index_all(xi_url, moodle_url, moodle_ws_token, ignore_errors)) diff --git a/src/api/core/warren/tests/test_cli.py b/src/api/core/warren/tests/test_cli.py index 33ceeb77..fe885f6e 100644 --- a/src/api/core/warren/tests/test_cli.py +++ b/src/api/core/warren/tests/test_cli.py @@ -463,6 +463,66 @@ def test_xi_index_course_content_command_with_unknown_course(monkeypatch): assert "Unknown course fake-course-id. It should be indexed first!" in result.output +def test_xi_index_all(monkeypatch): + """Test warren xi index all command.""" + runner = CliRunner() + + moodle_client_mock = MagicMock(return_value=None) + courses_indexer_execute_mock = AsyncMock() + + monkeypatch.setattr(Moodle, "__init__", moodle_client_mock) + monkeypatch.setattr(Courses, "execute", courses_indexer_execute_mock) + + xi_experience_read_mock = AsyncMock( + return_value=[ + ExperienceRead( + **ExperienceFactory.build_dict( + exclude=set(), id="ce0927fa-5f72-4623-9d29-37ef45c39609" + ) + ), + ExperienceRead( + **ExperienceFactory.build_dict( + exclude=set(), id="a0fb8abc-96d8-4b32-8551-f36a064a6a3f" + ) + ), + ] + ) + xi_experience_get_mock = AsyncMock( + return_value=ExperienceRead( + **ExperienceFactory.build_dict( + exclude=set(), id="ce0927fa-5f72-4623-9d29-37ef45c39609" + ) + ) + ) + content_indexer_execute_mock = AsyncMock() + + monkeypatch.setattr(CRUDExperience, "read", xi_experience_read_mock) + monkeypatch.setattr(CRUDExperience, "get", xi_experience_get_mock) + monkeypatch.setattr(CourseContent, "execute", content_indexer_execute_mock) + + result = runner.invoke( + cli, + [ + "xi", + "index", + "all", + "--xi-url", + "http://xi.foo.com", + "--moodle-url", + "http://moodle.foo.com", + "--moodle-ws-token", + "faketoken", + ], + ) + + assert result.exit_code == 0 + moodle_client_mock.assert_called_with( + url="http://moodle.foo.com", token="faketoken" + ) + courses_indexer_execute_mock.assert_called() + content_indexer_execute_mock.assert_called() + + def test_xi_list_courses_command_when_no_course_exists(monkeypatch): """Test warren xi list courses command with no indexed course.""" runner = CliRunner()