From 53969f206a1fc1378e5acf81cd0db5db3dad166f Mon Sep 17 00:00:00 2001 From: gounux Date: Sun, 17 Mar 2024 21:27:59 +0100 Subject: [PATCH 01/25] Add yaml header check wip --- geotribu_cli/cli.py | 11 +++ geotribu_cli/content/header_check.py | 141 +++++++++++++++++++++++++++ geotribu_cli/subcommands/__init__.py | 1 + 3 files changed, 153 insertions(+) create mode 100644 geotribu_cli/content/header_check.py diff --git a/geotribu_cli/cli.py b/geotribu_cli/cli.py index a26b0a1..4da2df7 100644 --- a/geotribu_cli/cli.py +++ b/geotribu_cli/cli.py @@ -29,6 +29,7 @@ parser_comments_broadcast, parser_comments_latest, parser_comments_read, + parser_header_check, parser_images_optimizer, parser_latest_content, parser_mastodon_export, @@ -223,6 +224,16 @@ def main(args: list[str] = None): add_common_arguments(subcmd_upgrade) parser_upgrade(subcmd_upgrade) + subcmd_header_check = subparsers.add_parser( + "header-check", + aliases=["header", "check", "header-check", "metadata"], + help="Vérifier entête markdown", + formatter_class=main_parser.formatter_class, + prog="header-check", + ) + add_common_arguments(subcmd_header_check) + parser_header_check(subcmd_header_check) + # -- NESTED SUBPARSER : CREATE --------------------------------------------------- subcmd_content_manager = subparsers.add_parser( "creer", diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py new file mode 100644 index 0000000..8da9123 --- /dev/null +++ b/geotribu_cli/content/header_check.py @@ -0,0 +1,141 @@ +import argparse +import logging +import os +import shutil +import uuid +from datetime import datetime +from typing import Any + +import requests +import yaml +from PIL import Image + +from geotribu_cli.__about__ import __executable_name__, __version__ +from geotribu_cli.constants import GeotribuDefaults +from geotribu_cli.utils.dates_manipulation import is_more_recent + +logger = logging.getLogger(__name__) +defaults_settings = GeotribuDefaults() + +# ############################################################################ +# ########## CLI ################# +# ################################ + + +def parser_header_check( + subparser: argparse.ArgumentParser, +) -> argparse.ArgumentParser: + """Set the argument parser subcommand. + + Args: + subparser (argparse.ArgumentParser): parser to set up + + Returns: + argparse.ArgumentParser: parser ready to use + """ + subparser.add_argument( + "content_path", + help="Chemin du fichier markdown dont l'entête est à vérifier", + type=str, + metavar="content", + ) + subparser.add_argument( + "-minr", + "--min-ratio", + dest="min_image_ratio", + default=1.2, + help="Ratio width/height minimum de l'image à vérifier", + ) + subparser.add_argument( + "-maxr", + "--max-ratio", + dest="max_image_ratio", + default=1.5, + help="Ratio width/height maximum de l'image à vérifier", + ) + subparser.add_argument( + "-r", + "--raise", + dest="raise_exceptions", + action="store_true", + default=False, + help="Lever des exceptions et donc arrêter le programme si des erreurs sont rencontrées", + ) + subparser.set_defaults(func=run) + return subparser + + +# ############################################################################ +# ########## MAIN ################ +# ################################ + + +def check_publish_date(date: Any) -> bool: + if isinstance(date, str): + publish_date = datetime.strptime(date.split(" ")[0], "%Y-%m-%d").date() + else: + publish_date = date + # TODO: check if date is another type and raise error + return is_more_recent(datetime.now().date(), publish_date) + + +def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> bool: + r = requests.get( + image_url, + headers={"User-Agent": f"{__executable_name__}v{__version__}"}, + stream=True, + ) + r.raise_for_status() + image_file_name = str(uuid.uuid4()) + with open(image_file_name, "wb") as image_file: + r.raw.decode_content = True + try: + shutil.copyfileobj(r.raw, image_file) + with Image.open(image_file_name) as image: + width, height = image.width, image.height + ratio = width / height + return min_ratio <= ratio <= max_ratio + finally: + os.remove(image_file_name) + + +def run(args: argparse.Namespace) -> None: + """Run the sub command logic. + + Download the RSS feed file and display results. + + Args: + args (argparse.Namespace): arguments passed to the subcommand + """ + logger.debug(f"Running {args.command} with {args}") + content_path = args.content_path + + if not os.path.exists(content_path): + raise ValueError(f"Mayday ! Le fichier {content_path} n'existe pas !") + + with open(content_path) as file: + content = file.read() + _, front_matter, _ = content.split("---", 2) + yaml_meta = yaml.safe_load(front_matter) + logger.debug(f"YAML metadata loaded : {yaml_meta}") + + # check that datetime is in the future + if not check_publish_date(yaml_meta["date"]): + msg = "La date de publication n'est pas dans le turfu !" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Date de publication ok") + + # check that image ratio is okayyy + if "image" in yaml_meta: + if not check_image_ratio( + yaml_meta["image"], args.min_image_ratio, args.max_image_ratio + ): + msg = f"Le ratio de l'image n'est pas dans l'interface autorisé ({args.minratio} - {args.maxratio})" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Ratio image ok") diff --git a/geotribu_cli/subcommands/__init__.py b/geotribu_cli/subcommands/__init__.py index a9908e9..c38d43d 100644 --- a/geotribu_cli/subcommands/__init__.py +++ b/geotribu_cli/subcommands/__init__.py @@ -4,6 +4,7 @@ from geotribu_cli.comments import parser_comments_broadcast # noqa: F401 from geotribu_cli.comments import parser_comments_latest # noqa: F401 from geotribu_cli.comments import parser_comments_read # noqa: F401 +from geotribu_cli.content.header_check import parser_header_check # noqa: F401 from geotribu_cli.content.new_article import parser_new_article # noqa: F401 from geotribu_cli.images.images_optimizer import parser_images_optimizer # noqa: F401 from geotribu_cli.rss.rss_reader import parser_latest_content # noqa: F401 From a43e560e06bb67b107114faefbcae013d2410fca Mon Sep 17 00:00:00 2001 From: gounux Date: Mon, 18 Mar 2024 08:59:57 +0100 Subject: [PATCH 02/25] Add tags check using JSON feed --- geotribu_cli/content/header_check.py | 26 +++++++++++++++++++++++++- geotribu_cli/content/json_feed.py | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 geotribu_cli/content/json_feed.py diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 8da9123..8ad07b4 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -12,6 +12,7 @@ from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults +from geotribu_cli.content.json_feed import JsonFeedClient from geotribu_cli.utils.dates_manipulation import is_more_recent logger = logging.getLogger(__name__) @@ -99,10 +100,23 @@ def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> boo os.remove(image_file_name) +def get_existing_tags() -> set[str]: + jfc = JsonFeedClient() + return jfc.get_tags(should_sort=True) + + +def check_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: + existing_tags = get_existing_tags() + all_exists = set(tags).issubset(existing_tags) + missing = set(tags).difference(existing_tags) + present = set(tags).intersection(existing_tags) + return all_exists, missing, present + + def run(args: argparse.Namespace) -> None: """Run the sub command logic. - Download the RSS feed file and display results. + Checks YAML header of a content Args: args (argparse.Namespace): arguments passed to the subcommand @@ -139,3 +153,13 @@ def run(args: argparse.Namespace) -> None: raise ValueError(msg) else: logger.info("Ratio image ok") + + # check that tags already exist + all_exists, missing, _ = check_tags(yaml_meta["tags"]) + if not all_exists: + msg = f"Les tags suivants n'existent pas dans les contenus Geotribu précédents : {','.join(missing)}" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Tags ok") diff --git a/geotribu_cli/content/json_feed.py b/geotribu_cli/content/json_feed.py new file mode 100644 index 0000000..c2e5aed --- /dev/null +++ b/geotribu_cli/content/json_feed.py @@ -0,0 +1,27 @@ +from typing import Any + +import requests +from requests import Response + +from geotribu_cli.__about__ import __executable_name__, __version__ + +HEADERS: dict = { + b"Accept": b"application/json", + b"User-Agent": bytes(f"{__executable_name__}/{__version__}", "utf8"), +} + + +class JsonFeedClient: + def __init__(self, url: str = "https://geotribu.fr/feed_json_created.json"): + """Class initialization.""" + self.url = url + + @property + def get_items(self) -> list[dict[str, Any]]: + r: Response = requests.get(self.url, headers=HEADERS) + r.raise_for_status() + return r.json()["items"] + + def get_tags(self, should_sort: bool = False) -> set[str]: + tags = set().union(*[i["tags"] for i in self.get_items]) + return sorted(tags) if should_sort else tags From 36dcb4221a4e3b8a4b5d322e54dbc93c19d04667 Mon Sep 17 00:00:00 2001 From: gounux Date: Sat, 23 Mar 2024 12:38:32 +0100 Subject: [PATCH 03/25] Add some tests --- .../content/2012-12-21_article_passe.md | 20 +++++++++ .../content/2044-04-01_article_futur.md | 21 +++++++++ tests/test_yaml_header_check.py | 44 +++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 tests/fixtures/content/2012-12-21_article_passe.md create mode 100644 tests/fixtures/content/2044-04-01_article_futur.md create mode 100644 tests/test_yaml_header_check.py diff --git a/tests/fixtures/content/2012-12-21_article_passe.md b/tests/fixtures/content/2012-12-21_article_passe.md new file mode 100644 index 0000000..2aa9b53 --- /dev/null +++ b/tests/fixtures/content/2012-12-21_article_passe.md @@ -0,0 +1,20 @@ +--- +title: Article supposément rédigé dans le passé +subtitle: Article supposément rédigé dans le passé pour tests +authors: + - Jane Doe +categories: + - article +comments: true +date: 2012-12-21 +description: Article supposément rédigé dans le passé +icon: octicons/server-16 +license: beerware +robots: index, follow +tags: + - Fromage + - QGIS + - OSM +--- + +# Article supposément rédigé dans le futur \ No newline at end of file diff --git a/tests/fixtures/content/2044-04-01_article_futur.md b/tests/fixtures/content/2044-04-01_article_futur.md new file mode 100644 index 0000000..d1c6ddc --- /dev/null +++ b/tests/fixtures/content/2044-04-01_article_futur.md @@ -0,0 +1,21 @@ +--- +title: Article supposément rédigé dans le futur +subtitle: Article supposément rédigé dans le futur pour tests +authors: + - Jane Doe +categories: + - article +comments: true +date: 2044-04-01 +description: Article supposément rédigé dans le futur +icon: octicons/server-16 +license: beerware +robots: index, follow +tags: + - Fromage + - IGN + - QGIS + - OSM +--- + +# Article supposément rédigé dans le futur \ No newline at end of file diff --git a/tests/test_yaml_header_check.py b/tests/test_yaml_header_check.py new file mode 100644 index 0000000..001409c --- /dev/null +++ b/tests/test_yaml_header_check.py @@ -0,0 +1,44 @@ +import unittest +from unittest.mock import patch + +import yaml + +from geotribu_cli.content.header_check import check_publish_date, check_tags + + +class TestYamlHeaderCheck(unittest.TestCase): + def setUp(self): + with open("tests/fixtures/content/2012-12-21_article_passe.md") as past_file: + past_content = past_file.read() + _, front_matter, _ = past_content.split("---", 2) + self.past_yaml_meta = yaml.safe_load(front_matter) + + with open("tests/fixtures/content/2044-04-01_article_futur.md") as future_file: + future_content = future_file.read() + _, front_matter, _ = future_content.split("---", 2) + self.future_yaml_meta = yaml.safe_load(front_matter) + + def test_past_publish_date(self): + self.assertFalse(check_publish_date(self.past_yaml_meta["date"])) + + def test_future_publish_date(self): + self.assertTrue(check_publish_date(self.future_yaml_meta["date"])) + + @patch("geotribu_cli.content.header_check.get_existing_tags") + def test_past_tags(self, get_existing_tags_mock): + get_existing_tags_mock.return_value = ["QGIS", "OSM"] + tags_ok, missing_tags, present_tags = check_tags(self.past_yaml_meta["tags"]) + self.assertFalse(tags_ok) + self.assertIn("Fromage", missing_tags) + self.assertIn("QGIS", present_tags) + self.assertIn("OSM", present_tags) + + @patch("geotribu_cli.content.header_check.get_existing_tags") + def test_future_tags(self, get_existing_tags_mock): + get_existing_tags_mock.return_value = ["Fromage", "IGN"] + tags_ok, missing_tags, present_tags = check_tags(self.future_yaml_meta["tags"]) + self.assertFalse(tags_ok) + self.assertIn("QGIS", missing_tags) + self.assertIn("OSM", missing_tags) + self.assertIn("Fromage", present_tags) + self.assertIn("IGN", present_tags) From bef103d8ccdfbf910b795dc6460ea07fc5bc1fae Mon Sep 17 00:00:00 2001 From: gounux Date: Sat, 23 Mar 2024 12:52:58 +0100 Subject: [PATCH 04/25] Add mandatory keys check --- geotribu_cli/content/header_check.py | 32 +++++++++++++++++++ .../content/2044-04-01_article_futur.md | 2 -- tests/test_yaml_header_check.py | 18 ++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 8ad07b4..1b845da 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -18,6 +18,18 @@ logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() +MANDATORY_KEYS = [ + "title", + "subtitle", + "authors", + "categories", + "date", + "description", + "icon", + "license", + "tags", +] + # ############################################################################ # ########## CLI ################# # ################################ @@ -113,6 +125,16 @@ def check_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: return all_exists, missing, present +def check_mandatory_keys( + keys: list[str], mandatory: list[str] = MANDATORY_KEYS +) -> tuple[bool, set[str]]: + missing = set() + for mk in mandatory: + if mk not in keys: + missing.add(mk) + return len(missing) == 0, missing + + def run(args: argparse.Namespace) -> None: """Run the sub command logic. @@ -163,3 +185,13 @@ def run(args: argparse.Namespace) -> None: raise ValueError(msg) else: logger.info("Tags ok") + + # check that mandatory keys are present + all_present, missing = check_mandatory_keys(yaml_meta.keys(), MANDATORY_KEYS) + if not all_present: + msg = f"Les clés suivantes ne sont pas présentes dans l'entête markdown : {','.join(missing)}" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Clés de l'entête ok") diff --git a/tests/fixtures/content/2044-04-01_article_futur.md b/tests/fixtures/content/2044-04-01_article_futur.md index d1c6ddc..ec73d39 100644 --- a/tests/fixtures/content/2044-04-01_article_futur.md +++ b/tests/fixtures/content/2044-04-01_article_futur.md @@ -7,9 +7,7 @@ categories: - article comments: true date: 2044-04-01 -description: Article supposément rédigé dans le futur icon: octicons/server-16 -license: beerware robots: index, follow tags: - Fromage diff --git a/tests/test_yaml_header_check.py b/tests/test_yaml_header_check.py index 001409c..8798df2 100644 --- a/tests/test_yaml_header_check.py +++ b/tests/test_yaml_header_check.py @@ -3,7 +3,11 @@ import yaml -from geotribu_cli.content.header_check import check_publish_date, check_tags +from geotribu_cli.content.header_check import ( + check_mandatory_keys, + check_publish_date, + check_tags, +) class TestYamlHeaderCheck(unittest.TestCase): @@ -42,3 +46,15 @@ def test_future_tags(self, get_existing_tags_mock): self.assertIn("OSM", missing_tags) self.assertIn("Fromage", present_tags) self.assertIn("IGN", present_tags) + + def test_past_mandatory_keys(self): + all_present, missing = check_mandatory_keys(self.past_yaml_meta.keys()) + self.assertTrue(all_present) + self.assertEqual(len(missing), 0) + + def test_future_mandatory_keys(self): + all_present, missing = check_mandatory_keys(self.future_yaml_meta.keys()) + self.assertFalse(all_present) + self.assertEqual(len(missing), 2) + self.assertIn("license", missing) + self.assertIn("description", missing) From 1a57ebb77307e1ae13d3d8ad3bcac42584389924 Mon Sep 17 00:00:00 2001 From: gounux Date: Mon, 1 Apr 2024 11:05:39 +0200 Subject: [PATCH 05/25] Check tags alphabetical order --- geotribu_cli/content/header_check.py | 22 ++++++++++++++++--- .../content/2012-12-21_article_passe.md | 2 +- tests/test_yaml_header_check.py | 21 +++++++++++++----- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 1b845da..dac8d4b 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -117,7 +117,7 @@ def get_existing_tags() -> set[str]: return jfc.get_tags(should_sort=True) -def check_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: +def check_existing_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: existing_tags = get_existing_tags() all_exists = set(tags).issubset(existing_tags) missing = set(tags).difference(existing_tags) @@ -125,6 +125,13 @@ def check_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: return all_exists, missing, present +def check_tags_order(tags: list[str]) -> bool: + for i in range(len(tags) - 1): + if tags[i] > tags[i + 1]: + return False + return True + + def check_mandatory_keys( keys: list[str], mandatory: list[str] = MANDATORY_KEYS ) -> tuple[bool, set[str]]: @@ -177,14 +184,23 @@ def run(args: argparse.Namespace) -> None: logger.info("Ratio image ok") # check that tags already exist - all_exists, missing, _ = check_tags(yaml_meta["tags"]) + all_exists, missing, _ = check_existing_tags(yaml_meta["tags"]) if not all_exists: msg = f"Les tags suivants n'existent pas dans les contenus Geotribu précédents : {','.join(missing)}" logger.error(msg) if args.raise_exceptions: raise ValueError(msg) else: - logger.info("Tags ok") + logger.info("Existence des tags ok") + + # check if tags are alphabetically sorted + if not check_tags_order(yaml_meta["tags"]): + msg = "Les tags ne sont pas triés par ordre alphabétique" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Ordre alphabétique des tags ok") # check that mandatory keys are present all_present, missing = check_mandatory_keys(yaml_meta.keys(), MANDATORY_KEYS) diff --git a/tests/fixtures/content/2012-12-21_article_passe.md b/tests/fixtures/content/2012-12-21_article_passe.md index 2aa9b53..9e1ccbc 100644 --- a/tests/fixtures/content/2012-12-21_article_passe.md +++ b/tests/fixtures/content/2012-12-21_article_passe.md @@ -13,8 +13,8 @@ license: beerware robots: index, follow tags: - Fromage - - QGIS - OSM + - QGIS --- # Article supposément rédigé dans le futur \ No newline at end of file diff --git a/tests/test_yaml_header_check.py b/tests/test_yaml_header_check.py index 8798df2..655524c 100644 --- a/tests/test_yaml_header_check.py +++ b/tests/test_yaml_header_check.py @@ -4,9 +4,10 @@ import yaml from geotribu_cli.content.header_check import ( + check_existing_tags, check_mandatory_keys, check_publish_date, - check_tags, + check_tags_order, ) @@ -29,24 +30,34 @@ def test_future_publish_date(self): self.assertTrue(check_publish_date(self.future_yaml_meta["date"])) @patch("geotribu_cli.content.header_check.get_existing_tags") - def test_past_tags(self, get_existing_tags_mock): + def test_past_tags_existence(self, get_existing_tags_mock): get_existing_tags_mock.return_value = ["QGIS", "OSM"] - tags_ok, missing_tags, present_tags = check_tags(self.past_yaml_meta["tags"]) + tags_ok, missing_tags, present_tags = check_existing_tags( + self.past_yaml_meta["tags"] + ) self.assertFalse(tags_ok) self.assertIn("Fromage", missing_tags) self.assertIn("QGIS", present_tags) self.assertIn("OSM", present_tags) @patch("geotribu_cli.content.header_check.get_existing_tags") - def test_future_tags(self, get_existing_tags_mock): + def test_future_tags_existence(self, get_existing_tags_mock): get_existing_tags_mock.return_value = ["Fromage", "IGN"] - tags_ok, missing_tags, present_tags = check_tags(self.future_yaml_meta["tags"]) + tags_ok, missing_tags, present_tags = check_existing_tags( + self.future_yaml_meta["tags"] + ) self.assertFalse(tags_ok) self.assertIn("QGIS", missing_tags) self.assertIn("OSM", missing_tags) self.assertIn("Fromage", present_tags) self.assertIn("IGN", present_tags) + def test_past_tags_order(self): + self.assertTrue(check_tags_order(self.past_yaml_meta["tags"])) + + def test_future_tags_order(self): + self.assertFalse(check_tags_order(self.future_yaml_meta["tags"])) + def test_past_mandatory_keys(self): all_present, missing = check_mandatory_keys(self.past_yaml_meta.keys()) self.assertTrue(all_present) From 274e435e7fa70b4358341fa9f4ce3f0eed1c8219 Mon Sep 17 00:00:00 2001 From: gounux Date: Wed, 17 Apr 2024 17:48:47 +0200 Subject: [PATCH 06/25] Remove publish date check --- geotribu_cli/content/header_check.py | 21 --------------------- tests/test_yaml_header_check.py | 7 ------- 2 files changed, 28 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index dac8d4b..fbf3d41 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -3,8 +3,6 @@ import os import shutil import uuid -from datetime import datetime -from typing import Any import requests import yaml @@ -13,7 +11,6 @@ from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.content.json_feed import JsonFeedClient -from geotribu_cli.utils.dates_manipulation import is_more_recent logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() @@ -83,15 +80,6 @@ def parser_header_check( # ################################ -def check_publish_date(date: Any) -> bool: - if isinstance(date, str): - publish_date = datetime.strptime(date.split(" ")[0], "%Y-%m-%d").date() - else: - publish_date = date - # TODO: check if date is another type and raise error - return is_more_recent(datetime.now().date(), publish_date) - - def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> bool: r = requests.get( image_url, @@ -162,15 +150,6 @@ def run(args: argparse.Namespace) -> None: yaml_meta = yaml.safe_load(front_matter) logger.debug(f"YAML metadata loaded : {yaml_meta}") - # check that datetime is in the future - if not check_publish_date(yaml_meta["date"]): - msg = "La date de publication n'est pas dans le turfu !" - logger.error(msg) - if args.raise_exceptions: - raise ValueError(msg) - else: - logger.info("Date de publication ok") - # check that image ratio is okayyy if "image" in yaml_meta: if not check_image_ratio( diff --git a/tests/test_yaml_header_check.py b/tests/test_yaml_header_check.py index 655524c..bdd452c 100644 --- a/tests/test_yaml_header_check.py +++ b/tests/test_yaml_header_check.py @@ -6,7 +6,6 @@ from geotribu_cli.content.header_check import ( check_existing_tags, check_mandatory_keys, - check_publish_date, check_tags_order, ) @@ -23,12 +22,6 @@ def setUp(self): _, front_matter, _ = future_content.split("---", 2) self.future_yaml_meta = yaml.safe_load(front_matter) - def test_past_publish_date(self): - self.assertFalse(check_publish_date(self.past_yaml_meta["date"])) - - def test_future_publish_date(self): - self.assertTrue(check_publish_date(self.future_yaml_meta["date"])) - @patch("geotribu_cli.content.header_check.get_existing_tags") def test_past_tags_existence(self, get_existing_tags_mock): get_existing_tags_mock.return_value = ["QGIS", "OSM"] From 892cc5c6ed2f742655a985055da8e466e28011e5 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 26 Apr 2024 10:51:46 +0200 Subject: [PATCH 07/25] Use utils function for checking if file exists --- geotribu_cli/content/header_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index fbf3d41..539b02f 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -11,6 +11,7 @@ from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.content.json_feed import JsonFeedClient +from geotribu_cli.utils.check_path import check_path_exists logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() @@ -141,8 +142,7 @@ def run(args: argparse.Namespace) -> None: logger.debug(f"Running {args.command} with {args}") content_path = args.content_path - if not os.path.exists(content_path): - raise ValueError(f"Mayday ! Le fichier {content_path} n'existe pas !") + check_path_exists(content_path, raise_error=True) with open(content_path) as file: content = file.read() From 45e776460aa10cee40af0a8b0e339da7cf0be1f7 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 3 May 2024 12:40:51 +0200 Subject: [PATCH 08/25] Use JSON feed client --- geotribu_cli/content/header_check.py | 6 +++--- geotribu_cli/content/json_feed.py | 27 --------------------------- 2 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 geotribu_cli/content/json_feed.py diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 539b02f..df14d41 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -10,7 +10,7 @@ from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults -from geotribu_cli.content.json_feed import JsonFeedClient +from geotribu_cli.json.json_client import JsonFeedClient from geotribu_cli.utils.check_path import check_path_exists logger = logging.getLogger(__name__) @@ -101,9 +101,9 @@ def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> boo os.remove(image_file_name) -def get_existing_tags() -> set[str]: +def get_existing_tags() -> list[str]: jfc = JsonFeedClient() - return jfc.get_tags(should_sort=True) + return jfc.tags(should_sort=True) def check_existing_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: diff --git a/geotribu_cli/content/json_feed.py b/geotribu_cli/content/json_feed.py deleted file mode 100644 index c2e5aed..0000000 --- a/geotribu_cli/content/json_feed.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Any - -import requests -from requests import Response - -from geotribu_cli.__about__ import __executable_name__, __version__ - -HEADERS: dict = { - b"Accept": b"application/json", - b"User-Agent": bytes(f"{__executable_name__}/{__version__}", "utf8"), -} - - -class JsonFeedClient: - def __init__(self, url: str = "https://geotribu.fr/feed_json_created.json"): - """Class initialization.""" - self.url = url - - @property - def get_items(self) -> list[dict[str, Any]]: - r: Response = requests.get(self.url, headers=HEADERS) - r.raise_for_status() - return r.json()["items"] - - def get_tags(self, should_sort: bool = False) -> set[str]: - tags = set().union(*[i["tags"] for i in self.get_items]) - return sorted(tags) if should_sort else tags From a2c0d57efd52a53bbe57eeedc9fafd593ef36c19 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 17 May 2024 10:17:24 +0200 Subject: [PATCH 09/25] Use frontmatter to load yaml metadata --- geotribu_cli/content/header_check.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index df14d41..69dedac 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -4,8 +4,8 @@ import shutil import uuid +import frontmatter import requests -import yaml from PIL import Image from geotribu_cli.__about__ import __executable_name__, __version__ @@ -145,9 +145,8 @@ def run(args: argparse.Namespace) -> None: check_path_exists(content_path, raise_error=True) with open(content_path) as file: - content = file.read() - _, front_matter, _ = content.split("---", 2) - yaml_meta = yaml.safe_load(front_matter) + content = frontmatter.load(file) + yaml_meta = content.metadata logger.debug(f"YAML metadata loaded : {yaml_meta}") # check that image ratio is okayyy From 536e82a96fd75b106cabd976d37aacef39d16a75 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 17 May 2024 10:42:49 +0200 Subject: [PATCH 10/25] Accept multiple path parameters --- geotribu_cli/content/header_check.py | 98 +++++++++++++++------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 69dedac..9b14990 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -7,6 +7,7 @@ import frontmatter import requests from PIL import Image +from unidecode import unidecode from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults @@ -49,6 +50,7 @@ def parser_header_check( help="Chemin du fichier markdown dont l'entête est à vérifier", type=str, metavar="content", + nargs="+", ) subparser.add_argument( "-minr", @@ -116,7 +118,7 @@ def check_existing_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: def check_tags_order(tags: list[str]) -> bool: for i in range(len(tags) - 1): - if tags[i] > tags[i + 1]: + if unidecode(tags[i].upper()) > unidecode(tags[i + 1].upper()): return False return True @@ -140,52 +142,58 @@ def run(args: argparse.Namespace) -> None: args (argparse.Namespace): arguments passed to the subcommand """ logger.debug(f"Running {args.command} with {args}") - content_path = args.content_path - - check_path_exists(content_path, raise_error=True) - - with open(content_path) as file: - content = frontmatter.load(file) - yaml_meta = content.metadata - logger.debug(f"YAML metadata loaded : {yaml_meta}") + content_paths = args.content_path + + for content_path in content_paths: + logger.info(f"Checking header of {content_path}") + check_path_exists(content_path, raise_error=True) + + with open(content_path) as file: + content = frontmatter.load(file) + yaml_meta = content.metadata + logger.debug(f"YAML metadata loaded : {yaml_meta}") + + # check that image ratio is okayyy + if "image" in yaml_meta: + if not yaml_meta["image"]: + logger.error("Pas d'URL pour l'image") + elif not check_image_ratio( + yaml_meta["image"], args.min_image_ratio, args.max_image_ratio + ): + msg = f"Le ratio de l'image n'est pas dans l'interface autorisé ({args.min_image_ratio} - {args.max_image_ratio})" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Ratio image ok") + + # check that tags already exist + all_exists, missing, _ = check_existing_tags(yaml_meta["tags"]) + if not all_exists: + msg = f"Les tags suivants n'existent pas dans les contenus Geotribu précédents : {','.join(missing)}" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Existence des tags ok") - # check that image ratio is okayyy - if "image" in yaml_meta: - if not check_image_ratio( - yaml_meta["image"], args.min_image_ratio, args.max_image_ratio - ): - msg = f"Le ratio de l'image n'est pas dans l'interface autorisé ({args.minratio} - {args.maxratio})" + # check if tags are alphabetically sorted + if not check_tags_order(yaml_meta["tags"]): + msg = f"Les tags ne sont pas triés par ordre alphabétique : {yaml_meta['tags']}" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Ordre alphabétique des tags ok") + + # check that mandatory keys are present + all_present, missing = check_mandatory_keys( + yaml_meta.keys(), MANDATORY_KEYS + ) + if not all_present: + msg = f"Les clés suivantes ne sont pas présentes dans l'entête markdown : {','.join(missing)}" logger.error(msg) if args.raise_exceptions: raise ValueError(msg) else: - logger.info("Ratio image ok") - - # check that tags already exist - all_exists, missing, _ = check_existing_tags(yaml_meta["tags"]) - if not all_exists: - msg = f"Les tags suivants n'existent pas dans les contenus Geotribu précédents : {','.join(missing)}" - logger.error(msg) - if args.raise_exceptions: - raise ValueError(msg) - else: - logger.info("Existence des tags ok") - - # check if tags are alphabetically sorted - if not check_tags_order(yaml_meta["tags"]): - msg = "Les tags ne sont pas triés par ordre alphabétique" - logger.error(msg) - if args.raise_exceptions: - raise ValueError(msg) - else: - logger.info("Ordre alphabétique des tags ok") - - # check that mandatory keys are present - all_present, missing = check_mandatory_keys(yaml_meta.keys(), MANDATORY_KEYS) - if not all_present: - msg = f"Les clés suivantes ne sont pas présentes dans l'entête markdown : {','.join(missing)}" - logger.error(msg) - if args.raise_exceptions: - raise ValueError(msg) - else: - logger.info("Clés de l'entête ok") + logger.info("Clés de l'entête ok") From 7224615c13feb40b1bcd611aa18309f2a015666e Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 17 May 2024 10:47:07 +0200 Subject: [PATCH 11/25] Add missing dependency --- requirements/base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/base.txt b/requirements/base.txt index 30f99dd..8ca2d6e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,3 +8,4 @@ rich_argparse>=1,<1.5 python-frontmatter>=1,<2 requests>=2.31,<3 typing-extensions>=4,<5 ; python_version < '3.11' +Unidecode==1.3.8 From 91176fd2ca5f888becaf0178deb7b604433d2743 Mon Sep 17 00:00:00 2001 From: gounux Date: Mon, 20 May 2024 09:34:45 +0200 Subject: [PATCH 12/25] Edit how image sizes are checked --- geotribu_cli/content/header_check.py | 68 +++++++++++++++++++++------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 9b14990..d6cb79f 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -53,18 +53,36 @@ def parser_header_check( nargs="+", ) subparser.add_argument( - "-minr", - "--min-ratio", - dest="min_image_ratio", - default=1.2, - help="Ratio width/height minimum de l'image à vérifier", + "-minw", + "--min-width", + dest="min_image_width", + default=400, + type=int, + help="Largeur minimum de l'image à vérifier", ) subparser.add_argument( - "-maxr", - "--max-ratio", - dest="max_image_ratio", - default=1.5, - help="Ratio width/height maximum de l'image à vérifier", + "-maxw", + "--max-width", + dest="max_image_width", + default=800, + type=int, + help="Largeur maximum de l'image à vérifier", + ) + subparser.add_argument( + "-minh", + "--min-height", + dest="min_image_height", + default=400, + type=int, + help="Hauteur minimum de l'image à vérifier", + ) + subparser.add_argument( + "-maxh", + "--max-height", + dest="max_image_height", + default=800, + type=int, + help="Hauteur maximum de l'image à vérifier", ) subparser.add_argument( "-r", @@ -83,7 +101,9 @@ def parser_header_check( # ################################ -def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> bool: +def check_image_size( + image_url: str, minw: int, maxw: int, minh: int, maxh: int +) -> bool: r = requests.get( image_url, headers={"User-Agent": f"{__executable_name__}v{__version__}"}, @@ -96,9 +116,15 @@ def check_image_ratio(image_url: str, min_ratio: float, max_ratio: float) -> boo try: shutil.copyfileobj(r.raw, image_file) with Image.open(image_file_name) as image: - width, height = image.width, image.height - ratio = width / height - return min_ratio <= ratio <= max_ratio + return minw <= image.width <= maxw and minh <= image.height <= maxh + # return check_image_dimensions( + # Path(image_file_name), + # min_width=minw, + # max_width=maxw, + # min_height=minh, + # max_height=maxh, + # allowed_images_extensions=(".jpg", ".jpeg", ".png", ".webp") + # ) finally: os.remove(image_file_name) @@ -157,10 +183,18 @@ def run(args: argparse.Namespace) -> None: if "image" in yaml_meta: if not yaml_meta["image"]: logger.error("Pas d'URL pour l'image") - elif not check_image_ratio( - yaml_meta["image"], args.min_image_ratio, args.max_image_ratio + elif not check_image_size( + yaml_meta["image"], + args.min_image_width, + args.max_image_width, + args.min_image_height, + args.max_image_height, ): - msg = f"Le ratio de l'image n'est pas dans l'interface autorisé ({args.min_image_ratio} - {args.max_image_ratio})" + msg = ( + f"Les dimensions de l'image ne sont pas dans l'intervalle autorisé " + f"(w:{args.min_image_width}-{args.max_image_width}," + f"h:{args.min_image_height}-{args.max_image_height})" + ) logger.error(msg) if args.raise_exceptions: raise ValueError(msg) From d45db583bbfebb9ce6974adfc759115cabadc772 Mon Sep 17 00:00:00 2001 From: gounux Date: Mon, 20 May 2024 09:43:39 +0200 Subject: [PATCH 13/25] Remove icon and subtitle from mandatory keys --- geotribu_cli/content/header_check.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index d6cb79f..1efe175 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -19,12 +19,10 @@ MANDATORY_KEYS = [ "title", - "subtitle", "authors", "categories", "date", "description", - "icon", "license", "tags", ] From 36339025b4656f7a7af991f6f1cf237690d89c1d Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 23 May 2024 09:10:35 +0200 Subject: [PATCH 14/25] Check author markdown file --- geotribu_cli/content/header_check.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 1efe175..489a913 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -3,6 +3,7 @@ import os import shutil import uuid +from pathlib import Path import frontmatter import requests @@ -50,6 +51,13 @@ def parser_header_check( metavar="content", nargs="+", ) + subparser.add_argument( + "-af", + "--authors-folder", + dest="authors_folder", + type=Path, + help="Chemin qui contient les presentations markdown des auteurs/autrices", + ) subparser.add_argument( "-minw", "--min-width", @@ -99,6 +107,11 @@ def parser_header_check( # ################################ +def check_author_md(author: str, folder: Path) -> bool: + p = os.path.join(folder, f"{author.lower().replace(' ', '-')}.md") + return os.path.exists(p) + + def check_image_size( image_url: str, minw: int, maxw: int, minh: int, maxh: int ) -> bool: @@ -199,6 +212,19 @@ def run(args: argparse.Namespace) -> None: else: logger.info("Ratio image ok") + # check that author md file is present + if args.authors_folder: + author_exists = check_author_md( + yaml_meta["author"], args.authors_folder + ) + if not author_exists: + msg = "Le fichier de l'auteur/autrice n'a pas pu être trouvé dans le répertoire" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info("Markdown de l'auteur/autrice ok") + # check that tags already exist all_exists, missing, _ = check_existing_tags(yaml_meta["tags"]) if not all_exists: From f983c6494551621d25234316c7d07c7ed3193ed2 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 23 May 2024 09:34:42 +0200 Subject: [PATCH 15/25] Fix author md check --- geotribu_cli/content/header_check.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 489a913..9b06cd8 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -108,7 +108,10 @@ def parser_header_check( def check_author_md(author: str, folder: Path) -> bool: - p = os.path.join(folder, f"{author.lower().replace(' ', '-')}.md") + if author == "Geotribu": + return True + formatted = author.translate(str.maketrans({"'": "", " ": "-"})) + p = os.path.join(folder, f"{unidecode(formatted.lower())}.md") return os.path.exists(p) @@ -190,7 +193,7 @@ def run(args: argparse.Namespace) -> None: yaml_meta = content.metadata logger.debug(f"YAML metadata loaded : {yaml_meta}") - # check that image ratio is okayyy + # check that image size is okay if "image" in yaml_meta: if not yaml_meta["image"]: logger.error("Pas d'URL pour l'image") @@ -214,16 +217,15 @@ def run(args: argparse.Namespace) -> None: # check that author md file is present if args.authors_folder: - author_exists = check_author_md( - yaml_meta["author"], args.authors_folder - ) - if not author_exists: - msg = "Le fichier de l'auteur/autrice n'a pas pu être trouvé dans le répertoire" - logger.error(msg) - if args.raise_exceptions: - raise ValueError(msg) - else: - logger.info("Markdown de l'auteur/autrice ok") + for author in yaml_meta["authors"]: + author_exists = check_author_md(author, args.authors_folder) + if not author_exists: + msg = f"Le fichier de l'auteur/autrice '{author}' n'a pas pu être trouvé dans le répertoire" + logger.error(msg) + if args.raise_exceptions: + raise ValueError(msg) + else: + logger.info(f"Markdown de l'auteur/autrice '{author}' ok") # check that tags already exist all_exists, missing, _ = check_existing_tags(yaml_meta["tags"]) From 2a0cc930d8a9bd95184721533fa908fe680f07a7 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 23 May 2024 11:11:29 +0200 Subject: [PATCH 16/25] Add image size by url function --- geotribu_cli/utils/check_image_size.py | 21 +++++++++++++++++++++ tests/test_utils_images_size.py | 15 ++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/geotribu_cli/utils/check_image_size.py b/geotribu_cli/utils/check_image_size.py index b879a27..00d7859 100644 --- a/geotribu_cli/utils/check_image_size.py +++ b/geotribu_cli/utils/check_image_size.py @@ -16,9 +16,11 @@ from decimal import Decimal from pathlib import Path from typing import Union +from urllib import request # 3rd party import imagesize +from PIL import ImageFile # ############################################################################# # ########## Globals ############### @@ -61,6 +63,25 @@ def get_image_size(image_filepath: Path) -> tuple[int, int]: return None +def get_image_size_by_url(url: str) -> tuple[int, int]: + """Get image dimensions as a tuple (width,height) of an image at an URL. Return None in case of error. + + :param str url: url of the image + + :return Tuple[int, int]: dimensions tuple (width,height) + """ + with request.urlopen(url) as file: + parser = ImageFile.Parser() + while True: + data = file.read(1024) + if not data: + break + parser.feed(data) + if parser.image: + return parser.image.size + return None + + def get_svg_size(image_filepath: Path) -> tuple[int, int]: """Extract SVG width and height from a SVG file and convert them into integers. \ Relevant and working only if the file root has width and height attributes. diff --git a/tests/test_utils_images_size.py b/tests/test_utils_images_size.py index 7a69c78..58c58f0 100644 --- a/tests/test_utils_images_size.py +++ b/tests/test_utils_images_size.py @@ -21,7 +21,11 @@ # project from geotribu_cli.__about__ import __title_clean__, __version__ -from geotribu_cli.utils.check_image_size import check_image_dimensions, get_image_size +from geotribu_cli.utils.check_image_size import ( + check_image_dimensions, + get_image_size, + get_image_size_by_url, +) # ############################################################################ # ########## Classes ############# @@ -120,6 +124,15 @@ def test_check_image_dimensions(self): ) ) + def test_image_url_dimensions(self): + for url, width, height in [ + ("https://cdn.geotribu.fr/img/coup_de_gueule.jpg", 74, 64), + ("https://cdn.geotribu.fr/img/pytroll.png", 100, 100), + ]: + w, h = get_image_size_by_url(url) + self.assertEqual(w, width) + self.assertEqual(h, height) + # ############################################################################ # ####### Stand-alone run ######## From 14c8a32c0a1b265af8308bdb3c98f06b60929598 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 23 May 2024 11:14:03 +0200 Subject: [PATCH 17/25] Add author md check tests --- geotribu_cli/content/header_check.py | 33 ++++---------------------- geotribu_cli/utils/check_image_size.py | 4 ++-- tests/fixtures/team/jane-doe.md | 26 ++++++++++++++++++++ tests/test_utils_images_size.py | 4 ++-- tests/test_yaml_header_check.py | 13 ++++++++++ 5 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 tests/fixtures/team/jane-doe.md diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 9b06cd8..08d4deb 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -1,18 +1,14 @@ import argparse import logging import os -import shutil -import uuid from pathlib import Path import frontmatter -import requests -from PIL import Image from unidecode import unidecode -from geotribu_cli.__about__ import __executable_name__, __version__ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.json.json_client import JsonFeedClient +from geotribu_cli.utils.check_image_size import get_image_dimensions_by_url from geotribu_cli.utils.check_path import check_path_exists logger = logging.getLogger(__name__) @@ -118,29 +114,8 @@ def check_author_md(author: str, folder: Path) -> bool: def check_image_size( image_url: str, minw: int, maxw: int, minh: int, maxh: int ) -> bool: - r = requests.get( - image_url, - headers={"User-Agent": f"{__executable_name__}v{__version__}"}, - stream=True, - ) - r.raise_for_status() - image_file_name = str(uuid.uuid4()) - with open(image_file_name, "wb") as image_file: - r.raw.decode_content = True - try: - shutil.copyfileobj(r.raw, image_file) - with Image.open(image_file_name) as image: - return minw <= image.width <= maxw and minh <= image.height <= maxh - # return check_image_dimensions( - # Path(image_file_name), - # min_width=minw, - # max_width=maxw, - # min_height=minh, - # max_height=maxh, - # allowed_images_extensions=(".jpg", ".jpeg", ".png", ".webp") - # ) - finally: - os.remove(image_file_name) + width, height = get_image_dimensions_by_url(image_url) + return minw <= width <= maxw and minh <= height <= maxh def get_existing_tags() -> list[str]: @@ -213,7 +188,7 @@ def run(args: argparse.Namespace) -> None: if args.raise_exceptions: raise ValueError(msg) else: - logger.info("Ratio image ok") + logger.info("Dimensions de l'image ok") # check that author md file is present if args.authors_folder: diff --git a/geotribu_cli/utils/check_image_size.py b/geotribu_cli/utils/check_image_size.py index 00d7859..594e372 100644 --- a/geotribu_cli/utils/check_image_size.py +++ b/geotribu_cli/utils/check_image_size.py @@ -63,8 +63,8 @@ def get_image_size(image_filepath: Path) -> tuple[int, int]: return None -def get_image_size_by_url(url: str) -> tuple[int, int]: - """Get image dimensions as a tuple (width,height) of an image at an URL. Return None in case of error. +def get_image_dimensions_by_url(url: str) -> tuple[int, int]: + """Get image dimensions as a tuple (width,height) of an image at an URL. Return None in case of error or no data. :param str url: url of the image diff --git a/tests/fixtures/team/jane-doe.md b/tests/fixtures/team/jane-doe.md new file mode 100644 index 0000000..27510ea --- /dev/null +++ b/tests/fixtures/team/jane-doe.md @@ -0,0 +1,26 @@ +--- +title: Jane Doe +categories: + - contributeur +social: + - bluesky: + - github: + - gitlab: + - linkedin: + - mail: + - mastodon: + - instance: + - username: + - openstreetmap: + - osgeo: + - twitter: + - website: +--- + +# Jane Doe + + + +Test + + diff --git a/tests/test_utils_images_size.py b/tests/test_utils_images_size.py index 58c58f0..a953a45 100644 --- a/tests/test_utils_images_size.py +++ b/tests/test_utils_images_size.py @@ -23,8 +23,8 @@ from geotribu_cli.__about__ import __title_clean__, __version__ from geotribu_cli.utils.check_image_size import ( check_image_dimensions, + get_image_dimensions_by_url, get_image_size, - get_image_size_by_url, ) # ############################################################################ @@ -129,7 +129,7 @@ def test_image_url_dimensions(self): ("https://cdn.geotribu.fr/img/coup_de_gueule.jpg", 74, 64), ("https://cdn.geotribu.fr/img/pytroll.png", 100, 100), ]: - w, h = get_image_size_by_url(url) + w, h = get_image_dimensions_by_url(url) self.assertEqual(w, width) self.assertEqual(h, height) diff --git a/tests/test_yaml_header_check.py b/tests/test_yaml_header_check.py index bdd452c..7cb7ecb 100644 --- a/tests/test_yaml_header_check.py +++ b/tests/test_yaml_header_check.py @@ -1,14 +1,18 @@ import unittest +from pathlib import Path from unittest.mock import patch import yaml from geotribu_cli.content.header_check import ( + check_author_md, check_existing_tags, check_mandatory_keys, check_tags_order, ) +TEAM_FOLDER = Path("tests/fixtures/team") + class TestYamlHeaderCheck(unittest.TestCase): def setUp(self): @@ -62,3 +66,12 @@ def test_future_mandatory_keys(self): self.assertEqual(len(missing), 2) self.assertIn("license", missing) self.assertIn("description", missing) + + def test_author_md_ok(self): + self.assertTrue(check_author_md("Jane Doe", TEAM_FOLDER)) + self.assertTrue(check_author_md("JaNe DoE", TEAM_FOLDER)) + self.assertTrue(check_author_md("Jàne Doe", TEAM_FOLDER)) + self.assertTrue(check_author_md("Jàne Döe", TEAM_FOLDER)) + self.assertTrue(check_author_md("Jàne Döé", TEAM_FOLDER)) + self.assertTrue(check_author_md("Jàne D'öé", TEAM_FOLDER)) + self.assertFalse(check_author_md("JaneDoe", TEAM_FOLDER)) From f4f5c6322eebd12fe1994b35e365b1f43845bc29 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman <40383801+gounux@users.noreply.github.com> Date: Mon, 27 May 2024 21:45:30 +0200 Subject: [PATCH 18/25] Update geotribu_cli/content/header_check.py Co-authored-by: Julien Signed-off-by: Guilhem Allaman <40383801+gounux@users.noreply.github.com> --- geotribu_cli/content/header_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 08d4deb..cd50f54 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -43,7 +43,7 @@ def parser_header_check( subparser.add_argument( "content_path", help="Chemin du fichier markdown dont l'entête est à vérifier", - type=str, + type=Path, metavar="content", nargs="+", ) From 275fc9d37f80119d10de08be7ee210ddffc304a3 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman <40383801+gounux@users.noreply.github.com> Date: Mon, 27 May 2024 21:45:40 +0200 Subject: [PATCH 19/25] Update geotribu_cli/content/header_check.py Co-authored-by: Julien Signed-off-by: Guilhem Allaman <40383801+gounux@users.noreply.github.com> --- geotribu_cli/content/header_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index cd50f54..9867cd3 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -9,7 +9,7 @@ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.json.json_client import JsonFeedClient from geotribu_cli.utils.check_image_size import get_image_dimensions_by_url -from geotribu_cli.utils.check_path import check_path_exists +from geotribu_cli.utils.check_path import check_path logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() From 2659b51ef20a714b513c3fa5de8243933c8a2501 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 19:45:47 +0000 Subject: [PATCH 20/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- geotribu_cli/content/header_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 9867cd3..5ac8da5 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -9,7 +9,6 @@ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.json.json_client import JsonFeedClient from geotribu_cli.utils.check_image_size import get_image_dimensions_by_url -from geotribu_cli.utils.check_path import check_path logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() From 856213dd930025de12cf6ee8efbdf79a6857703c Mon Sep 17 00:00:00 2001 From: Guilhem Allaman <40383801+gounux@users.noreply.github.com> Date: Mon, 27 May 2024 21:46:08 +0200 Subject: [PATCH 21/25] Update geotribu_cli/content/header_check.py Co-authored-by: Julien Signed-off-by: Guilhem Allaman <40383801+gounux@users.noreply.github.com> --- geotribu_cli/content/header_check.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 5ac8da5..295e99d 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -156,13 +156,20 @@ def run(args: argparse.Namespace) -> None: args (argparse.Namespace): arguments passed to the subcommand """ logger.debug(f"Running {args.command} with {args}") - content_paths = args.content_path + content_paths: list[Path] = args.content_path for content_path in content_paths: logger.info(f"Checking header of {content_path}") - check_path_exists(content_path, raise_error=True) + check_path( + input_path=content_path, + must_be_a_file=True, + must_be_a_folder=False, + must_be_readable=True, + raise_error=True, + ) - with open(content_path) as file: + + with content_path.open(mode="r", encoding="UTF-8") as file: content = frontmatter.load(file) yaml_meta = content.metadata logger.debug(f"YAML metadata loaded : {yaml_meta}") From 77b838c7d98efe820ffa16691a908cced59646fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 19:46:16 +0000 Subject: [PATCH 22/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- geotribu_cli/content/header_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 295e99d..0b1ca74 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -168,7 +168,6 @@ def run(args: argparse.Namespace) -> None: raise_error=True, ) - with content_path.open(mode="r", encoding="UTF-8") as file: content = frontmatter.load(file) yaml_meta = content.metadata From 61e8f5680cdd9ef0d3ce273e6b466b97ebfa41c5 Mon Sep 17 00:00:00 2001 From: gounux Date: Mon, 27 May 2024 21:50:42 +0200 Subject: [PATCH 23/25] Add missing imports --- geotribu_cli/content/header_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 0b1ca74..633cab8 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -9,6 +9,7 @@ from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.json.json_client import JsonFeedClient from geotribu_cli.utils.check_image_size import get_image_dimensions_by_url +from geotribu_cli.utils.check_path import check_path logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() From d4bbcbdd86a5a9cd546bfbc5595e70914d4ab74b Mon Sep 17 00:00:00 2001 From: gounux Date: Tue, 28 May 2024 09:42:51 +0200 Subject: [PATCH 24/25] Use slugger for author md file --- geotribu_cli/content/header_check.py | 4 ++-- requirements/base.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index 633cab8..b5b4901 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -10,6 +10,7 @@ from geotribu_cli.json.json_client import JsonFeedClient from geotribu_cli.utils.check_image_size import get_image_dimensions_by_url from geotribu_cli.utils.check_path import check_path +from geotribu_cli.utils.slugger import sluggy logger = logging.getLogger(__name__) defaults_settings = GeotribuDefaults() @@ -106,8 +107,7 @@ def parser_header_check( def check_author_md(author: str, folder: Path) -> bool: if author == "Geotribu": return True - formatted = author.translate(str.maketrans({"'": "", " ": "-"})) - p = os.path.join(folder, f"{unidecode(formatted.lower())}.md") + p = os.path.join(folder, f"{sluggy(author)}.md") return os.path.exists(p) diff --git a/requirements/base.txt b/requirements/base.txt index 8ca2d6e..30f99dd 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,4 +8,3 @@ rich_argparse>=1,<1.5 python-frontmatter>=1,<2 requests>=2.31,<3 typing-extensions>=4,<5 ; python_version < '3.11' -Unidecode==1.3.8 From 6a43fcbc9a5d2d6bcbe5a55d08bf1d9226b45cf3 Mon Sep 17 00:00:00 2001 From: gounux Date: Tue, 28 May 2024 09:46:20 +0200 Subject: [PATCH 25/25] No more unidecode --- geotribu_cli/content/header_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geotribu_cli/content/header_check.py b/geotribu_cli/content/header_check.py index b5b4901..cb8af9e 100644 --- a/geotribu_cli/content/header_check.py +++ b/geotribu_cli/content/header_check.py @@ -4,7 +4,6 @@ from pathlib import Path import frontmatter -from unidecode import unidecode from geotribu_cli.constants import GeotribuDefaults from geotribu_cli.json.json_client import JsonFeedClient @@ -133,7 +132,7 @@ def check_existing_tags(tags: list[str]) -> tuple[bool, set[str], set[str]]: def check_tags_order(tags: list[str]) -> bool: for i in range(len(tags) - 1): - if unidecode(tags[i].upper()) > unidecode(tags[i + 1].upper()): + if sluggy(tags[i].upper()) > sluggy(tags[i + 1].upper()): return False return True