From 1f97d86a8d2f7dbe5348f9a600aa9fe3f46bb0f4 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 10:14:44 +0800 Subject: [PATCH 01/12] update --- setup.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 94779b5..2db3561 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ """stac-check setup.py """ -from setuptools import setup, find_packages -__version__ = "1.3.3" +from setuptools import find_packages, setup + +__version__ = "1.4.0" with open("README.md", "r") as fh: long_description = fh.read() @@ -18,7 +19,7 @@ "click>=8.0.0", "requests>=2.19.1", "jsonschema>=3.1.2", - "stac-validator>=3.1.0", + "stac-validator==3.4.0", "PyYAML", "python-dotenv", ], @@ -28,14 +29,12 @@ "types-setuptools", ], }, - entry_points={ - 'console_scripts': ['stac-check=stac_check.cli:main'] - }, + entry_points={"console_scripts": ["stac-check=stac_check.cli:main"]}, author="Jonathan Healy", author_email="jonathan.d.healy@gmail.com", license="MIT", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.8", - tests_require=["pytest"] + tests_require=["pytest"], ) From 9145b276d85c0ad89afbac72fa0f22947295a087 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 10:14:57 +0800 Subject: [PATCH 02/12] add pre commit config --- .pre-commit-config.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..07d8ac2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/timothycrosley/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/psf/black + rev: 24.1.1 + hooks: + - id: black + language_version: python3.10 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + exclude: /tests/ + # --strict + args: + [ + --no-strict-optional, + --ignore-missing-imports, + --implicit-reexport, + --explicit-package-bases, + ] + additional_dependencies: ["types-attrs", "types-requests"] From fc9b7f2fa990a8a637132764584b21379a88856f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 10:27:18 +0800 Subject: [PATCH 03/12] pre-commit 1 --- docs/conf.py | 16 +-- stac_check/cli.py | 60 +++++---- stac_check/lint.py | 160 ++++++++++++++++-------- tests/test_cli.py | 9 +- tests/test_config.py | 7 +- tests/test_lint.py | 287 +++++++++++++++++++------------------------ 6 files changed, 284 insertions(+), 255 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3fad936..27428c3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,24 +6,24 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'stac-check' -author = 'Jonathan Healy' -release = '1.3.1' +project = "stac-check" +author = "Jonathan Healy" +release = "1.3.1" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [] -templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' -html_static_path = ['_static'] +html_theme = "alabaster" +html_static_path = ["_static"] html_css_files = [ - 'custom.css', + "custom.css", ] diff --git a/stac_check/cli.py b/stac_check/cli.py index dee6d90..e820aaf 100644 --- a/stac_check/cli.py +++ b/stac_check/cli.py @@ -1,8 +1,10 @@ import click -from .lint import Linter import pkg_resources -def link_asset_message(link_list:list, type: str, format: str) -> None: +from .lint import Linter + + +def link_asset_message(link_list: list, type: str, format: str) -> None: """Prints a list of links or assets and any errors associated with them. Args: @@ -20,6 +22,7 @@ def link_asset_message(link_list:list, type: str, format: str) -> None: else: click.secho(f"No {type.upper()} {format} errors!", fg="green") + def recursive_message(linter: Linter) -> None: """Displays messages related to the recursive validation of assets in a collection or catalog. @@ -36,18 +39,19 @@ def recursive_message(linter: Linter) -> None: for count, msg in enumerate(linter.validate_all): click.secho(f"Asset {count+1} Validated: {msg['path']}", bg="white", fg="black") click.secho() - if msg['valid_stac'] == True: + if msg["valid_stac"] == True: recursive_linter = Linter(msg["path"], recursive=0) cli_message(recursive_linter) else: - click.secho(f"Valid: {msg['valid_stac']}", fg='red') + click.secho(f"Valid: {msg['valid_stac']}", fg="red") click.secho("Schemas validated: ", fg="blue") for schema in msg["schema"]: click.secho(f" {schema}") - click.secho(f"Error Type: {msg['error_type']}", fg='red') - click.secho(f"Error Message: {msg['error_message']}", fg='red') + click.secho(f"Error Type: {msg['error_type']}", fg="red") + click.secho(f"Error Message: {msg['error_message']}", fg="red") click.secho("-------------------------") + def intro_message(linter: Linter) -> None: """Prints an introduction message for the stac-check tool. @@ -63,64 +67,69 @@ def intro_message(linter: Linter) -> None: Returns: None. """ - click.secho(""" + click.secho( + """ ____ ____ __ ___ ___ _ _ ____ ___ __ _ / ___)(_ _)/ _\ / __)___ / __)/ )( \( __)/ __)( / ) \___ \ )( / \( (__(___)( (__ ) __ ( ) _)( (__ ) ( (____/ (__)\_/\_/ \___) \___)\_)(_/(____)\___)(__\_) - """) + """ + ) click.secho("stac-check: STAC spec validaton and linting tool", bold=True) click.secho() if linter.version == "1.0.0": - click.secho(linter.set_update_message(), fg='green') + click.secho(linter.set_update_message(), fg="green") else: - click.secho(linter.set_update_message(), fg='red') + click.secho(linter.set_update_message(), fg="red") click.secho() - click.secho(f"Validator: stac-validator {linter.validator_version}", bg="blue", fg="white") + click.secho( + f"Validator: stac-validator {linter.validator_version}", bg="blue", fg="white" + ) click.secho() + def cli_message(linter: Linter) -> None: """Prints various messages about the STAC object being validated. Args: - linter: The `Linter` object containing information about + linter: The `Linter` object containing information about the STAC object to be validated. Returns: None """ if linter.valid_stac == True: - click.secho(f"Valid {linter.asset_type}: {linter.valid_stac}", fg='green') + click.secho(f"Valid {linter.asset_type}: {linter.valid_stac}", fg="green") else: - click.secho(f"Valid {linter.asset_type}: {linter.valid_stac}", fg='red') + click.secho(f"Valid {linter.asset_type}: {linter.valid_stac}", fg="red") - ''' schemas validated for core object ''' + """ schemas validated for core object """ click.secho() if len(linter.schema) > 0: click.secho("Schemas validated: ", fg="blue") for schema in linter.schema: click.secho(f" {schema}") - ''' best practices message''' + """ best practices message""" click.secho() for message in linter.best_practices_msg: if message == linter.best_practices_msg[0]: - click.secho(message, bg='blue') + click.secho(message, bg="blue") else: - click.secho(message, fg='red') + click.secho(message, fg="red") if linter.validate_all == True: click.secho() - click.secho(f"Recursive validation has passed!", fg='blue') + click.secho(f"Recursive validation has passed!", fg="blue") elif linter.validate_all == False and linter.recursive: click.secho() - click.secho(f"Recursive validation has failed!", fg='red') + click.secho(f"Recursive validation has failed!", fg="red") if linter.invalid_asset_format is not None: click.secho() @@ -143,7 +152,7 @@ def cli_message(linter: Linter) -> None: click.secho(f" {linter.error_type}") if linter.error_msg != "": - click.secho(f"Validation error message: ", fg='red') + click.secho(f"Validation error message: ", fg="red") click.secho(f" {linter.error_msg}") click.secho(f"This object has {len(linter.data['links'])} links") @@ -153,6 +162,7 @@ def cli_message(linter: Linter) -> None: ### Stac validator response for reference # click.secho(json.dumps(linter.message, indent=4)) + @click.option( "--recursive", "-r", @@ -172,12 +182,14 @@ def cli_message(linter: Linter) -> None: "-l", "--links", is_flag=True, help="Validate links for format and response." ) @click.command() -@click.argument('file') +@click.argument("file") @click.version_option(version=pkg_resources.require("stac-check")[0].version) def main(file, recursive, max_depth, assets, links): - linter = Linter(file, assets=assets, links=links, recursive=recursive, max_depth=max_depth) + linter = Linter( + file, assets=assets, links=links, recursive=recursive, max_depth=max_depth + ) intro_message(linter) if recursive > 0: recursive_message(linter) else: - cli_message(linter) \ No newline at end of file + cli_message(linter) diff --git a/stac_check/lint.py b/stac_check/lint.py index 30f7c4d..62421ce 100644 --- a/stac_check/lint.py +++ b/stac_check/lint.py @@ -1,17 +1,18 @@ -import pkg_resources -from stac_validator.validate import StacValidate -from stac_validator.utilities import is_valid_url import json -import yaml import os from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import pkg_resources import requests -from typing import Optional, Union, Dict, Any, List +import yaml from dotenv import load_dotenv -import pkg_resources +from stac_validator.utilities import is_valid_url +from stac_validator.validate import StacValidate load_dotenv() + @dataclass class Linter: """A class for linting STAC JSON files and generating validation messages. @@ -66,11 +67,11 @@ def get_asset_name(self, file: Union[str, Dict] = None) -> str: check_links_assets(self, num_links: int, url_type: str, format_type: str) -> List[str]: Checks whether the STAC JSON file has links or assets with invalid formats or requests. - check_error_type(self) -> str: + check_error_type(self) -> str: Checks whether the STAC JSON file has an error type. check_error_message(self) -> str: - Checks whether the STAC JSON file has an error message. + Checks whether the STAC JSON file has an error message. def check_summaries(self) -> bool: Checks whether the STAC JSON file has summaries. @@ -88,9 +89,9 @@ def check_summaries(self) -> bool: Checks whether the STAC JSON file has unlocated items. check_geometry_null(self) -> bool: - Checks whether the STAC JSON file has a null geometry. + Checks whether the STAC JSON file has a null geometry. - check_searchable_identifiers(self) -> bool: + check_searchable_identifiers(self) -> bool: Checks whether the STAC JSON file has searchable identifiers. check_percent_encoded(self) -> bool: @@ -117,7 +118,8 @@ def check_summaries(self) -> bool: create_best_practices_msg(self) -> List[str]: Creates a message with best practices recommendations for the STAC JSON file. """ - item: Union[str, dict] # url, file name, or dictionary + + item: Union[str, dict] # url, file name, or dictionary config_file: Optional[str] = None assets: bool = False links: bool = False @@ -128,17 +130,27 @@ def __post_init__(self): self.data = self.load_data(self.item) self.message = self.validate_file(self.item) self.config = self.parse_config(self.config_file) - self.asset_type = self.message["asset_type"] if "asset_type" in self.message else "" + self.asset_type = ( + self.message["asset_type"] if "asset_type" in self.message else "" + ) self.version = self.message["version"] if "version" in self.message else "" self.validator_version = pkg_resources.require("stac-validator")[0].version self.validate_all = self.recursive_validation(self.item) self.valid_stac = self.message["valid_stac"] self.error_type = self.check_error_type() self.error_msg = self.check_error_message() - self.invalid_asset_format = self.check_links_assets(10, "assets", "format") if self.assets else None - self.invalid_asset_request = self.check_links_assets(10, "assets", "request") if self.assets else None - self.invalid_link_format = self.check_links_assets(10, "links", "format") if self.links else None - self.invalid_link_request = self.check_links_assets(10, "links", "request") if self.links else None + self.invalid_asset_format = ( + self.check_links_assets(10, "assets", "format") if self.assets else None + ) + self.invalid_asset_request = ( + self.check_links_assets(10, "assets", "request") if self.assets else None + ) + self.invalid_link_format = ( + self.check_links_assets(10, "links", "format") if self.links else None + ) + self.invalid_link_request = ( + self.check_links_assets(10, "links", "request") if self.links else None + ) self.schema = self.message["schema"] if "schema" in self.message else [] self.object_id = self.data["id"] if "id" in self.data else "" self.file_name = self.get_asset_name(self.item) @@ -179,7 +191,7 @@ def parse_config(config_file: Optional[str] = None) -> Dict: with open(config_file) as f: config = yaml.load(f, Loader=yaml.FullLoader) default_config.update(config) - + return default_config def get_asset_name(self, file: Union[str, Dict] = None) -> str: @@ -196,7 +208,7 @@ def get_asset_name(self, file: Union[str, Dict] = None) -> str: TypeError: If the input `file` is not a string or a dictionary. """ if isinstance(file, str): - return os.path.basename(file).split('.')[0] + return os.path.basename(file).split(".")[0] else: return file["id"] @@ -285,7 +297,9 @@ def set_update_message(self) -> str: else: return "Thanks for using STAC version 1.0.0!" - def check_links_assets(self, num_links: int, url_type: str, format_type: str) -> List[str]: + def check_links_assets( + self, num_links: int, url_type: str, format_type: str + ) -> List[str]: """Checks the links and assets in the STAC catalog and returns a list of invalid links of a specified type and format. Args: @@ -298,8 +312,10 @@ def check_links_assets(self, num_links: int, url_type: str, format_type: str) -> """ links = [] if f"{url_type}_validated" in self.message: - for invalid_request_url in self.message[f"{url_type}_validated"][f"{format_type}_invalid"]: - if invalid_request_url not in links and 'http' in invalid_request_url: + for invalid_request_url in self.message[f"{url_type}_validated"][ + f"{format_type}_invalid" + ]: + if invalid_request_url not in links and "http" in invalid_request_url: links.append(invalid_request_url) num_links = num_links - 1 if num_links == 0: @@ -307,7 +323,7 @@ def check_links_assets(self, num_links: int, url_type: str, format_type: str) -> return links def check_error_type(self) -> str: - """Returns the error type of a STAC validation if it exists in the validation message, + """Returns the error type of a STAC validation if it exists in the validation message, and an empty string otherwise. Returns: @@ -338,6 +354,8 @@ def check_summaries(self) -> bool: """ if self.asset_type == "COLLECTION": return "summaries" in self.data + else: + return False def check_bloated_links(self, max_links: Optional[int] = 20) -> bool: """Checks if the number of links in the STAC data exceeds a certain maximum. @@ -350,6 +368,8 @@ def check_bloated_links(self, max_links: Optional[int] = 20) -> bool: """ if "links" in self.data: return len(self.data["links"]) > max_links + else: + return False def check_bloated_metadata(self, max_properties: Optional[int] = 20) -> bool: """Checks whether a STAC item's metadata contains too many properties. @@ -391,26 +411,33 @@ def check_unlocated(self) -> bool: def check_geometry_null(self) -> bool: """Checks if a STAC item has a null geometry property. - + Returns: - bool: A boolean indicating whether the geometry property is null (True) or not (False). + bool: A boolean indicating whether the geometry property is null (True) or not (False). """ if "geometry" in self.data: return self.data["geometry"] is None + else: + return False def check_searchable_identifiers(self) -> bool: - """Checks if the identifiers of a STAC item are searchable, i.e., + """Checks if the identifiers of a STAC item are searchable, i.e., they only contain lowercase letters, numbers, hyphens, and underscores. - + Returns: - bool: True if the identifiers are searchable, False otherwise. + bool: True if the identifiers are searchable, False otherwise. """ - if self.asset_type == "ITEM": + if self.asset_type == "ITEM": for letter in self.object_id: - if letter.islower() or letter.isnumeric() or letter == '-' or letter == '_': + if ( + letter.islower() + or letter.isnumeric() + or letter == "-" + or letter == "_" + ): pass else: - return False + return False return True def check_percent_encoded(self) -> bool: @@ -420,24 +447,30 @@ def check_percent_encoded(self) -> bool: Returns: bool: True if the identifiers are percent-encoded, False otherwise. """ - return self.asset_type == "ITEM" and "/" in self.object_id or ":" in self.object_id + return ( + self.asset_type == "ITEM" and "/" in self.object_id or ":" in self.object_id + ) def check_thumbnail(self) -> bool: """Checks if the thumbnail of a STAC item is valid, i.e., it has a valid format. - + Returns: bool: True if the thumbnail is valid, False otherwise. """ if "assets" in self.data: if "thumbnail" in self.data["assets"]: if "type" in self.data["assets"]["thumbnail"]: - if "png" in self.data["assets"]["thumbnail"]["type"] or "jpeg" in self.data["assets"]["thumbnail"]["type"] or \ - "jpg" in self.data["assets"]["thumbnail"]["type"] or "webp" in self.data["assets"]["thumbnail"]["type"]: + if ( + "png" in self.data["assets"]["thumbnail"]["type"] + or "jpeg" in self.data["assets"]["thumbnail"]["type"] + or "jpg" in self.data["assets"]["thumbnail"]["type"] + or "webp" in self.data["assets"]["thumbnail"]["type"] + ): return True else: return False return True - + def check_links_title_field(self) -> bool: """Checks if all links in a STAC collection or catalog have a 'title' field. The 'title' field is not required for the 'self' link. @@ -451,10 +484,9 @@ def check_links_title_field(self) -> bool: return False return True - def check_links_self(self) -> bool: """Checks whether the "self" link is present in the STAC collection or catalog or absent in STAC item. - + Returns: bool: True if the "self" link is present in STAC collection or catalog or absent in STAC item, False otherwise. """ @@ -474,14 +506,14 @@ def check_item_id_file_name(self) -> bool: def check_catalog_file_name(self) -> bool: """Checks whether the filename of a Catalog or Collection conforms to the STAC specification. - + Returns: bool: True if the filename is valid, False otherwise. """ if isinstance(self.item, str) and ".json" in self.item: - if self.asset_type == "CATALOG" and 'catalog.json' not in self.item: - return False - elif self.asset_type == "COLLECTION" and 'collection.json' not in self.item: + if self.asset_type == "CATALOG" and "catalog.json" not in self.item: + return False + elif self.asset_type == "COLLECTION" and "collection.json" not in self.item: return False return True else: @@ -502,7 +534,10 @@ def create_best_practices_dict(self) -> Dict: max_properties = self.config["settings"]["max_properties"] # best practices - item ids should only contain searchable identifiers - if self.check_searchable_identifiers() == False and config["searchable_identifiers"] == True: + if ( + self.check_searchable_identifiers() == False + and config["searchable_identifiers"] == True + ): msg_1 = f"Item name '{self.object_id}' should only contain Searchable identifiers" msg_2 = f"Identifiers should consist of only lowercase characters, numbers, '_', and '-'" best_practices_dict["searchable_identifiers"] = [msg_1, msg_2] @@ -518,8 +553,11 @@ def create_best_practices_dict(self) -> Dict: msg_1 = f"Item file names should match their ids: '{self.file_name}' not equal to '{self.object_id}" best_practices_dict["check_item_id"] = [msg_1] - # best practices - collection and catalog file names should be collection.json and catalog.json - if self.check_catalog_file_name() == False and config["catalog_id_file_name"] == True: + # best practices - collection and catalog file names should be collection.json and catalog.json + if ( + self.check_catalog_file_name() == False + and config["catalog_id_file_name"] == True + ): msg_1 = f"Object should be called '{self.asset_type.lower()}.json' not '{self.file_name}.json'" best_practices_dict["check_catalog_id"] = [msg_1] @@ -545,23 +583,37 @@ def create_best_practices_dict(self) -> Dict: best_practices_dict["null_geometry"] = [msg_1] # check to see if there are too many links - if self.check_bloated_links(max_links=max_links) and config["bloated_links"] == True: + if ( + self.check_bloated_links(max_links=max_links) + and config["bloated_links"] == True + ): msg_1 = f"You have {len(self.data['links'])} links. Please consider using sub-collections or sub-catalogs" best_practices_dict["bloated_links"] = [msg_1] # best practices - check for bloated metadata in properties - if self.check_bloated_metadata(max_properties=max_properties) and config["bloated_metadata"] == True: + if ( + self.check_bloated_metadata(max_properties=max_properties) + and config["bloated_metadata"] == True + ): msg_1 = f"You have {len(self.data['properties'])} properties. Please consider using links to avoid bloated metadata" best_practices_dict["bloated_metadata"] = [msg_1] # best practices - ensure thumbnail is a small file size ["png", "jpeg", "jpg", "webp"] - if not self.check_thumbnail() and self.asset_type == "ITEM" and config["check_thumbnail"] == True: - msg_1 = f"A thumbnail should have a small file size ie. png, jpeg, jpg, webp" + if ( + not self.check_thumbnail() + and self.asset_type == "ITEM" + and config["check_thumbnail"] == True + ): + msg_1 = ( + f"A thumbnail should have a small file size ie. png, jpeg, jpg, webp" + ) best_practices_dict["check_thumbnail"] = [msg_1] # best practices - ensure that links in catalogs and collections include a title field if not self.check_links_title_field() and config["links_title"] == True: - msg_1 = f"Links in catalogs and collections should always have a 'title' field" + msg_1 = ( + f"Links in catalogs and collections should always have a 'title' field" + ) best_practices_dict["check_links_title"] = [msg_1] # best practices - ensure that links in catalogs and collections include self link @@ -576,17 +628,17 @@ def create_best_practices_msg(self) -> List[str]: Generates a list of best practices messages based on the results of the 'create_best_practices_dict' method. Returns: - A list of strings, where each string contains a best practice message. Each message starts with the - 'STAC Best Practices:' base string and is followed by a specific recommendation. Each message is indented + A list of strings, where each string contains a best practice message. Each message starts with the + 'STAC Best Practices:' base string and is followed by a specific recommendation. Each message is indented with four spaces, and there is an empty string between each message for readability. """ best_practices = list() base_string = "STAC Best Practices: " best_practices.append(base_string) - for _,v in self.create_best_practices_dict().items(): + for _, v in self.create_best_practices_dict().items(): for value in v: - best_practices.extend([" " +value]) + best_practices.extend([" " + value]) best_practices.extend([""]) - return best_practices \ No newline at end of file + return best_practices diff --git a/tests/test_cli.py b/tests/test_cli.py index fc81efa..ec57ec6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ +import pytest from click.testing import CliRunner + from stac_check.cli import main -import pytest INTRO = "stac-check: STAC spec validaton and linting tool" VALID_ITEM = "Valid ITEM: True" @@ -8,6 +9,7 @@ VALIDATOR = "Validator: stac-validator 2.4.0" SCHEMA_MSG = "Schemas validated: " + @pytest.mark.skip(reason="cli output is changing constantly right now") def test_core_item_100(): runner = CliRunner() @@ -18,4 +20,7 @@ def test_core_item_100(): assert result.output.splitlines()[3] == VALIDATOR assert result.output.splitlines()[4] == VALID_ITEM assert result.output.splitlines()[5] == SCHEMA_MSG - assert result.output.splitlines()[6] == """ https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json""" + assert ( + result.output.splitlines()[6] + == """ https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json""" + ) diff --git a/tests/test_config.py b/tests/test_config.py index ebbbfc4..75906eb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ from stac_check.lint import Linter + def test_linter_config_file(): file = "sample_files/1.0.0/core-item.json" linter = Linter(file) @@ -8,7 +9,7 @@ def test_linter_config_file(): assert linter.config["linting"]["searchable_identifiers"] == True assert linter.create_best_practices_dict()["searchable_identifiers"] == [ f"Item name '{linter.object_id}' should only contain Searchable identifiers", - "Identifiers should consist of only lowercase characters, numbers, '_', and '-'" + "Identifiers should consist of only lowercase characters, numbers, '_', and '-'", ] # Load config file @@ -17,6 +18,7 @@ def test_linter_config_file(): assert linter.config["linting"]["searchable_identifiers"] == False assert "searchable_identifiers" not in linter.create_best_practices_dict() + def test_linter_max_links(): file = "sample_files/1.0.0/core-item-bloated.json" linter = Linter(file) @@ -27,6 +29,3 @@ def test_linter_max_links(): # Load config file linter = Linter(file, config_file="tests/test.config.yml") assert "bloated_links" not in linter.create_best_practices_dict() - - - \ No newline at end of file diff --git a/tests/test_lint.py b/tests/test_lint.py index a966c94..8b038d0 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,8 +1,11 @@ -from re import L -from stac_check.lint import Linter import pytest -@pytest.mark.skip(reason="test is ineffective - bad links are redirecting to a third party site") +from stac_check.lint import Linter + + +@pytest.mark.skip( + reason="test is ineffective - bad links are redirecting to a third party site" +) def test_linter_bad_asset_requests(): file = "sample_files/1.0.0/core-item.json" linter = Linter(file, assets=True) @@ -15,6 +18,7 @@ def test_linter_bad_asset_requests(): assert linter.invalid_asset_format == [] assert linter.invalid_asset_request == asset_request_errors + def test_linter_bad_assets(): file = "sample_files/1.0.0/core-item-bad-links.json" linter = Linter(file, assets=True) @@ -24,7 +28,7 @@ def test_linter_bad_assets(): asset_request_errors = [ "https:/storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", "http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json", - "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH" + "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH", ] assert linter.version == "1.0.0" assert linter.valid_stac == True @@ -38,8 +42,8 @@ def test_linter_bad_links(): linter = Linter(file, links=True) link_format_errors = ["http:/remotdata.io/catalog/20201211_223832_CS2/index.html"] link_request_errors = [ - "http://catalog/collection.json", - "http:/remotdata.io/catalog/20201211_223832_CS2/index.html" + "http://catalog/collection.json", + "http:/remotdata.io/catalog/20201211_223832_CS2/index.html", ] assert linter.version == "1.0.0" assert linter.valid_stac == True @@ -58,14 +62,14 @@ def test_linter_bad_links_assets(): asset_request_errors = [ "https:/storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", "http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json", - "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH" + "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH", ] link_format_errors = [ "http:/remotdata.io/catalog/20201211_223832_CS2/index.html", ] link_request_errors = [ - "http://catalog/collection.json", - "http:/remotdata.io/catalog/20201211_223832_CS2/index.html" + "http://catalog/collection.json", + "http:/remotdata.io/catalog/20201211_223832_CS2/index.html", ] assert linter.version == "1.0.0" assert linter.valid_stac == True @@ -76,6 +80,7 @@ def test_linter_bad_links_assets(): assert linter.invalid_link_format == link_format_errors assert linter.invalid_link_request == link_request_errors + def test_linter_collection(): file = "sample_files/1.0.0/collection.json" linter = Linter(file, assets=False, links=False) @@ -84,6 +89,7 @@ def test_linter_collection(): assert linter.asset_type == "COLLECTION" assert linter.check_summaries() == True + def test_linter_collection_no_summaries(): file = "sample_files/1.0.0/collection-no-summaries.json" linter = Linter(file, assets=False, links=False) @@ -97,9 +103,10 @@ def test_linter_collection_no_summaries(): "", " A STAC collection should contain a summaries field", " It is recommended to store information like eo:bands in summaries", - "" + "", ] + def test_linter_catalog(): file = "sample_files/1.0.0/catalog.json" linter = Linter(file, assets=False, links=False) @@ -108,6 +115,7 @@ def test_linter_catalog(): assert linter.asset_type == "CATALOG" assert linter.check_bloated_links() == False + def test_linter_collection_recursive(): file = "sample_files/1.0.0/catalog-with-bad-item.json" linter = Linter(file, assets=False, links=False, recursive=True) @@ -121,9 +129,10 @@ def test_linter_collection_recursive(): ], "valid_stac": True, "asset_type": "CATALOG", - "validation_method": "recursive" + "validation_method": "recursive", } + def test_linter_recursive_max_depth_1(): file = "https://radarstac.s3.amazonaws.com/stac/catalog.json" stac = Linter(file, assets=False, links=False, recursive=True, max_depth=1) @@ -138,6 +147,7 @@ def test_linter_recursive_max_depth_1(): } ] + def test_linter_recursive_max_depth_4(): file = "https://radarstac.s3.amazonaws.com/stac/catalog.json" stac = Linter(file, assets=False, links=False, recursive=True, max_depth=4) @@ -224,6 +234,7 @@ def test_linter_recursive_max_depth_4(): }, ] + def test_linter_item_id_not_matching_file_name(): file = "sample_files/1.0.0/core-item.json" linter = Linter(file) @@ -232,21 +243,25 @@ def test_linter_item_id_not_matching_file_name(): assert linter.file_name != linter.object_id assert linter.check_item_id_file_name() == False + def test_linter_collection_catalog_id(): file = "sample_files/1.0.0/collection-no-title.json" linter = Linter(file) assert linter.check_catalog_file_name() == False + def test_linter_item_id_format_best_practices(): file = "sample_files/1.0.0/core-item-invalid-id.json" linter = Linter(file) assert linter.check_searchable_identifiers() == False assert linter.check_percent_encoded() == True + def test_datetime_set_to_null(): file = "sample_files/1.0.0/core-item-null-datetime.json" linter = Linter(file) - assert linter.check_datetime_null()== True + assert linter.check_datetime_null() == True + def test_unlocated_item(): file = "sample_files/1.0.0/core-item-unlocated.json" @@ -254,6 +269,7 @@ def test_unlocated_item(): assert linter.check_unlocated() == True assert linter.check_geometry_null() == True + def test_bloated_item(): file = "sample_files/1.0.0/core-item-bloated.json" linter = Linter(file) @@ -264,6 +280,7 @@ def test_bloated_item(): assert linter.check_bloated_links() == True assert len(linter.data["links"]) > 20 + def test_small_thumbnail(): file = "sample_files/1.0.0/core-item-large-thumbnail.json" linter = Linter(file) @@ -275,17 +292,20 @@ def test_small_thumbnail(): assert linter.check_thumbnail() == True + def test_title_field(): file = "sample_files/1.0.0/collection-no-title.json" linter = Linter(file) assert linter.check_links_title_field() == False + def test_self_in_links(): file = "sample_files/1.0.0/collection-no-title.json" linter = Linter(file) assert linter.check_links_self() == False - + + def test_catalog_name(): file = "sample_files/1.0.0/catalog.json" linter = Linter(file) @@ -294,6 +314,7 @@ def test_catalog_name(): linter = Linter(file) assert linter.check_catalog_file_name() + def test_lint_dict_collection(): file = { "id": "simple-collection", @@ -301,111 +322,74 @@ def test_lint_dict_collection(): "stac_extensions": [ "https://stac-extensions.github.io/eo/v1.0.0/schema.json", "https://stac-extensions.github.io/projection/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json" + "https://stac-extensions.github.io/view/v1.0.0/schema.json", ], "stac_version": "1.0.0", "description": "A simple collection demonstrating core catalog fields with links to a couple of items", "title": "Simple Example Collection", "providers": [ { - "name": "Remote Data, Inc", - "description": "Producers of awesome spatiotemporal assets", - "roles": [ - "producer", - "processor" - ], - "url": "http://remotedata.io" + "name": "Remote Data, Inc", + "description": "Producers of awesome spatiotemporal assets", + "roles": ["producer", "processor"], + "url": "http://remotedata.io", } ], "extent": { "spatial": { - "bbox": [ - [ - 172.91173669923782, - 1.3438851951615003, - 172.95469614953714, - 1.3690476620161975 + "bbox": [ + [ + 172.91173669923782, + 1.3438851951615003, + 172.95469614953714, + 1.3690476620161975, + ] ] - ] }, "temporal": { - "interval": [ - [ - "2020-12-11T22:38:32.125Z", - "2020-12-14T18:02:31.437Z" - ] - ] - } + "interval": [["2020-12-11T22:38:32.125Z", "2020-12-14T18:02:31.437Z"]] + }, }, "license": "CC-BY-4.0", "summaries": { - "platform": [ - "cool_sat1", - "cool_sat2" - ], - "constellation": [ - "ion" - ], - "instruments": [ - "cool_sensor_v1", - "cool_sensor_v2" - ], - "gsd": { - "minimum": 0.512, - "maximum": 0.66 - }, - "eo:cloud_cover": { - "minimum": 1.2, - "maximum": 1.2 - }, - "proj:epsg": { - "minimum": 32659, - "maximum": 32659 - }, - "view:sun_elevation": { - "minimum": 54.9, - "maximum": 54.9 - }, - "view:off_nadir": { - "minimum": 3.8, - "maximum": 3.8 - }, - "view:sun_azimuth": { - "minimum": 135.7, - "maximum": 135.7 - } + "platform": ["cool_sat1", "cool_sat2"], + "constellation": ["ion"], + "instruments": ["cool_sensor_v1", "cool_sensor_v2"], + "gsd": {"minimum": 0.512, "maximum": 0.66}, + "eo:cloud_cover": {"minimum": 1.2, "maximum": 1.2}, + "proj:epsg": {"minimum": 32659, "maximum": 32659}, + "view:sun_elevation": {"minimum": 54.9, "maximum": 54.9}, + "view:off_nadir": {"minimum": 3.8, "maximum": 3.8}, + "view:sun_azimuth": {"minimum": 135.7, "maximum": 135.7}, }, "links": [ { - "rel": "root", - "href": "./collection.json", - "type": "application/json", - "title": "Simple Example Collection" + "rel": "root", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection", }, { - "rel": "item", - "href": "./simple-item.json", - "type": "application/geo+json", - "title": "Simple Item" + "rel": "item", + "href": "./simple-item.json", + "type": "application/geo+json", + "title": "Simple Item", }, + {"rel": "item", "href": "./core-item.json", "type": "application/geo+json"}, { - "rel": "item", - "href": "./core-item.json", - "type": "application/geo+json" + "rel": "item", + "href": "./extended-item.json", + "type": "application/geo+json", + "title": "Extended Item", }, - { - "rel": "item", - "href": "./extended-item.json", - "type": "application/geo+json", - "title": "Extended Item" - } - ] + ], } linter = Linter(file) assert linter.valid_stac == True assert linter.asset_type == "COLLECTION" assert linter.check_catalog_file_name() == True + def test_lint_dict_item(): file = { "stac_version": "1.0.0", @@ -416,34 +400,19 @@ def test_lint_dict_item(): 172.91173669923782, 1.3438851951615003, 172.95469614953714, - 1.3690476620161975 + 1.3690476620161975, ], "geometry": { "type": "Polygon", "coordinates": [ - [ - [ - 172.91173669923782, - 1.3438851951615003 - ], - [ - 172.95469614953714, - 1.3438851951615003 - ], [ - 172.95469614953714, - 1.3690476620161975 - ], - [ - 172.91173669923782, - 1.3690476620161975 - ], - [ - 172.91173669923782, - 1.3438851951615003 + [172.91173669923782, 1.3438851951615003], + [172.95469614953714, 1.3438851951615003], + [172.95469614953714, 1.3690476620161975], + [172.91173669923782, 1.3690476620161975], + [172.91173669923782, 1.3438851951615003], ] - ] - ] + ], }, "properties": { "title": "Core Item", @@ -454,86 +423,78 @@ def test_lint_dict_item(): "created": "2020-12-12T01:48:13.725Z", "updated": "2020-12-12T01:48:13.725Z", "platform": "cool_sat1", - "instruments": [ - "cool_sensor_v1" - ], + "instruments": ["cool_sensor_v1"], "constellation": "ion", "mission": "collection 5624", - "gsd": 0.512 + "gsd": 0.512, }, "collection": "simple-collection", "links": [ { - "rel": "collection", - "href": "./collection.json", - "type": "application/json", - "title": "Simple Example Collection" + "rel": "collection", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection", }, { - "rel": "root", - "href": "./collection.json", - "type": "application/json", - "title": "Simple Example Collection" + "rel": "root", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection", }, { - "rel": "parent", - "href": "./collection.json", - "type": "application/json", - "title": "Simple Example Collection" + "rel": "parent", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection", }, { - "rel": "alternate", - "type": "text/html", - "href": "http://remotedata.io/catalog/20201211_223832_CS2/index.html", - "title": "HTML version of this STAC Item" - } + "rel": "alternate", + "type": "text/html", + "href": "http://remotedata.io/catalog/20201211_223832_CS2/index.html", + "title": "HTML version of this STAC Item", + }, ], "assets": { "analytic": { - "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic.tif", - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "title": "4-Band Analytic", - "roles": [ - "data" - ] + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "4-Band Analytic", + "roles": ["data"], }, "thumbnail": { - "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", - "title": "Thumbnail", - "type": "image/png", - "roles": [ - "thumbnail" - ] + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", + "title": "Thumbnail", + "type": "image/png", + "roles": ["thumbnail"], }, "visual": { - "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif", - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "title": "3-Band Visual", - "roles": [ - "visual" - ] + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "3-Band Visual", + "roles": ["visual"], }, "udm": { - "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic_udm.tif", - "title": "Unusable Data Mask", - "type": "image/tiff; application=geotiff;" + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic_udm.tif", + "title": "Unusable Data Mask", + "type": "image/tiff; application=geotiff;", }, "json-metadata": { - "href": "http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json", - "title": "Extended Metadata", - "type": "application/json", - "roles": [ - "metadata" - ] + "href": "http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json", + "title": "Extended Metadata", + "type": "application/json", + "roles": ["metadata"], }, "ephemeris": { - "href": "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH", - "title": "Satellite Ephemeris Metadata" - } - } + "href": "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH", + "title": "Satellite Ephemeris Metadata", + }, + }, } linter = Linter(file) assert linter.valid_stac == True assert linter.asset_type == "ITEM" assert linter.check_datetime_null() == True - assert linter.create_best_practices_dict()["datetime_null"] == ['Please avoid setting the datetime field to null, many clients search on this field'] + assert linter.create_best_practices_dict()["datetime_null"] == [ + "Please avoid setting the datetime field to null, many clients search on this field" + ] From e0c85d9921f045c2ba08abaaa6853f60ab910c45 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:32:28 +0800 Subject: [PATCH 04/12] pre-commit 2 --- .pre-commit-config.yaml | 5 ++++- CHANGELOG.md | 50 ++++++++++++++++++++++++++++++++++++++++- docs/conf.py | 5 ++++- setup.py | 2 +- stac_check/cli.py | 24 ++++++++------------ stac_check/lint.py | 28 ++++++++++++----------- stac_check/logo.py | 8 +++++++ tests/test_cli.py | 26 --------------------- 8 files changed, 90 insertions(+), 58 deletions(-) create mode 100644 stac_check/logo.py delete mode 100644 tests/test_cli.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07d8ac2..a30e121 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,8 @@ repos: rev: 7.0.0 hooks: - id: flake8 + args: + - --ignore=E501,E712,W605,W503 - repo: https://github.com/timothycrosley/isort rev: 5.13.2 hooks: @@ -26,4 +28,5 @@ repos: --implicit-reexport, --explicit-package-bases, ] - additional_dependencies: ["types-attrs", "types-requests"] + additional_dependencies: + ["types-attrs", "types-requests", "types-setuptools", "types-PyYAML"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d99ed6..4563e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,53 +6,91 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ## Unreleased +## [v1.4.0] - 2024-10-09 + +### Added + +- Added pre-commit config ([#111](https://github.com/stac-utils/stac-check/pull/111)) + +### Changed + +- Updated stac-validator dependency to ensure STAC v1.1.0 compliance ([#111](https://github.com/stac-utils/stac-check/pull/111)) + ## [v1.3.3] - 2023-11-17 + ### Changed + - Development dependencies removed from runtime dependency list ([#109](https://github.com/stac-utils/stac-check/pull/109)) ## [v1.3.2] - 2023-03-23 + ### Added + - Ability to lint dictionaries https://github.com/stac-utils/stac-check/pull/94 - Docstrings and pdoc api documents + ### Fixed + - Fixed the check_catalog_file_name() method to only work on static catalogs https://github.com/stac-utils/stac-check/pull/94 - Jsonschema version to use a released version https://github.com/stac-utils/stac-check/pull/105 ## [v1.3.1] - 2022-10-05 + ### Changed + - Changed pin on stac-validator to >=3.1.0 from ==3.2.0 ## [v1.3.0] - 2022-09-20 + ### Added + - recursive mode lints assets https://github.com/stac-utils/stac-check/pull/84 + ### Changed + - recursive mode swaps pystac for stac-validator https://github.com/stac-utils/stac-check/pull/84 + ### Fixed + - fix catalog file name check https://github.com/stac-utils/stac-check/pull/83 ## [v1.2.0] - 2022-04-26 + ### Added + - Option to include a configuration file to ignore selected checks + ### Changed + - Change name from stac_check to stac-check in setup for cli + ### Fixed + - Fix thumbnail size check ## [v1.1.2] - 2022-03-03 + ### Changed + - Make it easier to export linting messages - Set stac-validator version to 2.4.0 + ### Fixed + - Fix self-link test ## [v1.0.1] - 2022-02-20 + ### Changed + - Update readme - Reorganized code for version 1.0.0 release ## [v0.2.0] - 2022-02-02 - 2022-02-19 + ### Added + - Import main validator as stac-validator was updated to 2.3.0 - Added best practices docuument to repo - Recommend 'self' link in links @@ -62,29 +100,39 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) - Check for small thumbnail image file type ## [v0.1.3] - 2022-01-23 + ### Added + - Check for bloated metadata, too many fields in properties - Check for geometry field, recommend that STAC not be used for non-spatial data ### Changed + - Changed bloated links check to a boolean to mirror bloated metadata ## [v0.1.2] - 2022-01-17 - 2022-01-22 + ### Added + - Check for null datetime - Check for unlocated items, bbox should be set to null if geometry is ## [v0.1.1] - 2021-11-26 - 2021-12-12 + ### Added + - Added github actions to test and push to pypi - Added makefile, dockerfile ### Changed + - Removed pipenv ## [v0.1.0] - 2021-11-26 - 2021-12-05 + ### Added -- Best practices - searchable identifiers - lowercase, numbers, '_' or '-' + +- Best practices - searchable identifiers - lowercase, numbers, '\_' or '-' for id names https://github.com/radiantearth/stac-spec/blob/master/best-practices.md#searchable-identifiers - Best practices ensure item ids don't contain ':' or '/' characters diff --git a/docs/conf.py b/docs/conf.py index 27428c3..13f6ad0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,5 @@ +from typing import List + # Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: @@ -10,10 +12,11 @@ author = "Jonathan Healy" release = "1.3.1" + # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions: List[str] = [] templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] diff --git a/setup.py b/setup.py index 2db3561..33c9578 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ "click>=8.0.0", "requests>=2.19.1", "jsonschema>=3.1.2", - "stac-validator==3.4.0", + "stac-validator>=3.4.0", "PyYAML", "python-dotenv", ], diff --git a/stac_check/cli.py b/stac_check/cli.py index e820aaf..491ceee 100644 --- a/stac_check/cli.py +++ b/stac_check/cli.py @@ -2,6 +2,7 @@ import pkg_resources from .lint import Linter +from .logo import logo def link_asset_message(link_list: list, type: str, format: str) -> None: @@ -33,14 +34,14 @@ def recursive_message(linter: Linter) -> None: None. """ click.secho() - click.secho(f"Recursive: Validate all assets in a collection or catalog", bold=True) + click.secho("Recursive: Validate all assets in a collection or catalog", bold=True) click.secho(f"Max-depth = {linter.max_depth}") click.secho("-------------------------") for count, msg in enumerate(linter.validate_all): click.secho(f"Asset {count+1} Validated: {msg['path']}", bg="white", fg="black") click.secho() if msg["valid_stac"] == True: - recursive_linter = Linter(msg["path"], recursive=0) + recursive_linter = Linter(msg["path"], recursive=True) cli_message(recursive_linter) else: click.secho(f"Valid: {msg['valid_stac']}", fg="red") @@ -67,14 +68,7 @@ def intro_message(linter: Linter) -> None: Returns: None. """ - click.secho( - """ - ____ ____ __ ___ ___ _ _ ____ ___ __ _ -/ ___)(_ _)/ _\ / __)___ / __)/ )( \( __)/ __)( / ) -\___ \ )( / \( (__(___)( (__ ) __ ( ) _)( (__ ) ( -(____/ (__)\_/\_/ \___) \___)\_)(_/(____)\___)(__\_) - """ - ) + click.secho(logo) click.secho("stac-check: STAC spec validaton and linting tool", bold=True) @@ -126,10 +120,10 @@ def cli_message(linter: Linter) -> None: if linter.validate_all == True: click.secho() - click.secho(f"Recursive validation has passed!", fg="blue") + click.secho("Recursive validation has passed!", fg="blue") elif linter.validate_all == False and linter.recursive: click.secho() - click.secho(f"Recursive validation has failed!", fg="red") + click.secho("Recursive validation has failed!", fg="red") if linter.invalid_asset_format is not None: click.secho() @@ -148,18 +142,18 @@ def cli_message(linter: Linter) -> None: link_asset_message(linter.invalid_link_request, "link", "request") if linter.error_type != "": - click.secho(f"Validation error type: ", fg="red") + click.secho("Validation error type: ", fg="red") click.secho(f" {linter.error_type}") if linter.error_msg != "": - click.secho(f"Validation error message: ", fg="red") + click.secho("Validation error message: ", fg="red") click.secho(f" {linter.error_msg}") click.secho(f"This object has {len(linter.data['links'])} links") click.secho() - ### Stac validator response for reference + # Stac validator response for reference # click.secho(json.dumps(linter.message, indent=4)) diff --git a/stac_check/lint.py b/stac_check/lint.py index 62421ce..0db65b0 100644 --- a/stac_check/lint.py +++ b/stac_check/lint.py @@ -285,6 +285,8 @@ def recursive_validation(self, file: Union[str, Dict[str, Any]]) -> str: stac = StacValidate(recursive=True, max_depth=self.max_depth) stac.validate_dict(file) return stac.message + else: + return "Recursive validation is disabled." def set_update_message(self) -> str: """Returns a message for users to update their STAC version. @@ -394,7 +396,7 @@ def check_datetime_null(self) -> bool: """ if "properties" in self.data: if "datetime" in self.data["properties"]: - if self.data["properties"]["datetime"] == None: + if self.data["properties"]["datetime"] is None: return True else: return False @@ -408,6 +410,8 @@ def check_unlocated(self) -> bool: """ if "geometry" in self.data: return self.data["geometry"] is None and self.data["bbox"] is not None + else: + return False def check_geometry_null(self) -> bool: """Checks if a STAC item has a null geometry property. @@ -539,13 +543,13 @@ def create_best_practices_dict(self) -> Dict: and config["searchable_identifiers"] == True ): msg_1 = f"Item name '{self.object_id}' should only contain Searchable identifiers" - msg_2 = f"Identifiers should consist of only lowercase characters, numbers, '_', and '-'" + msg_2 = "Identifiers should consist of only lowercase characters, numbers, '_', and '-'" best_practices_dict["searchable_identifiers"] = [msg_1, msg_2] # best practices - item ids should not contain ':' or '/' characters if self.check_percent_encoded() and config["percent_encoded"] == True: msg_1 = f"Item name '{self.object_id}' should not contain ':' or '/'" - msg_2 = f"https://github.com/radiantearth/stac-spec/blob/master/best-practices.md#item-ids" + msg_2 = "https://github.com/radiantearth/stac-spec/blob/master/best-practices.md#item-ids" best_practices_dict["percent_encoded"] = [msg_1, msg_2] # best practices - item ids should match file names @@ -563,23 +567,23 @@ def create_best_practices_dict(self) -> Dict: # best practices - collections should contain summaries if self.check_summaries() == False and config["check_summaries"] == True: - msg_1 = f"A STAC collection should contain a summaries field" - msg_2 = f"It is recommended to store information like eo:bands in summaries" + msg_1 = "A STAC collection should contain a summaries field" + msg_2 = "It is recommended to store information like eo:bands in summaries" best_practices_dict["check_summaries"] = [msg_1, msg_2] # best practices - datetime fields should not be set to null if self.check_datetime_null() and config["null_datetime"] == True: - msg_1 = f"Please avoid setting the datetime field to null, many clients search on this field" + msg_1 = "Please avoid setting the datetime field to null, many clients search on this field" best_practices_dict["datetime_null"] = [msg_1] # best practices - check unlocated items to make sure bbox field is not set if self.check_unlocated() and config["check_unlocated"] == True: - msg_1 = f"Unlocated item. Please avoid setting the bbox field when geometry is set to null" + msg_1 = "Unlocated item. Please avoid setting the bbox field when geometry is set to null" best_practices_dict["check_unlocated"] = [msg_1] # best practices - recommend items have a geometry if self.check_geometry_null() and config["check_geometry"] == True: - msg_1 = f"All items should have a geometry field. STAC is not meant for non-spatial data" + msg_1 = "All items should have a geometry field. STAC is not meant for non-spatial data" best_practices_dict["null_geometry"] = [msg_1] # check to see if there are too many links @@ -604,21 +608,19 @@ def create_best_practices_dict(self) -> Dict: and self.asset_type == "ITEM" and config["check_thumbnail"] == True ): - msg_1 = ( - f"A thumbnail should have a small file size ie. png, jpeg, jpg, webp" - ) + msg_1 = "A thumbnail should have a small file size ie. png, jpeg, jpg, webp" best_practices_dict["check_thumbnail"] = [msg_1] # best practices - ensure that links in catalogs and collections include a title field if not self.check_links_title_field() and config["links_title"] == True: msg_1 = ( - f"Links in catalogs and collections should always have a 'title' field" + "Links in catalogs and collections should always have a 'title' field" ) best_practices_dict["check_links_title"] = [msg_1] # best practices - ensure that links in catalogs and collections include self link if not self.check_links_self() and config["links_self"] == True: - msg_1 = f"A link to 'self' in links is strongly recommended" + msg_1 = "A link to 'self' in links is strongly recommended" best_practices_dict["check_links_self"] = [msg_1] return best_practices_dict diff --git a/stac_check/logo.py b/stac_check/logo.py new file mode 100644 index 0000000..796b14a --- /dev/null +++ b/stac_check/logo.py @@ -0,0 +1,8 @@ +# flake8: noqa + +logo = """ + ____ ____ __ ___ ___ _ _ ____ ___ __ _ +/ ___)(_ _)/ _\ / __)___ / __)/ )( \( __)/ __)( / ) +\___ \ )( / \( (__(___)( (__ ) __ ( ) _)( (__ ) ( +(____/ (__)\_/\_/ \___) \___)\_)(_/(____)\___)(__\_) + """ \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index ec57ec6..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from click.testing import CliRunner - -from stac_check.cli import main - -INTRO = "stac-check: STAC spec validaton and linting tool" -VALID_ITEM = "Valid ITEM: True" -VERSION_MSG_1 = "Thanks for using STAC version 1.0.0!" -VALIDATOR = "Validator: stac-validator 2.4.0" -SCHEMA_MSG = "Schemas validated: " - - -@pytest.mark.skip(reason="cli output is changing constantly right now") -def test_core_item_100(): - runner = CliRunner() - result = runner.invoke(main, ["sample_files/1.0.0/core-item.json"]) - assert result.exit_code == 0 - assert result.output.splitlines()[1] == INTRO - assert result.output.splitlines()[2] == VERSION_MSG_1 - assert result.output.splitlines()[3] == VALIDATOR - assert result.output.splitlines()[4] == VALID_ITEM - assert result.output.splitlines()[5] == SCHEMA_MSG - assert ( - result.output.splitlines()[6] - == """ https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json""" - ) From e1e49302dfcaae6d5602c878afcf836ee0f182cd Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:33:39 +0800 Subject: [PATCH 05/12] unignore W605 --- .pre-commit-config.yaml | 2 +- stac_check/logo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a30e121..4cfc8dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: flake8 args: - - --ignore=E501,E712,W605,W503 + - --ignore=E501,E712,W503 - repo: https://github.com/timothycrosley/isort rev: 5.13.2 hooks: diff --git a/stac_check/logo.py b/stac_check/logo.py index 796b14a..0ea5b17 100644 --- a/stac_check/logo.py +++ b/stac_check/logo.py @@ -5,4 +5,4 @@ / ___)(_ _)/ _\ / __)___ / __)/ )( \( __)/ __)( / ) \___ \ )( / \( (__(___)( (__ ) __ ( ) _)( (__ ) ( (____/ (__)\_/\_/ \___) \___)\_)(_/(____)\___)(__\_) - """ \ No newline at end of file + """ From f4232768a9e8d52a8ef2e7d89fc518aa253e6039 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:38:55 +0800 Subject: [PATCH 06/12] add pre-commit to test runner --- .github/workflows/test-runner.yml | 38 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 011f62a..ccc9574 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -11,24 +11,50 @@ on: - dev jobs: + pre-commit: + name: Run pre-commit checks + runs-on: ubuntu-latest - test: + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install '.[dev]' + + - name: Run pre-commit checks + uses: pre-commit/action@v3.5.0 + with: + extra_args: --all-files + env: + PRE_COMMIT_HOME: ~/.cache/pre-commit + test: + needs: pre-commit # This ensures tests run after pre-commit checks name: Execute tests runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@main + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Run unit tests + - name: Install dependencies run: | + python -m pip install --upgrade pip pip install '.[dev]' - pytest -v + + - name: Run unit tests + run: pytest -v From 16d2921014581f084c71cee979963d17ec391862 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:41:26 +0800 Subject: [PATCH 07/12] change pre-commit action version --- .github/workflows/test-runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index ccc9574..3512b31 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -29,7 +29,7 @@ jobs: pip install '.[dev]' - name: Run pre-commit checks - uses: pre-commit/action@v3.5.0 + uses: pre-commit/action@v3.0.1 with: extra_args: --all-files env: From 1b565a63412e9ef39a28260a623ddff930ab8ad2 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:45:30 +0800 Subject: [PATCH 08/12] add setuptools --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 33c9578..529881f 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ "stac-validator>=3.4.0", "PyYAML", "python-dotenv", + "setuptools" ], extras_require={ "dev": [ From 18f29679ad5152f60287179456e79c812cffa5b3 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 11:46:41 +0800 Subject: [PATCH 09/12] pre-commit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 529881f..a830685 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ "stac-validator>=3.4.0", "PyYAML", "python-dotenv", - "setuptools" + "setuptools", ], extras_require={ "dev": [ From b557622a36d0e909f5413a4c9a8ae774db1b8812 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 12:05:30 +0800 Subject: [PATCH 10/12] add publish config --- .github/workflows/publish.yml | 35 +++++++++++++++++++++++++++++++++++ CHANGELOG.md | 1 + 2 files changed, 36 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..4ee172d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +name: Publish + +on: + push: + tags: + - "v*.*.*" # Triggers when a tag starting with 'v' followed by version numbers is pushed + +jobs: + build-and-publish: + name: Build and Publish to PyPI + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build package + run: | + python setup.py sdist bdist_wheel + + - name: Publish package to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + twine upload dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 4563e81..a83f1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ### Added - Added pre-commit config ([#111](https://github.com/stac-utils/stac-check/pull/111)) +- Added publish.yml to automatically publish new releases ([#111](https://github.com/stac-utils/stac-check/pull/111)) ### Changed From 1c5ea82becbf5236ef166c7ad28c9b9ce0da4f1f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 13:02:12 +0800 Subject: [PATCH 11/12] use token --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4ee172d..1d89ac3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,7 +29,7 @@ jobs: - name: Publish package to PyPI env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | twine upload dist/* From 1b604b301fbd9d702c2cf6a30d25b565a3d77537 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 9 Oct 2024 13:05:48 +0800 Subject: [PATCH 12/12] small edit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a83f1d4..57f9aee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ### Added - Added pre-commit config ([#111](https://github.com/stac-utils/stac-check/pull/111)) -- Added publish.yml to automatically publish new releases ([#111](https://github.com/stac-utils/stac-check/pull/111)) +- Added publish.yml to automatically publish new releases to PyPI ([#111](https://github.com/stac-utils/stac-check/pull/111)) ### Changed