From 6ac7a1b3570b852d83e61e4d5a36d5562c2a2c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 7 Mar 2024 23:14:07 +0100 Subject: [PATCH 1/6] Add a crappy script that finds deprecated apps tl;dr, only working for github and gitlab upstreams, checks if the latest commit is more than 1 year old. It also removes deprecated-software if required. --- find_deprecated.py | 140 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 find_deprecated.py diff --git a/find_deprecated.py b/find_deprecated.py new file mode 100755 index 00000000..d860ac20 --- /dev/null +++ b/find_deprecated.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import argparse +import tomlkit +import multiprocessing +import datetime +import json +import sys +from functools import cache +import logging +from pathlib import Path +from typing import Optional + +import toml +import tqdm +import github + +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from appslib.utils import REPO_APPS_ROOT, get_catalog # noqa: E402 pylint: disable=import-error,wrong-import-position +from app_caches import app_cache_folder # noqa: E402 pylint: disable=import-error,wrong-import-position +from autoupdate_app_sources.rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType # noqa: E402,E501 pylint: disable=import-error,wrong-import-position + + +@cache +def get_github() -> tuple[Optional[tuple[str, str]], Optional[github.Github], Optional[github.InputGitAuthor]]: + try: + github_login = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip() + github_token = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip() + github_email = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip() + + auth = (github_login, github_token) + github_api = github.Github(github_token) + author = github.InputGitAuthor(github_login, github_email) + return auth, github_api, author + except Exception as e: + logging.warning(f"Could not get github: {e}") + return None, None, None + + + +def upstream_last_update_ago(app: str) -> tuple[str, int | None]: + manifest_toml = app_cache_folder(app) / "manifest.toml" + manifest_json = app_cache_folder(app) / "manifest.json" + + if manifest_toml.exists(): + manifest = toml.load(manifest_toml.open("r", encoding="utf-8")) + upstream = manifest.get("upstream", {}).get("code") + + elif manifest_json.exists(): + manifest = json.load(manifest_json.open("r", encoding="utf-8")) + upstream = manifest.get("upstream", {}).get("code") + else: + raise RuntimeError(f"App {app} doesn't have a manifest!") + + if upstream is None: + raise RuntimeError(f"App {app} doesn't have an upstream code link!") + + api = None + if upstream.startswith("https://github.com/"): + api = GithubAPI(upstream, auth=get_github()[0]) + + if upstream.startswith("https://gitlab."): + api = GitlabAPI(upstream) + + if upstream.startswith("https://codeberg.org") or upstream.startswith("https://framagit.org"): + api = GiteaForgejoAPI(upstream) + + if not api: + autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate") + if autoupdate: + strat = autoupdate["strategy"] + if "gitea" in strat or "forgejo" in strat: + api = GiteaForgejoAPI(upstream) + + if api: + last_commit = api.commits()[0] + date = last_commit["commit"]["author"]["date"] + date = datetime.datetime.fromisoformat(date) + ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None) + return app, ago.days + + raise RuntimeError(f"App {app} not handled (not github, gitlab or gitea with autoupdate). Upstream is {upstream}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("apps", nargs="*", type=str, + help="If not passed, the script will run on the catalog. Github keys required.") + parser.add_argument("-j", "--processes", type=int, default=multiprocessing.cpu_count()) + args = parser.parse_args() + + apps_dict = get_catalog() + if args.apps: + apps_dict = {app: info for app, info in apps_dict.items() if app in args.apps} + + deprecated: list[str] = [] + not_deprecated: list[str] = [] + # for app, info in apps_dict.items(): + with multiprocessing.Pool(processes=args.processes) as pool: + tasks = pool.imap_unordered(upstream_last_update_ago, apps_dict.keys()) + + for _ in tqdm.tqdm(range(len(apps_dict)), ascii=" ยท#"): + try: + app, result = next(tasks) + except Exception as e: + print(e) + continue + + if result is None: + continue + + if result > 365: + deprecated.append(app) + else: + not_deprecated.append(app) + + catalog = tomlkit.load(open("apps.toml")) + for app, info in catalog.items(): + antifeatures = info.get("antifeatures", []) + if app in deprecated: + if "deprecated-software" not in antifeatures: + antifeatures.append("deprecated-software") + elif app in not_deprecated: + if "deprecated-software" in antifeatures: + antifeatures.remove("deprecated-software") + else: + continue + # unique the keys + if antifeatures: + info["antifeatures"] = antifeatures + else: + if "antifeatures" in info.keys(): + info.pop("antifeatures") + tomlkit.dump(catalog, open("apps.toml", "w")) + + +if __name__ == "__main__": + main() From 99196389b5a49f9f8d3ab423bdbc26d63bb3cdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 8 Mar 2024 19:02:33 +0100 Subject: [PATCH 2/6] find_deprecated: Read the `archived` status of the upstream repository --- autoupdate_app_sources/rest_api.py | 12 ++++++++++++ find_deprecated.py | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 9c33c489..0aa14b5b 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -42,6 +42,10 @@ def releases(self) -> list[dict[str, Any]]: """Get a list of releases for project.""" return self.internal_api(f"repos/{self.upstream_repo}/releases?per_page=100") + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"repos/{self.upstream_repo}")["archived"] + def url_for_ref(self, ref: str, ref_type: RefType) -> str: """Get a URL for a ref.""" if ref_type == RefType.tags or ref_type == RefType.releases: @@ -145,6 +149,10 @@ def releases(self) -> list[dict[str, Any]]: return retval + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"projects/{self.project_id}")["archived"] + def url_for_ref(self, ref: str, _: RefType) -> str: name = self.project_path.split("/")[-1] clean_ref = ref.replace("/", "-") @@ -196,6 +204,10 @@ def releases(self) -> list[dict[str, Any]]: """Get a list of releases for project.""" return self.internal_api(f"repos/{self.project_path}/releases") + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"repos/{self.project_path}")["archived"] + def url_for_ref(self, ref: str, _: RefType) -> str: """Get a URL for a ref.""" return f"{self.forge_root}/{self.project_path}/archive/{ref}.tar.gz" diff --git a/find_deprecated.py b/find_deprecated.py index d860ac20..b81cd038 100755 --- a/find_deprecated.py +++ b/find_deprecated.py @@ -39,7 +39,6 @@ def get_github() -> tuple[Optional[tuple[str, str]], Optional[github.Github], Op return None, None, None - def upstream_last_update_ago(app: str) -> tuple[str, int | None]: manifest_toml = app_cache_folder(app) / "manifest.toml" manifest_json = app_cache_folder(app) / "manifest.json" @@ -75,6 +74,10 @@ def upstream_last_update_ago(app: str) -> tuple[str, int | None]: api = GiteaForgejoAPI(upstream) if api: + if api.archived(): + # A stupid value that we know to be higher than the trigger value + return app, 1000 + last_commit = api.commits()[0] date = last_commit["commit"]["author"]["date"] date = datetime.datetime.fromisoformat(date) From 533e0022c67e48e209bb8a4de37cdff5939b020d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 8 Mar 2024 19:20:55 +0100 Subject: [PATCH 3/6] Add find_deprecated github action --- .github/workflows/deprecated.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/deprecated.yml diff --git a/.github/workflows/deprecated.yml b/.github/workflows/deprecated.yml new file mode 100644 index 00000000..7ce64471 --- /dev/null +++ b/.github/workflows/deprecated.yml @@ -0,0 +1,30 @@ +name: Find deprecated softwares + +on: + schedule: + - cron: '0 20 * * 1' + +jobs: + black: + name: Find deprecated softwares + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Install toml python lib + run: | + pip3 install toml tomlkit gitpython + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Flag deprecated apps in the catalog" + commit-message: ":coffin: Flag deprecated apps in the catalog" + body: | + This was done with find_deprecated.py + base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch + branch: actions/deprecated From 59a2417003e238cc4929d8020ec85795244e603c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 9 Mar 2024 14:26:23 +0100 Subject: [PATCH 4/6] rest_api: deprecated sometimes isn't present? --- autoupdate_app_sources/rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 0aa14b5b..159c3fcf 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -151,7 +151,7 @@ def releases(self) -> list[dict[str, Any]]: def archived(self) -> bool: """Return the archival status for the repository""" - return self.internal_api(f"projects/{self.project_id}")["archived"] + return self.internal_api(f"projects/{self.project_id}").get("archived", False) def url_for_ref(self, ref: str, _: RefType) -> str: name = self.project_path.split("/")[-1] From 83d472e5ce7dc62371353e1491b47b87d09778be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 9 Mar 2024 14:26:47 +0100 Subject: [PATCH 5/6] find_deprecated: better handling of exceptions --- find_deprecated.py | 59 +++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/find_deprecated.py b/find_deprecated.py index b81cd038..17f9cb11 100755 --- a/find_deprecated.py +++ b/find_deprecated.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import traceback import argparse import tomlkit import multiprocessing @@ -57,32 +58,36 @@ def upstream_last_update_ago(app: str) -> tuple[str, int | None]: raise RuntimeError(f"App {app} doesn't have an upstream code link!") api = None - if upstream.startswith("https://github.com/"): - api = GithubAPI(upstream, auth=get_github()[0]) - - if upstream.startswith("https://gitlab."): - api = GitlabAPI(upstream) - - if upstream.startswith("https://codeberg.org") or upstream.startswith("https://framagit.org"): - api = GiteaForgejoAPI(upstream) - - if not api: - autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate") - if autoupdate: - strat = autoupdate["strategy"] - if "gitea" in strat or "forgejo" in strat: - api = GiteaForgejoAPI(upstream) - - if api: - if api.archived(): - # A stupid value that we know to be higher than the trigger value - return app, 1000 - - last_commit = api.commits()[0] - date = last_commit["commit"]["author"]["date"] - date = datetime.datetime.fromisoformat(date) - ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None) - return app, ago.days + try: + if upstream.startswith("https://github.com/"): + api = GithubAPI(upstream, auth=get_github()[0]) + + if upstream.startswith("https://gitlab."): + api = GitlabAPI(upstream) + + if upstream.startswith("https://codeberg.org") or upstream.startswith("https://framagit.org"): + api = GiteaForgejoAPI(upstream) + + if not api: + autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate") + if autoupdate: + strat = autoupdate["strategy"] + if "gitea" in strat or "forgejo" in strat: + api = GiteaForgejoAPI(upstream) + + if api: + if api.archived(): + # A stupid value that we know to be higher than the trigger value + return app, 1000 + + last_commit = api.commits()[0] + date = last_commit["commit"]["author"]["date"] + date = datetime.datetime.fromisoformat(date) + ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None) + return app, ago.days + except Exception: + logging.error(f"Exception while handling {app}", traceback.format_exc()) + raise raise RuntimeError(f"App {app} not handled (not github, gitlab or gitea with autoupdate). Upstream is {upstream}") @@ -108,7 +113,7 @@ def main() -> None: try: app, result = next(tasks) except Exception as e: - print(e) + print(f"Exception found: {e}") continue if result is None: From 468427109751bb69d6b94ba02e1d4acf9ec060bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 9 Mar 2024 15:47:01 +0100 Subject: [PATCH 6/6] better handling of exceptions, again --- autoupdate_app_sources/rest_api.py | 1 + find_deprecated.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 159c3fcf..a7f56d41 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -66,6 +66,7 @@ def changelog_for_ref(self, new_ref: str, old_ref: str, ref_type: RefType) -> st class GitlabAPI: def __init__(self, upstream: str): # Find gitlab api root... + upstream = upstream.rstrip("/") self.forge_root = self.get_forge_root(upstream).rstrip("/") self.project_path = upstream.replace(self.forge_root, "").lstrip("/") self.project_id = self.find_project_id(self.project_path) diff --git a/find_deprecated.py b/find_deprecated.py index 17f9cb11..cb302f64 100755 --- a/find_deprecated.py +++ b/find_deprecated.py @@ -60,12 +60,16 @@ def upstream_last_update_ago(app: str) -> tuple[str, int | None]: api = None try: if upstream.startswith("https://github.com/"): - api = GithubAPI(upstream, auth=get_github()[0]) + try: + api = GithubAPI(upstream, auth=get_github()[0]) + except AssertionError as e: + logging.error(f"Exception while handling {app}: {e}") + return app, None - if upstream.startswith("https://gitlab."): + if upstream.startswith("https://gitlab.") or upstream.startswith("https://framagit.org"): api = GitlabAPI(upstream) - if upstream.startswith("https://codeberg.org") or upstream.startswith("https://framagit.org"): + if upstream.startswith("https://codeberg.org"): api = GiteaForgejoAPI(upstream) if not api: