From 56817e69c879c1151d92bb21f2704cf4051fd1bb Mon Sep 17 00:00:00 2001 From: ginwakeup <> Date: Thu, 29 Sep 2022 22:39:05 +0100 Subject: [PATCH 1/4] Adding requests --- poetry.lock | 65 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 52b5e19..d1326a3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,22 @@ +[[package]] +name = "certifi" +version = "2022.9.24" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "charset-normalizer" +version = "2.1.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + [[package]] name = "click" version = "8.1.3" @@ -50,6 +69,14 @@ python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "loguru" version = "0.6.0" @@ -73,6 +100,24 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "smmap" version = "5.0.0" @@ -81,6 +126,19 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "urllib3" +version = "1.26.12" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "win32-setctime" version = "1.1.0" @@ -95,9 +153,11 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "45f337e8927eb8bc4876fe875e7436b548323508aa18e008b93714463667a2da" +content-hash = "c34bc7cf551c0fe6593b86383b1ab3b4cc8b1c76afb8e7f3549feaf35dfcf4d7" [metadata.files] +certifi = [] +charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, @@ -106,6 +166,7 @@ colorama = [] dacite = [] gitdb = [] gitpython = [] +idna = [] loguru = [] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -142,5 +203,7 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +requests = [] smmap = [] +urllib3 = [] win32-setctime = [] diff --git a/pyproject.toml b/pyproject.toml index 136ff0b..a53f096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ PyYAML = "^6.0" loguru = "^0.6.0" GitPython = "^3.1.27" dacite = "^1.6.0" +requests = "^2.28.1" [tool.poetry.dev-dependencies] From 6af385e7969a37540377238aeee2fd6b1bdb6896 Mon Sep 17 00:00:00 2001 From: ginwakeup <> Date: Thu, 29 Sep 2022 22:42:14 +0100 Subject: [PATCH 2/4] Removed TODOs, adding organization key explanation --- README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index afc220e..881c590 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,6 @@ ![Alt text](resources/gh/header.png?raw=true "Submoduler") An app that iterates a list of repositories and updates each of their submodules to the latest commit. - -# TODOs - -- [x] Run Process for 1 or more repositories using HTTPS and PAT. -- [ ] Run process for entire organization. ## Build the Docker Image @@ -52,4 +47,17 @@ interval: 3 - **init**: if not initialised, init the submodules. - **force_reset**: remove any local change and force reset on the submodules. - **recursive**: update the children submodules. +- organization: this key can contain a organization name and configuration for submoduler. + e.g. + ```yaml + organization: + test-org-name: + to_latest_revision: true + init: true + force_reset: true + recursive: true + interval: 3 + ``` + When specifying a Organization, all the repos living under it will be pulled and monitored/updated. + Right now, only one Submoduler Configuration is supported for all the Org repos, and only 1 repo is supported. - **interval**: this defines how often the check is performed on your repository submodules in seconds. From 154f1e4ad47b5cfdbaafd15415956527cb286077 Mon Sep 17 00:00:00 2001 From: ginwakeup <> Date: Thu, 29 Sep 2022 22:42:34 +0100 Subject: [PATCH 3/4] Adding org example config --- submoduler-example-config.yaml | 2 +- submoduler-organization-example-config.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 submoduler-organization-example-config.yaml diff --git a/submoduler-example-config.yaml b/submoduler-example-config.yaml index 9ac4c6d..515b804 100644 --- a/submoduler-example-config.yaml +++ b/submoduler-example-config.yaml @@ -5,4 +5,4 @@ repos: init: true force_reset: true recursive: true -interval: 3 \ No newline at end of file +interval: 3 diff --git a/submoduler-organization-example-config.yaml b/submoduler-organization-example-config.yaml new file mode 100644 index 0000000..e2e4632 --- /dev/null +++ b/submoduler-organization-example-config.yaml @@ -0,0 +1,7 @@ +organization: + test-org-name: + to_latest_revision: true + init: true + force_reset: true + recursive: true +interval: 3 From 869e175121be8f1d08b2045d5a5e0e839837cb0c Mon Sep 17 00:00:00 2001 From: ginwakeup <> Date: Thu, 29 Sep 2022 22:42:43 +0100 Subject: [PATCH 4/4] Adding support for organization repos --- submoduler/main.py | 2 +- submoduler/submoduler.py | 75 +++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/submoduler/main.py b/submoduler/main.py index 23d42fd..ce484ee 100644 --- a/submoduler/main.py +++ b/submoduler/main.py @@ -9,7 +9,7 @@ @click.command @click.option('--config_path', - help="Path to the configuration file. This supports relative path, e.g.: ../submoduler.yaml.", + help="Path to the configuration file. This supports relative path, e.g.: ../submoduler-example-config.yaml.", default="/opt/submoduler.yaml") @click.option('--user', help="Username.", diff --git a/submoduler/submoduler.py b/submoduler/submoduler.py index ef68776..81410af 100644 --- a/submoduler/submoduler.py +++ b/submoduler/submoduler.py @@ -3,6 +3,7 @@ import threading import subprocess import traceback +import requests import git @@ -16,6 +17,7 @@ class Submoduler: """Core class with the only responsibility to parse repositories and perform the Git Operations.""" _CACHE_DIR = os.path.join(os.path.expanduser("~/"), "submoduler", "repos") + _GH_API_URL = " https://api.github.com" def __init__(self, configuration: dict, user: str, pat: str, email: str): """Init. @@ -35,12 +37,19 @@ def __init__(self, configuration: dict, user: str, pat: str, email: str): if user is None: raise Exception("No Username defined.") + self._user = user + self._pat = pat self._set_credentials(user, email, pat) self._interval = configuration.get("interval") logger.info(f"Interval set to {self._interval}") - self._repos_configs = configuration.get("repos") + self._repos_configs = configuration.get("repos", {}) + self._organization = configuration.get("organization", {}) + if len(self._organization) > 1: + error = "Only one Organization is supported at this time. Picking the first and discarding the others." + logger.error(error) + raise Exception(error) self._repos: list[RepoMeta] = [] os.makedirs(self._CACHE_DIR, exist_ok=True) @@ -88,28 +97,54 @@ def _make_repo(self, repo_path: str, repo_meta: dict): self._repos.append(repo_meta_class) + def _get_org_repos(self, org_name: str) -> list[dict]: + """Given an organization name, return all its repositories. + + Args: + org_name: organization name. + + Returns: + list: list of dict. + """ + return requests.get(f"{self._GH_API_URL}/orgs/{org_name}/repos", auth=(self._user, self._pat)).json() + + def _clone_repo(self, repo_url, repo_clone_path): + if repo_url.lower().startswith(("https")): + try: + Repo.clone_from(repo_url, repo_clone_path) + except git.GitCommandError as git_error: + if "already exists" in git_error.stderr: + pass + else: + logger.error(traceback.format_exc()) + raise Exception(git_error.stderr) + + else: + # Not a URL, unknown format. + logger.warning(f"Couldn't recognize a format for url: {repo_url}") + + def _process_repo(self, repo_url: str, repo_clone_path: str, repo_meta: dict): + """Clones a Repo and creates a RepoMeta object stored in the class. + + Args: + repo_url: https url of the repository to clone. + repo_clone_path: local path where the repository should be cloned. + repo_meta: dictionary of metadata for the RepoMeta object. + """ + self._clone_repo(repo_url, repo_clone_path) + self._make_repo(repo_clone_path, repo_meta) + def _parse_repos(self): """Parse repositories listed in the configuration and populates self._repos with RepoMeta objects.""" for repo_name, repo_meta in self._repos_configs.items(): - repo_url = repo_meta.get("url") - - if repo_url.lower().startswith(("https")): - repo_clone_path = os.path.join(self._CACHE_DIR, repo_name) - try: - #repo_url = repo_url.replace("https://", f"https://{self._pat}@") - Repo.clone_from(repo_url, repo_clone_path) - except git.GitCommandError as git_error: - if "already exists" in git_error.stderr: - pass - else: - logger.error(traceback.format_exc()) - raise Exception(git_error.stderr) - - self._make_repo(repo_clone_path, repo_meta) - - else: - # Not a URL, unknown format. - logger.warning(f"Couldn't recognize a format for url: {repo_url}") + repo_clone_path = os.path.join(self._CACHE_DIR, repo_name) + self._process_repo(repo_meta.get("url"), repo_clone_path, repo_meta) + + for org_name, org_meta in self._organization.items(): + repos = self._get_org_repos(org_name) + for repo_dict in repos: + repo_clone_path = os.path.join(self._CACHE_DIR, org_name, repo_dict.get("name")) + self._process_repo(repo_dict.get("html_url"), repo_clone_path, org_meta) def _start(self): """Starts a thread for each repository to update its submodules every 'interval' as specified in the configuration."""