Merge pull request #3 from ginwakeup/feat/org-support

Feature: Adding Organisation support

README.md

@@ -2,11 +2,6 @@
 ![Alt text](resources/gh/header.png?raw=true "Submoduler")
 
 An app that iterates a list of repositories and updates each of their submodules to the latest commit.
-
-# TODOs
-
-- [x] Run Process for 1 or more repositories using HTTPS and PAT.
-- [ ] Run process for entire organization.
 
 ## Build the Docker Image
 
@@ -52,4 +47,17 @@ interval: 3
   - **init**: if not initialised, init the submodules.
   - **force_reset**: remove any local change and force reset on the submodules.
   - **recursive**: update the children submodules.
+- organization: this key can contain a organization name and configuration for submoduler.
+  e.g.
+  ```yaml
+  organization:
+    test-org-name:
+      to_latest_revision: true
+      init: true
+      force_reset: true
+      recursive: true
+  interval: 3
+  ```
+  When specifying a Organization, all the repos living under it will be pulled and monitored/updated.
+  Right now, only one Submoduler Configuration is supported for all the Org repos, and only 1 repo is supported.
 - **interval**: this defines how often the check is performed on your repository submodules in seconds.

pyproject.toml

@@ -11,6 +11,7 @@ PyYAML = "^6.0"
 loguru = "^0.6.0"
 GitPython = "^3.1.27"
 dacite = "^1.6.0"
+requests = "^2.28.1"
 
 [tool.poetry.dev-dependencies]
 

submoduler-example-config.yaml

@@ -5,4 +5,4 @@ repos:
     init: true
     force_reset: true
     recursive: true
-interval: 3
+interval: 3

submoduler-organization-example-config.yaml

@@ -0,0 +1,7 @@
+organization:
+  test-org-name:
+    to_latest_revision: true
+    init: true
+    force_reset: true
+    recursive: true
+interval: 3

submoduler/submoduler.py

@@ -3,6 +3,7 @@
 import threading
 import subprocess
 import traceback
+import requests
 
 import git
 
@@ -16,6 +17,7 @@
 class Submoduler:
     """Core class with the only responsibility to parse repositories and perform the Git Operations."""
     _CACHE_DIR = os.path.join(os.path.expanduser("~/"), "submoduler", "repos")
+    _GH_API_URL = " https://api.github.com"
 
     def __init__(self, configuration: dict, user: str, pat: str, email: str):
         """Init.
@@ -35,12 +37,19 @@ def __init__(self, configuration: dict, user: str, pat: str, email: str):
         if user is None:
             raise Exception("No Username defined.")
 
+        self._user = user
+        self._pat = pat
         self._set_credentials(user, email, pat)
 
         self._interval = configuration.get("interval")
         logger.info(f"Interval set to {self._interval}")
 
-        self._repos_configs = configuration.get("repos")
+        self._repos_configs = configuration.get("repos", {})
+        self._organization = configuration.get("organization", {})
+        if len(self._organization) > 1:
+            error = "Only one Organization is supported at this time. Picking the first and discarding the others."
+            logger.error(error)
+            raise Exception(error)
 
         self._repos: list[RepoMeta] = []
         os.makedirs(self._CACHE_DIR, exist_ok=True)
@@ -88,28 +97,54 @@ def _make_repo(self, repo_path: str, repo_meta: dict):
 
         self._repos.append(repo_meta_class)
 
+    def _get_org_repos(self, org_name: str) -> list[dict]:
+        """Given an organization name, return all its repositories.
+
+        Args:
+            org_name: organization name.
+
+        Returns:
+            list: list of dict.
+        """
+        return requests.get(f"{self._GH_API_URL}/orgs/{org_name}/repos", auth=(self._user, self._pat)).json()
+
+    def _clone_repo(self, repo_url, repo_clone_path):
+        if repo_url.lower().startswith(("https")):
+            try:
+                Repo.clone_from(repo_url, repo_clone_path)
+            except git.GitCommandError as git_error:
+                if "already exists" in git_error.stderr:
+                    pass
+                else:
+                    logger.error(traceback.format_exc())
+                    raise Exception(git_error.stderr)
+
+        else:
+            # Not a URL, unknown format.
+            logger.warning(f"Couldn't recognize a format for url: {repo_url}")
+
+    def _process_repo(self, repo_url: str, repo_clone_path: str, repo_meta: dict):
+        """Clones a Repo and creates a RepoMeta object stored in the class.
+
+        Args:
+            repo_url: https url of the repository to clone.
+            repo_clone_path: local path where the repository should be cloned.
+            repo_meta: dictionary of metadata for the RepoMeta object.
+        """
+        self._clone_repo(repo_url, repo_clone_path)
+        self._make_repo(repo_clone_path, repo_meta)
+
     def _parse_repos(self):
         """Parse repositories listed in the configuration and populates self._repos with RepoMeta objects."""
         for repo_name, repo_meta in self._repos_configs.items():
-            repo_url = repo_meta.get("url")
-
-            if repo_url.lower().startswith(("https")):
-                repo_clone_path = os.path.join(self._CACHE_DIR, repo_name)
-                try:
-                    #repo_url = repo_url.replace("https://", f"https://{self._pat}@")
-                    Repo.clone_from(repo_url, repo_clone_path)
-                except git.GitCommandError as git_error:
-                    if "already exists" in git_error.stderr:
-                        pass
-                    else:
-                        logger.error(traceback.format_exc())
-                        raise Exception(git_error.stderr)
-
-                self._make_repo(repo_clone_path, repo_meta)
-
-            else:
-                # Not a URL, unknown format.
-                logger.warning(f"Couldn't recognize a format for url: {repo_url}")
+            repo_clone_path = os.path.join(self._CACHE_DIR, repo_name)
+            self._process_repo(repo_meta.get("url"), repo_clone_path, repo_meta)
+
+        for org_name, org_meta in self._organization.items():
+            repos = self._get_org_repos(org_name)
+            for repo_dict in repos:
+                repo_clone_path = os.path.join(self._CACHE_DIR, org_name, repo_dict.get("name"))
+                self._process_repo(repo_dict.get("html_url"), repo_clone_path, org_meta)
 
     def _start(self):
         """Starts a thread for each repository to update its submodules every 'interval' as specified in the configuration."""