#12: add option --no-web-ui to skip processing web only settings.

CHANGELOG.md

@@ -0,0 +1,7 @@
+# Change Log
+
+## [0.1.0-SNAPSHOT] - unreleased
+
+### Added
+
+- Added option `--no-web-ui` to skip processing settings accessed via the GitHub Web UI. ([#12](https://gitlab.eclipse.org/eclipsefdn/security/otterdog/-/issues/12))

otterdog/config.py

@@ -210,14 +210,15 @@ def from_dict(cls, data: dict[str, Any]):
 
 
 class OtterdogConfig:
-    def __init__(self, config_file: str, force_processing: bool, local_only: bool):
+    def __init__(self, config_file: str, force_processing: bool, local_only: bool, no_web_ui: bool):
         if not os.path.exists(config_file):
             raise RuntimeError(f"configuration file '{config_file}' not found")
 
         self._config_file = os.path.realpath(config_file)
         self._data_dir = os.path.dirname(self._config_file)
         self._credential_providers = {}
         self._force_processing = force_processing
+        self._no_web_ui = no_web_ui
 
         with open(config_file) as f:
             self._configuration = json.load(f)
@@ -241,6 +242,10 @@ def config_file(self) -> str:
     def force_processing(self) -> bool:
         return self._force_processing
 
+    @property
+    def no_web_ui(self) -> bool:
+        return self._no_web_ui
+
     @property
     def data_dir(self) -> str:
         return self._data_dir
@@ -310,5 +315,5 @@ def __repr__(self):
         return f"OtterdogConfig('{self.data_dir}')"
 
     @classmethod
-    def from_file(cls, config_file: str, force_processing: bool, local_only: bool):
-        return cls(config_file, force_processing, local_only)
+    def from_file(cls, config_file: str, force_processing: bool, local_only: bool, no_web_ui: bool):
+        return cls(config_file, force_processing, local_only, no_web_ui)

otterdog/diff_operation.py

@@ -112,15 +112,23 @@ def _process_settings(self, github_id: str, expected_org: org.Organization, diff
         start = datetime.now()
         self.printer.print(f"organization settings: Reading...")
 
+        # filter out web settings if --no-web-ui is used
+        expected_settings_keys = expected_settings.keys()
+        if self.config.no_web_ui:
+            expected_settings_keys = {x for x in expected_settings_keys if not self.gh_client.is_web_setting(x)}
+
         # determine differences for settings.
-        current_github_org_settings = self.gh_client.get_org_settings(github_id, expected_settings.keys())
+        current_github_org_settings = self.gh_client.get_org_settings(github_id, expected_settings_keys)
         current_otterdog_org_settings = mapping.map_github_org_settings_data_to_otterdog(current_github_org_settings)
 
         end = datetime.now()
         self.printer.print(f"organization settings: Read complete after {(end - start).total_seconds()}s")
 
         modified_settings = {}
         for key, expected_value in sorted(expected_settings.items()):
+            if key not in expected_settings_keys:
+                continue
+
             if key not in current_otterdog_org_settings:
                 self.printer.print_warn(f"unexpected key '{key}' found in configuration, skipping")
                 continue

otterdog/github.py

@@ -33,6 +33,9 @@ def __init__(self, credentials: Credentials):
         self.web_client = GithubWeb(self.credentials)
         self.graphql_client = GithubGraphQL(credentials.github_token)
 
+    def is_web_setting(self, setting_key: str) -> bool:
+        return setting_key in self.settings_web_keys
+
     def is_readonly_org_setting(self, setting_key: str) -> bool:
         setting_entry = self.settings_schema["properties"].get(setting_key)
         return setting_entry.get("readonly", False)

otterdog/main.py

@@ -59,8 +59,10 @@ def main(arguments=None):
         subparser.add_argument("organization", nargs="*", help="the github id of the organization")
         subparser.add_argument("--config", "-c", help=f"configuration file, defaults to '{CONFIG_FILE}'",
                                action="store", default=CONFIG_FILE)
-        subparser.add_argument("--local", action="store_true", default=0, help="does not update the default config")
-        subparser.add_argument("--force", "-f", action="store_true", default=0, help="skips interactive approvals")
+        subparser.add_argument("--local", action="store_true", default=False, help="does not update the default config")
+        subparser.add_argument("--force", "-f", action="store_true", default=False, help="skips interactive approvals")
+        subparser.add_argument("--no-web-ui", "-n", action="store_true", default=False,
+                               help="skip settings retrieved via web ui")
         subparser.add_argument("--verbose", "-v", action="count", default=0, help="enable more verbose output")
 
     args = parser.parse_args(arguments)
@@ -71,7 +73,7 @@ def main(arguments=None):
     printer.print()
 
     try:
-        config = OtterdogConfig.from_file(args.config, args.force, args.local)
+        config = OtterdogConfig.from_file(args.config, args.force, args.local, args.no_web_ui)
 
         exit_code = 0
 

otterdog/organization.py

@@ -149,7 +149,10 @@ def __str__(self) -> str:
         return f"Organization(id={self.github_id})"
 
 
-def load_from_file(github_id: str, config_file: str, config: OtterdogConfig) -> Organization:
+def load_from_file(github_id: str,
+                   config_file: str,
+                   config: OtterdogConfig,
+                   resolve_secrets: bool = True) -> Organization:
     if not os.path.exists(config_file):
         msg = f"configuration file '{config_file}' for organization '{github_id}' does not exist"
         raise RuntimeError(msg)
@@ -158,9 +161,10 @@ def load_from_file(github_id: str, config_file: str, config: OtterdogConfig) ->
     org_data = utils.jsonnet_evaluate_file(config_file)
 
     # resolve webhook secrets
-    for webhook in org_data["webhooks"]:
-        if "secret" in webhook:
-            webhook["secret"] = config.get_secret(webhook["secret"])
+    if resolve_secrets:
+        for webhook in org_data["webhooks"]:
+            if "secret" in webhook:
+                webhook["secret"] = config.get_secret(webhook["secret"])
 
     org = Organization(github_id)
     org.load_config(org_data)

otterdog/show_operation.py

@@ -50,7 +50,8 @@ def execute(self, org_config: OrganizationConfig) -> int:
             try:
                 organization = org.load_from_file(github_id,
                                                   self.jsonnet_config.get_org_config_file(github_id),
-                                                  self.config)
+                                                  self.config,
+                                                  False)
             except RuntimeError as ex:
                 self.printer.print_warn(f"failed to load configuration: {str(ex)}")
                 return 1

otterdog/validate_operation.py

@@ -50,7 +50,8 @@ def execute(self, org_config: OrganizationConfig) -> int:
             try:
                 organization = org.load_from_file(github_id,
                                                   self.jsonnet_config.get_org_config_file(github_id),
-                                                  self.config)
+                                                  self.config,
+                                                  False)
             except RuntimeError as ex:
                 self.printer.print_error(f"Validation failed\nfailed to load configuration: {str(ex)}")
                 return 1

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "otterdog"
-version = "0.1.0"
+version = "0.1.0-SNAPSHOT"
 description = "Tool to manage GitHub organizations and their repositories."
 authors = ["Thomas Neidhart <thomass.neidhart@eclipse-foundation.org>"]
 readme = "README.md"