Use data stored in Cloudflare R2 instead of GitHub API for scheduled refresh

custom_components/hacs/base.py

@@ -32,6 +32,11 @@
 from homeassistant.loader import Integration
 from homeassistant.util import dt
 
+from custom_components.hacs.repositories.base import (
+    HACS_MANIFEST_KEYS_TO_EXPORT,
+    REPOSITORY_KEYS_TO_EXPORT,
+)
+
 from .const import DOMAIN, TV, URL_BASE
 from .data_client import HacsDataClient
 from .enums import (
@@ -266,7 +271,7 @@ def mark_default(self, repository: HacsRepository) -> None:
 
         self._default_repositories.add(repo_id)
 
-    def set_repository_id(self, repository, repo_id):
+    def set_repository_id(self, repository: HacsRepository, repo_id: str):
         """Update a repository id."""
         existing_repo_id = str(repository.data.id)
         if existing_repo_id == repo_id:
@@ -548,8 +553,6 @@ async def async_register_repository(
         if check:
             try:
                 await repository.async_registration(ref)
-                if self.status.new:
-                    repository.data.new = False
                 if repository.validate.errors:
                     self.common.skip.append(repository.data.full_name)
                     if not self.status.startup:
@@ -571,6 +574,9 @@ async def async_register_repository(
                     f"Validation for {repository_full_name} failed with {exception}."
                 ) from exception
 
+        if self.status.new:
+            repository.data.new = False
+
         if repository_id is not None:
             repository.data.id = repository_id
 
@@ -628,16 +634,32 @@ async def startup_tasks(self, _=None) -> None:
                     )
                     break
 
+        if not self.configuration.experimental:
+            self.recuring_tasks.append(
+                self.hass.helpers.event.async_track_time_interval(
+                    self.async_update_downloaded_repositories, timedelta(hours=48)
+                )
+            )
+            self.recuring_tasks.append(
+                self.hass.helpers.event.async_track_time_interval(
+                    self.async_update_all_repositories,
+                    timedelta(hours=96),
+                )
+            )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
-                self.async_get_all_category_repositories, timedelta(hours=3)
+                self.async_update_downloaded_custom_repositories, timedelta(hours=48)
             )
         )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
-                self.async_update_all_repositories, timedelta(hours=96)
+                self.async_get_all_category_repositories,
+                timedelta(hours=6 if self.configuration.experimental else 3),
             )
         )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
                 self.async_check_rate_limit, timedelta(minutes=5)
@@ -648,11 +670,7 @@ async def startup_tasks(self, _=None) -> None:
                 self.async_prosess_queue, timedelta(minutes=10)
             )
         )
-        self.recuring_tasks.append(
-            self.hass.helpers.event.async_track_time_interval(
-                self.async_update_downloaded_repositories, timedelta(hours=48)
-            )
-        )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
                 self.async_handle_critical_repositories, timedelta(hours=2)
@@ -767,11 +785,40 @@ async def async_get_all_category_repositories(self, _=None) -> None:
         self.log.info("Loading known repositories")
         await asyncio.gather(
             *[
-                self.async_get_category_repositories(HacsCategory(category))
+                self.async_get_category_repositories_experimental(category)
+                if self.configuration.experimental
+                else self.async_get_category_repositories(HacsCategory(category))
                 for category in self.common.categories or []
             ]
         )
 
+    async def async_get_category_repositories_experimental(self, category: str) -> None:
+        """Update all category repositories."""
+        self.log.info("Fetching updated content for %s", category)
+        category_data = await self.data_client.get_data(category)
+
+        await self.data.register_unknown_repositories(category_data, category)
+
+        for repo_id, repo_data in category_data.items():
+            repo = repo_data["full_name"]
+            if self.common.renamed_repositories.get(repo):
+                repo = self.common.renamed_repositories[repo]
+            if self.repositories.is_removed(repo):
+                continue
+            if repo in self.common.archived_repositories:
+                continue
+            if repository := self.repositories.get_by_full_name(repo):
+                self.repositories.set_repository_id(repository, repo_id)
+                self.repositories.mark_default(repository)
+                if repository.data.last_fetched is None or (
+                    repository.data.last_fetched.timestamp() < repo_data["last_fetched"]
+                ):
+                    repository.data.update_data({**dict(REPOSITORY_KEYS_TO_EXPORT), **repo_data})
+                    if (manifest := repo_data.get("manifest")) is not None:
+                        repository.repository_manifest.update_data(
+                            {**dict(HACS_MANIFEST_KEYS_TO_EXPORT), **manifest}
+                        )
+
     async def async_get_category_repositories(self, category: HacsCategory) -> None:
         """Get repositories from category."""
         if self.system.disabled:
@@ -869,9 +916,12 @@ async def async_handle_removed_repositories(self, _=None) -> None:
         self.log.info("Loading removed repositories")
 
         try:
-            removed_repositories = await self.async_github_get_hacs_default_file(
-                HacsCategory.REMOVED
-            )
+            if self.configuration.experimental:
+                removed_repositories = await self.data_client.get_data("removed")
+            else:
+                removed_repositories = await self.async_github_get_hacs_default_file(
+                    HacsCategory.REMOVED
+                )
         except HacsException:
             return
 
@@ -927,6 +977,21 @@ async def async_update_downloaded_repositories(self, _=None) -> None:
 
         self.log.debug("Recurring background task for downloaded repositories done")
 
+    async def async_update_downloaded_custom_repositories(self, _=None) -> None:
+        """Execute the task."""
+        if self.system.disabled or not self.configuration.experimental:
+            return
+        self.log.info("Starting recurring background task for downloaded custom repositories")
+
+        for repository in self.repositories.list_downloaded:
+            if (
+                repository.data.category in self.common.categories
+                and not self.repositories.is_default(repository.data.id)
+            ):
+                self.queue.add(repository.update_repository(ignore_issues=True))
+
+        self.log.debug("Recurring background task for downloaded custom repositories done")
+
     async def async_handle_critical_repositories(self, _=None) -> None:
         """Handle critical repositories."""
         critical_queue = QueueManager(hass=self.hass)

custom_components/hacs/repositories/base.py

@@ -77,10 +77,36 @@
     "home-assistant-frontend",
     "home-assistant-hacs",
     "home-assistant-custom",
+    "home-assistant-sensor",
     "lovelace-ui",
 )
 
 
+REPOSITORY_KEYS_TO_EXPORT = (
+    # Keys can not be removed from this list until v3
+    # If keys are added, the action need to be re-run with force
+    ("description", ""),
+    ("downloads", 0),
+    ("domain", None),
+    ("etag_repository", None),
+    ("full_name", ""),
+    ("last_commit", None),
+    ("last_updated", 0),
+    ("last_version", None),
+    ("manifest_name", None),
+    ("open_issues", 0),
+    ("stargazers_count", 0),
+    ("topics", []),
+)
+
+HACS_MANIFEST_KEYS_TO_EXPORT = (
+    # Keys can not be removed from this list until v3
+    # If keys are added, the action need to be re-run with force
+    ("country", []),
+    ("name", None),
+)
+
+
 class FileInformation:
     """FileInformation."""
 
@@ -146,21 +172,24 @@ def create_from_dict(source: dict, action: bool = False) -> RepositoryData:
 
     def update_data(self, data: dict, action: bool = False) -> None:
         """Update data of the repository."""
-        for key in data:
+        for key, value in data.items():
             if key not in self.__dict__:
                 continue
+
+            if key == "last_fetched" and isinstance(value, float):
+                setattr(self, key, datetime.fromtimestamp(value))
             elif key == "id":
-                setattr(self, key, str(data[key]))
+                setattr(self, key, str(value))
             elif key == "country":
-                if isinstance(data[key], str):
-                    setattr(self, key, [data[key]])
+                if isinstance(value, str):
+                    setattr(self, key, [value])
                 else:
-                    setattr(self, key, data[key])
+                    setattr(self, key, value)
             elif key == "topics" and not action:
-                setattr(self, key, [topic for topic in data[key] if topic not in TOPIC_FILTER])
+                setattr(self, key, [topic for topic in value if topic not in TOPIC_FILTER])
 
             else:
-                setattr(self, key, data[key])
+                setattr(self, key, value)
 
 
 @attr.s(auto_attribs=True)
@@ -203,6 +232,20 @@ def from_dict(manifest: dict):
                 setattr(manifest_data, key, value)
         return manifest_data
 
+    def update_data(self, data: dict) -> None:
+        """Update the manifest data."""
+        for key, value in data.items():
+            if key not in self.__dict__:
+                continue
+
+            if key == "country":
+                if isinstance(value, str):
+                    setattr(self, key, [value])
+                else:
+                    setattr(self, key, value)
+            else:
+                setattr(self, key, value)
+
 
 class RepositoryReleases:
     """RepositoyReleases."""

custom_components/hacs/system_health.py

@@ -7,6 +7,7 @@
 from .const import DOMAIN
 
 GITHUB_STATUS = "https://www.githubstatus.com/"
+CLOUDFLARE_STATUS = "https://www.cloudflarestatus.com/"
 
 
 @callback
@@ -29,6 +30,9 @@ async def system_health_info(hass):
         "GitHub Web": system_health.async_check_can_reach_url(
             hass, "https://github.com/", GITHUB_STATUS
         ),
+        "HACS Data": system_health.async_check_can_reach_url(
+            hass, "https://data-v2.hacs.xyz/data.json", CLOUDFLARE_STATUS
+        ),
         "GitHub API Calls Remaining": response.data.resources.core.remaining,
         "Installed Version": hacs.version,
         "Stage": hacs.stage,

custom_components/hacs/utils/data.py

@@ -15,26 +15,27 @@
 from .path import is_safe
 from .store import async_load_from_store, async_save_to_store
 
-DEFAULT_BASE_REPOSITORY_DATA = (
+EXPORTED_BASE_DATA = (("new", False),)
+
+EXPORTED_REPOSITORY_DATA = EXPORTED_BASE_DATA + (
     ("authors", []),
     ("category", ""),
     ("description", ""),
     ("domain", None),
     ("downloads", 0),
     ("etag_repository", None),
     ("full_name", ""),
-    ("last_updated", 0),
     ("hide", False),
+    ("last_updated", 0),
     ("new", False),
     ("stargazers_count", 0),
     ("topics", []),
 )
 
-DEFAULT_EXTENDED_REPOSITORY_DATA = (
+EXPORTED_DOWNLOADED_REPOSITORY_DATA = EXPORTED_REPOSITORY_DATA + (
     ("archived", False),
     ("config_flow", False),
     ("default_branch", None),
-    ("description", ""),
     ("first_install", False),
     ("installed_commit", None),
     ("installed", False),
@@ -46,8 +47,6 @@
     ("releases", False),
     ("selected_tag", None),
     ("show_beta", False),
-    ("stargazers_count", 0),
-    ("topics", []),
 )
 
 
@@ -100,16 +99,16 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict:
         """Store the repository data."""
         data = {"repository_manifest": repository.repository_manifest.manifest}
 
-        for key, default_value in DEFAULT_BASE_REPOSITORY_DATA:
-            if (value := repository.data.__getattribute__(key)) != default_value:
+        for key, default in (
+            EXPORTED_DOWNLOADED_REPOSITORY_DATA
+            if repository.data.installed
+            else EXPORTED_REPOSITORY_DATA
+        ):
+            if (value := getattr(repository.data, key, default)) != default:
                 data[key] = value
 
-        if repository.data.installed:
-            for key, default_value in DEFAULT_EXTENDED_REPOSITORY_DATA:
-                if (value := repository.data.__getattribute__(key)) != default_value:
-                    data[key] = value
+        if repository.data.installed_version:
             data["version_installed"] = repository.data.installed_version
-
         if repository.data.last_fetched:
             data["last_fetched"] = repository.data.last_fetched.timestamp()
 
@@ -137,6 +136,8 @@ async def restore(self):
         if not hacs and not repositories:
             # Assume new install
             self.hacs.status.new = True
+            if self.hacs.configuration.experimental:
+                return True
             self.logger.info("<HacsData restore> Loading base repository information")
             repositories = await self.hacs.hass.async_add_executor_job(
                 json_util.load_json,
@@ -207,9 +208,10 @@ async def register_unknown_repositories(self, repositories, category: str | None
     @callback
     def async_restore_repository(self, entry, repository_data):
         """Restore repository."""
-        full_name = repository_data["full_name"]
-        if not (repository := self.hacs.repositories.get_by_full_name(full_name)):
-            self.logger.error("<HacsData restore> Did not find %s (%s)", full_name, entry)
+        full_name = repository_data.get("full_name")
+        if full_name is None or not (
+            repository := self.hacs.repositories.get_by_full_name(full_name)
+        ):
             return
         # Restore repository attributes
         self.hacs.repositories.set_repository_id(repository, entry)