Use data stored in Cloudflare R2 instead of GitHub API for scheduled …

…refresh (#2991)

custom_components/hacs/base.py

@@ -32,6 +32,11 @@
 from homeassistant.loader import Integration
 from homeassistant.util import dt
 
+from custom_components.hacs.repositories.base import (
+ HACS_MANIFEST_KEYS_TO_EXPORT,
+ REPOSITORY_KEYS_TO_EXPORT,
+)
+
 from .const import DOMAIN, TV, URL_BASE
 from .data_client import HacsDataClient
 from .enums import (
@@ -266,7 +271,7 @@ def mark_default(self, repository: HacsRepository) -> None:
 
  self._default_repositories.add(repo_id)
 
- def set_repository_id(self, repository, repo_id):
+ def set_repository_id(self, repository: HacsRepository, repo_id: str):
  """Update a repository id."""
  existing_repo_id = str(repository.data.id)
  if existing_repo_id == repo_id:
@@ -548,8 +553,6 @@ async def async_register_repository(
  if check:
  try:
  await repository.async_registration(ref)
- if self.status.new:
- repository.data.new = False
  if repository.validate.errors:
  self.common.skip.append(repository.data.full_name)
  if not self.status.startup:
@@ -571,6 +574,9 @@ async def async_register_repository(
  f"Validation for {repository_full_name} failed with {exception}."
  ) from exception
 
+ if self.status.new:
+ repository.data.new = False
+
  if repository_id is not None:
  repository.data.id = repository_id
 
@@ -628,16 +634,32 @@ async def startup_tasks(self, _=None) -> None:
  )
  break
 
+ if not self.configuration.experimental:
+ self.recuring_tasks.append(
+ self.hass.helpers.event.async_track_time_interval(
+ self.async_update_downloaded_repositories, timedelta(hours=48)
+ )
+ )
+ self.recuring_tasks.append(
+ self.hass.helpers.event.async_track_time_interval(
+ self.async_update_all_repositories,
+ timedelta(hours=96),
+ )
+ )
+
  self.recuring_tasks.append(
  self.hass.helpers.event.async_track_time_interval(
- self.async_get_all_category_repositories, timedelta(hours=3)
+ self.async_update_downloaded_custom_repositories, timedelta(hours=48)
  )
  )
+
  self.recuring_tasks.append(
  self.hass.helpers.event.async_track_time_interval(
- self.async_update_all_repositories, timedelta(hours=96)
+ self.async_get_all_category_repositories,
+ timedelta(hours=6 if self.configuration.experimental else 3),
  )
  )
+
  self.recuring_tasks.append(
  self.hass.helpers.event.async_track_time_interval(
  self.async_check_rate_limit, timedelta(minutes=5)
@@ -648,11 +670,7 @@ async def startup_tasks(self, _=None) -> None:
  self.async_prosess_queue, timedelta(minutes=10)
  )
  )
- self.recuring_tasks.append(
- self.hass.helpers.event.async_track_time_interval(
- self.async_update_downloaded_repositories, timedelta(hours=48)
- )
- )
+
  self.recuring_tasks.append(
  self.hass.helpers.event.async_track_time_interval(
  self.async_handle_critical_repositories, timedelta(hours=2)
@@ -767,11 +785,40 @@ async def async_get_all_category_repositories(self, _=None) -> None:
  self.log.info("Loading known repositories")
  await asyncio.gather(
  *[
- self.async_get_category_repositories(HacsCategory(category))
+ self.async_get_category_repositories_experimental(category)
+ if self.configuration.experimental
+ else self.async_get_category_repositories(HacsCategory(category))
  for category in self.common.categories or []
  ]
  )
 
+ async def async_get_category_repositories_experimental(self, category: str) -> None:
+ """Update all category repositories."""
+ self.log.info("Fetching updated content for %s", category)
+ category_data = await self.data_client.get_data(category)
+
+ await self.data.register_unknown_repositories(category_data, category)
+
+ for repo_id, repo_data in category_data.items():
+ repo = repo_data["full_name"]
+ if self.common.renamed_repositories.get(repo):
+ repo = self.common.renamed_repositories[repo]
+ if self.repositories.is_removed(repo):
+ continue
+ if repo in self.common.archived_repositories:
+ continue
+ if repository := self.repositories.get_by_full_name(repo):
+ self.repositories.set_repository_id(repository, repo_id)
+ self.repositories.mark_default(repository)
+ if repository.data.last_fetched is None or (
+ repository.data.last_fetched.timestamp() < repo_data["last_fetched"]
+ ):
+ repository.data.update_data({**dict(REPOSITORY_KEYS_TO_EXPORT), **repo_data})
+ if (manifest := repo_data.get("manifest")) is not None:
+ repository.repository_manifest.update_data(
+ {**dict(HACS_MANIFEST_KEYS_TO_EXPORT), **manifest}
+ )
+
  async def async_get_category_repositories(self, category: HacsCategory) -> None:
  """Get repositories from category."""
  if self.system.disabled:
@@ -869,9 +916,12 @@ async def async_handle_removed_repositories(self, _=None) -> None:
  self.log.info("Loading removed repositories")
 
  try:
- removed_repositories = await self.async_github_get_hacs_default_file(
- HacsCategory.REMOVED
- )
+ if self.configuration.experimental:
+ removed_repositories = await self.data_client.get_data("removed")
+ else:
+ removed_repositories = await self.async_github_get_hacs_default_file(
+ HacsCategory.REMOVED
+ )
  except HacsException:
  return
 
@@ -927,6 +977,21 @@ async def async_update_downloaded_repositories(self, _=None) -> None:
 
  self.log.debug("Recurring background task for downloaded repositories done")
 
+ async def async_update_downloaded_custom_repositories(self, _=None) -> None:
+ """Execute the task."""
+ if self.system.disabled or not self.configuration.experimental:
+ return
+ self.log.info("Starting recurring background task for downloaded custom repositories")
+
+ for repository in self.repositories.list_downloaded:
+ if (
+ repository.data.category in self.common.categories
+ and not self.repositories.is_default(repository.data.id)
+ ):
+ self.queue.add(repository.update_repository(ignore_issues=True))
+
+ self.log.debug("Recurring background task for downloaded custom repositories done")
+
  async def async_handle_critical_repositories(self, _=None) -> None:
  """Handle critical repositories."""
  critical_queue = QueueManager(hass=self.hass)

custom_components/hacs/repositories/base.py

@@ -77,10 +77,36 @@
  "home-assistant-frontend",
  "home-assistant-hacs",
  "home-assistant-custom",
+ "home-assistant-sensor",
  "lovelace-ui",
 )
 
 
+REPOSITORY_KEYS_TO_EXPORT = (
+ # Keys can not be removed from this list until v3
+ # If keys are added, the action need to be re-run with force
+ ("description", ""),
+ ("downloads", 0),
+ ("domain", None),
+ ("etag_repository", None),
+ ("full_name", ""),
+ ("last_commit", None),
+ ("last_updated", 0),
+ ("last_version", None),
+ ("manifest_name", None),
+ ("open_issues", 0),
+ ("stargazers_count", 0),
+ ("topics", []),
+)
+
+HACS_MANIFEST_KEYS_TO_EXPORT = (
+ # Keys can not be removed from this list until v3
+ # If keys are added, the action need to be re-run with force
+ ("country", []),
+ ("name", None),
+)
+
+
 class FileInformation:
  """FileInformation."""
 
@@ -146,21 +172,24 @@ def create_from_dict(source: dict, action: bool = False) -> RepositoryData:
 
  def update_data(self, data: dict, action: bool = False) -> None:
  """Update data of the repository."""
- for key in data:
+ for key, value in data.items():
  if key not in self.__dict__:
  continue
+
+ if key == "last_fetched" and isinstance(value, float):
+ setattr(self, key, datetime.fromtimestamp(value))
  elif key == "id":
- setattr(self, key, str(data[key]))
+ setattr(self, key, str(value))
  elif key == "country":
- if isinstance(data[key], str):
- setattr(self, key, [data[key]])
+ if isinstance(value, str):
+ setattr(self, key, [value])
  else:
- setattr(self, key, data[key])
+ setattr(self, key, value)
  elif key == "topics" and not action:
- setattr(self, key, [topic for topic in data[key] if topic not in TOPIC_FILTER])
+ setattr(self, key, [topic for topic in value if topic not in TOPIC_FILTER])
 
  else:
- setattr(self, key, data[key])
+ setattr(self, key, value)
 
 
 @attr.s(auto_attribs=True)
@@ -203,6 +232,20 @@ def from_dict(manifest: dict):
  setattr(manifest_data, key, value)
  return manifest_data
 
+ def update_data(self, data: dict) -> None:
+ """Update the manifest data."""
+ for key, value in data.items():
+ if key not in self.__dict__:
+ continue
+
+ if key == "country":
+ if isinstance(value, str):
+ setattr(self, key, [value])
+ else:
+ setattr(self, key, value)
+ else:
+ setattr(self, key, value)
+
 
 class RepositoryReleases:
  """RepositoyReleases."""

custom_components/hacs/system_health.py

@@ -7,6 +7,7 @@
 from .const import DOMAIN
 
 GITHUB_STATUS = "https://www.githubstatus.com/"
+CLOUDFLARE_STATUS = "https://www.cloudflarestatus.com/"
 
 
 @callback
@@ -29,6 +30,9 @@ async def system_health_info(hass):
  "GitHub Web": system_health.async_check_can_reach_url(
  hass, "https://github.com/", GITHUB_STATUS
  ),
+ "HACS Data": system_health.async_check_can_reach_url(
+ hass, "https://data-v2.hacs.xyz/data.json", CLOUDFLARE_STATUS
+ ),
  "GitHub API Calls Remaining": response.data.resources.core.remaining,
  "Installed Version": hacs.version,
  "Stage": hacs.stage,

custom_components/hacs/utils/data.py

@@ -15,26 +15,27 @@
 from .path import is_safe
 from .store import async_load_from_store, async_save_to_store
 
-DEFAULT_BASE_REPOSITORY_DATA = (
+EXPORTED_BASE_DATA = (("new", False),)
+
+EXPORTED_REPOSITORY_DATA = EXPORTED_BASE_DATA + (
  ("authors", []),
  ("category", ""),
  ("description", ""),
  ("domain", None),
  ("downloads", 0),
  ("etag_repository", None),
  ("full_name", ""),
- ("last_updated", 0),
  ("hide", False),
+ ("last_updated", 0),
  ("new", False),
  ("stargazers_count", 0),
  ("topics", []),
 )
 
-DEFAULT_EXTENDED_REPOSITORY_DATA = (
+EXPORTED_DOWNLOADED_REPOSITORY_DATA = EXPORTED_REPOSITORY_DATA + (
  ("archived", False),
  ("config_flow", False),
  ("default_branch", None),
- ("description", ""),
  ("first_install", False),
  ("installed_commit", None),
  ("installed", False),
@@ -46,8 +47,6 @@
  ("releases", False),
  ("selected_tag", None),
  ("show_beta", False),
- ("stargazers_count", 0),
- ("topics", []),
 )
 
 
@@ -100,16 +99,16 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict:
  """Store the repository data."""
  data = {"repository_manifest": repository.repository_manifest.manifest}
 
- for key, default_value in DEFAULT_BASE_REPOSITORY_DATA:
- if (value := repository.data.__getattribute__(key)) != default_value:
+ for key, default in (
+ EXPORTED_DOWNLOADED_REPOSITORY_DATA
+ if repository.data.installed
+ else EXPORTED_REPOSITORY_DATA
+ ):
+ if (value := getattr(repository.data, key, default)) != default:
  data[key] = value
 
- if repository.data.installed:
- for key, default_value in DEFAULT_EXTENDED_REPOSITORY_DATA:
- if (value := repository.data.__getattribute__(key)) != default_value:
- data[key] = value
+ if repository.data.installed_version:
  data["version_installed"] = repository.data.installed_version
-
  if repository.data.last_fetched:
  data["last_fetched"] = repository.data.last_fetched.timestamp()
 
@@ -137,6 +136,8 @@ async def restore(self):
  if not hacs and not repositories:
  # Assume new install
  self.hacs.status.new = True
+ if self.hacs.configuration.experimental:
+ return True
  self.logger.info("<HacsData restore> Loading base repository information")
  repositories = await self.hacs.hass.async_add_executor_job(
  json_util.load_json,
@@ -207,9 +208,10 @@ async def register_unknown_repositories(self, repositories, category: str | None
  @callback
  def async_restore_repository(self, entry, repository_data):
  """Restore repository."""
- full_name = repository_data["full_name"]
- if not (repository := self.hacs.repositories.get_by_full_name(full_name)):
- self.logger.error("<HacsData restore> Did not find %s (%s)", full_name, entry)
+ full_name = repository_data.get("full_name")
+ if full_name is None or not (
+ repository := self.hacs.repositories.get_by_full_name(full_name)
+ ):
  return
  # Restore repository attributes
  self.hacs.repositories.set_repository_id(repository, entry)