Limit stored data (#3007)

* Limit stored data

* Add initial "migration"

custom_components/hacs/const.py

@@ -17,6 +17,8 @@
 DEFAULT_CONCURRENT_TASKS = 15
 DEFAULT_CONCURRENT_BACKOFF_TIME = 1
 
+HACS_REPOSITORY_ID = "172733314"
+
 HACS_ACTION_GITHUB_API_HEADERS = {
  "User-Agent": "HACS/action",
  "Accept": ACCEPT_HEADERS["preview"],

custom_components/hacs/utils/data.py

@@ -3,19 +3,24 @@
 
 import asyncio
 from datetime import datetime
+from typing import Any
 
 from homeassistant.core import callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.util import json as json_util
 
 from ..base import HacsBase
-from ..enums import HacsDisabledReason, HacsDispatchEvent, HacsGitHubRepo
+from ..const import HACS_REPOSITORY_ID
+from ..enums import HacsDisabledReason, HacsDispatchEvent
 from ..repositories.base import TOPIC_FILTER, HacsManifest, HacsRepository
 from .logger import LOGGER
 from .path import is_safe
 from .store import async_load_from_store, async_save_to_store
 
-EXPORTED_BASE_DATA = (("new", False),)
+EXPORTED_BASE_DATA = (
+ ("new", False),
+ ("full_name", ""),
+)
 
 EXPORTED_REPOSITORY_DATA = EXPORTED_BASE_DATA + (
  ("authors", []),
@@ -24,7 +29,6 @@
  ("domain", None),
  ("downloads", 0),
  ("etag_repository", None),
- ("full_name", ""),
  ("hide", False),
  ("last_updated", 0),
  ("new", False),
@@ -80,6 +84,8 @@ async def async_write(self, force: bool = False) -> None:
  "ignored_repositories": self.hacs.common.ignored_repositories,
  },
  )
+ if self.hacs.configuration.experimental:
+ await self._async_store_experimental_content_and_repos()
  await self._async_store_content_and_repos()
 
  async def _async_store_content_and_repos(self, _=None): # bb: ignore
@@ -94,6 +100,16 @@ async def _async_store_content_and_repos(self, _=None): # bb: ignore
  for event in (HacsDispatchEvent.REPOSITORY, HacsDispatchEvent.CONFIG):
  self.hacs.async_dispatch(event, {})
 
+ async def _async_store_experimental_content_and_repos(self, _=None): # bb: ignore
+ """Store the main repos file and each repo that is out of date."""
+ # Repositories
+ self.content = {}
+ for repository in self.hacs.repositories.list_all:
+ if repository.data.category in self.hacs.common.categories:
+ self.async_store_experimental_repository_data(repository)
+
+ await async_save_to_store(self.hacs.hass, "data", {"repositories": self.content})
+
  @callback
  def async_store_repository_data(self, repository: HacsRepository) -> dict:
  """Store the repository data."""
@@ -114,20 +130,64 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict:
 
  self.content[str(repository.data.id)] = data
 
+ @callback
+ def async_store_experimental_repository_data(self, repository: HacsRepository) -> None:
+ """Store the experimental repository data for non downloaded repositories."""
+ data = {}
+ self.content.setdefault(repository.data.category, [])
+
+ if repository.data.installed:
+ data["repository_manifest"] = repository.repository_manifest.manifest
+ for key, default in EXPORTED_DOWNLOADED_REPOSITORY_DATA:
+ if (value := getattr(repository.data, key, default)) != default:
+ data[key] = value
+
+ if repository.data.installed_version:
+ data["version_installed"] = repository.data.installed_version
+ if repository.data.last_fetched:
+ data["last_fetched"] = repository.data.last_fetched.timestamp()
+ else:
+ for key, default in EXPORTED_BASE_DATA:
+ if (value := getattr(repository.data, key, default)) != default:
+ data[key] = value
+
+ self.content[repository.data.category].append({"id": str(repository.data.id), **data})
+
  async def restore(self):
  """Restore saved data."""
  self.hacs.status.new = False
+ repositories = {}
+ hacs = {}
+
  try:
  hacs = await async_load_from_store(self.hacs.hass, "hacs") or {}
  except HomeAssistantError:
- hacs = {}
+ pass
 
  try:
- repositories = await async_load_from_store(self.hacs.hass, "repositories") or {}
+ data = (
+ await async_load_from_store(
+ self.hacs.hass,
+ "data" if self.hacs.configuration.experimental else "repositories",
+ )
+ or {}
+ )
+ if data and self.hacs.configuration.experimental:
+ for category, entries in data.get("repositories", {}).items():
+ for repository in entries:
+ repositories[repository["id"]] = {"category": category, **repository}
+ else:
+ repositories = (
+ data or await async_load_from_store(self.hacs.hass, "repositories") or {}
+ )
  except HomeAssistantError as exception:
  self.hacs.log.error(
  "Could not read %s, restore the file from a backup - %s",
- self.hacs.hass.config.path(".storage/hacs.repositories"),
+ self.hacs.hass.config.path(
+ ".storage/hacs.data"
+ if self.hacs.configuration.experimental
+ else ".storage/hacs.repositories"
+ ),
  exception,
  )
  self.hacs.disable_hacs(HacsDisabledReason.RESTORE)
@@ -206,13 +266,16 @@ async def register_unknown_repositories(self, repositories, category: str | None
  await asyncio.gather(*register_tasks)
 
  @callback
- def async_restore_repository(self, entry, repository_data):
+ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
  """Restore repository."""
- full_name = repository_data.get("full_name")
- if full_name is None or not (
- repository := self.hacs.repositories.get_by_full_name(full_name)
- ):
+ repository: HacsRepository | None = None
+ if full_name := repository_data.get("full_name"):
+ repository = self.hacs.repositories.get_by_full_name(full_name)
+ if not repository:
+ repository = self.hacs.repositories.get_by_id(entry)
+ if not repository:
  return
+
  # Restore repository attributes
  self.hacs.repositories.set_repository_id(repository, entry)
  repository.data.authors = repository_data.get("authors", [])
@@ -253,6 +316,6 @@ def async_restore_repository(self, entry, repository_data):
  if repository.data.installed:
  repository.data.first_install = False
 
- if full_name == HacsGitHubRepo.INTEGRATION:
+ if entry == HACS_REPOSITORY_ID:
  repository.data.installed_version = self.hacs.version
  repository.data.installed = True