From f697bdecd287ae22a6f73208abd3f02c3bd1b483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= Date: Sun, 15 Jan 2023 14:59:15 +0100 Subject: [PATCH] Limit stored data (#3007) * Limit stored data * Add initial "migration" --- custom_components/hacs/const.py | 2 + custom_components/hacs/utils/data.py | 87 ++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/custom_components/hacs/const.py b/custom_components/hacs/const.py index f4c285c690b..39491af9f13 100644 --- a/custom_components/hacs/const.py +++ b/custom_components/hacs/const.py @@ -17,6 +17,8 @@ DEFAULT_CONCURRENT_TASKS = 15 DEFAULT_CONCURRENT_BACKOFF_TIME = 1 +HACS_REPOSITORY_ID = "172733314" + HACS_ACTION_GITHUB_API_HEADERS = { "User-Agent": "HACS/action", "Accept": ACCEPT_HEADERS["preview"], diff --git a/custom_components/hacs/utils/data.py b/custom_components/hacs/utils/data.py index e480c5db380..067b71d19ad 100644 --- a/custom_components/hacs/utils/data.py +++ b/custom_components/hacs/utils/data.py @@ -3,19 +3,24 @@ import asyncio from datetime import datetime +from typing import Any from homeassistant.core import callback from homeassistant.exceptions import HomeAssistantError from homeassistant.util import json as json_util from ..base import HacsBase -from ..enums import HacsDisabledReason, HacsDispatchEvent, HacsGitHubRepo +from ..const import HACS_REPOSITORY_ID +from ..enums import HacsDisabledReason, HacsDispatchEvent from ..repositories.base import TOPIC_FILTER, HacsManifest, HacsRepository from .logger import LOGGER from .path import is_safe from .store import async_load_from_store, async_save_to_store -EXPORTED_BASE_DATA = (("new", False),) +EXPORTED_BASE_DATA = ( + ("new", False), + ("full_name", ""), +) EXPORTED_REPOSITORY_DATA = EXPORTED_BASE_DATA + ( ("authors", []), @@ -24,7 +29,6 @@ ("domain", None), ("downloads", 0), ("etag_repository", None), - ("full_name", ""), ("hide", False), ("last_updated", 0), ("new", False), @@ -80,6 +84,8 @@ async def async_write(self, force: bool = False) -> None: "ignored_repositories": self.hacs.common.ignored_repositories, }, ) + if self.hacs.configuration.experimental: + await self._async_store_experimental_content_and_repos() await self._async_store_content_and_repos() async def _async_store_content_and_repos(self, _=None): # bb: ignore @@ -94,6 +100,16 @@ async def _async_store_content_and_repos(self, _=None): # bb: ignore for event in (HacsDispatchEvent.REPOSITORY, HacsDispatchEvent.CONFIG): self.hacs.async_dispatch(event, {}) + async def _async_store_experimental_content_and_repos(self, _=None): # bb: ignore + """Store the main repos file and each repo that is out of date.""" + # Repositories + self.content = {} + for repository in self.hacs.repositories.list_all: + if repository.data.category in self.hacs.common.categories: + self.async_store_experimental_repository_data(repository) + + await async_save_to_store(self.hacs.hass, "data", {"repositories": self.content}) + @callback def async_store_repository_data(self, repository: HacsRepository) -> dict: """Store the repository data.""" @@ -114,20 +130,64 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict: self.content[str(repository.data.id)] = data + @callback + def async_store_experimental_repository_data(self, repository: HacsRepository) -> None: + """Store the experimental repository data for non downloaded repositories.""" + data = {} + self.content.setdefault(repository.data.category, []) + + if repository.data.installed: + data["repository_manifest"] = repository.repository_manifest.manifest + for key, default in EXPORTED_DOWNLOADED_REPOSITORY_DATA: + if (value := getattr(repository.data, key, default)) != default: + data[key] = value + + if repository.data.installed_version: + data["version_installed"] = repository.data.installed_version + if repository.data.last_fetched: + data["last_fetched"] = repository.data.last_fetched.timestamp() + else: + for key, default in EXPORTED_BASE_DATA: + if (value := getattr(repository.data, key, default)) != default: + data[key] = value + + self.content[repository.data.category].append({"id": str(repository.data.id), **data}) + async def restore(self): """Restore saved data.""" self.hacs.status.new = False + repositories = {} + hacs = {} + try: hacs = await async_load_from_store(self.hacs.hass, "hacs") or {} except HomeAssistantError: - hacs = {} + pass try: - repositories = await async_load_from_store(self.hacs.hass, "repositories") or {} + data = ( + await async_load_from_store( + self.hacs.hass, + "data" if self.hacs.configuration.experimental else "repositories", + ) + or {} + ) + if data and self.hacs.configuration.experimental: + for category, entries in data.get("repositories", {}).items(): + for repository in entries: + repositories[repository["id"]] = {"category": category, **repository} + else: + repositories = ( + data or await async_load_from_store(self.hacs.hass, "repositories") or {} + ) except HomeAssistantError as exception: self.hacs.log.error( "Could not read %s, restore the file from a backup - %s", - self.hacs.hass.config.path(".storage/hacs.repositories"), + self.hacs.hass.config.path( + ".storage/hacs.data" + if self.hacs.configuration.experimental + else ".storage/hacs.repositories" + ), exception, ) self.hacs.disable_hacs(HacsDisabledReason.RESTORE) @@ -206,13 +266,16 @@ async def register_unknown_repositories(self, repositories, category: str | None await asyncio.gather(*register_tasks) @callback - def async_restore_repository(self, entry, repository_data): + def async_restore_repository(self, entry: str, repository_data: dict[str, Any]): """Restore repository.""" - full_name = repository_data.get("full_name") - if full_name is None or not ( - repository := self.hacs.repositories.get_by_full_name(full_name) - ): + repository: HacsRepository | None = None + if full_name := repository_data.get("full_name"): + repository = self.hacs.repositories.get_by_full_name(full_name) + if not repository: + repository = self.hacs.repositories.get_by_id(entry) + if not repository: return + # Restore repository attributes self.hacs.repositories.set_repository_id(repository, entry) repository.data.authors = repository_data.get("authors", []) @@ -253,6 +316,6 @@ def async_restore_repository(self, entry, repository_data): if repository.data.installed: repository.data.first_install = False - if full_name == HacsGitHubRepo.INTEGRATION: + if entry == HACS_REPOSITORY_ID: repository.data.installed_version = self.hacs.version repository.data.installed = True