|
| 1 | +""" |
| 2 | +FileSizeEnrichmentRepository component for adding file size information to project pages. |
| 3 | +
|
| 4 | +This component wraps another repository and automatically enriches file metadata |
| 5 | +with size information by making HTTP HEAD requests to files that don't already |
| 6 | +have size information. |
| 7 | +""" |
| 8 | + |
| 9 | +import asyncio |
| 10 | +from dataclasses import replace |
| 11 | +import logging |
| 12 | +import typing |
| 13 | + |
| 14 | +import httpx |
| 15 | +from simple_repository import SimpleRepository, model |
| 16 | +from simple_repository.components.core import RepositoryContainer |
| 17 | + |
| 18 | +from ._typing_compat import override |
| 19 | + |
| 20 | +logger = logging.getLogger(__name__) |
| 21 | + |
| 22 | + |
| 23 | +class FileSizeEnrichmentRepository(RepositoryContainer): |
| 24 | + """ |
| 25 | + Repository component that enriches file metadata with size information. |
| 26 | +
|
| 27 | + This component automatically adds size information to files that don't already |
| 28 | + have it by making HTTP HEAD requests. It maintains parallelism for efficiency |
| 29 | + while respecting concurrency limits. |
| 30 | + """ |
| 31 | + |
| 32 | + def __init__( |
| 33 | + self, |
| 34 | + source: SimpleRepository, |
| 35 | + http_client: httpx.AsyncClient, |
| 36 | + *, |
| 37 | + max_concurrent_requests: int = 10, |
| 38 | + ) -> None: |
| 39 | + """ |
| 40 | + Initialize the FileSizeEnrichmentRepository. |
| 41 | +
|
| 42 | + Args: |
| 43 | + source: The underlying repository to wrap |
| 44 | + http_client: HTTP client for making HEAD requests |
| 45 | + max_concurrent_requests: Maximum number of concurrent HEAD requests |
| 46 | + """ |
| 47 | + super().__init__(source) |
| 48 | + self.http_client = http_client |
| 49 | + self.semaphore = asyncio.Semaphore(max_concurrent_requests) |
| 50 | + |
| 51 | + @override |
| 52 | + async def get_project_page( |
| 53 | + self, |
| 54 | + project_name: str, |
| 55 | + *, |
| 56 | + request_context: typing.Optional[model.RequestContext] = None, |
| 57 | + ) -> model.ProjectDetail: |
| 58 | + """ |
| 59 | + Get project page with file sizes enriched. |
| 60 | +
|
| 61 | + Files that don't have size information will have their sizes fetched |
| 62 | + via HTTP HEAD requests in parallel. |
| 63 | + """ |
| 64 | + project_page = await super().get_project_page( |
| 65 | + project_name, request_context=request_context |
| 66 | + ) |
| 67 | + |
| 68 | + # Identify files that need size information |
| 69 | + files_needing_size = [ |
| 70 | + file for file in project_page.files if not file.size and file.url |
| 71 | + ] |
| 72 | + |
| 73 | + if not files_needing_size: |
| 74 | + # No files need size information, return as-is |
| 75 | + return project_page |
| 76 | + |
| 77 | + # Fetch sizes for files that need them |
| 78 | + size_info = await self._fetch_file_sizes(files_needing_size) |
| 79 | + |
| 80 | + # Create new files with updated size information |
| 81 | + enriched_files = [] |
| 82 | + for file in project_page.files: |
| 83 | + if file.filename in size_info: |
| 84 | + file = replace(file, size=size_info[file.filename]) |
| 85 | + enriched_files.append(file) |
| 86 | + |
| 87 | + return replace(project_page, files=tuple(enriched_files)) |
| 88 | + |
| 89 | + async def _fetch_file_sizes( |
| 90 | + self, files: typing.List[model.File] |
| 91 | + ) -> typing.Dict[str, int]: |
| 92 | + """ |
| 93 | + Fetch file sizes for multiple files in parallel. |
| 94 | +
|
| 95 | + Args: |
| 96 | + files: List of files to fetch sizes for |
| 97 | +
|
| 98 | + Returns: |
| 99 | + Dictionary mapping filename to size in bytes |
| 100 | + """ |
| 101 | + |
| 102 | + async def fetch_single_file_size( |
| 103 | + file: model.File, |
| 104 | + ) -> typing.Tuple[str, typing.Optional[int]]: |
| 105 | + """Fetch size for a single file with semaphore protection.""" |
| 106 | + async with self.semaphore: |
| 107 | + try: |
| 108 | + logger.debug(f"Fetching size for {file.filename} from {file.url}") |
| 109 | + |
| 110 | + # Make HEAD request to get Content-Length |
| 111 | + response = await self.http_client.head( |
| 112 | + file.url, follow_redirects=True, headers={} |
| 113 | + ) |
| 114 | + response.raise_for_status() |
| 115 | + |
| 116 | + content_length = response.headers.get("Content-Length") |
| 117 | + if content_length: |
| 118 | + return file.filename, int(content_length) |
| 119 | + else: |
| 120 | + logger.warning(f"No Content-Length header for {file.filename}") |
| 121 | + return file.filename, None |
| 122 | + |
| 123 | + except Exception as e: |
| 124 | + logger.warning(f"Failed to get size for {file.filename}: {e}") |
| 125 | + return file.filename, None |
| 126 | + |
| 127 | + # Create tasks for all files |
| 128 | + tasks = [fetch_single_file_size(file) for file in files] |
| 129 | + |
| 130 | + # Wait for all tasks to complete |
| 131 | + results = await asyncio.gather(*tasks, return_exceptions=True) |
| 132 | + |
| 133 | + # Process results, filtering out failures |
| 134 | + size_info = {} |
| 135 | + for result in results: |
| 136 | + if isinstance(result, Exception): |
| 137 | + logger.warning(f"Exception occurred during size fetching: {result}") |
| 138 | + continue |
| 139 | + |
| 140 | + filename, size = result |
| 141 | + if size is not None: |
| 142 | + size_info[filename] = size |
| 143 | + |
| 144 | + return size_info |
0 commit comments