diff --git a/archive_query_log/cli/captures.py b/archive_query_log/cli/captures.py index 4ac6cd68..306f3120 100644 --- a/archive_query_log/cli/captures.py +++ b/archive_query_log/cli/captures.py @@ -155,8 +155,7 @@ def fetch(config: Config) -> None: changed_sources = tqdm(changed_sources, total=num_changed_sources, desc="Fetching captures", unit="source") for source in changed_sources: - if "web.archive.org" in source.archive.cdx_api_url: - _add_captures(config, source) + _add_captures(config, source) else: echo("No new/changed sources.") @@ -166,7 +165,6 @@ def import_() -> None: pass - _CEPH_DIR = Path("/mnt/ceph/storage/") _DEFAULT_DATA_DIR = ( _CEPH_DIR / "data-in-progress/data-research/web-search/" @@ -200,4 +198,4 @@ def aql_22( check_memento=check_memento, search_provider=search_provider, search_provider_index=search_provider_index, - ) \ No newline at end of file + )