add internal parameters

adbar · Jun 2, 2023 · cf46917 · cf46917
1 parent f9b6e93
commit cf46917
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 2 deletions.
diff --git a/courlan/core.py b/courlan/core.py
@@ -207,6 +207,7 @@ def filter_links(
     lang: Optional[str] = None,
     rules: Optional[RobotFileParser] = None,
     external: bool = False,
+    strict: bool = False,
     with_nav: bool = True,
 ) -> Tuple[List[str], List[str]]:
     "Find links in a HTML document, filter them and add them to the data store."
@@ -216,7 +217,7 @@ def filter_links(
         base_url=base_url,
         external_bool=external,
         language=lang,
-        strict=False,
+        strict=strict,
         with_nav=with_nav,
     ):
         # sanity check

diff --git a/courlan/urlstore.py b/courlan/urlstore.py
@@ -237,8 +237,9 @@ def add_from_html(
             htmlstring=htmlstring,
             base_url=base_url,
             external=external,
-            lang=lang,
+            lang=lang or self.language,
             rules=rules,
+            strict=self.strict,
             with_nav=with_nav,
         )
         self.add_urls(urls=links, appendleft=links_priority)

diff --git a/tests/urlstore_tests.py b/tests/urlstore_tests.py
@@ -370,3 +370,10 @@ def test_from_html():
     todo = url_store.find_unvisited_urls(base_url)
     known_links = url_store.find_known_urls(base_url)
     assert "https://example.org/en/page2" not in todo and len(known_links) == 4
+
+    # strict + language as URL store parameters
+    url_store = UrlStore(strict=True, language="de")
+    base_url = "https://example.org"
+    htmlstring = '<html><body><a href="https://example.org/en/page2"/><a href="https://example.org/imprint.html"/></body></html>'
+    url_store.add_from_html(htmlstring, base_url)
+    assert not url_store.find_known_urls(base_url)