fix: account for special chars in normalization

courlan/clean.py

@@ -148,6 +148,14 @@ def decode_punycode(string: str) -> str:
  return ".".join(parts)
 
 
+def normalize_part(url_part: str) -> str:
+ """Normalize URLs parts (specifically path and fragment) while
+ accounting for certain characters."""
+ if not "%" in url_part and not "!" in url_part:
+ url_part = quote(url_part)
+ return url_part
+
+
 def normalize_url(
  parsed_url: Union[SplitResult, str],
  strict: bool = False,
@@ -166,12 +174,12 @@ def normalize_url(
  netloc = decode_punycode(netloc.lower())
  # path: https://github.com/saintamh/alcazar/blob/master/alcazar/utils/urls.py
  # leading /../'s in the path are removed
- newpath = quote(PATH2.sub("", PATH1.sub("/", parsed_url.path)))
+ newpath = normalize_part(PATH2.sub("", PATH1.sub("/", parsed_url.path)))
  # strip unwanted query elements
  newquery = clean_query(parsed_url, strict, language) or ""
  if newquery and newpath == "":
  newpath = "/"
  # fragment
- newfragment = "" if strict else quote(parsed_url.fragment)
+ newfragment = "" if strict else normalize_part(parsed_url.fragment)
  # rebuild
  return urlunsplit([scheme, netloc, newpath, newquery, newfragment])

tests/unit_tests.py

@@ -493,6 +493,17 @@ def test_normalization():
  assert normalize_url("http://xn--Mnchen-3ya.de") == "http://münchen.de"
  assert normalize_url("http://Mnchen-3ya.de") == "http://mnchen-3ya.de"
  assert normalize_url("http://xn--München.de") == "http://xn--münchen.de"
+ # account for particular characters
+ assert (
+ normalize_url(
+ "https://www.deutschlandfunknova.de/beitrag/nord--und-s%C3%BCdgaza-israels-armee-verk%C3%BCndet-teilung-des-gazastreifens"
+ )
+ == "https://www.deutschlandfunknova.de/beitrag/nord--und-s%C3%BCdgaza-israels-armee-verk%C3%BCndet-teilung-des-gazastreifens"
+ )
+ assert (
+ normalize_url("https://taz.de/Zukunft-des-49-Euro-Tickets/!5968518/")
+ == "https://taz.de/Zukunft-des-49-Euro-Tickets/!5968518/"
+ )
 
 
 def test_qelems():