Check .shtml files for noindex directives

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased] - 2022-06-25
+## [Unreleased] - 2022-07-25
 
 ### Added
 
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 
 ### Fixed
+* Checks .shtml files for noindex directives, excluding those that do from the sitemap.
 
 ### CI/CD
 

generatesitemap.py

@@ -113,7 +113,7 @@ def getFileExtension(f) :
     i = f.rfind(".")
     return f[i+1:].lower() if i >= 0 and f.rfind("/") < i else None
 
-HTML_EXTENSIONS = { "html", "htm" }
+HTML_EXTENSIONS = { "html", "htm", "shtml" }
 
 def isHTMLFile(f) :
     """Checks if the file is an HTML file,
@@ -233,12 +233,7 @@ def urlstring(f, baseUrl, dropExtension=False) :
         u = f[1:]
     else :
         u = f
-    if len(u) >= 11 and u[-11:] == "/index.html" :
-        u = u[:-10]
-    elif u == "index.html" :
-        u = ""
-    elif dropExtension and len(u) >= 5 and u[-5:] == ".html" :
-        u = u[:-5]
+    u = sortname(u, dropExtension)
     if len(u) >= 1 and u[0]=="/" and len(baseUrl) >= 1 and baseUrl[-1]=="/" :
         u = u[1:]
     elif (len(u)==0 or u[0]!="/") and (len(baseUrl)==0 or baseUrl[-1]!="/") :

tests/tests.py

@@ -46,22 +46,22 @@ def validateDate(s) :
 class TestGenerateSitemap(unittest.TestCase) :
 
     def test_createExtensionSet_htmlOnly(self):
-        self.assertEqual({"html", "htm"}, gs.createExtensionSet(True, False, set()))
+        self.assertEqual({"html", "htm", "shtml"}, gs.createExtensionSet(True, False, set()))
 
     def test_createExtensionSet_pdfOnly(self):
         self.assertEqual({"pdf"}, gs.createExtensionSet(False, True, set()))
 
     def test_createExtensionSet_htmlAndPdf(self):
-        self.assertEqual({"html", "htm", "pdf"}, gs.createExtensionSet(True, True, set()))
+        self.assertEqual({"html", "htm", "shtml", "pdf"}, gs.createExtensionSet(True, True, set()))
 
     def test_createExtensionSet_html_and_more(self):
-        self.assertEqual({"html", "htm", "abc"}, gs.createExtensionSet(True, False, {"abc"}))
+        self.assertEqual({"html", "htm", "shtml", "abc"}, gs.createExtensionSet(True, False, {"abc"}))
 
     def test_createExtensionSet_pdf_and_more(self):
         self.assertEqual({"pdf", "abc", "def"}, gs.createExtensionSet(False, True, {"abc", "def"}))
 
     def test_createExtensionSet_htmlAndPdf_and_more(self):
-        self.assertEqual({"html", "htm", "pdf", "abc"}, gs.createExtensionSet(True, True, {"abc"}))
+        self.assertEqual({"html", "htm", "shtml", "pdf", "abc"}, gs.createExtensionSet(True, True, {"abc"}))
 
     def test_createExtensionSet_only_additional(self):
         self.assertEqual({"abc", "def"}, gs.createExtensionSet(False, False, {"abc", "def"}))
@@ -133,7 +133,16 @@ def test_isHTMLFile(self) :
                           "b/a.html",
                           "b/a.htm",
                           "b/index.html",
-                          "b/index.htm"
+                          "b/index.htm",
+                          ".shtml",
+                          "a.shtml",
+                          "index.shtml",
+                          "/.shtml",
+                          "/a.shtml",
+                          "/index.shtml",
+                          "b/.shtml",
+                          "b/a.shtml",
+                          "b/index.shtml"
                           ]
         nonHtmlFilenames = [ ".0html",
                           ".0htm",