feat: add better support for additional mimetypes, especially *.odd a…

…nd *.rng (#384)

shttps/Connection.cpp

@@ -1475,7 +1475,7 @@ namespace shttps {
             throw Error(__file__, __LINE__, "File not readable:" + path);
         }
 
-        struct stat fstatbuf;
+        struct stat fstatbuf{};
 
         if (stat(path.c_str(), &fstatbuf) != 0) {
             throw Error(__file__, __LINE__, "Cannot fstat file: " + path);

shttps/LuaServer.cpp

@@ -2058,7 +2058,6 @@ namespace shttps {
 
         lua_pushboolean(L, true);
         lua_pushnil(L);
-
         return 2;
     }
     //=========================================================================

shttps/Parsing.cpp

@@ -78,8 +78,8 @@ namespace shttps {
                 {"xhtml",   {"application/xhtml+xml"}},
                 {"css",     {"text/css"}},
                 {"xml",     {"text/xml", "application/xml"}},
-                {"odd",     {"text/xml"}},
-                {"rng",     {"text/xml", "application/xml"}},
+                {"odd",     {"text/xml", "text/html"}},
+                {"rng",     {"text/xml", "application/xml", "text/html"}},
                 {"z",       {"application/x-compress"}},
                 {"tgz",     {"application/x-compress"}},
                 {"dvi",     {"application/x-dvi"}},

src/SipiHttpServer.cpp

@@ -1497,11 +1497,11 @@ namespace Sipi {
                     }
                 }
                 if (access(infile.c_str(), R_OK) == 0) {
-                    std::string actual_mimetype = shttps::Parsing::getFileMimetype(infile).first;
+                    std::string actual_mimetype = shttps::Parsing::getBestFileMimetype(infile);
                     //
                     // first we get the filesize and time using fstat
                     //
-                    struct stat fstatbuf;
+                    struct stat fstatbuf{};
 
                     if (stat(infile.c_str(), &fstatbuf) != 0) {
                         syslog(LOG_ERR, "Cannot fstat file %s ", infile.c_str());
@@ -1518,11 +1518,11 @@ namespace Sipi {
 
                     std::string range = conn_obj.header("range");
                     if (range.empty()) {
+                        // no "Content-Length" since send_file() will add this
                         conn_obj.header("Content-Type", actual_mimetype);
                         conn_obj.header("Cache-Control", "public, must-revalidate, max-age=0");
                         conn_obj.header("Pragma", "no-cache");
                         conn_obj.header("Accept-Ranges", "bytes");
-                        conn_obj.header("Content-Length", std::to_string(fsize));
                         conn_obj.header("Last-Modified", timebuf);
                         conn_obj.header("Content-Transfer-Encoding: binary");
                         conn_obj.sendFile(infile);
@@ -1548,13 +1548,12 @@ namespace Sipi {
                             throw Error(__file__, __LINE__, "Range expression invalid!");
                         }
 
+                        // no "Content-Length" since send_file() will add this
                         conn_obj.status(Connection::PARTIAL_CONTENT);
                         conn_obj.header("Content-Type", actual_mimetype);
                         conn_obj.header("Cache-Control", "public, must-revalidate, max-age=0");
                         conn_obj.header("Pragma", "no-cache");
                         conn_obj.header("Accept-Ranges", "bytes");
-                        conn_obj.header("Content-Length", std::to_string(end - start + 1));
-                        conn_obj.header("Content-Length", std::to_string(end - start + 1));
                         std::stringstream ss;
                         ss << "bytes " << start << "-" << end << "/" << fsize;
                         conn_obj.header("Content-Range", ss.str());

test/_test_data/config/sipi.init-knora.lua

@@ -22,10 +22,10 @@
 require "get_knora_session"
 
 function file_pre_flight(filepath, cookie)
-    if filepath == "./images/unit/test.csv" then
-        return "allow", filepath
-    else
+    if filepath == "./images/unit/test2.csv" then
         return "deny", ""
+    else
+        return "allow", filepath
     end
 end
 

test/_test_data/scripts/upload.lua

@@ -48,6 +48,7 @@ for imgindex,imgparam in pairs(server.uploads) do
     --
     -- first we check the mimetype consistency
     --
+    local success
     success, mime_ok = server.file_mimeconsistency(imgindex)
     if not success then
         server.log(newfilepath, server.loglevel.error)
@@ -71,7 +72,8 @@ for imgindex,imgparam in pairs(server.uploads) do
     --
     -- generate a UUID
     --
-    local success, uuid62 = server.uuid62()
+    local uuid62
+    success, uuid62 = server.uuid62()
     if not success then
         server.log(uuid62, server.loglevel.error)
         send_error(500, uuid62)
@@ -109,7 +111,7 @@ for imgindex,imgparam in pairs(server.uploads) do
         success, newfilepath = helper.filename_hash(newfilename[imgindex]);
         if not success then
             server.log(newfilepath, server.loglevel.error)
-            server.send_error(500, newfilepath)
+            send_error(500, newfilepath)
             return false
         end
 
@@ -132,19 +134,26 @@ for imgindex,imgparam in pairs(server.uploads) do
             server.print('Error converting image to j2k: ', filename, ' ** ', errmsg)
         end
     else
-        filename = imgparam["origname"]
+        filename = "_" .. imgparam["origname"]
         --
         -- here we add the subdirs that are necessary if Sipi is configured to use subdirs
         --
         success, newfilepath = helper.filename_hash(filename)
         if not success then
             server.log(newfilepath, server.loglevel.error)
-            server.send_error(500, newfilepath)
+            send_error(500, newfilepath)
             return false
         end
 
         fullfilepath = config.imgroot .. '/unit/' .. newfilepath
-                server.copyTmpfile(index, fullfilepath)
+        local errormsg
+        success, errormsg = server.copyTmpfile(imgindex, fullfilepath)
+        if not success then
+            server.log(errormsg, server.loglevel.error)
+            send_error(500,errormsg)
+            return false
+        end
+        iiifurls[" fullfilepath"] =  fullfilepath
 
         iiifurls[filename] = protocol .. server.host .. '/unit/' .. newfilepath
         iiifurls["filename"] = filename

test/e2e/conftest.py

@@ -526,7 +526,6 @@ def run_ab(self, concurrent_requests, total_requests, timeout, url_path):
 
     def cleanup(self):
         """Cleanup all files created"""
-        return
         fileList = glob.glob(self.data_dir + '/knora/_*')
         for filePath in fileList:
             os.remove(filePath)

test/e2e/test_02_server.py

@@ -18,7 +18,9 @@
 # See the GNU Affero General Public License for more details.
 # You should have received a copy of the GNU Affero General Public
 # License along with Sipi.  If not, see <http://www.gnu.org/licenses/>.
+import filecmp
 import pprint
+import shutil
 
 import pytest
 from pathlib import Path
@@ -119,7 +121,7 @@ def test_read_write(self, manager):
 
         assert response_json == expected_result
 
-    def test_jpg_with_comment(selfself, manager):
+    def test_jpg_with_comment(self, manager):
         """process an uploaded jpeg file with comment block properly"""
 
         response_json = manager.post_file(
@@ -140,6 +142,26 @@ def test_jpg_with_comment(selfself, manager):
 
         assert response_json == expected_result
 
+    def test_odd_file(self, manager):
+        """Up- and download a ODD file"""
+        response_json = manager.post_file(
+            "/api/upload", manager.data_dir_path("knora/test_odd.odd"), "text/xml")
+
+        filename = response_json["filename"]
+        response_json = manager.get_json("/unit/{}/knora.json".format(filename), False)
+        expected_result = {
+            '@context': 'http://sipi.io/api/file/3/context.json',
+            'id': 'http://127.0.0.1:1024/unit/_test_odd.odd',
+            'internalMimeType': 'text/xml',
+            'fileSize': 48002,
+            'originalFilename': ''
+        }
+        assert response_json == expected_result
+
+        downloaded_file_path = manager.download_file("/unit/{}/file".format(filename))
+        assert filecmp.cmp(manager.data_dir_path("knora/test_odd.odd"), downloaded_file_path)
+
+
     def test_mimeconsistency(self, manager):
         """upload any file and check mimetype consistency"""
 
@@ -195,8 +217,7 @@ def test_mimeconsistency(self, manager):
         ]
 
         for test in testdata:
-            response_json = manager.post_file(
-                "/api/mimetest", manager.data_dir_path(test["filepath"]), test["mimetype"])
+            response_json = manager.post_file("/api/mimetest", manager.data_dir_path(test["filepath"]), test["mimetype"])
             assert response_json == test["expected_result"]
 
     def test_thumbnail(self, manager):
@@ -245,8 +266,6 @@ def test_image_conversion(self, manager):
     def test_knora_info_validation(self, manager):
         """return a valid knora.json response"""
 
-        time.sleep(3)
-
         testdata = [
             {
                 "filepath": "unit/lena512.tif",
@@ -256,13 +275,14 @@ def test_knora_info_validation(self, manager):
                     "id": "http://127.0.0.1:1024/unit/",
                     "width": 512,
                     "height": 512,
-                    "originalFilename": "lena512.tif",
+                    "internalMimeType": "image/jp2",
                     "originalMimeType": "image/tiff",
-                    "internalMimeType": "image/jp2"
+                    "originalFilename": "lena512.tif"
                 }
             }, {
                 "filepath": "unit/test.csv",
                 "mimetype": "text/csv",
+                "sidecar": ["unit/test.info", "unit/_test.info"],
                 "expected_result": {
                     "@context": "http://sipi.io/api/file/3/context.json",
                     "id": "http://127.0.0.1:1024/unit/",
@@ -276,9 +296,14 @@ def test_knora_info_validation(self, manager):
         ]
 
         for test in testdata:
+            print("$$$$$==================================================")
+            if test.get("sidecar") is not None:
+                shutil.copyfile(manager.data_dir_path(test["sidecar"][0]), manager.data_dir_path(test["sidecar"][1]))
             response_json = manager.post_file(
                 "/api/upload", manager.data_dir_path(test["filepath"]), test["mimetype"])
+            print(response_json)
             filename = response_json["filename"]
+            print("filename=", filename)
             if test["mimetype"] == "image/tiff":
                 manager.expect_status_code(
                     "/unit/{}/full/max/0/default.jpg".format(filename), 200)