/vsicurl/: add a VSICURL_QUERY_STRING path specific option

This can for example be used when managing Shared Access Signatures (SAS) on application side, and not wanting to include the signature as part of the filename propagated through GDAL.
rouault · Dec 16, 2024 · 6043224 · 6043224
1 parent acdb4bd
commit 6043224
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 9 deletions.
diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py
@@ -1573,3 +1573,39 @@ def test_vsicurl_cache_control_no_cache(server):
         data = gdal.VSIFReadL(1, 6, f).decode("ascii")
         gdal.VSIFCloseL(f)
         assert data == "barbaz"
+
+
+###############################################################################
+# Test VSICURL_QUERY_STRING path specific option.
+
+
+@gdaltest.enable_exceptions()
+@pytest.mark.parametrize(
+    "filename,query_string",
+    [
+        ("test_vsicurl_VSICURL_QUERY_STRING.bin", "foo=bar"),
+        ("test_vsicurl_VSICURL_QUERY_STRING.bin?", "foo=bar"),
+        ("test_vsicurl_VSICURL_QUERY_STRING.bin", "?foo=bar"),
+        ("test_vsicurl_VSICURL_QUERY_STRING.bin?", "?foo=bar"),
+    ],
+)
+def test_vsicurl_VSICURL_QUERY_STRING(server, filename, query_string):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "HEAD",
+        "/test_vsicurl_VSICURL_QUERY_STRING.bin?foo=bar",
+        200,
+        {"Content-Length": "3"},
+    )
+
+    with webserver.install_http_handler(handler):
+        full_filename = f"/vsicurl/http://localhost:{server.port}/{filename}"
+        gdal.SetPathSpecificOption(full_filename, "VSICURL_QUERY_STRING", query_string)
+        try:
+            statres = gdal.VSIStatL(full_filename)
+            assert statres.size == 3
+        finally:
+            gdal.SetPathSpecificOption(full_filename, "VSICURL_QUERY_STRING", None)
diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst
@@ -382,7 +382,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following
 - retry_delay=number_in_seconds: default to 30. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_DELAY` configuration option.
 - retry_codes=``ALL`` or comma-separated list of HTTP error codes. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_CODES` configuration option. (GDAL >= 3.10)
 - list_dir=yes/no: whether an attempt to read the file list of the directory where the file is located should be done. Default to YES.
-- empty_dir=yes/no: whether to disable directory listing and disable logic in drivers to probe for individual side-car files. Default to NO. 
+- empty_dir=yes/no: whether to disable directory listing and disable logic in drivers to probe for individual side-car files. Default to NO.
 - useragent=value: HTTP UserAgent header
 - referer=value: HTTP Referer header
 - cookie=value: HTTP Cookie header
@@ -419,6 +419,11 @@ forwarded when redirections are followed.
 That behavior can be configured by setting the
 :config:`CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT` configuration option.
 
+Starting with GDAL 3.11, a query string can be appended to a given /vsicurl/ filename by taking its value from the
+``VSICURL_QUERY_STRING`` path-specific option set with :cpp:func:`VSISetPathSpecificOption`.
+This can for example be used when managing Shared Access Signatures (SAS) on application side, and not
+wanting to include the signature as part of the filename propagated through GDAL.
+
 Starting with GDAL 2.3, the :config:`GDAL_HTTP_MAX_RETRY` (number of attempts) and :config:`GDAL_HTTP_RETRY_DELAY` (in seconds) configuration option can be set, so that request retries are done in case of HTTP errors 429, 502, 503 or 504.
 
 Starting with GDAL 3.6, the following configuration options control the TCP keep-alive functionality (cf https://daniel.haxx.se/blog/2020/02/10/curl-ootw-keepalive-time/ for a detailed explanation):

diff --git a/port/cpl_known_config_options.h b/port/cpl_known_config_options.h
@@ -1031,6 +1031,7 @@ constexpr static const char* const apszKnownConfigOptions[] =
    "VSICURL_PC_SAS_SIGN_HREF_URL", // from cpl_vsil_curl.cpp
    "VSICURL_PC_SAS_TOKEN_URL", // from cpl_vsil_curl.cpp
    "VSICURL_PC_URL_SIGNING", // from cpl_vsil_curl.cpp
+   "VSICURL_QUERY_STRING", // from cpl_vsil_curl.cpp
    "VSIS3_COPYFILE_USE_STREAMING_SOURCE", // from cpl_vsil_s3.cpp
    "VSIS3_SIMULATE_THREADING", // from cpl_vsil_s3.cpp
    "VSIS3_SYNC_MULTITHREADING", // from cpl_vsil_s3.cpp

diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp
@@ -948,9 +948,6 @@ namespace cpl
 
 void VSICurlHandle::ManagePlanetaryComputerSigning() const
 {
-    if (!m_bPlanetaryComputerURLSigning)
-        return;
-
     // Take global lock
     static std::mutex goMutex;
     std::lock_guard<std::mutex> oLock(goMutex);
@@ -1066,6 +1063,43 @@ void VSICurlHandle::ManagePlanetaryComputerSigning() const
     }
 }
 
+/************************************************************************/
+/*                        UpdateQueryString()                           */
+/************************************************************************/
+
+void VSICurlHandle::UpdateQueryString() const
+{
+    if (m_bPlanetaryComputerURLSigning)
+    {
+        ManagePlanetaryComputerSigning();
+    }
+    else
+    {
+        const char *pszQueryString = VSIGetPathSpecificOption(
+            m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr);
+        if (pszQueryString)
+        {
+            if (m_osFilename.back() == '?')
+            {
+                if (pszQueryString[0] == '?')
+                    m_osQueryString = pszQueryString + 1;
+                else
+                    m_osQueryString = pszQueryString;
+            }
+            else
+            {
+                if (pszQueryString[0] == '?')
+                    m_osQueryString = pszQueryString;
+                else
+                {
+                    m_osQueryString = "?";
+                    m_osQueryString.append(pszQueryString);
+                }
+            }
+        }
+    }
+}
+
 /************************************************************************/
 /*                     GetFileSizeOrHeaders()                           */
 /************************************************************************/
@@ -1084,7 +1118,7 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError,
 
     CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL);
 
-    ManagePlanetaryComputerSigning();
+    UpdateQueryString();
 
     std::string osURL(m_pszURL + m_osQueryString);
     bool bRetryWithGet = false;
@@ -1857,7 +1891,7 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset,
 begin:
     CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL);
 
-    ManagePlanetaryComputerSigning();
+    UpdateQueryString();
 
     bool bHasExpired = false;
 
@@ -2432,7 +2466,7 @@ int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData,
                                                 panSizes);
     }
 
-    ManagePlanetaryComputerSigning();
+    UpdateQueryString();
 
     bool bHasExpired = false;
 
@@ -3098,7 +3132,7 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize,
     std::string osURL;
     {
         std::lock_guard<std::mutex> oLock(m_oMutex);
-        ManagePlanetaryComputerSigning();
+        UpdateQueryString();
         bool bHasExpired;
         osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions);
     }
@@ -3263,7 +3297,7 @@ void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
         nMaxSize += panSizes[i];
     }
 
-    ManagePlanetaryComputerSigning();
+    UpdateQueryString();
 
     bool bHasExpired = false;
     CPLStringList aosHTTPOptions(m_aosHTTPOptions);

diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h
@@ -406,6 +406,8 @@ class VSICurlHandle : public VSIVirtualHandle
     mutable std::string m_osPlanetaryComputerCollection{};
     void ManagePlanetaryComputerSigning() const;
 
+    void UpdateQueryString() const;
+
     int ReadMultiRangeSingleGet(int nRanges, void **ppData,
                                 const vsi_l_offset *panOffsets,
                                 const size_t *panSizes);