From 60432246f813a34ae07f9e7eb4e25bdc81e375c6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Dec 2024 20:02:37 +0100 Subject: [PATCH] /vsicurl/: add a VSICURL_QUERY_STRING path specific option This can for example be used when managing Shared Access Signatures (SAS) on application side, and not wanting to include the signature as part of the filename propagated through GDAL. --- autotest/gcore/vsicurl.py | 36 +++++++++++++++++ doc/source/user/virtual_file_systems.rst | 7 +++- port/cpl_known_config_options.h | 1 + port/cpl_vsil_curl.cpp | 50 ++++++++++++++++++++---- port/cpl_vsil_curl_class.h | 2 + 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 3e8c9df5cd0e..2bc0f2f2b4b4 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -1573,3 +1573,39 @@ def test_vsicurl_cache_control_no_cache(server): data = gdal.VSIFReadL(1, 6, f).decode("ascii") gdal.VSIFCloseL(f) assert data == "barbaz" + + +############################################################################### +# Test VSICURL_QUERY_STRING path specific option. + + +@gdaltest.enable_exceptions() +@pytest.mark.parametrize( + "filename,query_string", + [ + ("test_vsicurl_VSICURL_QUERY_STRING.bin", "foo=bar"), + ("test_vsicurl_VSICURL_QUERY_STRING.bin?", "foo=bar"), + ("test_vsicurl_VSICURL_QUERY_STRING.bin", "?foo=bar"), + ("test_vsicurl_VSICURL_QUERY_STRING.bin?", "?foo=bar"), + ], +) +def test_vsicurl_VSICURL_QUERY_STRING(server, filename, query_string): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add( + "HEAD", + "/test_vsicurl_VSICURL_QUERY_STRING.bin?foo=bar", + 200, + {"Content-Length": "3"}, + ) + + with webserver.install_http_handler(handler): + full_filename = f"/vsicurl/http://localhost:{server.port}/{filename}" + gdal.SetPathSpecificOption(full_filename, "VSICURL_QUERY_STRING", query_string) + try: + statres = gdal.VSIStatL(full_filename) + assert statres.size == 3 + finally: + gdal.SetPathSpecificOption(full_filename, "VSICURL_QUERY_STRING", None) diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst index 9ee74bbc1ed6..1b1eb0526dd2 100644 --- a/doc/source/user/virtual_file_systems.rst +++ b/doc/source/user/virtual_file_systems.rst @@ -382,7 +382,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following - retry_delay=number_in_seconds: default to 30. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_DELAY` configuration option. - retry_codes=``ALL`` or comma-separated list of HTTP error codes. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_CODES` configuration option. (GDAL >= 3.10) - list_dir=yes/no: whether an attempt to read the file list of the directory where the file is located should be done. Default to YES. -- empty_dir=yes/no: whether to disable directory listing and disable logic in drivers to probe for individual side-car files. Default to NO. +- empty_dir=yes/no: whether to disable directory listing and disable logic in drivers to probe for individual side-car files. Default to NO. - useragent=value: HTTP UserAgent header - referer=value: HTTP Referer header - cookie=value: HTTP Cookie header @@ -419,6 +419,11 @@ forwarded when redirections are followed. That behavior can be configured by setting the :config:`CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT` configuration option. +Starting with GDAL 3.11, a query string can be appended to a given /vsicurl/ filename by taking its value from the +``VSICURL_QUERY_STRING`` path-specific option set with :cpp:func:`VSISetPathSpecificOption`. +This can for example be used when managing Shared Access Signatures (SAS) on application side, and not +wanting to include the signature as part of the filename propagated through GDAL. + Starting with GDAL 2.3, the :config:`GDAL_HTTP_MAX_RETRY` (number of attempts) and :config:`GDAL_HTTP_RETRY_DELAY` (in seconds) configuration option can be set, so that request retries are done in case of HTTP errors 429, 502, 503 or 504. Starting with GDAL 3.6, the following configuration options control the TCP keep-alive functionality (cf https://daniel.haxx.se/blog/2020/02/10/curl-ootw-keepalive-time/ for a detailed explanation): diff --git a/port/cpl_known_config_options.h b/port/cpl_known_config_options.h index ddb077899615..3d57944c2ce5 100644 --- a/port/cpl_known_config_options.h +++ b/port/cpl_known_config_options.h @@ -1031,6 +1031,7 @@ constexpr static const char* const apszKnownConfigOptions[] = "VSICURL_PC_SAS_SIGN_HREF_URL", // from cpl_vsil_curl.cpp "VSICURL_PC_SAS_TOKEN_URL", // from cpl_vsil_curl.cpp "VSICURL_PC_URL_SIGNING", // from cpl_vsil_curl.cpp + "VSICURL_QUERY_STRING", // from cpl_vsil_curl.cpp "VSIS3_COPYFILE_USE_STREAMING_SOURCE", // from cpl_vsil_s3.cpp "VSIS3_SIMULATE_THREADING", // from cpl_vsil_s3.cpp "VSIS3_SYNC_MULTITHREADING", // from cpl_vsil_s3.cpp diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index 52166956b771..e259b4078aa0 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -948,9 +948,6 @@ namespace cpl void VSICurlHandle::ManagePlanetaryComputerSigning() const { - if (!m_bPlanetaryComputerURLSigning) - return; - // Take global lock static std::mutex goMutex; std::lock_guard oLock(goMutex); @@ -1066,6 +1063,43 @@ void VSICurlHandle::ManagePlanetaryComputerSigning() const } } +/************************************************************************/ +/* UpdateQueryString() */ +/************************************************************************/ + +void VSICurlHandle::UpdateQueryString() const +{ + if (m_bPlanetaryComputerURLSigning) + { + ManagePlanetaryComputerSigning(); + } + else + { + const char *pszQueryString = VSIGetPathSpecificOption( + m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr); + if (pszQueryString) + { + if (m_osFilename.back() == '?') + { + if (pszQueryString[0] == '?') + m_osQueryString = pszQueryString + 1; + else + m_osQueryString = pszQueryString; + } + else + { + if (pszQueryString[0] == '?') + m_osQueryString = pszQueryString; + else + { + m_osQueryString = "?"; + m_osQueryString.append(pszQueryString); + } + } + } + } +} + /************************************************************************/ /* GetFileSizeOrHeaders() */ /************************************************************************/ @@ -1084,7 +1118,7 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); - ManagePlanetaryComputerSigning(); + UpdateQueryString(); std::string osURL(m_pszURL + m_osQueryString); bool bRetryWithGet = false; @@ -1857,7 +1891,7 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, begin: CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); - ManagePlanetaryComputerSigning(); + UpdateQueryString(); bool bHasExpired = false; @@ -2432,7 +2466,7 @@ int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, panSizes); } - ManagePlanetaryComputerSigning(); + UpdateQueryString(); bool bHasExpired = false; @@ -3098,7 +3132,7 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, std::string osURL; { std::lock_guard oLock(m_oMutex); - ManagePlanetaryComputerSigning(); + UpdateQueryString(); bool bHasExpired; osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); } @@ -3263,7 +3297,7 @@ void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, nMaxSize += panSizes[i]; } - ManagePlanetaryComputerSigning(); + UpdateQueryString(); bool bHasExpired = false; CPLStringList aosHTTPOptions(m_aosHTTPOptions); diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h index 6b47050dd638..7b36e2f9adb9 100644 --- a/port/cpl_vsil_curl_class.h +++ b/port/cpl_vsil_curl_class.h @@ -406,6 +406,8 @@ class VSICurlHandle : public VSIVirtualHandle mutable std::string m_osPlanetaryComputerCollection{}; void ManagePlanetaryComputerSigning() const; + void UpdateQueryString() const; + int ReadMultiRangeSingleGet(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes);