Implement directory listing

.gitmodules

@@ -1,3 +1,7 @@
 [submodule "vendor/gtest"]
 	path = vendor/gtest
 	url = https://github.com/google/googletest.git
+
+[submodule "vendor/tinyxml2"]
+	path = vendor/tinyxml2
+	url = https://github.com/leethomason/tinyxml2.git

CMakeLists.txt

@@ -47,12 +47,19 @@ if(NOT APPLE)
   SET( CMAKE_MODULE_LINKER_FLAGS "-Wl,--no-undefined")
 endif()
 
+if( NOT XROOTD_EXTERNAL_TINYXML2 )
+	set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+	add_subdirectory(vendor/tinyxml2)
+else()
+	find_package(tinyxml2::tinyxml2)
+endif()
+
 include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS})
 
-add_library(XrdS3 SHARED src/S3File.cc src/S3AccessInfo.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdS3 SHARED src/S3File.cc src/S3Directory.cc src/S3AccessInfo.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 
-target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})
+target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} tinyxml2::tinyxml2)
 target_link_libraries(XrdHTTPServer -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})
 
 # The CMake documentation strongly advises against using these macros; instead, the pkg_check_modules

src/HTTPCommands.cc

@@ -168,6 +168,8 @@ bool HTTPRequest::sendPreparedRequest(const std::string &protocol,
 									  const std::string &uri,
 									  const std::string &payload) {
 
+	m_log.Log(XrdHTTPServer::Debug, "SendRequest", "Sending HTTP request",
+			  uri.c_str());
 	CURLcode rv = curl_global_init(CURL_GLOBAL_ALL);
 	if (rv != 0) {
 		this->errorCode = "E_CURL_LIB";
@@ -317,20 +319,6 @@ bool HTTPRequest::sendPreparedRequest(const std::string &protocol,
 		CAFile = x509_ca_file;
 	}
 
-	if (CAPath.empty()) {
-		char *soap_ssl_ca_dir = getenv("GAHP_SSL_CADIR");
-		if (soap_ssl_ca_dir != NULL) {
-			CAPath = soap_ssl_ca_dir;
-		}
-	}
-
-	if (CAFile.empty()) {
-		char *soap_ssl_ca_file = getenv("GAHP_SSL_CAFILE");
-		if (soap_ssl_ca_file != NULL) {
-			CAFile = soap_ssl_ca_file;
-		}
-	}
-
 	if (!CAPath.empty()) {
 		SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_CAPATH, CAPath.c_str());
 	}

src/S3AccessInfo.cc

@@ -49,3 +49,9 @@ const std::string &S3AccessInfo::getS3SecretKeyFile() const {
 void S3AccessInfo::setS3SecretKeyFile(const std::string &s3SecretKeyFile) {
 	s3_secret_key_file = s3SecretKeyFile;
 }
+
+const std::string &S3AccessInfo::getS3UrlStyle() const { return s3_url_style; }
+
+void S3AccessInfo::setS3UrlStyle(const std::string &s3UrlStyle) {
+	s3_url_style = s3UrlStyle;
+}

src/S3AccessInfo.hh

@@ -2,8 +2,7 @@
 // Created by Rich Wellner on 2/29/24.
 //
 
-#ifndef XROOTD_S3_HTTP_S3ACCESSINFO_HH
-#define XROOTD_S3_HTTP_S3ACCESSINFO_HH
+#pragma once
 
 #include <string>
 
@@ -33,13 +32,18 @@ class S3AccessInfo {
 
 	void setS3SecretKeyFile(const std::string &s3SecretKeyFile);
 
+	const std::string &getS3UrlStyle() const;
+
+	void setS3UrlStyle(const std::string &s3UrlStyle);
+
+	const int getS3SignatureVersion() const {return 4;}
+
   private:
 	std::string s3_bucket_name;
 	std::string s3_service_name;
 	std::string s3_region;
 	std::string s3_service_url;
 	std::string s3_access_key_file;
 	std::string s3_secret_key_file;
+	std::string s3_url_style;
 };
-
-#endif // XROOTD_S3_HTTP_S3ACCESSINFO_HH

src/S3Commands.cc

@@ -24,6 +24,7 @@
 #include <XrdSys/XrdSysError.hh>
 #include <curl/curl.h>
 #include <openssl/hmac.h>
+#include <tinyxml2.h>
 
 #include <algorithm>
 #include <cassert>
@@ -44,8 +45,6 @@
 	default:
 		this->errorCode = "E_INTERNAL";
 		this->errorMessage = "Invalid signature version.";
-		// dprintf( D_ALWAYS, "Invalid signature version (%d), failing.\n",
-		// signatureVersion );
 		return false;
 	}
 }
@@ -57,7 +56,8 @@
 // Takes in the configured `s3.service_url` and uses the bucket/object requested
 // to generate the host URL, as well as the canonical URI (which is the path to
 // the object).
-bool AmazonRequest::parseURL(const std::string &url, std::string &path) {
+bool AmazonRequest::parseURL(const std::string &url, std::string &bucket_path,
+							 std::string &path) {
 	auto i = url.find("://");
 	if (i == std::string::npos) {
 		return false;
@@ -71,20 +71,23 @@
 			// :// and the last /
 			host = substring(url, i + 3);
 			// Likewise, the path is going to be /bucket/object
 			// Sometimes we intentionally configure the plugin with no bucket because we
 			// assume the incoming object request already encodes the bucket. This is used
 			// for exporting many buckets from a single endpoint.
 			if (bucket.empty()) {
 				path = "/" + object;
+				bucket_path = "/" + object.substr(0, object.find('/'));
 			} else {
 				path = "/" + bucket + "/" + object;
+				bucket_path = bucket;
 			}
 		} else {
 			// In virtual-style requests, the host should be determined as
 			// everything between
 			// :// up until the last /, but with <bucket> appended to the front.
 			host = bucket + "." + substring(url, i + 3);
 			path = "/" + object;
+			bucket_path = "/";
 		}
 
 		return true;
@@ -136,6 +139,8 @@
 		}
 		trim(saKey);
 	} else {
+		canonicalQueryString = canonicalizeQueryString();
+
 		requiresSignature =
 			false;	 // If we don't create a signature, it must not be needed...
 		return true; // If there was no saKey, we need not generate a signature
@@ -176,14 +181,8 @@
 	canonicalURI = pathEncode(canonicalURI);
 
 	// The canonical query string is the alphabetically sorted list of
-	// URI-encoded parameter names '=' values, separated by '&'s.  That
-	// wouldn't be hard to do, but we don't need to, since we send
-	// everything in the POST body, instead.
-	std::string canonicalQueryString;
-
-	// This function doesn't (currently) support query parameters,
-	// but no current caller attempts to use them.
-	assert((httpVerb != "GET") || query_parameters.size() == 0);
+	// URI-encoded parameter names '=' values, separated by '&'s.
+	canonicalQueryString = canonicalizeQueryString();
 
 	// The canonical headers must include the Host header, so add that
 	// now if we don't have it.
@@ -209,7 +208,6 @@
 	if (!doSha256(payload, messageDigest, &mdLength)) {
 		this->errorCode = "E_INTERNAL";
 		this->errorMessage = "Unable to hash payload.";
-		// dprintf( D_ALWAYS, "Unable to hash payload, failing.\n" );
 		return false;
 	}
 	std::string payloadHash;
@@ -396,10 +394,6 @@
 		return false;
 	}
 
-	if (!sendContentSHA) {
-		// dprintf( D_FULLDEBUG, "Payload is '%s'\n", payload.c_str() );
-	}
-
 	std::string authorizationValue;
 	if (!createV4Signature(payload, authorizationValue, sendContentSHA)) {
 		if (this->errorCode.empty()) {
@@ -417,7 +411,12 @@
 		headers["Authorization"] = authorizationValue;
 	}
 
-	return sendPreparedRequest(protocol, hostUrl, payload);
+	// This operation is on the bucket itself; alter the URL
+	auto url = hostUrl;
+	if (!canonicalQueryString.empty()) {
+		url += "?" + canonicalQueryString;
+	}
+	return sendPreparedRequest(protocol, url, payload);
 }
 
 // It's stated in the API documentation that you can upload to any region
@@ -484,3 +483,94 @@
 }
 
 // ---------------------------------------------------------------------------
+
+bool AmazonS3List::SendRequest(const std::string &continuationToken,
+							   size_t max_keys) {
+	query_parameters["list-type"] = "2";
+	query_parameters["delimiter"] = "/";
+	query_parameters["prefix"] = urlquote(object);
+	if (!continuationToken.empty()) {
+		query_parameters["continuation-token"] = urlquote(continuationToken);
+	}
+	query_parameters["max-keys"] = std::to_string(max_keys);
+	httpVerb = "GET";
+
+	// Operation is on the bucket itself; alter the URL to remove the object
+	hostUrl = protocol + "://" + host + bucketPath;
+
+	return SendS3Request("");
+}
+
+bool AmazonS3List::Results(std::vector<S3ObjectInfo> &objInfo,
+						   std::vector<std::string> &commonPrefixes,
+						   std::string &ct, std::string &errMsg) {
+	tinyxml2::XMLDocument doc;
+	auto err = doc.Parse(resultString.c_str());
+	if (err != tinyxml2::XML_SUCCESS) {
+		errMsg = doc.ErrorStr();
+		return false;
+	}
+
+	auto elem = doc.RootElement();
+	if (strcmp(elem->Name(), "ListBucketResult")) {
+		errMsg = "S3 ListBucket response is not rooted with ListBucketResult "
+				 "element";
+		return false;
+	}
+
+	bool isTruncated = false;
+	for (auto child = elem->FirstChildElement(); child != nullptr;
+		 child = child->NextSiblingElement()) {
+		if (!strcmp(child->Name(), "IsTruncated")) {
+			bool isTrunc;
+			if (child->QueryBoolText(&isTrunc) == tinyxml2::XML_SUCCESS) {
+				isTruncated = true;
+			}
+		} else if (!strcmp(child->Name(), "CommonPrefixes")) {
+			auto prefix = child->FirstChildElement("Prefix");
+			if (prefix != nullptr) {
+				auto prefixChar = prefix->GetText();
+				if (prefixChar != nullptr) {
+					auto prefixStr = std::string(prefixChar);
+					trim(prefixStr);
+					if (!prefixStr.empty()) {
+						commonPrefixes.emplace_back(prefixStr);
+					}
+				}
+			}
+		} else if (!strcmp(child->Name(), "Contents")) {
+			std::string keyStr;
+			int64_t size;
+			bool goodSize = false;
+			auto key = child->FirstChildElement("Key");
+			if (key != nullptr) {
+				auto keyChar = key->GetText();
+				if (keyChar != nullptr) {
+					keyStr = std::string(keyChar);
+					trim(keyStr);
+				}
+			}
+			auto sizeElem = child->FirstChildElement("Size");
+			if (sizeElem != nullptr) {
+				goodSize =
+					(sizeElem->QueryInt64Text(&size) == tinyxml2::XML_SUCCESS);
+			}
+			if (goodSize && !keyStr.empty()) {
+				S3ObjectInfo obj;
+				obj.m_key = keyStr;
+				obj.m_size = size;
+				objInfo.emplace_back(obj);
+			}
+		} else if (!strcmp(child->Name(), "NextContinuationToken")) {
+			auto ctChar = child->GetText();
+			if (ctChar) {
+				ct = ctChar;
+				trim(ct);
+			}
+		}
+	}
+	if (!isTruncated) {
+		ct = "";
+	}
+	return true;
+}