diff --git a/INSTALL b/INSTALL index 29e8b96..59deacd 100644 --- a/INSTALL +++ b/INSTALL @@ -1,6 +1,7 @@ BUILDING ======== +* Read the README that applies to your OS * Build the required prerequisite libraries. * Build and install the AFF4 library: @@ -16,30 +17,3 @@ TESTING $ make check -BUILDING OS X -================ - -Install the following via macports -* zlib -* raptor2 -* google-glog -* pcrexx -* tclap (missing *.pc file - place in /opt/local/lib/pkgconfig/) - -Install the following manually as they aren't in macports -* snappy -* uuid - -$ cd ~/git -$ git clone ssh://bastion/srv/git/aff4.git -$ cd aff4 -$ git submodule update --init third_party/gtest - Ignore the error about tree reference. -$ cd third_party/gtest -$ git reset --hard -$ cd ../.. -$ ./autogen.sh -$ ./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib" -$ make - -For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local") \ No newline at end of file diff --git a/README.linux b/README.linux index 44190b0..be68823 100644 --- a/README.linux +++ b/README.linux @@ -82,7 +82,7 @@ make -j4 install cd .. apt-get source libspdlog-dev -cd spdlog-0.11.0/ +cd spdlog-0.17.0/ cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make all install cd .. diff --git a/README.macos b/README.macos new file mode 100644 index 0000000..4c3814f --- /dev/null +++ b/README.macos @@ -0,0 +1,29 @@ +BUILDING MacOS +================ + +Install the following via brew + +brew install snappy lz4 raptor pkgconfig tclap uriparser spdlog + +Then do the following + +cd ~/git +git clone https://github.com/gabime/spdlog.git +cd spdlog +git checkout v0.17.0 +cmake . && make all install +cd .. +git clone https://github.com/Velocidex/c-aff4.git +cd c-aff4 +git submodule update --init third_party/gtest + + Ignore the error about tree reference. + +cd third_party/gtest +git reset --hard +cd ../.. +./autogen.sh +./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib" +make + +For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local") \ No newline at end of file diff --git a/aff4/aff4_base.h b/aff4/aff4_base.h index 522a3fd..e9369ee 100644 --- a/aff4/aff4_base.h +++ b/aff4/aff4_base.h @@ -52,6 +52,32 @@ specific language governing permissions and limitations under the License. // Windows defines this macro which interfers with glog's version. #undef ERROR +#ifdef __APPLE__ + +#include +#include + +#define pread64 pread +#define O_LARGEFILE 0 +#include + +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) + +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) + +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) + +#endif + #include "aff4/lexicon.h" #include "aff4/aff4_errors.h" #include "aff4/rdf.h" diff --git a/aff4/aff4_file.cc b/aff4/aff4_file.cc index 9824d6b..db5f0e4 100644 --- a/aff4/aff4_file.cc +++ b/aff4/aff4_file.cc @@ -327,9 +327,8 @@ FileBackedObject::~FileBackedObject() { std::string mode, AFF4Flusher &result ) { - auto new_object = AFF4Flusher(new FileBackedObject(resolver)); + auto new_object = AFF4Flusher(new FileBackedObject(resolver, filename)); new_object->urn = URN::NewURNFromFilename(filename, false); - std::vector directory_components = split(filename, PATH_SEP); directory_components.pop_back(); diff --git a/aff4/aff4_file.h b/aff4/aff4_file.h index ba0f17f..3d4932b 100644 --- a/aff4/aff4_file.h +++ b/aff4/aff4_file.h @@ -47,7 +47,7 @@ class FileBackedObject: public AFF4Stream { // The filename for this object. std::string filename; - explicit FileBackedObject(DataStore* resolver): AFF4Stream(resolver), fd(0){} + explicit FileBackedObject(DataStore* resolver, std::string filename): AFF4Stream(resolver), filename(filename), fd(0){} virtual ~FileBackedObject(); AFF4Status ReadBuffer(char* data, size_t *length) override; diff --git a/aff4/aff4_image.cc b/aff4/aff4_image.cc index ae189a8..773deec 100644 --- a/aff4/aff4_image.cc +++ b/aff4/aff4_image.cc @@ -419,6 +419,7 @@ AFF4Status AFF4Image::OpenAFF4Image( resolver->logger->error( "ImageStream {} does not specify a size. " "Is this part of a split image set?", new_obj->urn); + return NOT_FOUND; } } else { diff --git a/aff4/aff4_imager_utils.cc b/aff4/aff4_imager_utils.cc index 6b2fb92..40037ae 100644 --- a/aff4/aff4_imager_utils.cc +++ b/aff4/aff4_imager_utils.cc @@ -248,7 +248,8 @@ AFF4Status BasicImager::handle_aff4_volumes() { AFF4Flusher(backing_stream.release()), volume)); - volume_objs.AddVolume(AFF4Flusher(volume.release())); + volume_objs.AddVolume(std::move(AFF4Flusher(volume.release()))); + volume_objs.AddSearchPath(volume_to_load); } } } diff --git a/aff4/aff4_map.cc b/aff4/aff4_map.cc index 6f2bd4e..506ef28 100644 --- a/aff4/aff4_map.cc +++ b/aff4/aff4_map.cc @@ -99,8 +99,20 @@ AFF4Status AFF4Map::OpenAFF4Map( line.erase(line.length()-1, 1); } AFF4Flusher target; - RETURN_IF_ERROR(volumes->GetStream(URN(line), target)); - + if((volumes->GetStream(URN(line), target)) != STATUS_OK){ + // Search in the repository for this target. + resolver->logger->info("Attempting to locate resource {} ", line); + if(resolver->HasURNWithAttribute(URN(line), AFF4_STORED)){ + // Get the ID of the container with this stream. + URN targetContainer; + RETURN_IF_ERROR(resolver->Get(URN(line), AFF4_STORED, targetContainer)); + RETURN_IF_ERROR(volumes->LocateAndAdd(targetContainer)); + RETURN_IF_ERROR(volumes->GetStream(URN(line), target)); + } else { + return NOT_FOUND; + } + } + resolver->logger->debug("MAP: Opened {} {} for target {}", target->urn, line, map_obj->targets.size()); diff --git a/aff4/aff4_utils.h b/aff4/aff4_utils.h index 57084cc..44a2a8e 100644 --- a/aff4/aff4_utils.h +++ b/aff4/aff4_utils.h @@ -11,7 +11,7 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ + */ #ifndef SRC_AFF4_UTILS_H_ #define SRC_AFF4_UTILS_H_ @@ -22,19 +22,22 @@ specific language governing permissions and limitations under the License. #include #include +#include +#include + #include "spdlog/spdlog.h" namespace aff4 { #define UNUSED(x) (void)x -std::string aff4_sprintf(std::string fmt, ...); + std::string aff4_sprintf(std::string fmt, ...); -std::string GetLastErrorMessage(); + std::string GetLastErrorMessage(); -std::vector split(const std::string& s, char delim); + std::vector split(const std::string& s, char delim); -std::shared_ptr get_logger(); + std::shared_ptr get_logger(); #define RETURN_IF_ERROR(expr) \ do { \ @@ -45,19 +48,62 @@ std::shared_ptr get_logger(); }; \ } while (0); -// A portable version of fnmatch. -int fnmatch(const char *pattern, const char *string); + // A portable version of fnmatch. + int fnmatch(const char *pattern, const char *string); -inline bool hasEnding(std::string const &fullString, std::string const &ending) { - if (fullString.length() >= ending.length()) { - return (0 == fullString.compare( + inline bool hasEnding(std::string const &fullString, std::string const &ending) { + if (fullString.length() >= ending.length()) { + return (0 == fullString.compare( fullString.length() - ending.length(), ending.length(), ending)); - } else { - return false; + } else { + return false; + } + } + + inline bool IsAFF4Container(std::string filename) noexcept { + // Cheap nasty not really unicode transformation to lower case. + std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower); + return hasEnding(filename, ".af4") || hasEnding(filename, ".aff4"); + } + + /** + * Does the file entity exist, and is a regular file. + * @param name The filename to check + * @return TRUE if the file entity exists and is a regular file. + */ + inline bool IsFile(const std::string& name) { +#ifndef _WIN32 + /* + * POSIX based systems. + */ + struct stat buffer; + if (::stat(name.c_str(), &buffer) == 0) { + return S_ISREG(buffer.st_mode); + } + return false; +#else + /* + * Windows based systems + */ + std::wstring filename = s2ws(name); + DWORD dwAttrib = GetFileAttributes(filename.c_str()); + return (dwAttrib != INVALID_FILE_ATTRIBUTES && !(dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); + +#endif + } + + inline std::wstring s2ws(const std::string& str) { + using convert_typeX = std::codecvt_utf8; + std::wstring_convert converterX; + return converterX.from_bytes(str); } -} + inline std::string ws2s(const std::wstring& wstr) { + using convert_typeX = std::codecvt_utf8; + std::wstring_convert converterX; + return converterX.to_bytes(wstr); + } } // namespace aff4 diff --git a/aff4/libaff4-c.cc b/aff4/libaff4-c.cc index 7f970bd..d341323 100644 --- a/aff4/libaff4-c.cc +++ b/aff4/libaff4-c.cc @@ -153,6 +153,7 @@ AFF4_Handle* AFF4_open(const char* filename, AFF4_Message** msg) { }; h->volumes.AddVolume(std::move(zip)); + h->volumes.AddSearchPath(filename); // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format. images = h->resolver.Query(aff4::AFF4_TYPE, &type); diff --git a/aff4/volume_group.cc b/aff4/volume_group.cc index dd937bb..af16c47 100644 --- a/aff4/volume_group.cc +++ b/aff4/volume_group.cc @@ -1,123 +1,304 @@ +#include + #include "aff4/volume_group.h" + +#include "aff4/aff4_errors.h" +#include "aff4/data_store.h" #include "aff4/aff4_image.h" #include "aff4/aff4_map.h" +#include "aff4/aff4_file.h" +#include "aff4/aff4_directory.h" #include "aff4/zip.h" #include "aff4/aff4_symstream.h" +#include +#include + +#ifndef _WIN32 +#include +#include +#endif namespace aff4 { + void VolumeGroup::AddVolume(AFF4Flusher &&volume) { + /* + * If the volume is a ZipFile, add the filename to the search path. + */ + AFF4Volume* vol = volume.get(); + if (ZipFile * v = dynamic_cast (vol)) { + AFF4Stream* stream = v->backing_stream.get(); + if (FileBackedObject * f = dynamic_cast (stream)) { + std::string filename = f->filename; + AddSearchPath(filename); + } + } + volume_objs.insert(std::make_pair(volume->urn, std::move(volume))); + } + void VolumeGroup::AddSearchPath(std::string path) { + // Convert to absolute. +#ifdef _WIN32 + char resolved[_PATH_MAX + 1]; + memset(&resolved, 0, _PATH_MAX + 1); + if (_fullpath((char*)&resolved, path.c_str(), _PATH_MAX) != NULL) { + path = std::string(resolved); + } +#else + char resolved[PATH_MAX + 1]; + memset(&resolved, 0, PATH_MAX + 1); + if (realpath(path.c_str(), (char*)&resolved) != NULL) { + path = std::string(resolved); + } +#endif + if (AFF4Directory::IsDirectory(path, /* must_exist= */ true)) { + searchPaths.insert(path); + } else { + // get the parent path. + path = path.substr(0, path.find_last_of("/\\")); + searchPaths.insert(path); + } + } + void VolumeGroup::RemoveSearchPath(std::string path) { + searchPaths.erase(path); + } -void VolumeGroup::AddVolume(AFF4Flusher &&volume) { - volume_objs.insert(std::make_pair(volume->urn, std::move(volume))); -} + // Construct the appropriate stream and return it. -// Construct the appropriate stream and return it. -AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &result) { - // Get all the type attrbutes of the URN. - std::vector> types; - if (STATUS_OK == resolver->Get(stream_urn, AFF4_TYPE, types)) { - for (auto &type : types) { - std::string type_str(type->SerializeToString()); - - if (type_str == AFF4_IMAGESTREAM_TYPE || - type_str == AFF4_LEGACY_IMAGESTREAM_TYPE) { - AFF4Flusher image_stream; - RETURN_IF_ERROR( - AFF4Image::OpenAFF4Image( - resolver, stream_urn, this, image_stream)); - - result.reset(image_stream.release()); - - resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); - return STATUS_OK; - } + AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &result) { + // Get all the type attrbutes of the URN. + std::vector> types; + if (STATUS_OK == resolver->Get(stream_urn, AFF4_TYPE, types)) { + for (auto &type : types) { + std::string type_str(type->SerializeToString()); + + if (type_str == AFF4_IMAGESTREAM_TYPE || + type_str == AFF4_LEGACY_IMAGESTREAM_TYPE) { + AFF4Flusher image_stream; + RETURN_IF_ERROR( + AFF4Image::OpenAFF4Image( + resolver, stream_urn, this, image_stream)); - // The AFF4 Standard specifies an "AFF4 Image" as an abstract - // container for image related properties. It is not actually a - // concrete stream but it refers to a storage stream using its - // aff4:dataStream property. - - // Note that to create such a stream, you can simply create a - // regular stream with NewAFF4Image or NewAFF4Map and then set - // the aff4:dataStream of a new object to a concerete Map or - // ImageStream. - if (type_str == AFF4_IMAGE_TYPE || - type_str == AFF4_DISK_IMAGE_TYPE || - type_str == AFF4_VOLUME_IMAGE_TYPE || - type_str == AFF4_MEMORY_IMAGE_TYPE || - type_str == AFF4_CONTIGUOUS_IMAGE_TYPE || - type_str == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { - URN delegate; - - if (STATUS_OK == resolver->Get(stream_urn, AFF4_DATASTREAM, delegate)) { - // TODO: This can get recursive. Protect against abuse. - return GetStream(delegate, result); + result.reset(image_stream.release()); + + resolver->logger->debug("Openning {} as type {}", + stream_urn, type_str); + return STATUS_OK; } - } - if (type_str == AFF4_MAP_TYPE) { - AFF4Flusher map_stream; - RETURN_IF_ERROR( - AFF4Map::OpenAFF4Map( - resolver, stream_urn, this, map_stream)); + // The AFF4 Standard specifies an "AFF4 Image" as an abstract + // container for image related properties. It is not actually a + // concrete stream but it refers to a storage stream using its + // aff4:dataStream property. - result.reset(map_stream.release()); - resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); + // Note that to create such a stream, you can simply create a + // regular stream with NewAFF4Image or NewAFF4Map and then set + // the aff4:dataStream of a new object to a concerete Map or + // ImageStream. + if (type_str == AFF4_IMAGE_TYPE || + type_str == AFF4_DISK_IMAGE_TYPE || + type_str == AFF4_VOLUME_IMAGE_TYPE || + type_str == AFF4_MEMORY_IMAGE_TYPE || + type_str == AFF4_CONTIGUOUS_IMAGE_TYPE || + type_str == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { + URN delegate; - return STATUS_OK; - } + if (STATUS_OK == resolver->Get(stream_urn, AFF4_DATASTREAM, delegate)) { + // TODO: This can get recursive. Protect against abuse. + return GetStream(delegate, result); + } + } + + if (type_str == AFF4_MAP_TYPE) { + AFF4Flusher map_stream; + RETURN_IF_ERROR( + AFF4Map::OpenAFF4Map( + resolver, stream_urn, this, map_stream)); + + result.reset(map_stream.release()); + resolver->logger->debug("Openning {} as type {}", + stream_urn, type_str); + + return STATUS_OK; + } + + // Zip segments are stored directly in each volume. We use + // the resolver to figure out which volume has each + // segment. + if (type_str == AFF4_ZIP_SEGMENT_TYPE || + type_str == AFF4_FILE_TYPE) { + URN owner; + RETURN_IF_ERROR(resolver->Get(stream_urn, AFF4_STORED, owner)); - // Zip segments are stored directly in each volume. We use - // the resolver to figure out which volume has each - // segment. - if (type_str == AFF4_ZIP_SEGMENT_TYPE || - type_str == AFF4_FILE_TYPE) { - URN owner; - RETURN_IF_ERROR(resolver->Get(stream_urn, AFF4_STORED, owner)); - - resolver->logger->debug("Openning {} as type {}", stream_urn, type_str); - auto it = volume_objs.find(owner); - if (it != volume_objs.end()) { - return (it->second->OpenMemberStream(stream_urn, result)); + resolver->logger->debug("Openning {} as type {}", stream_urn, type_str); + auto it = volume_objs.find(owner); + if (it != volume_objs.end()) { + return (it->second->OpenMemberStream(stream_urn, result)); + } } } } - } - // Handle symbolic streams now. - if (stream_urn == AFF4_IMAGESTREAM_ZERO) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0)); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_FF) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0xff)); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_UNKNOWN) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNKNOWN")); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_UNREADABLE) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNREADABLEDATA")); - return STATUS_OK; + // Handle symbolic streams now. + if (stream_urn == AFF4_IMAGESTREAM_ZERO) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0)); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_FF) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0xff)); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_UNKNOWN) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNKNOWN")); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_UNREADABLE) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNREADABLEDATA")); + return STATUS_OK; + } + + for (int i = 0; i < 256; i++) { + std::string urn = aff4_sprintf( + "%s%02X", AFF4_IMAGESTREAM_SYMBOLIC_PREFIX, i); + + if (stream_urn == urn) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, i)); + return STATUS_OK; + } + } + + return NOT_FOUND; } - for (int i = 0; i < 256; i++) { - std::string urn = aff4_sprintf( - "%s%02X", AFF4_IMAGESTREAM_SYMBOLIC_PREFIX, i); + AFF4Status VolumeGroup::LocateAndAdd(URN& urn) { + if (volume_objs.find(urn) != volume_objs.end()) { + // Already loaded.... + return STATUS_OK; + } + + // Check known volume URNs. + if (foundVolumes.find(urn) != foundVolumes.end()) { + + std::string volume_to_load = foundVolumes.find(urn)->second; + // load and return; + AFF4Flusher backing_stream; + RETURN_IF_ERROR(NewFileBackedObject( + resolver, volume_to_load, + "read", backing_stream)); + + AFF4Flusher volume; + RETURN_IF_ERROR(ZipFile::OpenZipFile( + resolver, + std::move(AFF4Flusher( + backing_stream.release())), + volume)); + + AddVolume(std::move(AFF4Flusher(volume.release()))); + return STATUS_OK; + } + + // Search for container. + resolver->logger->info("Searching for container {}", urn); + for (std::string path : searchPaths) { + /* + * Look for all AFF4 in this path, and check it's URN. + */ + ScanForAFF4Volumes(path); + } + // Check known volume URNs. + if (foundVolumes.find(urn) != foundVolumes.end()) { - if (stream_urn == urn) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, i)); + std::string volume_to_load = foundVolumes.find(urn)->second; + + resolver->logger->info("Searching for container {} = {}", urn, volume_to_load); + + // load and return; + AFF4Flusher backing_stream; + RETURN_IF_ERROR(NewFileBackedObject( + resolver, volume_to_load, + "read", backing_stream)); + + AFF4Flusher volume; + RETURN_IF_ERROR(ZipFile::OpenZipFile( + resolver, + std::move(AFF4Flusher( + backing_stream.release())), + volume)); + + AddVolume(std::move(AFF4Flusher(volume.release()))); return STATUS_OK; } + + return NOT_FOUND; } - return NOT_FOUND; + bool VolumeGroup::FoundVolumesContains(const std::string& filename) { + for (auto it = foundVolumes.begin(); it != foundVolumes.end(); it++) { + if (it->second.compare(filename) == 0) { + return true; + } + } + return false; } + void VolumeGroup::ScanForAFF4Volumes(const std::string& path) { + // It is expected that the map is LOCKED prior to this call. + if (path.empty()) { + return; + } + resolver->logger->info("Scanning path {}", path); +#ifdef _WIN32 + /* + * Windows based systems + */ + std::wstring wpath = s2ws(path); + + std::wstring pattern(wpath); + pattern.append(L"\\*"); + + WIN32_FIND_DATAW data; + HANDLE hFind; + if ((hFind = FindFirstFile(pattern.c_str(), &data)) != INVALID_HANDLE_VALUE) { + do { + std::wstring filename(data.cFileName); + if (filename.compare(L".") && filename.compare(L"..")) { + std::string absoluteFilename = path + "\\" + ws2s(filename); + if ((IsFile(absoluteFilename)) && (IsAFF4Container(ws2s(filename)))) { + if (!FoundVolumesContains(absoluteFilename)) { + // We don't have this file + std::string resID = aff4::ZipFile::GetResourceID(absoluteFilename, resolver->logger); + if (!resID.empty()) { + foundVolumes[resID] = absoluteFilename; + } + } + } + } + while (FindNextFile(hFind, &data) != 0); + FindClose(hFind); + } +#else + /* + * POSIX based systems. + */ + DIR* dirp = ::opendir(path.c_str()); + struct dirent * dp; + while ((dp = readdir(dirp)) != NULL) { + std::string filename(dp->d_name); + if (filename.compare(".") && filename.compare("..")) { + std::string absoluteFilename = path + "/" + filename; + if ((IsFile(absoluteFilename)) && (IsAFF4Container(filename))) { + if (!FoundVolumesContains(absoluteFilename)) { + // We don't have this file + std::string resID = aff4::ZipFile::GetResourceID(absoluteFilename, resolver->logger); + if (!resID.empty()) { + foundVolumes[resID] = absoluteFilename; + } + } + } + } + } + closedir(dirp); +#endif + } -} // namespace aff4 + } // namespace aff4 diff --git a/aff4/volume_group.h b/aff4/volume_group.h index 671c74b..6282188 100644 --- a/aff4/volume_group.h +++ b/aff4/volume_group.h @@ -2,17 +2,22 @@ #define AFF4_VOLUME_GROUP_H_ #include +#include + #include "aff4/aff4_io.h" #include "aff4/data_store.h" - - namespace aff4 { class VolumeGroup { protected: - std::unordered_map> volume_objs; - DataStore *resolver; + std::unordered_map> volume_objs; + DataStore *resolver; + + std::set searchPaths; + std::unordered_map foundVolumes; + + bool FoundVolumesContains(const std::string& filename); public: VolumeGroup(DataStore *resolver) : resolver(resolver) {} @@ -21,6 +26,13 @@ namespace aff4 { void AddVolume(AFF4Flusher &&volume); AFF4Status GetStream(URN segment_urn, AFF4Flusher &result); + + void AddSearchPath(std::string path); + void RemoveSearchPath(std::string path); + + AFF4Status LocateAndAdd(URN& urn); + void ScanForAFF4Volumes(const std::string& path); + }; } // namespace aff4 diff --git a/aff4/zip.cc b/aff4/zip.cc index 6458233..332088c 100644 --- a/aff4/zip.cc +++ b/aff4/zip.cc @@ -1035,6 +1035,160 @@ AFF4Status ZipFile::StreamAddMember(URN member_urn, AFF4Stream& stream, } return STATUS_OK; + } + + std::string ZipFile::GetResourceID(std::string& filename, std::shared_ptr logger) { + /* + * The container.description file is the first file in the container. + * So rather than opening the whole container, simply stream in the first + * part of the file, extract the Zip entry details, and read the contents of + * the first file. + * This is MUCH quicker than a full open/scan/read operation. + * If any of these fail, switch to a slow path execution. + */ + std::unique_ptr buffer(new uint8_t[4096]); + uint64_t segmentEntrySize = 0; + uint16_t filenameLength = 0; + std::string segmentName; +#ifndef _WIN32 + /* + * POSIX based systems. + */ + int fileHandle = ::open(filename.c_str(), O_RDONLY | O_LARGEFILE); + if (fileHandle == -1) { + // we failed, so return nothing. (error will be in errno). + logger->critical("Unable to open container {} = {}", filename, GetLastErrorMessage()); + return ""; + } + int read = ::pread64(fileHandle, buffer.get(), 4096, 0); + ::close(fileHandle); + if (read != 4096) { + logger->critical("Unable to read {} bytes from the container : {} = {}", 4096, filename, read); + return ""; + } +#else + /* + * Windows based systems + */ + // Note: DO NOT ADD OVERLAPPED ATTRIBUTE for opening the file. + std::wstring wpath = aff4::s2ws(filename); // Convert the filename to UTF-16/WString for Win32 API + HANDLE fileHandle = CreateFile(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, NULL); + if (fileHandle == INVALID_HANDLE_VALUE) { + logger->critical("Unable to open container {} = {}", filename, GetLastErrorMessage()); + return ""; + } + DWORD read = 0; + BOOL res = ReadFile(fileHandle, buffer.get(), 4096, &read, NULL); + ::CloseHandle(fileHandle); + if (res == FALSE || read != 4096) { + logger->critical("Unable to read {} bytes from the container : {} = {}", 4096, filename, read); + return ""; + } +#endif + + /* + * The buffer should now contain the first 512bytes of the container. + */ + aff4::ZipFileHeader* header = (aff4::ZipFileHeader*)(buffer.get()); + // Start sanity check. + if (le32toh(header->magic) != 0x4034b50) { + logger->critical("Invalid PKZIP Magic number : {} = {}", filename, header->magic); + return ""; + } + if (le16toh(header->compression_method) != 0) { + logger->info("Unexpected PKZIP compression method : {} = {}", filename, header->compression_method); + goto slowPath; + } + if (le16toh(header->file_name_length) != 0x15) { // 'container.description' + logger->info("Unexpected segment name length : {} = {} != 0x15", filename, header->file_name_length); + goto slowPath; + } + // Get the filename. + filenameLength = le16toh(header->file_name_length); + segmentName = std::string((char*) (buffer.get() + sizeof (aff4::ZipFileHeader)), filenameLength); + if (segmentName.compare("container.description") != 0) { + logger->info("Unexpected segment name : {} = {}", filename, segmentName); + goto slowPath; + } + + // At this point the segment name checks out, so let get the content. + if (le32toh(header->file_size) == -1) { + // Check for extra headers. + if (le16toh(header->extra_field_len) > 0) { + aff4::ZipExtraFieldHeader* extra = (aff4::ZipExtraFieldHeader*)(buffer.get() + sizeof (aff4::ZipFileHeader) + filenameLength); + if (le16toh(extra->header_id) == 1) { + uint64_t exHeaderOffset = sizeof(aff4::ZipExtraFieldHeader); + uint16_t dataSize = extra->data_size; + if((le32toh(header->file_size) == -1) && (dataSize >= 8)){ + uint64_t* off = (uint64_t*) ((uint8_t*)extra + exHeaderOffset); + segmentEntrySize = le64toh(*off); + exHeaderOffset += 8; + dataSize -=8; + } + if((le32toh(header->compress_size) == -1) && (dataSize >= 8)){ + if(segmentEntrySize == 0 || segmentEntrySize == (size_t)-1){ + uint64_t* off = (uint64_t*) ((uint8_t*)extra + exHeaderOffset); + segmentEntrySize = le64toh(*off); + } + exHeaderOffset += 8; + dataSize -=8; + } + } + } + if(segmentEntrySize == 0 || segmentEntrySize == (uint64_t)-1){ + // We need to scan for the ZipDataDescriptor64 structure... this should be in the first 512 bytes that we have read. + int dd64magic = 0x08074b50; + size_t dd64Size = sizeof (aff4::Zip64DataDescriptorHeader); + uint8_t* offset = buffer.get(); + aff4::Zip64DataDescriptorHeader* dd64 = (Zip64DataDescriptorHeader*)offset; + while (le32toh(dd64->magic) != dd64magic) { + offset++; + if (offset > (buffer.get() + 4096 - dd64Size)) { + break; + } + dd64 = (aff4::Zip64DataDescriptorHeader*)offset; + } + if (le32toh(dd64->magic) == dd64magic) { + segmentEntrySize = le64toh(dd64->file_size); + } + } + } else { + segmentEntrySize = le32toh(header->file_size); + } + if ((segmentEntrySize == 0) || // + (segmentEntrySize > (4096 - (sizeof (aff4::ZipFileHeader) - filenameLength - le16toh(header->extra_field_len))))) { + // Too large... + logger->critical("Segment appears too large for container.description? {} = {}", filename, segmentEntrySize); + goto slowPath; + } + // And get our data. + goto slowPath; + { + char* data = (char*) buffer.get() + sizeof (aff4::ZipFileHeader) + filenameLength + le16toh(header->extra_field_len); + std::string resource(data, segmentEntrySize); + return resource; + } + + /* + * Slow path, open as Zip file and read the container.description file. + */ + slowPath: + MemoryDataStore resolver; + aff4::AFF4Flusher file; + aff4::AFF4Flusher zip(new aff4::ZipFile(&resolver)); + aff4::AFF4Flusher stream; + if (aff4::STATUS_OK != aff4::NewFileBackedObject(&resolver, filename, "read", file)) { + return ""; + } + zip->backing_stream.swap(file); + zip->parse_cd(); + + // Open the 'container.description' file and read it's contents. + if (aff4::STATUS_OK != zip->OpenMemberStream(URN("container.description"), stream)) { + return ""; + } + std::string value = stream->Read(stream->Size()); + return value; } // Register ZipFile as an AFF4 object. diff --git a/aff4/zip.h b/aff4/zip.h index 889e29b..3e75b41 100644 --- a/aff4/zip.h +++ b/aff4/zip.h @@ -312,6 +312,10 @@ class ZipFile: public AFF4Volume { AFF4Flusher &&backing_stream, AFF4Flusher &result); + static std::string GetResourceID( + std::string& filename, + std::shared_ptr logger); + // Generic volume interface. AFF4Status CreateMemberStream( URN segment_urn,