From 79d4d3eb6e905e92738f4375407de1db5bc48de9 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Sat, 25 May 2019 15:41:35 +1000 Subject: [PATCH 1/8] Bug: Zip data descriptor was not written on some code paths. --- aff4/aff4_imager_utils.cc | 7 +++++++ aff4/zip.cc | 2 ++ 2 files changed, 9 insertions(+) diff --git a/aff4/aff4_imager_utils.cc b/aff4/aff4_imager_utils.cc index b2d9d7e..1f880db 100644 --- a/aff4/aff4_imager_utils.cc +++ b/aff4/aff4_imager_utils.cc @@ -643,6 +643,13 @@ AFF4Status BasicImager::handle_compression() { // We only allow a wild card in the last component. std::vector BasicImager::GlobFilename(std::string glob) const { std::vector result; + + // Devices do not glob. + if (glob.substr(0,2) == "\\\\") { + result.push_back(glob); + return result; + } + WIN32_FIND_DATA ffd; unsigned int found = glob.find_last_of("/\\"); std::string path = ""; diff --git a/aff4/zip.cc b/aff4/zip.cc index 3acf962..3dd04cf 100644 --- a/aff4/zip.cc +++ b/aff4/zip.cc @@ -737,6 +737,7 @@ AFF4Status ZipFileSegment::Flush() { RETURN_IF_ERROR(zip_info->WriteFileHeader(*backing_store)); RETURN_IF_ERROR(backing_store->Write(cdata)); + RETURN_IF_ERROR(zip_info->WriteDataDescriptor(*backing_store)); // Compression method not known - ignore and store uncompressed. } else { @@ -744,6 +745,7 @@ AFF4Status ZipFileSegment::Flush() { RETURN_IF_ERROR(zip_info->WriteFileHeader(*backing_store)); RETURN_IF_ERROR(backing_store->Write(buffer)); + RETURN_IF_ERROR(zip_info->WriteDataDescriptor(*backing_store)); } // Replace ourselves in the members map. From 9e057c87817e6a4abd780cb638bfc8dfcf235686 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Fri, 31 May 2019 22:42:42 +1000 Subject: [PATCH 2/8] Fix bug in map handling Fixes issue #92 --- aff4/aff4_map.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aff4/aff4_map.cc b/aff4/aff4_map.cc index 1969be6..46c524f 100644 --- a/aff4/aff4_map.cc +++ b/aff4/aff4_map.cc @@ -566,7 +566,7 @@ AFF4Status AFF4Map::WriteStream(AFF4Stream* source, ProgressContext* progress) { RETURN_IF_ERROR(last_target->WriteStream(source, progress)); // Add a single range to cover the bulk of the image. - AddRange(0, last_target->Size(), last_target->Size(), last_target); + AddRange(0, 0, last_target->Size(), last_target); return STATUS_OK; } From 987bcf525d6854398a45a80dbadd6e35b785f00e Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:32:45 +1000 Subject: [PATCH 3/8] Correct and refactor build instructions. --- INSTALL | 28 +--------------------------- README.linux | 2 +- README.macos | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 README.macos diff --git a/INSTALL b/INSTALL index 29e8b96..59deacd 100644 --- a/INSTALL +++ b/INSTALL @@ -1,6 +1,7 @@ BUILDING ======== +* Read the README that applies to your OS * Build the required prerequisite libraries. * Build and install the AFF4 library: @@ -16,30 +17,3 @@ TESTING $ make check -BUILDING OS X -================ - -Install the following via macports -* zlib -* raptor2 -* google-glog -* pcrexx -* tclap (missing *.pc file - place in /opt/local/lib/pkgconfig/) - -Install the following manually as they aren't in macports -* snappy -* uuid - -$ cd ~/git -$ git clone ssh://bastion/srv/git/aff4.git -$ cd aff4 -$ git submodule update --init third_party/gtest - Ignore the error about tree reference. -$ cd third_party/gtest -$ git reset --hard -$ cd ../.. -$ ./autogen.sh -$ ./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib" -$ make - -For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local") \ No newline at end of file diff --git a/README.linux b/README.linux index 44190b0..be68823 100644 --- a/README.linux +++ b/README.linux @@ -82,7 +82,7 @@ make -j4 install cd .. apt-get source libspdlog-dev -cd spdlog-0.11.0/ +cd spdlog-0.17.0/ cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make all install cd .. diff --git a/README.macos b/README.macos new file mode 100644 index 0000000..1d9b33e --- /dev/null +++ b/README.macos @@ -0,0 +1,42 @@ +brew install +brew install +brew install snappy lz4 raptor pkgconfig tclap uriparser spdlog + +BUILDING OS X +================ + +Install the following via macports +* zlib +* raptor2 +* google-glog +* pcrexx +* tclap (missing *.pc file - place in /opt/local/lib/pkgconfig/) +* yaml-cpp +* snappy + +Install the following manually as they aren't in macports + +* uuid + +Then do the following + +cd ~/git +git clone https://github.com/gabime/spdlog.git +cd spdlog +git checkout v0.17.0 +cmake . && make all install +cd .. +git clone https://github.com/Velocidex/c-aff4.git +cd c-aff4 +git submodule update --init third_party/gtest + + Ignore the error about tree reference. + +cd third_party/gtest +git reset --hard +cd ../.. +./autogen.sh +./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib" +make + +For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local") \ No newline at end of file From 4c4a4c54ffe4cc75b566b0e678e6df2bdbd78ce5 Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:38:35 +1000 Subject: [PATCH 4/8] MacOS specific defines and includes. --- aff4/aff4_base.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/aff4/aff4_base.h b/aff4/aff4_base.h index 522a3fd..e9369ee 100644 --- a/aff4/aff4_base.h +++ b/aff4/aff4_base.h @@ -52,6 +52,32 @@ specific language governing permissions and limitations under the License. // Windows defines this macro which interfers with glog's version. #undef ERROR +#ifdef __APPLE__ + +#include +#include + +#define pread64 pread +#define O_LARGEFILE 0 +#include + +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) + +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) + +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) + +#endif + #include "aff4/lexicon.h" #include "aff4/aff4_errors.h" #include "aff4/rdf.h" From b971e29508765544ffd2f532462f6cd83c6095a7 Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:40:43 +1000 Subject: [PATCH 5/8] Bugfix for sizeless ImageStream --- aff4/aff4_image.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/aff4/aff4_image.cc b/aff4/aff4_image.cc index 0c6dd16..d17fe5b 100644 --- a/aff4/aff4_image.cc +++ b/aff4/aff4_image.cc @@ -353,6 +353,7 @@ AFF4Status AFF4Image::OpenAFF4Image( resolver->logger->error( "ImageStream {} does not specify a size. " "Is this part of a split image set?", new_obj->urn); + return NOT_FOUND; } } else { From 4d37cdf280e1ddd4474f08a1ab271878ae5b1713 Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:42:17 +1000 Subject: [PATCH 6/8] Add fast Volume URN/ARN identification from Zip file by carving the start of file for "container.description" --- aff4/zip.cc | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++++ aff4/zip.h | 4 ++ 2 files changed, 158 insertions(+) diff --git a/aff4/zip.cc b/aff4/zip.cc index 3dd04cf..6cfe8a5 100644 --- a/aff4/zip.cc +++ b/aff4/zip.cc @@ -1009,6 +1009,160 @@ AFF4Status ZipFile::StreamAddMember(URN member_urn, AFF4Stream& stream, } return STATUS_OK; + } + + std::string ZipFile::GetResourceID(std::string& filename, std::shared_ptr logger) { + /* + * The container.description file is the first file in the container. + * So rather than opening the whole container, simply stream in the first + * part of the file, extract the Zip entry details, and read the contents of + * the first file. + * This is MUCH quicker than a full open/scan/read operation. + * If any of these fail, switch to a slow path execution. + */ + std::unique_ptr buffer(new uint8_t[4096]); + uint64_t segmentEntrySize = 0; + uint16_t filenameLength = 0; + std::string segmentName; +#ifndef _WIN32 + /* + * POSIX based systems. + */ + int fileHandle = ::open(filename.c_str(), O_RDONLY | O_LARGEFILE); + if (fileHandle == -1) { + // we failed, so return nothing. (error will be in errno). + logger->critical("Unable to open container {} = {}", filename, GetLastErrorMessage()); + return ""; + } + int read = ::pread64(fileHandle, buffer.get(), 4096, 0); + ::close(fileHandle); + if (read != 4096) { + logger->critical("Unable to read {} bytes from the container : {} = {}", 4096, filename, read); + return ""; + } +#else + /* + * Windows based systems + */ + // Note: DO NOT ADD OVERLAPPED ATTRIBUTE for opening the file. + std::wstring wpath = aff4::s2ws(filename); // Convert the filename to UTF-16/WString for Win32 API + HANDLE fileHandle = CreateFile(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, NULL); + if (fileHandle == INVALID_HANDLE_VALUE) { + logger->critical("Unable to open container {} = {}", filename, GetLastErrorMessage()); + return ""; + } + DWORD read = 0; + BOOL res = ReadFile(fileHandle, buffer.get(), 4096, &read, NULL); + ::CloseHandle(fileHandle); + if (res == FALSE || read != 4096) { + logger->critical("Unable to read {} bytes from the container : {} = {}", 4096, filename, read); + return ""; + } +#endif + + /* + * The buffer should now contain the first 512bytes of the container. + */ + aff4::ZipFileHeader* header = (aff4::ZipFileHeader*)(buffer.get()); + // Start sanity check. + if (le32toh(header->magic) != 0x4034b50) { + logger->critical("Invalid PKZIP Magic number : {} = {}", filename, header->magic); + return ""; + } + if (le16toh(header->compression_method) != 0) { + logger->info("Unexpected PKZIP compression method : {} = {}", filename, header->compression_method); + goto slowPath; + } + if (le16toh(header->file_name_length) != 0x15) { // 'container.description' + logger->info("Unexpected segment name length : {} = {} != 0x15", filename, header->file_name_length); + goto slowPath; + } + // Get the filename. + filenameLength = le16toh(header->file_name_length); + segmentName = std::string((char*) (buffer.get() + sizeof (aff4::ZipFileHeader)), filenameLength); + if (segmentName.compare("container.description") != 0) { + logger->info("Unexpected segment name : {} = {}", filename, segmentName); + goto slowPath; + } + + // At this point the segment name checks out, so let get the content. + if (le32toh(header->file_size) == -1) { + // Check for extra headers. + if (le16toh(header->extra_field_len) > 0) { + aff4::ZipExtraFieldHeader* extra = (aff4::ZipExtraFieldHeader*)(buffer.get() + sizeof (aff4::ZipFileHeader) + filenameLength); + if (le16toh(extra->header_id) == 1) { + uint64_t exHeaderOffset = sizeof(aff4::ZipExtraFieldHeader); + uint16_t dataSize = extra->data_size; + if((le32toh(header->file_size) == -1) && (dataSize >= 8)){ + uint64_t* off = (uint64_t*) ((uint8_t*)extra + exHeaderOffset); + segmentEntrySize = le64toh(*off); + exHeaderOffset += 8; + dataSize -=8; + } + if((le32toh(header->compress_size) == -1) && (dataSize >= 8)){ + if(segmentEntrySize == 0 || segmentEntrySize == (size_t)-1){ + uint64_t* off = (uint64_t*) ((uint8_t*)extra + exHeaderOffset); + segmentEntrySize = le64toh(*off); + } + exHeaderOffset += 8; + dataSize -=8; + } + } + } + if(segmentEntrySize == 0 || segmentEntrySize == (uint64_t)-1){ + // We need to scan for the ZipDataDescriptor64 structure... this should be in the first 512 bytes that we have read. + int dd64magic = 0x08074b50; + size_t dd64Size = sizeof (aff4::Zip64DataDescriptorHeader); + uint8_t* offset = buffer.get(); + aff4::Zip64DataDescriptorHeader* dd64 = (Zip64DataDescriptorHeader*)offset; + while (le32toh(dd64->magic) != dd64magic) { + offset++; + if (offset > (buffer.get() + 4096 - dd64Size)) { + break; + } + dd64 = (aff4::Zip64DataDescriptorHeader*)offset; + } + if (le32toh(dd64->magic) == dd64magic) { + segmentEntrySize = le64toh(dd64->file_size); + } + } + } else { + segmentEntrySize = le32toh(header->file_size); + } + if ((segmentEntrySize == 0) || // + (segmentEntrySize > (4096 - (sizeof (aff4::ZipFileHeader) - filenameLength - le16toh(header->extra_field_len))))) { + // Too large... + logger->critical("Segment appears too large for container.description? {} = {}", filename, segmentEntrySize); + goto slowPath; + } + // And get our data. + goto slowPath; + { + char* data = (char*) buffer.get() + sizeof (aff4::ZipFileHeader) + filenameLength + le16toh(header->extra_field_len); + std::string resource(data, segmentEntrySize); + return resource; + } + + /* + * Slow path, open as Zip file and read the container.description file. + */ + slowPath: + MemoryDataStore resolver; + aff4::AFF4Flusher file; + aff4::AFF4Flusher zip(new aff4::ZipFile(&resolver)); + aff4::AFF4Flusher stream; + if (aff4::STATUS_OK != aff4::NewFileBackedObject(&resolver, filename, "read", file)) { + return ""; + } + zip->backing_stream.swap(file); + zip->parse_cd(); + + // Open the 'container.description' file and read it's contents. + if (aff4::STATUS_OK != zip->OpenMemberStream(URN("container.description"), stream)) { + return ""; + } + std::string value = stream->Read(stream->Size()); + return value; } // Register ZipFile as an AFF4 object. diff --git a/aff4/zip.h b/aff4/zip.h index 889e29b..3e75b41 100644 --- a/aff4/zip.h +++ b/aff4/zip.h @@ -312,6 +312,10 @@ class ZipFile: public AFF4Volume { AFF4Flusher &&backing_stream, AFF4Flusher &result); + static std::string GetResourceID( + std::string& filename, + std::shared_ptr logger); + // Generic volume interface. AFF4Status CreateMemberStream( URN segment_urn, From 3851234e776f3c1c18f04d704acf93e0a2bd9bbc Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:44:04 +1000 Subject: [PATCH 7/8] Add multi-container autoload for striped image support --- aff4/aff4_file.cc | 3 +- aff4/aff4_file.h | 2 +- aff4/aff4_imager_utils.cc | 1 + aff4/aff4_map.cc | 18 +- aff4/aff4_utils.h | 72 ++++++-- aff4/libaff4-c.cc | 1 + aff4/volume_group.cc | 363 ++++++++++++++++++++++++++++---------- aff4/volume_group.h | 20 ++- 8 files changed, 366 insertions(+), 114 deletions(-) diff --git a/aff4/aff4_file.cc b/aff4/aff4_file.cc index 2f407ae..1647549 100644 --- a/aff4/aff4_file.cc +++ b/aff4/aff4_file.cc @@ -263,9 +263,8 @@ FileBackedObject::~FileBackedObject() { std::string mode, AFF4Flusher &result ) { - auto new_object = AFF4Flusher(new FileBackedObject(resolver)); + auto new_object = AFF4Flusher(new FileBackedObject(resolver, filename)); new_object->urn = URN::NewURNFromFilename(filename, false); - std::vector directory_components = split(filename, PATH_SEP); directory_components.pop_back(); diff --git a/aff4/aff4_file.h b/aff4/aff4_file.h index 2c6adb2..b4a210d 100644 --- a/aff4/aff4_file.h +++ b/aff4/aff4_file.h @@ -46,7 +46,7 @@ class FileBackedObject: public AFF4Stream { // The filename for this object. std::string filename; - explicit FileBackedObject(DataStore* resolver): AFF4Stream(resolver), fd(0){} + explicit FileBackedObject(DataStore* resolver, std::string filename): AFF4Stream(resolver), filename(filename), fd(0){} virtual ~FileBackedObject(); AFF4Status ReadBuffer(char* data, size_t *length) override; diff --git a/aff4/aff4_imager_utils.cc b/aff4/aff4_imager_utils.cc index 1f880db..1456fc2 100644 --- a/aff4/aff4_imager_utils.cc +++ b/aff4/aff4_imager_utils.cc @@ -252,6 +252,7 @@ AFF4Status BasicImager::handle_aff4_volumes() { volume_objs.AddVolume( std::move(AFF4Flusher(volume.release()))); + volume_objs.AddSearchPath(volume_to_load); } } } diff --git a/aff4/aff4_map.cc b/aff4/aff4_map.cc index 1969be6..c0f3d38 100644 --- a/aff4/aff4_map.cc +++ b/aff4/aff4_map.cc @@ -99,8 +99,20 @@ AFF4Status AFF4Map::OpenAFF4Map( line.erase(line.length()-1, 1); } AFF4Flusher target; - RETURN_IF_ERROR(volumes->GetStream(URN(line), target)); - + if((volumes->GetStream(URN(line), target)) != STATUS_OK){ + // Search in the repository for this target. + resolver->logger->info("Attempting to locate resource {} ", line); + if(resolver->HasURNWithAttribute(URN(line), AFF4_STORED)){ + // Get the ID of the container with this stream. + URN targetContainer; + RETURN_IF_ERROR(resolver->Get(URN(line), AFF4_STORED, targetContainer)); + RETURN_IF_ERROR(volumes->LocateAndAdd(targetContainer)); + RETURN_IF_ERROR(volumes->GetStream(URN(line), target)); + } else { + return NOT_FOUND; + } + } + resolver->logger->debug("MAP: Opened {} {} for target {}", target->urn, line, map_obj->targets.size()); @@ -566,7 +578,7 @@ AFF4Status AFF4Map::WriteStream(AFF4Stream* source, ProgressContext* progress) { RETURN_IF_ERROR(last_target->WriteStream(source, progress)); // Add a single range to cover the bulk of the image. - AddRange(0, last_target->Size(), last_target->Size(), last_target); + AddRange(0, 0, last_target->Size(), last_target); return STATUS_OK; } diff --git a/aff4/aff4_utils.h b/aff4/aff4_utils.h index 57084cc..44a2a8e 100644 --- a/aff4/aff4_utils.h +++ b/aff4/aff4_utils.h @@ -11,7 +11,7 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ + */ #ifndef SRC_AFF4_UTILS_H_ #define SRC_AFF4_UTILS_H_ @@ -22,19 +22,22 @@ specific language governing permissions and limitations under the License. #include #include +#include +#include + #include "spdlog/spdlog.h" namespace aff4 { #define UNUSED(x) (void)x -std::string aff4_sprintf(std::string fmt, ...); + std::string aff4_sprintf(std::string fmt, ...); -std::string GetLastErrorMessage(); + std::string GetLastErrorMessage(); -std::vector split(const std::string& s, char delim); + std::vector split(const std::string& s, char delim); -std::shared_ptr get_logger(); + std::shared_ptr get_logger(); #define RETURN_IF_ERROR(expr) \ do { \ @@ -45,19 +48,62 @@ std::shared_ptr get_logger(); }; \ } while (0); -// A portable version of fnmatch. -int fnmatch(const char *pattern, const char *string); + // A portable version of fnmatch. + int fnmatch(const char *pattern, const char *string); -inline bool hasEnding(std::string const &fullString, std::string const &ending) { - if (fullString.length() >= ending.length()) { - return (0 == fullString.compare( + inline bool hasEnding(std::string const &fullString, std::string const &ending) { + if (fullString.length() >= ending.length()) { + return (0 == fullString.compare( fullString.length() - ending.length(), ending.length(), ending)); - } else { - return false; + } else { + return false; + } + } + + inline bool IsAFF4Container(std::string filename) noexcept { + // Cheap nasty not really unicode transformation to lower case. + std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower); + return hasEnding(filename, ".af4") || hasEnding(filename, ".aff4"); + } + + /** + * Does the file entity exist, and is a regular file. + * @param name The filename to check + * @return TRUE if the file entity exists and is a regular file. + */ + inline bool IsFile(const std::string& name) { +#ifndef _WIN32 + /* + * POSIX based systems. + */ + struct stat buffer; + if (::stat(name.c_str(), &buffer) == 0) { + return S_ISREG(buffer.st_mode); + } + return false; +#else + /* + * Windows based systems + */ + std::wstring filename = s2ws(name); + DWORD dwAttrib = GetFileAttributes(filename.c_str()); + return (dwAttrib != INVALID_FILE_ATTRIBUTES && !(dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); + +#endif + } + + inline std::wstring s2ws(const std::string& str) { + using convert_typeX = std::codecvt_utf8; + std::wstring_convert converterX; + return converterX.from_bytes(str); } -} + inline std::string ws2s(const std::wstring& wstr) { + using convert_typeX = std::codecvt_utf8; + std::wstring_convert converterX; + return converterX.to_bytes(wstr); + } } // namespace aff4 diff --git a/aff4/libaff4-c.cc b/aff4/libaff4-c.cc index 7f970bd..d341323 100644 --- a/aff4/libaff4-c.cc +++ b/aff4/libaff4-c.cc @@ -153,6 +153,7 @@ AFF4_Handle* AFF4_open(const char* filename, AFF4_Message** msg) { }; h->volumes.AddVolume(std::move(zip)); + h->volumes.AddSearchPath(filename); // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format. images = h->resolver.Query(aff4::AFF4_TYPE, &type); diff --git a/aff4/volume_group.cc b/aff4/volume_group.cc index dd937bb..af16c47 100644 --- a/aff4/volume_group.cc +++ b/aff4/volume_group.cc @@ -1,123 +1,304 @@ +#include + #include "aff4/volume_group.h" + +#include "aff4/aff4_errors.h" +#include "aff4/data_store.h" #include "aff4/aff4_image.h" #include "aff4/aff4_map.h" +#include "aff4/aff4_file.h" +#include "aff4/aff4_directory.h" #include "aff4/zip.h" #include "aff4/aff4_symstream.h" +#include +#include + +#ifndef _WIN32 +#include +#include +#endif namespace aff4 { + void VolumeGroup::AddVolume(AFF4Flusher &&volume) { + /* + * If the volume is a ZipFile, add the filename to the search path. + */ + AFF4Volume* vol = volume.get(); + if (ZipFile * v = dynamic_cast (vol)) { + AFF4Stream* stream = v->backing_stream.get(); + if (FileBackedObject * f = dynamic_cast (stream)) { + std::string filename = f->filename; + AddSearchPath(filename); + } + } + volume_objs.insert(std::make_pair(volume->urn, std::move(volume))); + } + void VolumeGroup::AddSearchPath(std::string path) { + // Convert to absolute. +#ifdef _WIN32 + char resolved[_PATH_MAX + 1]; + memset(&resolved, 0, _PATH_MAX + 1); + if (_fullpath((char*)&resolved, path.c_str(), _PATH_MAX) != NULL) { + path = std::string(resolved); + } +#else + char resolved[PATH_MAX + 1]; + memset(&resolved, 0, PATH_MAX + 1); + if (realpath(path.c_str(), (char*)&resolved) != NULL) { + path = std::string(resolved); + } +#endif + if (AFF4Directory::IsDirectory(path, /* must_exist= */ true)) { + searchPaths.insert(path); + } else { + // get the parent path. + path = path.substr(0, path.find_last_of("/\\")); + searchPaths.insert(path); + } + } + void VolumeGroup::RemoveSearchPath(std::string path) { + searchPaths.erase(path); + } -void VolumeGroup::AddVolume(AFF4Flusher &&volume) { - volume_objs.insert(std::make_pair(volume->urn, std::move(volume))); -} + // Construct the appropriate stream and return it. -// Construct the appropriate stream and return it. -AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &result) { - // Get all the type attrbutes of the URN. - std::vector> types; - if (STATUS_OK == resolver->Get(stream_urn, AFF4_TYPE, types)) { - for (auto &type : types) { - std::string type_str(type->SerializeToString()); - - if (type_str == AFF4_IMAGESTREAM_TYPE || - type_str == AFF4_LEGACY_IMAGESTREAM_TYPE) { - AFF4Flusher image_stream; - RETURN_IF_ERROR( - AFF4Image::OpenAFF4Image( - resolver, stream_urn, this, image_stream)); - - result.reset(image_stream.release()); - - resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); - return STATUS_OK; - } + AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &result) { + // Get all the type attrbutes of the URN. + std::vector> types; + if (STATUS_OK == resolver->Get(stream_urn, AFF4_TYPE, types)) { + for (auto &type : types) { + std::string type_str(type->SerializeToString()); + + if (type_str == AFF4_IMAGESTREAM_TYPE || + type_str == AFF4_LEGACY_IMAGESTREAM_TYPE) { + AFF4Flusher image_stream; + RETURN_IF_ERROR( + AFF4Image::OpenAFF4Image( + resolver, stream_urn, this, image_stream)); - // The AFF4 Standard specifies an "AFF4 Image" as an abstract - // container for image related properties. It is not actually a - // concrete stream but it refers to a storage stream using its - // aff4:dataStream property. - - // Note that to create such a stream, you can simply create a - // regular stream with NewAFF4Image or NewAFF4Map and then set - // the aff4:dataStream of a new object to a concerete Map or - // ImageStream. - if (type_str == AFF4_IMAGE_TYPE || - type_str == AFF4_DISK_IMAGE_TYPE || - type_str == AFF4_VOLUME_IMAGE_TYPE || - type_str == AFF4_MEMORY_IMAGE_TYPE || - type_str == AFF4_CONTIGUOUS_IMAGE_TYPE || - type_str == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { - URN delegate; - - if (STATUS_OK == resolver->Get(stream_urn, AFF4_DATASTREAM, delegate)) { - // TODO: This can get recursive. Protect against abuse. - return GetStream(delegate, result); + result.reset(image_stream.release()); + + resolver->logger->debug("Openning {} as type {}", + stream_urn, type_str); + return STATUS_OK; } - } - if (type_str == AFF4_MAP_TYPE) { - AFF4Flusher map_stream; - RETURN_IF_ERROR( - AFF4Map::OpenAFF4Map( - resolver, stream_urn, this, map_stream)); + // The AFF4 Standard specifies an "AFF4 Image" as an abstract + // container for image related properties. It is not actually a + // concrete stream but it refers to a storage stream using its + // aff4:dataStream property. - result.reset(map_stream.release()); - resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); + // Note that to create such a stream, you can simply create a + // regular stream with NewAFF4Image or NewAFF4Map and then set + // the aff4:dataStream of a new object to a concerete Map or + // ImageStream. + if (type_str == AFF4_IMAGE_TYPE || + type_str == AFF4_DISK_IMAGE_TYPE || + type_str == AFF4_VOLUME_IMAGE_TYPE || + type_str == AFF4_MEMORY_IMAGE_TYPE || + type_str == AFF4_CONTIGUOUS_IMAGE_TYPE || + type_str == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { + URN delegate; - return STATUS_OK; - } + if (STATUS_OK == resolver->Get(stream_urn, AFF4_DATASTREAM, delegate)) { + // TODO: This can get recursive. Protect against abuse. + return GetStream(delegate, result); + } + } + + if (type_str == AFF4_MAP_TYPE) { + AFF4Flusher map_stream; + RETURN_IF_ERROR( + AFF4Map::OpenAFF4Map( + resolver, stream_urn, this, map_stream)); + + result.reset(map_stream.release()); + resolver->logger->debug("Openning {} as type {}", + stream_urn, type_str); + + return STATUS_OK; + } + + // Zip segments are stored directly in each volume. We use + // the resolver to figure out which volume has each + // segment. + if (type_str == AFF4_ZIP_SEGMENT_TYPE || + type_str == AFF4_FILE_TYPE) { + URN owner; + RETURN_IF_ERROR(resolver->Get(stream_urn, AFF4_STORED, owner)); - // Zip segments are stored directly in each volume. We use - // the resolver to figure out which volume has each - // segment. - if (type_str == AFF4_ZIP_SEGMENT_TYPE || - type_str == AFF4_FILE_TYPE) { - URN owner; - RETURN_IF_ERROR(resolver->Get(stream_urn, AFF4_STORED, owner)); - - resolver->logger->debug("Openning {} as type {}", stream_urn, type_str); - auto it = volume_objs.find(owner); - if (it != volume_objs.end()) { - return (it->second->OpenMemberStream(stream_urn, result)); + resolver->logger->debug("Openning {} as type {}", stream_urn, type_str); + auto it = volume_objs.find(owner); + if (it != volume_objs.end()) { + return (it->second->OpenMemberStream(stream_urn, result)); + } } } } - } - // Handle symbolic streams now. - if (stream_urn == AFF4_IMAGESTREAM_ZERO) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0)); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_FF) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0xff)); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_UNKNOWN) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNKNOWN")); - return STATUS_OK; - } - if (stream_urn == AFF4_IMAGESTREAM_UNREADABLE) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNREADABLEDATA")); - return STATUS_OK; + // Handle symbolic streams now. + if (stream_urn == AFF4_IMAGESTREAM_ZERO) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0)); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_FF) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, 0xff)); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_UNKNOWN) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNKNOWN")); + return STATUS_OK; + } + if (stream_urn == AFF4_IMAGESTREAM_UNREADABLE) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, "UNREADABLEDATA")); + return STATUS_OK; + } + + for (int i = 0; i < 256; i++) { + std::string urn = aff4_sprintf( + "%s%02X", AFF4_IMAGESTREAM_SYMBOLIC_PREFIX, i); + + if (stream_urn == urn) { + result.reset(new AFF4SymbolicStream(resolver, stream_urn, i)); + return STATUS_OK; + } + } + + return NOT_FOUND; } - for (int i = 0; i < 256; i++) { - std::string urn = aff4_sprintf( - "%s%02X", AFF4_IMAGESTREAM_SYMBOLIC_PREFIX, i); + AFF4Status VolumeGroup::LocateAndAdd(URN& urn) { + if (volume_objs.find(urn) != volume_objs.end()) { + // Already loaded.... + return STATUS_OK; + } + + // Check known volume URNs. + if (foundVolumes.find(urn) != foundVolumes.end()) { + + std::string volume_to_load = foundVolumes.find(urn)->second; + // load and return; + AFF4Flusher backing_stream; + RETURN_IF_ERROR(NewFileBackedObject( + resolver, volume_to_load, + "read", backing_stream)); + + AFF4Flusher volume; + RETURN_IF_ERROR(ZipFile::OpenZipFile( + resolver, + std::move(AFF4Flusher( + backing_stream.release())), + volume)); + + AddVolume(std::move(AFF4Flusher(volume.release()))); + return STATUS_OK; + } + + // Search for container. + resolver->logger->info("Searching for container {}", urn); + for (std::string path : searchPaths) { + /* + * Look for all AFF4 in this path, and check it's URN. + */ + ScanForAFF4Volumes(path); + } + // Check known volume URNs. + if (foundVolumes.find(urn) != foundVolumes.end()) { - if (stream_urn == urn) { - result.reset(new AFF4SymbolicStream(resolver, stream_urn, i)); + std::string volume_to_load = foundVolumes.find(urn)->second; + + resolver->logger->info("Searching for container {} = {}", urn, volume_to_load); + + // load and return; + AFF4Flusher backing_stream; + RETURN_IF_ERROR(NewFileBackedObject( + resolver, volume_to_load, + "read", backing_stream)); + + AFF4Flusher volume; + RETURN_IF_ERROR(ZipFile::OpenZipFile( + resolver, + std::move(AFF4Flusher( + backing_stream.release())), + volume)); + + AddVolume(std::move(AFF4Flusher(volume.release()))); return STATUS_OK; } + + return NOT_FOUND; } - return NOT_FOUND; + bool VolumeGroup::FoundVolumesContains(const std::string& filename) { + for (auto it = foundVolumes.begin(); it != foundVolumes.end(); it++) { + if (it->second.compare(filename) == 0) { + return true; + } + } + return false; } + void VolumeGroup::ScanForAFF4Volumes(const std::string& path) { + // It is expected that the map is LOCKED prior to this call. + if (path.empty()) { + return; + } + resolver->logger->info("Scanning path {}", path); +#ifdef _WIN32 + /* + * Windows based systems + */ + std::wstring wpath = s2ws(path); + + std::wstring pattern(wpath); + pattern.append(L"\\*"); + + WIN32_FIND_DATAW data; + HANDLE hFind; + if ((hFind = FindFirstFile(pattern.c_str(), &data)) != INVALID_HANDLE_VALUE) { + do { + std::wstring filename(data.cFileName); + if (filename.compare(L".") && filename.compare(L"..")) { + std::string absoluteFilename = path + "\\" + ws2s(filename); + if ((IsFile(absoluteFilename)) && (IsAFF4Container(ws2s(filename)))) { + if (!FoundVolumesContains(absoluteFilename)) { + // We don't have this file + std::string resID = aff4::ZipFile::GetResourceID(absoluteFilename, resolver->logger); + if (!resID.empty()) { + foundVolumes[resID] = absoluteFilename; + } + } + } + } + while (FindNextFile(hFind, &data) != 0); + FindClose(hFind); + } +#else + /* + * POSIX based systems. + */ + DIR* dirp = ::opendir(path.c_str()); + struct dirent * dp; + while ((dp = readdir(dirp)) != NULL) { + std::string filename(dp->d_name); + if (filename.compare(".") && filename.compare("..")) { + std::string absoluteFilename = path + "/" + filename; + if ((IsFile(absoluteFilename)) && (IsAFF4Container(filename))) { + if (!FoundVolumesContains(absoluteFilename)) { + // We don't have this file + std::string resID = aff4::ZipFile::GetResourceID(absoluteFilename, resolver->logger); + if (!resID.empty()) { + foundVolumes[resID] = absoluteFilename; + } + } + } + } + } + closedir(dirp); +#endif + } -} // namespace aff4 + } // namespace aff4 diff --git a/aff4/volume_group.h b/aff4/volume_group.h index 671c74b..6282188 100644 --- a/aff4/volume_group.h +++ b/aff4/volume_group.h @@ -2,17 +2,22 @@ #define AFF4_VOLUME_GROUP_H_ #include +#include + #include "aff4/aff4_io.h" #include "aff4/data_store.h" - - namespace aff4 { class VolumeGroup { protected: - std::unordered_map> volume_objs; - DataStore *resolver; + std::unordered_map> volume_objs; + DataStore *resolver; + + std::set searchPaths; + std::unordered_map foundVolumes; + + bool FoundVolumesContains(const std::string& filename); public: VolumeGroup(DataStore *resolver) : resolver(resolver) {} @@ -21,6 +26,13 @@ namespace aff4 { void AddVolume(AFF4Flusher &&volume); AFF4Status GetStream(URN segment_urn, AFF4Flusher &result); + + void AddSearchPath(std::string path); + void RemoveSearchPath(std::string path); + + AFF4Status LocateAndAdd(URN& urn); + void ScanForAFF4Volumes(const std::string& path); + }; } // namespace aff4 From 1dc699f4487e9b95ab6a150aa883e45bc3523a3e Mon Sep 17 00:00:00 2001 From: Bradley Date: Sat, 8 Jun 2019 06:56:04 +1000 Subject: [PATCH 8/8] Fix MacOS install instructions --- README.macos | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/README.macos b/README.macos index 1d9b33e..4c3814f 100644 --- a/README.macos +++ b/README.macos @@ -1,22 +1,9 @@ -brew install -brew install -brew install snappy lz4 raptor pkgconfig tclap uriparser spdlog - -BUILDING OS X +BUILDING MacOS ================ -Install the following via macports -* zlib -* raptor2 -* google-glog -* pcrexx -* tclap (missing *.pc file - place in /opt/local/lib/pkgconfig/) -* yaml-cpp -* snappy +Install the following via brew -Install the following manually as they aren't in macports - -* uuid +brew install snappy lz4 raptor pkgconfig tclap uriparser spdlog Then do the following