Autoload of striped images

INSTALL

@@ -1,6 +1,7 @@
 BUILDING
 ========
 
+* Read the README that applies to your OS
 * Build the required prerequisite libraries.
 
 * Build and install the AFF4 library:
@@ -16,30 +17,3 @@ TESTING
   $ make check
 
 
-BUILDING OS X
-================
-
-Install the following via macports
-* zlib
-* raptor2
-* google-glog
-* pcrexx
-* tclap (missing *.pc file - place in /opt/local/lib/pkgconfig/)
-
-Install the following manually as they aren't in macports
-* snappy 
-* uuid 
-
-$ cd ~/git
-$ git clone ssh://bastion/srv/git/aff4.git
-$ cd aff4
-$ git submodule update --init third_party/gtest
-   Ignore the error about tree reference.
-$ cd third_party/gtest
-$ git reset --hard
-$ cd ../..
-$ ./autogen.sh
-$ ./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib"
-$ make
-
-For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local")

README.linux

@@ -82,7 +82,7 @@ make -j4 install
 cd ..
 
 apt-get source libspdlog-dev
-cd spdlog-0.11.0/
+cd spdlog-0.17.0/
 cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make all install
 cd ..
 

README.macos

@@ -0,0 +1,29 @@
+BUILDING MacOS
+================
+
+Install the following via brew
+
+brew install snappy lz4 raptor pkgconfig tclap uriparser spdlog
+
+Then do the following
+
+cd ~/git
+git clone https://github.com/gabime/spdlog.git
+cd spdlog
+git checkout v0.17.0
+cmake  . && make all install 
+cd ..
+git clone https://github.com/Velocidex/c-aff4.git
+cd c-aff4
+git submodule update --init third_party/gtest
+
+   Ignore the error about tree reference.
+
+cd third_party/gtest
+git reset --hard
+cd ../..
+./autogen.sh
+./configure CC=clang CXX=clang++ CXXFLAGS="-std=c++11 -stdlib=libc++ -O2 -g0 -I/opt/local/include" LDFLAGS="-stdlib=libc++ -L/opt/local/lib"
+make
+
+For the manual installations, need to ensure they are installed in /opt/local. ("./configure --prefix=/opt/local")

aff4/aff4_base.h

@@ -52,6 +52,32 @@ specific language governing permissions and limitations under the License.
 // Windows defines this macro which interfers with glog's version.
 #undef ERROR
 
+#ifdef __APPLE__
+
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#define pread64 pread
+#define O_LARGEFILE 0
+#include <libkern/OSByteOrder.h>
+
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+
+#endif
+
 #include "aff4/lexicon.h"
 #include "aff4/aff4_errors.h"
 #include "aff4/rdf.h"

aff4/aff4_file.cc

@@ -327,9 +327,8 @@ FileBackedObject::~FileBackedObject() {
      std::string mode,
      AFF4Flusher<FileBackedObject> &result
  ) {
-     auto new_object = AFF4Flusher<FileBackedObject>(new FileBackedObject(resolver));
+     auto new_object = AFF4Flusher<FileBackedObject>(new FileBackedObject(resolver, filename));
      new_object->urn = URN::NewURNFromFilename(filename, false);
-
      std::vector<std::string> directory_components = split(filename, PATH_SEP);
      directory_components.pop_back();
 

aff4/aff4_file.h

@@ -47,7 +47,7 @@ class FileBackedObject: public AFF4Stream {
     // The filename for this object.
     std::string filename;
 
-    explicit FileBackedObject(DataStore* resolver): AFF4Stream(resolver), fd(0){}
+    explicit FileBackedObject(DataStore* resolver, std::string filename): AFF4Stream(resolver), filename(filename), fd(0){}
     virtual ~FileBackedObject();
 
     AFF4Status ReadBuffer(char* data, size_t *length) override;

aff4/aff4_image.cc

@@ -419,6 +419,7 @@ AFF4Status AFF4Image::OpenAFF4Image(
             resolver->logger->error(
                 "ImageStream {} does not specify a size. "
                 "Is this part of a split image set?", new_obj->urn);
+            return NOT_FOUND;
         }
 
     } else {

aff4/aff4_imager_utils.cc

@@ -248,7 +248,8 @@ AFF4Status BasicImager::handle_aff4_volumes() {
                                     AFF4Flusher<AFF4Stream>(backing_stream.release()),
                                     volume));
 
-                volume_objs.AddVolume(AFF4Flusher<AFF4Volume>(volume.release()));
+                volume_objs.AddVolume(std::move(AFF4Flusher<AFF4Volume>(volume.release())));
+                volume_objs.AddSearchPath(volume_to_load);
             }
         }
     }

aff4/aff4_map.cc

@@ -99,8 +99,20 @@ AFF4Status AFF4Map::OpenAFF4Map(
             line.erase(line.length()-1, 1);
         }
         AFF4Flusher<AFF4Stream> target;
-        RETURN_IF_ERROR(volumes->GetStream(URN(line), target));
-
+        if((volumes->GetStream(URN(line), target)) != STATUS_OK){
+            // Search in the repository for this target.
+            resolver->logger->info("Attempting to locate resource {} ", line);
+            if(resolver->HasURNWithAttribute(URN(line), AFF4_STORED)){
+                // Get the ID of the container with this stream.
+                URN targetContainer;
+                RETURN_IF_ERROR(resolver->Get(URN(line), AFF4_STORED, targetContainer));
+                RETURN_IF_ERROR(volumes->LocateAndAdd(targetContainer));
+                RETURN_IF_ERROR(volumes->GetStream(URN(line), target));
+            } else {
+                return NOT_FOUND;
+            }
+        }
+
         resolver->logger->debug("MAP: Opened {} {} for target {}",
                                 target->urn, line,  map_obj->targets.size());
 

aff4/aff4_utils.h

@@ -11,7 +11,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 CONDITIONS OF ANY KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations under the License.
-*/
+ */
 
 #ifndef SRC_AFF4_UTILS_H_
 #define SRC_AFF4_UTILS_H_
@@ -22,19 +22,22 @@ specific language governing permissions and limitations under the License.
 #include <string>
 #include <sstream>
 
+#include <codecvt>
+#include <locale>
+
 #include "spdlog/spdlog.h"
 
 namespace aff4 {
 
 #define UNUSED(x) (void)x
 
-std::string aff4_sprintf(std::string fmt, ...);
+    std::string aff4_sprintf(std::string fmt, ...);
 
-std::string GetLastErrorMessage();
+    std::string GetLastErrorMessage();
 
-std::vector<std::string> split(const std::string& s, char delim);
+    std::vector<std::string> split(const std::string& s, char delim);
 
-std::shared_ptr<spdlog::logger> get_logger();
+    std::shared_ptr<spdlog::logger> get_logger();
 
 #define RETURN_IF_ERROR(expr)                   \
     do {                                        \
@@ -45,19 +48,62 @@ std::shared_ptr<spdlog::logger> get_logger();
         };                                      \
     } while (0);
 
-// A portable version of fnmatch.
-int fnmatch(const char *pattern, const char *string);
+    // A portable version of fnmatch.
+    int fnmatch(const char *pattern, const char *string);
 
-inline bool hasEnding(std::string const &fullString, std::string const &ending) {
-    if (fullString.length() >= ending.length()) {
-        return (0 == fullString.compare(
+    inline bool hasEnding(std::string const &fullString, std::string const &ending) {
+        if (fullString.length() >= ending.length()) {
+            return (0 == fullString.compare(
                     fullString.length() - ending.length(),
                     ending.length(), ending));
-    } else {
-         return false;
+        } else {
+            return false;
+        }
+    }
+
+    inline bool IsAFF4Container(std::string filename) noexcept {
+        // Cheap nasty not really unicode transformation to lower case.
+        std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower);
+        return hasEnding(filename, ".af4") || hasEnding(filename, ".aff4");
+    }
+
+    /**
+     * Does the file entity exist, and is a regular file.
+     * @param name The filename to check
+     * @return TRUE if the file entity exists and is a regular file.
+     */
+    inline bool IsFile(const std::string& name) {
+#ifndef _WIN32
+        /*
+         * POSIX based systems.
+         */
+        struct stat buffer;
+        if (::stat(name.c_str(), &buffer) == 0) {
+            return S_ISREG(buffer.st_mode);
+        }
+        return false;
+#else 
+        /*
+         * Windows based systems
+         */
+        std::wstring filename = s2ws(name);
+        DWORD dwAttrib = GetFileAttributes(filename.c_str());
+        return (dwAttrib != INVALID_FILE_ATTRIBUTES && !(dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
+
+#endif
+    }
+
+    inline std::wstring s2ws(const std::string& str) {
+        using convert_typeX = std::codecvt_utf8<wchar_t>;
+        std::wstring_convert<convert_typeX, wchar_t> converterX;
+        return converterX.from_bytes(str);
     }
-}
 
+    inline std::string ws2s(const std::wstring& wstr) {
+        using convert_typeX = std::codecvt_utf8<wchar_t>;
+        std::wstring_convert<convert_typeX, wchar_t> converterX;
+        return converterX.to_bytes(wstr);
+    }
 
 } // namespace aff4
 

aff4/libaff4-c.cc

@@ -153,6 +153,7 @@ AFF4_Handle* AFF4_open(const char* filename, AFF4_Message** msg) {
     };
 
     h->volumes.AddVolume(std::move(zip));
+    h->volumes.AddSearchPath(filename);
 
     // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format.
     images = h->resolver.Query(aff4::AFF4_TYPE, &type);