From 41289fa5bc212a01c2d3fb3d6cc49c361811a182 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 13:41:15 -0700 Subject: [PATCH 1/8] Update libFuzzer to llvm/llvm-project's 70cbc6d --- libfuzzer/CREDITS.TXT | 36 -- libfuzzer/FuzzerBuiltins.h | 1 - libfuzzer/FuzzerBuiltinsMsvc.h | 6 - libfuzzer/FuzzerCorpus.h | 38 ++- libfuzzer/FuzzerDataFlowTrace.cpp | 4 +- libfuzzer/FuzzerDataFlowTrace.h | 14 +- libfuzzer/FuzzerDictionary.h | 8 +- libfuzzer/FuzzerDriver.cpp | 12 +- libfuzzer/FuzzerFork.cpp | 11 +- libfuzzer/FuzzerIO.cpp | 11 +- libfuzzer/FuzzerIO.h | 5 +- libfuzzer/FuzzerLoop.cpp | 30 +- libfuzzer/FuzzerMerge.cpp | 8 +- libfuzzer/FuzzerMutate.cpp | 48 ++- libfuzzer/FuzzerRandom.h | 23 +- libfuzzer/FuzzerSHA1.cpp | 9 +- libfuzzer/FuzzerTracePC.cpp | 11 +- libfuzzer/FuzzerTracePC.h | 39 ++- libfuzzer/FuzzerUtil.cpp | 9 +- libfuzzer/FuzzerUtil.h | 6 +- libfuzzer/FuzzerUtilFuchsia.cpp | 2 +- libfuzzer/FuzzerUtilPosix.cpp | 11 +- libfuzzer/LICENSE.TXT | 311 ----------------- .../tests/FuzzedDataProviderUnittest.cpp | 14 + libfuzzer/tests/FuzzerUnittest.cpp | 314 ++++++++++++------ 25 files changed, 411 insertions(+), 570 deletions(-) delete mode 100644 libfuzzer/CREDITS.TXT delete mode 100644 libfuzzer/LICENSE.TXT diff --git a/libfuzzer/CREDITS.TXT b/libfuzzer/CREDITS.TXT deleted file mode 100644 index 6964eba..0000000 --- a/libfuzzer/CREDITS.TXT +++ /dev/null @@ -1,36 +0,0 @@ -This file is a partial list of people who have contributed to the LLVM/CompilerRT -project. If you have contributed a patch or made some other contribution to -LLVM/CompilerRT, please submit a patch to this file to add yourself, and it will be -done! - -The list is sorted by surname and formatted to allow easy grepping and -beautification by scripts. The fields are: name (N), email (E), web-address -(W), PGP key ID and fingerprint (P), description (D), and snail-mail address -(S). - -N: Craig van Vliet -E: cvanvliet@auroraux.org -W: http://www.auroraux.org -D: Code style and Readability fixes. - -N: Edward O'Callaghan -E: eocallaghan@auroraux.org -W: http://www.auroraux.org -D: CMake'ify Compiler-RT build system -D: Maintain Solaris & AuroraUX ports of Compiler-RT - -N: Howard Hinnant -E: hhinnant@apple.com -D: Architect and primary author of compiler-rt - -N: Guan-Hong Liu -E: koviankevin@hotmail.com -D: IEEE Quad-precision functions - -N: Joerg Sonnenberger -E: joerg@NetBSD.org -D: Maintains NetBSD port. - -N: Matt Thomas -E: matt@NetBSD.org -D: ARM improvements. diff --git a/libfuzzer/FuzzerBuiltins.h b/libfuzzer/FuzzerBuiltins.h index 4c0ada8..ce0bd5c 100644 --- a/libfuzzer/FuzzerBuiltins.h +++ b/libfuzzer/FuzzerBuiltins.h @@ -26,7 +26,6 @@ inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } -inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } } // namespace fuzzer diff --git a/libfuzzer/FuzzerBuiltinsMsvc.h b/libfuzzer/FuzzerBuiltinsMsvc.h index c5bec97..ab191b6 100644 --- a/libfuzzer/FuzzerBuiltinsMsvc.h +++ b/libfuzzer/FuzzerBuiltinsMsvc.h @@ -52,12 +52,6 @@ inline uint32_t Clzll(uint64_t X) { return 64; } -inline uint32_t Clz(uint32_t X) { - unsigned long LeadZeroIdx = 0; - if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; - return 32; -} - inline int Popcountll(unsigned long long X) { #if !defined(_M_ARM) && !defined(_M_X64) return __popcnt(X) + __popcnt(X >> 32); diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h index daea4f5..f8c1260 100644 --- a/libfuzzer/FuzzerCorpus.h +++ b/libfuzzer/FuzzerCorpus.h @@ -44,7 +44,7 @@ struct InputInfo { // Power schedule. bool NeedsEnergyUpdate = false; double Energy = 0.0; - size_t SumIncidence = 0; + double SumIncidence = 0.0; Vector> FeatureFreqs; // Delete feature Idx and its frequency from FeatureFreqs. @@ -74,27 +74,28 @@ struct InputInfo { void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime, std::chrono::microseconds AverageUnitExecutionTime) { Energy = 0.0; - SumIncidence = 0; + SumIncidence = 0.0; // Apply add-one smoothing to locally discovered features. for (auto F : FeatureFreqs) { - size_t LocalIncidence = F.second + 1; - Energy -= LocalIncidence * logl(LocalIncidence); + double LocalIncidence = F.second + 1; + Energy -= LocalIncidence * log(LocalIncidence); SumIncidence += LocalIncidence; } // Apply add-one smoothing to locally undiscovered features. - // PreciseEnergy -= 0; // since logl(1.0) == 0) - SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + // PreciseEnergy -= 0; // since log(1.0) == 0) + SumIncidence += + static_cast(GlobalNumberOfFeatures - FeatureFreqs.size()); // Add a single locally abundant feature apply add-one smoothing. - size_t AbdIncidence = NumExecutedMutations + 1; - Energy -= AbdIncidence * logl(AbdIncidence); + double AbdIncidence = static_cast(NumExecutedMutations + 1); + Energy -= AbdIncidence * log(AbdIncidence); SumIncidence += AbdIncidence; // Normalize. if (SumIncidence != 0) - Energy = (Energy / SumIncidence) + logl(SumIncidence); + Energy = Energy / SumIncidence + log(SumIncidence); if (ScalePerExecTime) { // Scaling to favor inputs with lower execution time. @@ -213,6 +214,8 @@ class InputCorpus { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); + // Inputs.size() is cast to uint32_t below. + assert(Inputs.size() < std::numeric_limits::max()); Inputs.push_back(new InputInfo()); InputInfo &II = *Inputs.back(); II.U = U; @@ -224,7 +227,7 @@ class InputCorpus { II.HasFocusFunction = HasFocusFunction; // Assign maximal energy to the new seed. II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); - II.SumIncidence = RareFeatures.size(); + II.SumIncidence = static_cast(RareFeatures.size()); II.NeedsEnergyUpdate = false; std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); ComputeSHA1(U.data(), U.size(), II.Sha1); @@ -399,7 +402,7 @@ class InputCorpus { // Zero energy seeds will never be fuzzed and remain zero energy. if (II->Energy > 0.0) { II->SumIncidence += 1; - II->Energy += logl(II->SumIncidence) / II->SumIncidence; + II->Energy += log(II->SumIncidence) / II->SumIncidence; } } @@ -426,7 +429,8 @@ class InputCorpus { NumUpdatedFeatures++; if (FeatureDebug) Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); - SmallestElementPerFeature[Idx] = Inputs.size(); + // Inputs.size() is guaranteed to be less than UINT32_MAX by AddToCorpus. + SmallestElementPerFeature[Idx] = static_cast(Inputs.size()); InputSizesPerFeature[Idx] = NewSize; return true; } @@ -464,7 +468,7 @@ class InputCorpus { static const bool FeatureDebug = false; - size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } + uint32_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } void ValidateFeatureSet() { if (FeatureDebug) @@ -539,9 +543,11 @@ class InputCorpus { if (VanillaSchedule) { for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures - ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) - : 0.; + Weights[i] = + Inputs[i]->NumFeatures + ? static_cast((i + 1) * + (Inputs[i]->HasFocusFunction ? 1000 : 1)) + : 0.; } if (FeatureDebug) { diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp index 0e9cdf7..23d4225 100644 --- a/libfuzzer/FuzzerDataFlowTrace.cpp +++ b/libfuzzer/FuzzerDataFlowTrace.cpp @@ -60,6 +60,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { CoveredBlocks.push_back(BB); } if (CoveredBlocks.empty()) return false; + // Ensures no CoverageVector is longer than UINT32_MAX. uint32_t NumBlocks = CoveredBlocks.back(); CoveredBlocks.pop_back(); for (auto BB : CoveredBlocks) @@ -200,7 +201,8 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, FunctionNames[FocusFuncIdx].c_str()); for (size_t i = 0; i < NumFunctions; i++) { - if (!Weights[i]) continue; + if (Weights[i] == 0.0) + continue; Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, Weights[i], Coverage.GetNumberOfBlocks(i), Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), diff --git a/libfuzzer/FuzzerDataFlowTrace.h b/libfuzzer/FuzzerDataFlowTrace.h index d6e3de3..07c03bb 100644 --- a/libfuzzer/FuzzerDataFlowTrace.h +++ b/libfuzzer/FuzzerDataFlowTrace.h @@ -42,7 +42,8 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, const Vector &CorporaFiles); class BlockCoverage { - public: +public: + // These functions guarantee no CoverageVector is longer than UINT32_MAX. bool AppendCoverage(std::istream &IN); bool AppendCoverage(const std::string &S); @@ -50,7 +51,8 @@ class BlockCoverage { uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { auto It = Functions.find(FunctionId); - if (It == Functions.end()) return 0; + if (It == Functions.end()) + return 0; const auto &Counters = It->second; if (BasicBlockId < Counters.size()) return Counters[BasicBlockId]; @@ -61,7 +63,7 @@ class BlockCoverage { auto It = Functions.find(FunctionId); if (It == Functions.end()) return 0; const auto &Counters = It->second; - return Counters.size(); + return static_cast(Counters.size()); } uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { @@ -78,8 +80,7 @@ class BlockCoverage { Vector FunctionWeights(size_t NumFunctions) const; void clear() { Functions.clear(); } - private: - +private: typedef Vector CoverageVector; uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { @@ -91,7 +92,8 @@ class BlockCoverage { } uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { - return Counters.size() - NumberOfCoveredBlocks(Counters); + return static_cast(Counters.size()) - + NumberOfCoveredBlocks(Counters); } uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { diff --git a/libfuzzer/FuzzerDictionary.h b/libfuzzer/FuzzerDictionary.h index 301c5d9..db55907 100644 --- a/libfuzzer/FuzzerDictionary.h +++ b/libfuzzer/FuzzerDictionary.h @@ -23,12 +23,14 @@ template class FixedWord { public: static const size_t kMaxSize = kMaxSizeT; FixedWord() {} - FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); } + FixedWord(const uint8_t *B, size_t S) { Set(B, S); } - void Set(const uint8_t *B, uint8_t S) { + void Set(const uint8_t *B, size_t S) { + static_assert(kMaxSizeT <= std::numeric_limits::max(), + "FixedWord::kMaxSizeT cannot fit in a uint8_t."); assert(S <= kMaxSize); memcpy(Data, B, S); - Size = S; + Size = static_cast(S); } bool operator==(const FixedWord &w) const { diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp index 447cafc..ceaa907 100644 --- a/libfuzzer/FuzzerDriver.cpp +++ b/libfuzzer/FuzzerDriver.cpp @@ -159,14 +159,14 @@ static bool ParseOneFlag(const char *Param) { const char *Str = FlagValue(Param, Name); if (Str) { if (FlagDescriptions[F].IntFlag) { - int Val = MyStol(Str); - *FlagDescriptions[F].IntFlag = Val; + auto Val = MyStol(Str); + *FlagDescriptions[F].IntFlag = static_cast(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %d\n", Name, Val); return true; } else if (FlagDescriptions[F].UIntFlag) { - unsigned int Val = std::stoul(Str); - *FlagDescriptions[F].UIntFlag = Val; + auto Val = std::stoul(Str); + *FlagDescriptions[F].UIntFlag = static_cast(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %u\n", Name, Val); return true; @@ -789,8 +789,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { unsigned Seed = Flags.seed; // Initialize Seed. if (Seed == 0) - Seed = - std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); + Seed = static_cast( + std::chrono::system_clock::now().time_since_epoch().count() + GetPid()); if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp index 84725d2..5134a5d 100644 --- a/libfuzzer/FuzzerFork.cpp +++ b/libfuzzer/FuzzerFork.cpp @@ -142,7 +142,9 @@ struct GlobalEnv { CollectDFT(SF); } auto Time2 = std::chrono::system_clock::now(); - Job->DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + assert(DftTimeInSeconds < std::numeric_limits::max()); + Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); } if (!Seeds.empty()) { Job->SeedListPath = @@ -314,8 +316,11 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Files.push_back(File.File); } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); - CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, - {}, &Env.Cov, CFPath, false); + Set NewFeatures, NewCov; + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features, + &NewFeatures, Env.Cov, &NewCov, CFPath, false); + Env.Features.insert(NewFeatures.begin(), NewFeatures.end()); + Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp index 54a7219..7f149ac 100644 --- a/libfuzzer/FuzzerIO.cpp +++ b/libfuzzer/FuzzerIO.cpp @@ -90,8 +90,9 @@ void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) { fclose(Out); } -void ReadDirToVectorOfUnits(const char *Path, Vector *V, - long *Epoch, size_t MaxSize, bool ExitOnError) { +void ReadDirToVectorOfUnits(const char *Path, Vector *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + Vector *VPaths) { long E = Epoch ? *Epoch : 0; Vector Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); @@ -103,12 +104,14 @@ void ReadDirToVectorOfUnits(const char *Path, Vector *V, if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); auto S = FileToVector(X, MaxSize, ExitOnError); - if (!S.empty()) + if (!S.empty()) { V->push_back(S); + if (VPaths) + VPaths->push_back(X); + } } } - void GetSizedFilesFromDir(const std::string &Dir, Vector *V) { Vector Files; ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h index abd2511..bde1826 100644 --- a/libfuzzer/FuzzerIO.h +++ b/libfuzzer/FuzzerIO.h @@ -32,8 +32,9 @@ void WriteToFile(const Unit &U, const std::string &Path); void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path); void AppendToFile(const std::string &Data, const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, Vector *V, - long *Epoch, size_t MaxSize, bool ExitOnError); +void ReadDirToVectorOfUnits(const char *Path, Vector *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + Vector *VPaths = 0); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp index 6e3bf44..86a78ab 100644 --- a/libfuzzer/FuzzerLoop.cpp +++ b/libfuzzer/FuzzerLoop.cpp @@ -414,19 +414,25 @@ void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; Vector AdditionalCorpus; - ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, - &EpochOfLastReadOfOutputCorpus, MaxSize, - /*ExitOnError*/ false); + Vector AdditionalCorpusPaths; + ReadDirToVectorOfUnits( + Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false, + (Options.Verbosity >= 2 ? &AdditionalCorpusPaths : nullptr)); if (Options.Verbosity >= 2) Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); bool Reloaded = false; - for (auto &U : AdditionalCorpus) { + for (size_t i = 0; i != AdditionalCorpus.size(); ++i) { + auto &U = AdditionalCorpus[i]; if (U.size() > MaxSize) U.resize(MaxSize); if (!Corpus.HasUnit(U)) { if (RunOne(U.data(), U.size())) { CheckExitOnSrcPosOrItem(); Reloaded = true; + if (Options.Verbosity >= 2) + Printf("Reloaded %s\n", AdditionalCorpusPaths[i].c_str()); } } } @@ -440,8 +446,9 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && secondsSinceProcessStartUp() >= 2) PrintStats("pulse "); - if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && - TimeOfUnit >= Options.ReportSlowUnits) { + auto Threshhold = + static_cast(static_cast(TimeOfLongestUnitInSeconds) * 1.1); + if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) { TimeOfLongestUnitInSeconds = TimeOfUnit; Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); @@ -501,6 +508,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, bool *FoundUniqFeatures) { if (!Size) return false; + // Largest input length should be INT_MAX. + assert(Size < std::numeric_limits::max()); ExecuteCallback(Data, Size); auto TimeOfUnit = duration_cast(UnitStopTime - UnitStartTime); @@ -508,8 +517,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + TPC.CollectFeatures([&](uint32_t Feature) { + if (Corpus.AddFeature(Feature, static_cast(Size), Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); if (Options.Entropic) Corpus.UpdateFeatureFrequency(II, Feature); @@ -575,7 +584,10 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } -void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { +// This method is not inlined because it would cause a test to fail where it +// is part of the stack unwinding. See D97975 for details. +ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data, + size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); diff --git a/libfuzzer/FuzzerMerge.cpp b/libfuzzer/FuzzerMerge.cpp index e3ad8b3..162453c 100644 --- a/libfuzzer/FuzzerMerge.cpp +++ b/libfuzzer/FuzzerMerge.cpp @@ -82,9 +82,9 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; - size_t N; - ISS1 >> Marker; - ISS1 >> N; + uint32_t N; + if (!(ISS1 >> Marker) || !(ISS1 >> N)) + return false; if (Marker == "STARTED") { // STARTED FILE_ID FILE_SIZE if (ExpectedStartMarker != N) @@ -137,6 +137,8 @@ size_t Merger::Merge(const Set &InitialFeatures, const Set &InitialCov, Set *NewCov, Vector *NewFiles) { NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); assert(NumFilesInFirstCorpus <= Files.size()); Set AllFeatures = InitialFeatures; diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp index cf34a9f..4650f1b 100644 --- a/libfuzzer/FuzzerMutate.cpp +++ b/libfuzzer/FuzzerMutate.cpp @@ -61,14 +61,20 @@ MutationDispatcher::MutationDispatcher(Random &Rand, } static char RandCh(Random &Rand) { - if (Rand.RandBool()) return Rand(256); + if (Rand.RandBool()) + return static_cast(Rand(256)); const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; return Special[Rand(sizeof(Special) - 1)]; } size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize) { - return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); + if (EF->__msan_unpoison) + EF->__msan_unpoison(Data, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(4); + return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, + Rand.Rand()); } size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, @@ -81,8 +87,18 @@ size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, return 0; CustomCrossOverInPlaceHere.resize(MaxSize); auto &U = CustomCrossOverInPlaceHere; + + if (EF->__msan_unpoison) { + EF->__msan_unpoison(Data, Size); + EF->__msan_unpoison(Other.data(), Other.size()); + EF->__msan_unpoison(U.data(), U.size()); + } + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(7); size_t NewSize = EF->LLVMFuzzerCustomCrossOver( - Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand()); + Data, Size, Other.data(), Other.size(), U.data(), U.size(), + Rand.Rand()); + if (!NewSize) return 0; assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); @@ -135,7 +151,8 @@ size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data, // Insert new values at Data[Idx]. memmove(Data + Idx + N, Data + Idx, Size - Idx); // Give preference to 0x00 and 0xff. - uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); + uint8_t Byte = static_cast( + Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255)); for (size_t i = 0; i < N; i++) Data[Idx + i] = Byte; return Size + N; @@ -178,7 +195,8 @@ size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, Size += W.size(); } else { // Overwrite some bytes with W. if (W.size() > Size) return 0; - size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + size_t Idx = + UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1 - W.size()); memcpy(Data + Idx, W.data(), W.size()); } return Size; @@ -227,8 +245,8 @@ DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( T Arg1, T Arg2, const uint8_t *Data, size_t Size) { if (Rand.RandBool()) Arg1 = Bswap(Arg1); if (Rand.RandBool()) Arg2 = Bswap(Arg2); - T Arg1Mutation = Arg1 + Rand(-1, 1); - T Arg2Mutation = Arg2 + Rand(-1, 1); + T Arg1Mutation = static_cast(Arg1 + Rand(-1, 1)); + T Arg2Mutation = static_cast(Arg2 + Rand(-1, 1)); return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, sizeof(Arg1), Data, Size); } @@ -245,23 +263,23 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC( DictionaryEntry DE; switch (Rand(4)) { case 0: { - auto X = TPC.TORC8.Get(Rand.Rand()); + auto X = TPC.TORC8.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 1: { - auto X = TPC.TORC4.Get(Rand.Rand()); + auto X = TPC.TORC4.Get(Rand.Rand()); if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); else DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 2: { - auto X = TPC.TORCW.Get(Rand.Rand()); + auto X = TPC.TORCW.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 3: if (Options.UseMemmem) { - auto X = TPC.MMT.Get(Rand.Rand()); - DE = DictionaryEntry(X); + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); } break; default: assert(0); @@ -387,12 +405,12 @@ size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { assert(Off + sizeof(T) <= Size); T Val; if (Off < 64 && !Rand(4)) { - Val = Size; + Val = static_cast(Size); if (Rand.RandBool()) Val = Bswap(Val); } else { memcpy(&Val, Data + Off, sizeof(Val)); - T Add = Rand(21); + T Add = static_cast(Rand(21)); Add -= 10; if (Rand.RandBool()) Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes. @@ -462,7 +480,7 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { assert(DE->GetW().size()); // Linear search is fine here as this happens seldom. if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) - PersistentAutoDictionary.push_back({DE->GetW(), 1}); + PersistentAutoDictionary.push_back(*DE); } } diff --git a/libfuzzer/FuzzerRandom.h b/libfuzzer/FuzzerRandom.h index 659283e..ad6c07e 100644 --- a/libfuzzer/FuzzerRandom.h +++ b/libfuzzer/FuzzerRandom.h @@ -18,18 +18,27 @@ class Random : public std::minstd_rand { public: Random(unsigned int seed) : std::minstd_rand(seed) {} result_type operator()() { return this->std::minstd_rand::operator()(); } - size_t Rand() { return this->operator()(); } - size_t RandBool() { return Rand() % 2; } + template + typename std::enable_if::value, T>::type Rand() { + return static_cast(this->operator()()); + } + size_t RandBool() { return this->operator()() % 2; } size_t SkewTowardsLast(size_t n) { size_t T = this->operator()(n * n); - size_t Res = sqrt(T); + size_t Res = static_cast(sqrt(T)); return Res; } - size_t operator()(size_t n) { return n ? Rand() % n : 0; } - intptr_t operator()(intptr_t From, intptr_t To) { + template + typename std::enable_if::value, T>::type operator()(T n) { + return n ? Rand() % n : 0; + } + template + typename std::enable_if::value, T>::type + operator()(T From, T To) { assert(From < To); - intptr_t RangeSize = To - From + 1; - return operator()(RangeSize) + From; + auto RangeSize = static_cast(To) - + static_cast(From) + 1; + return static_cast(this->operator()(RangeSize) + From); } }; diff --git a/libfuzzer/FuzzerSHA1.cpp b/libfuzzer/FuzzerSHA1.cpp index 2005dc7..b05655c 100644 --- a/libfuzzer/FuzzerSHA1.cpp +++ b/libfuzzer/FuzzerSHA1.cpp @@ -134,12 +134,13 @@ void sha1_hashBlock(sha1nfo *s) { s->state[4] += e; } -void sha1_addUncounted(sha1nfo *s, uint8_t data) { - uint8_t * const b = (uint8_t*) s->buffer; +// Adds the least significant byte of |data|. +void sha1_addUncounted(sha1nfo *s, uint32_t data) { + uint8_t *const b = (uint8_t *)s->buffer; #ifdef SHA_BIG_ENDIAN - b[s->bufferOffset] = data; + b[s->bufferOffset] = static_cast(data); #else - b[s->bufferOffset ^ 3] = data; + b[s->bufferOffset ^ 3] = static_cast(data); #endif s->bufferOffset++; if (s->bufferOffset == BLOCK_LENGTH) { diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp index 91e94d8..d808b9b 100644 --- a/libfuzzer/FuzzerTracePC.cpp +++ b/libfuzzer/FuzzerTracePC.cpp @@ -106,6 +106,15 @@ void TracePC::PrintModuleInfo() { } if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin()) Printf("INFO: %zd Extra Counters\n", NumExtraCounters); + + size_t MaxFeatures = CollectFeatures([](uint32_t) {}); + if (MaxFeatures > std::numeric_limits::max()) + Printf("WARNING: The coverage PC tables may produce up to %zu features.\n" + "This exceeds the maximum 32-bit value. Some features may be\n" + "ignored, and fuzzing may become less precise. If possible,\n" + "consider refactoring the fuzzer into several smaller fuzzers\n" + "linked against only a portion of the current target.\n", + MaxFeatures); } ATTRIBUTE_NO_SANITIZE_ALL @@ -356,7 +365,7 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, uint8_t HammingDistance = 0; for (; I < Len; I++) { if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { - HammingDistance = Popcountll(B1[I] ^ B2[I]); + HammingDistance = static_cast(Popcountll(B1[I] ^ B2[I])); break; } } diff --git a/libfuzzer/FuzzerTracePC.h b/libfuzzer/FuzzerTracePC.h index 0090923..a937329 100644 --- a/libfuzzer/FuzzerTracePC.h +++ b/libfuzzer/FuzzerTracePC.h @@ -54,7 +54,7 @@ struct MemMemTable { void Add(const uint8_t *Data, size_t Size) { if (Size <= 2) return; Size = std::min(Size, Word::GetMaxSize()); - size_t Idx = SimpleFastHash(Data, Size) % kSize; + auto Idx = SimpleFastHash(Data, Size) % kSize; MemMemWords[Idx].Set(Data, Size); } const Word &Get(size_t Idx) { @@ -79,7 +79,7 @@ class TracePC { void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } void UpdateObservedPCs(); - template void CollectFeatures(Callback CB) const; + template size_t CollectFeatures(Callback CB) const; void ResetMaps() { ValueProfileMap.Reset(); @@ -193,7 +193,7 @@ size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, Handle8bitCounter(FirstFeature, P - Begin, V); // Iterate by Step bytes at a time. - for (; P < End; P += Step) + for (; P + Step <= End; P += Step) if (LargeType Bundle = *reinterpret_cast(P)) { Bundle = HostToLE(Bundle); for (size_t I = 0; I < Step; I++, Bundle >>= 8) @@ -234,16 +234,16 @@ unsigned CounterToFeature(T Counter) { return Bit; } -template // void Callback(size_t Feature) -ATTRIBUTE_NO_SANITIZE_ADDRESS -ATTRIBUTE_NOINLINE -void TracePC::CollectFeatures(Callback HandleFeature) const { +template // void Callback(uint32_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS ATTRIBUTE_NOINLINE size_t +TracePC::CollectFeatures(Callback HandleFeature) const { auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) - HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + HandleFeature(static_cast(FirstFeature + Idx * 8 + + CounterToFeature(Counter))); else - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast(FirstFeature + Idx)); }; size_t FirstFeature = 0; @@ -263,16 +263,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast(FirstFeature + Idx)); }); FirstFeature += ValueProfileMap.SizeInBits(); } // Step function, grows similar to 8 * Log_2(A). - auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { - if (!A) return A; - uint32_t Log2 = Log(A); - if (Log2 < 3) return A; + auto StackDepthStepFunction = [](size_t A) -> size_t { + if (!A) + return A; + auto Log2 = Log(A); + if (Log2 < 3) + return A; Log2 -= 3; return (Log2 + 1) * 8 + ((A >> Log2) & 7); }; @@ -280,8 +282,13 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { assert(StackDepthStepFunction(1024 * 4) == 80); assert(StackDepthStepFunction(1024 * 1024) == 144); - if (auto MaxStackOffset = GetMaxStackOffset()) - HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); + if (auto MaxStackOffset = GetMaxStackOffset()) { + HandleFeature(static_cast( + FirstFeature + StackDepthStepFunction(MaxStackOffset / 8))); + FirstFeature += StackDepthStepFunction(std::numeric_limits::max()); + } + + return FirstFeature; } extern TracePC TPC; diff --git a/libfuzzer/FuzzerUtil.cpp b/libfuzzer/FuzzerUtil.cpp index 7eecb68..0518549 100644 --- a/libfuzzer/FuzzerUtil.cpp +++ b/libfuzzer/FuzzerUtil.cpp @@ -111,7 +111,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { char Hex[] = "0xAA"; Hex[2] = Str[Pos + 2]; Hex[3] = Str[Pos + 3]; - U->push_back(strtol(Hex, nullptr, 16)); + U->push_back(static_cast(strtol(Hex, nullptr, 16))); Pos += 3; continue; } @@ -226,10 +226,11 @@ unsigned NumberOfCpuCores() { return N; } -size_t SimpleFastHash(const uint8_t *Data, size_t Size) { - size_t Res = 0; +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial) { + uint64_t Res = Initial; + const uint8_t *Bytes = static_cast(Data); for (size_t i = 0; i < Size; i++) - Res = Res * 11 + Data[i]; + Res = Res * 11 + Bytes[i]; return Res; } diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h index e90be08..a188a7b 100644 --- a/libfuzzer/FuzzerUtil.h +++ b/libfuzzer/FuzzerUtil.h @@ -88,9 +88,11 @@ std::string DisassembleCmd(const std::string &FileName); std::string SearchRegexCmd(const std::string &Regex); -size_t SimpleFastHash(const uint8_t *Data, size_t Size); +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial = 0); -inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } +inline size_t Log(size_t X) { + return static_cast((sizeof(unsigned long long) * 8) - Clzll(X) - 1); +} inline size_t PageSize() { return 4096; } inline uint8_t *RoundUpByPage(uint8_t *P) { diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp index af43946..5034b4a 100644 --- a/libfuzzer/FuzzerUtilFuchsia.cpp +++ b/libfuzzer/FuzzerUtilFuchsia.cpp @@ -515,7 +515,7 @@ int ExecuteCommand(const Command &Cmd) { return rc; } - return Info.return_code; + return static_cast(Info.return_code); } bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) { diff --git a/libfuzzer/FuzzerUtilPosix.cpp b/libfuzzer/FuzzerUtilPosix.cpp index afb7334..0446d73 100644 --- a/libfuzzer/FuzzerUtilPosix.cpp +++ b/libfuzzer/FuzzerUtilPosix.cpp @@ -77,10 +77,13 @@ static void SetSigaction(int signum, return; } - sigact = {}; - sigact.sa_flags = SA_SIGINFO; - sigact.sa_sigaction = callback; - if (sigaction(signum, &sigact, 0)) { + struct sigaction new_sigact = {}; + // Address sanitizer needs SA_ONSTACK (causing the signal handler to run on a + // dedicated stack) in order to be able to detect stack overflows; keep the + // flag if it's set. + new_sigact.sa_flags = SA_SIGINFO | (sigact.sa_flags & SA_ONSTACK); + new_sigact.sa_sigaction = callback; + if (sigaction(signum, &new_sigact, nullptr)) { Printf("libFuzzer: sigaction failed with %d\n", errno); exit(1); } diff --git a/libfuzzer/LICENSE.TXT b/libfuzzer/LICENSE.TXT deleted file mode 100644 index 5a79a1b..0000000 --- a/libfuzzer/LICENSE.TXT +++ /dev/null @@ -1,311 +0,0 @@ -============================================================================== -The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: -============================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - -============================================================================== -Software from third parties included in the LLVM Project: -============================================================================== -The LLVM Project contains third party software which is under different license -terms. All such code will be identified clearly using at least one of two -mechanisms: -1) It will be in a separate directory tree with its own `LICENSE.txt` or - `LICENSE` file at the top containing the specific license and restrictions - which apply to that software, or -2) It will contain specific license and restriction terms at the top of every - file. - -============================================================================== -Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): -============================================================================== - -The compiler_rt library is dual licensed under both the University of Illinois -"BSD-Like" license and the MIT license. As a user of this code you may choose -to use it under either license. As a contributor, you agree to allow your code -to be used under both. - -Full text of the relevant licenses is included below. - -============================================================================== - -University of Illinois/NCSA -Open Source License - -Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT - -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== - -Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/libfuzzer/tests/FuzzedDataProviderUnittest.cpp b/libfuzzer/tests/FuzzedDataProviderUnittest.cpp index 99d9d8e..ea6774e 100644 --- a/libfuzzer/tests/FuzzedDataProviderUnittest.cpp +++ b/libfuzzer/tests/FuzzedDataProviderUnittest.cpp @@ -283,6 +283,20 @@ TEST(FuzzedDataProvider, ConsumeBool) { EXPECT_EQ(false, DataProv.ConsumeBool()); } +TEST(FuzzedDataProvider, PickValueInStdArray) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + const std::array Array = {1, 2, 3, 4, 5}; + EXPECT_EQ(5, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(1, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); +} + TEST(FuzzedDataProvider, PickValueInArray) { FuzzedDataProvider DataProv(Data, sizeof(Data)); const int Array[] = {1, 2, 3, 4, 5}; diff --git a/libfuzzer/tests/FuzzerUnittest.cpp b/libfuzzer/tests/FuzzerUnittest.cpp index d2b5cbb..974a01f 100644 --- a/libfuzzer/tests/FuzzerUnittest.cpp +++ b/libfuzzer/tests/FuzzerUnittest.cpp @@ -614,73 +614,80 @@ TEST(Corpus, Distribution) { } } -TEST(Merge, Bad) { - const char *kInvalidInputs[] = { - "", - "x", - "3\nx", - "2\n3", - "2\n2", - "2\n2\nA\n", - "2\n2\nA\nB\nC\n", - "0\n0\n", - "1\n1\nA\nFT 0", - "1\n1\nA\nSTARTED 1", - }; - Merger M; - for (auto S : kInvalidInputs) { - // fprintf(stderr, "TESTING:\n%s\n", S); - EXPECT_FALSE(M.Parse(S, false)); - } +template void EQ(const Vector &A, const Vector &B) { + EXPECT_EQ(A, B); } -void EQ(const Vector &A, const Vector &B) { - EXPECT_EQ(A, B); +template void EQ(const Set &A, const Vector &B) { + EXPECT_EQ(A, Set(B.begin(), B.end())); } -void EQ(const Vector &A, const Vector &B) { - Set a(A.begin(), A.end()); +void EQ(const Vector &A, const Vector &B) { + Set a; + for (const auto &File : A) + a.insert(File.Name); Set b(B.begin(), B.end()); EXPECT_EQ(a, b); } -static void Merge(const std::string &Input, - const Vector Result, - size_t NumNewFeatures) { - Merger M; - Vector NewFiles; - Set NewFeatures, NewCov; - EXPECT_TRUE(M.Parse(Input, true)); - EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); - EQ(NewFiles, Result); -} +#define TRACED_EQ(A, ...) \ + { \ + SCOPED_TRACE(#A); \ + EQ(A, __VA_ARGS__); \ + } -TEST(Merge, Good) { +TEST(Merger, Parse) { Merger M; + const char *kInvalidInputs[] = { + // Bad file numbers + "", + "x", + "0\n0", + "3\nx", + "2\n3", + "2\n2", + // Bad file names + "2\n2\nA\n", + "2\n2\nA\nB\nC\n", + // Unknown markers + "2\n1\nA\nSTARTED 0\nBAD 0 0x0", + // Bad file IDs + "1\n1\nA\nSTARTED 1", + "2\n1\nA\nSTARTED 0\nFT 1 0x0", + }; + for (auto S : kInvalidInputs) { + SCOPED_TRACE(S); + EXPECT_FALSE(M.Parse(S, false)); + } + + // Parse initial control file EXPECT_TRUE(M.Parse("1\n0\nAA\n", false)); - EXPECT_EQ(M.Files.size(), 1U); + ASSERT_EQ(M.Files.size(), 1U); EXPECT_EQ(M.NumFilesInFirstCorpus, 0U); EXPECT_EQ(M.Files[0].Name, "AA"); EXPECT_TRUE(M.LastFailure.empty()); EXPECT_EQ(M.FirstNotProcessedFile, 0U); + // Parse control file that failed on first attempt EXPECT_TRUE(M.Parse("2\n1\nAA\nBB\nSTARTED 0 42\n", false)); - EXPECT_EQ(M.Files.size(), 2U); + ASSERT_EQ(M.Files.size(), 2U); EXPECT_EQ(M.NumFilesInFirstCorpus, 1U); EXPECT_EQ(M.Files[0].Name, "AA"); EXPECT_EQ(M.Files[1].Name, "BB"); EXPECT_EQ(M.LastFailure, "AA"); EXPECT_EQ(M.FirstNotProcessedFile, 1U); + // Parse control file that failed on later attempt EXPECT_TRUE(M.Parse("3\n1\nAA\nBB\nC\n" - "STARTED 0 1000\n" - "FT 0 1 2 3\n" - "STARTED 1 1001\n" - "FT 1 4 5 6 \n" - "STARTED 2 1002\n" - "", true)); - EXPECT_EQ(M.Files.size(), 3U); + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "", + true)); + ASSERT_EQ(M.Files.size(), 3U); EXPECT_EQ(M.NumFilesInFirstCorpus, 1U); EXPECT_EQ(M.Files[0].Name, "AA"); EXPECT_EQ(M.Files[0].Size, 1000U); @@ -690,83 +697,172 @@ TEST(Merge, Good) { EXPECT_EQ(M.Files[2].Size, 1002U); EXPECT_EQ(M.LastFailure, "C"); EXPECT_EQ(M.FirstNotProcessedFile, 3U); - EQ(M.Files[0].Features, {1, 2, 3}); - EQ(M.Files[1].Features, {4, 5, 6}); - - - Vector NewFiles; - Set NewFeatures, NewCov; + TRACED_EQ(M.Files[0].Features, {1, 2, 3}); + TRACED_EQ(M.Files[1].Features, {4, 5, 6}); + + // Parse control file without features or PCs + EXPECT_TRUE(M.Parse("2\n0\nAA\nBB\n" + "STARTED 0 1000\n" + "FT 0\n" + "COV 0\n" + "STARTED 1 1001\n" + "FT 1\n" + "COV 1\n" + "", + true)); + ASSERT_EQ(M.Files.size(), 2U); + EXPECT_EQ(M.NumFilesInFirstCorpus, 0U); + EXPECT_TRUE(M.LastFailure.empty()); + EXPECT_EQ(M.FirstNotProcessedFile, 2U); + EXPECT_TRUE(M.Files[0].Features.empty()); + EXPECT_TRUE(M.Files[0].Cov.empty()); + EXPECT_TRUE(M.Files[1].Features.empty()); + EXPECT_TRUE(M.Files[1].Cov.empty()); + // Parse features and PCs EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n" - "STARTED 0 1000\nFT 0 1 2 3\n" - "STARTED 1 1001\nFT 1 4 5 6 \n" - "STARTED 2 1002\nFT 2 6 1 3 \n" - "", true)); - EXPECT_EQ(M.Files.size(), 3U); + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "COV 0 11 12 13\n" + "STARTED 1 1001\n" + "FT 1 4 5 6\n" + "COV 1 7 8 9\n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "COV 2 16 11 13\n" + "", + true)); + ASSERT_EQ(M.Files.size(), 3U); EXPECT_EQ(M.NumFilesInFirstCorpus, 2U); EXPECT_TRUE(M.LastFailure.empty()); EXPECT_EQ(M.FirstNotProcessedFile, 3U); - EQ(M.Files[0].Features, {1, 2, 3}); - EQ(M.Files[1].Features, {4, 5, 6}); - EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(0U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); - EQ(NewFiles, {}); + TRACED_EQ(M.Files[0].Features, {1, 2, 3}); + TRACED_EQ(M.Files[0].Cov, {11, 12, 13}); + TRACED_EQ(M.Files[1].Features, {4, 5, 6}); + TRACED_EQ(M.Files[1].Cov, {7, 8, 9}); + TRACED_EQ(M.Files[2].Features, {1, 3, 6}); + TRACED_EQ(M.Files[2].Cov, {16}); +} + +TEST(Merger, Merge) { + Merger M; + Set Features, NewFeatures; + Set Cov, NewCov; + Vector NewFiles; + // Adds new files and features + EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 6U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {"A", "B"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6}); + + // Doesn't return features or files in the initial corpus. EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nFT 0 1 2 3\n" - "STARTED 1 1001\nFT 1 4 5 6 \n" - "STARTED 2 1002\nFT 2 6 1 3\n" - "", true)); - EQ(M.Files[0].Features, {1, 2, 3}); - EQ(M.Files[1].Features, {4, 5, 6}); - EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(3U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); - EQ(NewFiles, {"B"}); - - // Same as the above, but with InitialFeatures. - EXPECT_TRUE(M.Parse("2\n0\nB\nC\n" - "STARTED 0 1001\nFT 0 4 5 6 \n" - "STARTED 1 1002\nFT 1 6 1 3\n" - "", true)); - EQ(M.Files[0].Features, {4, 5, 6}); - EQ(M.Files[1].Features, {1, 3, 6}); - Set InitialFeatures; - InitialFeatures.insert(1); - InitialFeatures.insert(2); - InitialFeatures.insert(3); - EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, {}, &NewCov, &NewFiles)); - EQ(NewFiles, {"B"}); -} + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {"B"}); + TRACED_EQ(NewFeatures, {4, 5, 6}); + + // No new features, so no new files + EXPECT_TRUE(M.Parse("3\n2\nA\nB\nC\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3\n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 0U); + TRACED_EQ(M.Files, {"A", "B", "C"}); + TRACED_EQ(NewFiles, {}); + TRACED_EQ(NewFeatures, {}); + + // Can pass initial features and coverage. + Features = {1, 2, 3}; + Cov = {}; + EXPECT_TRUE(M.Parse("2\n0\nA\nB\n" + "STARTED 0 1000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6\n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U); + TRACED_EQ(M.Files, {"A", "B"}); + TRACED_EQ(NewFiles, {"B"}); + TRACED_EQ(NewFeatures, {4, 5, 6}); + Features.clear(); + Cov.clear(); -TEST(Merge, Merge) { - - Merge("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nFT 0 1 2 3\n" - "STARTED 1 1001\nFT 1 4 5 6 \n" - "STARTED 2 1002\nFT 2 6 1 3 \n", - {"B"}, 3); - - Merge("3\n0\nA\nB\nC\n" - "STARTED 0 2000\nFT 0 1 2 3\n" - "STARTED 1 1001\nFT 1 4 5 6 \n" - "STARTED 2 1002\nFT 2 6 1 3 \n", - {"A", "B", "C"}, 6); - - Merge("4\n0\nA\nB\nC\nD\n" - "STARTED 0 2000\nFT 0 1 2 3\n" - "STARTED 1 1101\nFT 1 4 5 6 \n" - "STARTED 2 1102\nFT 2 6 1 3 100 \n" - "STARTED 3 1000\nFT 3 1 \n", - {"A", "B", "C", "D"}, 7); - - Merge("4\n1\nA\nB\nC\nD\n" - "STARTED 0 2000\nFT 0 4 5 6 7 8\n" - "STARTED 1 1100\nFT 1 1 2 3 \n" - "STARTED 2 1100\nFT 2 2 3 \n" - "STARTED 3 1000\nFT 3 1 \n", - {"B", "D"}, 3); + // Parse smaller files first + EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n" + "STARTED 0 2000\n" + "FT 0 1 2 3\n" + "STARTED 1 1001\n" + "FT 1 4 5 6 \n" + "STARTED 2 1002\n" + "FT 2 6 1 3 \n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 6U); + TRACED_EQ(M.Files, {"B", "C", "A"}); + TRACED_EQ(NewFiles, {"B", "C", "A"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6}); + + EXPECT_TRUE(M.Parse("4\n0\nA\nB\nC\nD\n" + "STARTED 0 2000\n" + "FT 0 1 2 3\n" + "STARTED 1 1101\n" + "FT 1 4 5 6 \n" + "STARTED 2 1102\n" + "FT 2 6 1 3 100 \n" + "STARTED 3 1000\n" + "FT 3 1 \n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 7U); + TRACED_EQ(M.Files, {"A", "B", "C", "D"}); + TRACED_EQ(NewFiles, {"D", "B", "C", "A"}); + TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6, 100}); + + // For same sized file, parse more features first + EXPECT_TRUE(M.Parse("4\n1\nA\nB\nC\nD\n" + "STARTED 0 2000\n" + "FT 0 4 5 6 7 8\n" + "STARTED 1 1100\n" + "FT 1 1 2 3 \n" + "STARTED 2 1100\n" + "FT 2 2 3 \n" + "STARTED 3 1000\n" + "FT 3 1 \n" + "", + true)); + EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U); + TRACED_EQ(M.Files, {"A", "B", "C", "D"}); + TRACED_EQ(NewFiles, {"D", "B"}); + TRACED_EQ(NewFeatures, {1, 2, 3}); } +#undef TRACED_EQ + TEST(DFT, BlockCoverage) { BlockCoverage Cov; // Assuming C0 has 5 instrumented blocks, From 84e5ca2110634c253f5dbe7bbfe76fc8db01d282 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 13:45:01 -0700 Subject: [PATCH 2/8] Put example crates in this workspace --- Cargo.toml | 6 ++++++ example/Cargo.toml | 3 --- example_arbitrary/Cargo.toml | 3 --- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db1d7b3..3afb2e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,9 @@ cc = "1.0" [features] arbitrary-derive = ["arbitrary/derive"] + +[workspace] +members = [ + "./example", + "./example_arbitrary", +] diff --git a/example/Cargo.toml b/example/Cargo.toml index ccd9977..b338d95 100644 --- a/example/Cargo.toml +++ b/example/Cargo.toml @@ -4,8 +4,5 @@ version = "0.1.0" authors = ["Simonas Kazlauskas "] edition = "2018" -[workspace] -members = ["."] - [dependencies] libfuzzer-sys = { path = ".." } diff --git a/example_arbitrary/Cargo.toml b/example_arbitrary/Cargo.toml index a40a776..4593fd0 100644 --- a/example_arbitrary/Cargo.toml +++ b/example_arbitrary/Cargo.toml @@ -4,8 +4,5 @@ version = "0.1.0" authors = ["Simonas Kazlauskas "] edition = "2018" -[workspace] -members = ["."] - [dependencies] libfuzzer-sys = { path = "..", features = ["arbitrary-derive"] } From c8fedd2601a01250e8658d622affd0963cf61450 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 17:49:53 -0700 Subject: [PATCH 3/8] Update the `arbitrary` example in the docs for `arbitrary` 1.0 --- src/lib.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4b0622a..c5ea4e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,8 +84,9 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize /// /// ```no_run /// #![no_main] +/// # mod foo { /// -/// use libfuzzer_sys::{arbitrary::{Arbitrary, Unstructured}, fuzz_target}; +/// use libfuzzer_sys::{arbitrary::{Arbitrary, Error, Unstructured}, fuzz_target}; /// /// #[derive(Debug)] /// pub struct Rgb { @@ -94,11 +95,8 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize /// b: u8, /// } /// -/// impl Arbitrary for Rgb { -/// fn arbitrary(raw: &mut U) -> Result -/// where -/// U: Unstructured + ?Sized -/// { +/// impl<'a> Arbitrary<'a> for Rgb { +/// fn arbitrary(raw: &mut Unstructured<'a>) -> Result { /// let mut buf = [0; 3]; /// raw.fill_buffer(&mut buf)?; /// let r = buf[0]; @@ -112,7 +110,12 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize /// fuzz_target!(|color: Rgb| { /// my_crate::convert_color(color); /// }); -/// # mod my_crate { fn convert_color(_: super::Rgb) {} } +/// # mod my_crate { +/// # use super::Rgb; +/// # pub fn convert_color(_: Rgb) {} +/// # } +/// # } +/// ``` #[macro_export] macro_rules! fuzz_target { (|$bytes:ident| $body:block) => { From b882dd6436d577acf5395a7af2dcdbb1a376db29 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 17:50:21 -0700 Subject: [PATCH 4/8] Mention the `arbitrary-derive` feature in the docs for greater discoverability --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index c5ea4e0..c2f39f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,6 +116,9 @@ pub fn initialize(_argc: *const isize, _argv: *const *const *const u8) -> isize /// # } /// # } /// ``` +/// +/// You can also enable the `arbitrary` crate's custom derive via this crate's +/// `"arbitrary-derive"` cargo feature. #[macro_export] macro_rules! fuzz_target { (|$bytes:ident| $body:block) => { From 70d77fb84c810b35ed83296cce8e83826aff6524 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 17:56:29 -0700 Subject: [PATCH 5/8] Add support for defining custom mutators This adds support for defining custom mutators, as described in https://github.com/google/fuzzing/blob/master/docs/structure-aware-fuzzing.md --- Cargo.toml | 5 + ci/script.sh | 15 +++ example_mutator/Cargo.toml | 11 ++ example_mutator/src/main.rs | 55 ++++++++++ src/lib.rs | 208 ++++++++++++++++++++++++++++++++++++ 5 files changed, 294 insertions(+) create mode 100644 example_mutator/Cargo.toml create mode 100755 example_mutator/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 3afb2e0..edd761b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,4 +21,9 @@ arbitrary-derive = ["arbitrary/derive"] members = [ "./example", "./example_arbitrary", + "./example_mutator", ] + +[dev-dependencies] +flate2 = "1.0.20" +rand = "0.8.3" diff --git a/ci/script.sh b/ci/script.sh index 25914fd..de649a8 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -39,3 +39,18 @@ RUST_LIBFUZZER_DEBUG_PATH=$(pwd)/debug_output \ cat $(pwd)/debug_output grep -q Rgb $(pwd)/debug_output popd + +pushd ./example_mutator +cargo rustc \ + --release \ + -- \ + -Cpasses='sancov' \ + -Cllvm-args=-sanitizer-coverage-level=3 \ + -Cllvm-args=-sanitizer-coverage-trace-compares \ + -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \ + -Cllvm-args=-sanitizer-coverage-stack-depth \ + -Cllvm-args=-sanitizer-coverage-trace-geps \ + -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \ + -Zsanitizer=address +(! $CARGO_TARGET_DIR/release/example_mutator -runs=10000000) +popd diff --git a/example_mutator/Cargo.toml b/example_mutator/Cargo.toml new file mode 100644 index 0000000..a74f650 --- /dev/null +++ b/example_mutator/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "example_mutator" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +flate2 = "1.0.20" +libfuzzer-sys = { path = ".." } diff --git a/example_mutator/src/main.rs b/example_mutator/src/main.rs new file mode 100755 index 0000000..41914ab --- /dev/null +++ b/example_mutator/src/main.rs @@ -0,0 +1,55 @@ +#![no_main] + +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use libfuzzer_sys::{fuzz_mutator, fuzz_target}; +use std::io::{Read, Write}; + +fuzz_target!(|data: &[u8]| { + // Decompress the input data and crash if it starts with "boom". + if let Some(data) = decompress(data) { + if data.starts_with(b"boom") { + panic!(); + } + } +}); + +fuzz_mutator!( + |data: &mut [u8], size: usize, max_size: usize, _seed: u32| { + // Decompress the input data. If that fails, use a dummy value. + let mut decompressed = decompress(&data[..size]).unwrap_or_else(|| b"hi".to_vec()); + + // Mutate the decompressed data with `libFuzzer`'s default mutator. Make + // the `decompressed` vec's extra capacity available for insertion + // mutations via `resize`. + let len = decompressed.len(); + let cap = decompressed.capacity(); + decompressed.resize(cap, 0); + let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len); + + // Recompress the mutated data. + let compressed = compress(&decompressed[..new_decompressed_size]); + + // Copy the recompressed mutated data into `data` and return the new size. + let new_size = std::cmp::min(max_size, compressed.len()); + data[..new_size].copy_from_slice(&compressed[..new_size]); + new_size + } +); + +fn decompress(data: &[u8]) -> Option> { + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + if decoder.read_to_end(&mut decompressed).is_ok() { + Some(decompressed) + } else { + None + } +} + +fn compress(data: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(data) + .expect("writing into a vec is infallible"); + encoder.finish().expect("writing into a vec is infallible") +} diff --git a/src/lib.rs b/src/lib.rs index c2f39f2..7e516f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,8 @@ extern "C" { // We do not actually cross the FFI bound here. #[allow(improper_ctypes)] fn rust_fuzzer_test_input(input: &[u8]); + + fn LLVMFuzzerMutate(data: *mut u8, size: usize, max_size: usize) -> usize; } #[doc(hidden)] @@ -188,3 +190,209 @@ macro_rules! fuzz_target { } }; } + +/// Define a custom mutator. +/// +/// This is optional, and libFuzzer will use its own, default mutation strategy +/// if this is not provided. +/// +/// You might consider using a custom mutator when your fuzz target is very +/// particular about the shape of its input: +/// +/// * You want to fuzz "deeper" than just the parser. +/// * The input contains checksums that have to match the hash of some subset of +/// the data or else the whole thing is invalid, and therefore mutating any of +/// that subset means you need to recompute the checksums. +/// * Small random changes to the input buffer make it invalid. +/// +/// That is, a custom mutator is useful in similar situations where [a `T: +/// Arbitrary` input type](macro.fuzz_target.html#arbitrary-input-types) is +/// useful. Note that the two approaches are not mutually exclusive; you can use +/// whichever is easier for your problem domain or both! +/// +/// ## Implementation Contract +/// +/// The original, unmodified input is given in `data[..size]`. +/// +/// You must modify the data in place and return the new size. +/// +/// The new size should not be greater than `max_size`. If this is not the case, +/// then the `data` will be truncated to fit within `max_size`. Note that +/// `max_size < size` is possible when shrinking test cases. +/// +/// You must produce the same mutation given the same `seed`. Generally, when +/// choosing what kind of mutation to make or where to mutate, you should start +/// by creating a random number generator (RNG) that is seeded with the given +/// `seed` and then consult the RNG whenever making a decision: +/// +/// ```no_run +/// #![no_main] +/// +/// use rand::{rngs::StdRng, Rng, SeedableRng}; +/// +/// libfuzzer_sys::fuzz_mutator!(|data: &mut [u8], size: usize, max_size: usize, seed: u32| { +/// let mut rng = StdRng::seed_from_u64(seed as u64); +/// +/// # let first_mutation = |_, _, _, _| todo!(); +/// # let second_mutation = |_, _, _, _| todo!(); +/// # let third_mutation = |_, _, _, _| todo!(); +/// # let fourth_mutation = |_, _, _, _| todo!(); +/// // Choose which of our four supported kinds of mutations we want to make. +/// match rng.gen_range(0..4) { +/// 0 => first_mutation(rng, data, size, max_size), +/// 1 => second_mutation(rng, data, size, max_size), +/// 2 => third_mutation(rng, data, size, max_size), +/// 3 => fourth_mutation(rng, data, size, max_size), +/// _ => unreachable!() +/// } +/// }); +/// ``` +/// +/// ## Example: Compression +/// +/// Consider a simple fuzz target that takes compressed data as input, +/// decompresses it, and then asserts that the decompressed data doesn't begin +/// with "boom". It is difficult for `libFuzzer` (or any other fuzzer) to crash +/// this fuzz target because nearly all mutations it makes will invalidate the +/// compression format. Therefore, we use a custom mutator that decompresses the +/// raw input, mutates the decompressed data, and then recompresses it. This +/// allows `libFuzzer` to quickly discover crashing inputs. +/// +/// ```no_run +/// #![no_main] +/// +/// use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +/// use libfuzzer_sys::{fuzz_mutator, fuzz_target}; +/// use std::io::{Read, Write}; +/// +/// fuzz_target!(|data: &[u8]| { +/// // Decompress the input data and crash if it starts with "boom". +/// if let Some(data) = decompress(data) { +/// if data.starts_with(b"boom") { +/// panic!(); +/// } +/// } +/// }); +/// +/// fuzz_mutator!( +/// |data: &mut [u8], size: usize, max_size: usize, _seed: u32| { +/// // Decompress the input data. If that fails, use a dummy value. +/// let mut decompressed = decompress(&data[..size]).unwrap_or_else(|| b"hi".to_vec()); +/// +/// // Mutate the decompressed data with `libFuzzer`'s default mutator. Make +/// // the `decompressed` vec's extra capacity available for insertion +/// // mutations via `resize`. +/// let len = decompressed.len(); +/// let cap = decompressed.capacity(); +/// decompressed.resize(cap, 0); +/// let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len); +/// +/// // Recompress the mutated data. +/// let compressed = compress(&decompressed[..new_decompressed_size]); +/// +/// // Copy the recompressed mutated data into `data` and return the new size. +/// let new_size = std::cmp::min(max_size, compressed.len()); +/// data[..new_size].copy_from_slice(&compressed[..new_size]); +/// new_size +/// } +/// ); +/// +/// fn decompress(compressed_data: &[u8]) -> Option> { +/// let mut decoder = GzDecoder::new(compressed_data); +/// let mut decompressed = Vec::new(); +/// if decoder.read_to_end(&mut decompressed).is_ok() { +/// Some(decompressed) +/// } else { +/// None +/// } +/// } +/// +/// fn compress(data: &[u8]) -> Vec { +/// let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); +/// encoder +/// .write_all(data) +/// .expect("writing into a vec is infallible"); +/// encoder.finish().expect("writing into a vec is infallible") +/// } +/// ``` +/// +/// This example is inspired by [a similar example from the official `libFuzzer` +/// docs](https://github.com/google/fuzzing/blob/master/docs/structure-aware-fuzzing.md#example-compression). +/// +/// ## More Example Ideas +/// +/// * A PNG custom mutator that decodes a PNG, mutates the image, and then +/// re-encodes the mutated image as a new PNG. +/// +/// * A [`serde`](https://serde.rs/) custom mutator that deserializes your +/// structure, mutates it, and then reserializes it. +/// +/// * A Wasm binary custom mutator that inserts, replaces, and removes a +/// bytecode instruction in a function's body. +/// +/// * An HTTP request custom mutator that inserts, replaces, and removes a +/// header from an HTTP request. +#[macro_export] +macro_rules! fuzz_mutator { + ( + | + $data:ident : &mut [u8] , + $size:ident : usize , + $max_size:ident : usize , + $seed:ident : u32 $(,)* + | + $body:block + ) => { + /// Auto-generated function. + #[export_name = "LLVMFuzzerCustomMutator"] + pub fn rust_fuzzer_custom_mutator( + $data: *mut u8, + $size: usize, + $max_size: usize, + $seed: std::os::raw::c_uint, + ) -> usize { + // Depending on if we are growing or shrinking the test case, `size` + // might be larger or smaller than `max_size`. The `data`'s capacity + // is the maximum of the two. + let len = std::cmp::max($max_size, $size); + let $data: &mut [u8] = unsafe { std::slice::from_raw_parts_mut($data, len) }; + + // `unsigned int` is generally a `u32`, but not on all targets. Do + // an infallible (and potentially lossy, but that's okay because it + // preserves determinism) conversion. + let $seed = $seed as u32; + + // Truncate the new size if it is larger than the max. + let new_size = { $body }; + std::cmp::min(new_size, $max_size) + } + }; +} + +/// The default `libFuzzer` mutator. +/// +/// You generally don't have to use this at all unless you're defining a +/// custom mutator with [the `fuzz_mutator!` macro][crate::fuzz_mutator]. +/// +/// Mutates `data[..size]` in place and returns the new size of the mutated +/// data. +/// +/// # Example +/// +/// ```no_run +/// // Create some data in a buffer. +/// let mut data = vec![0; 128]; +/// data[..5].copy_from_slice(b"hello"); +/// +/// // Ask `libFuzzer` to mutate the data. +/// let new_size = libfuzzer_sys::fuzzer_mutate(&mut data, 5); +/// +/// // Get the mutated data out of the buffer. +/// let mutated_data = &data[..new_size]; +/// ``` +pub fn fuzzer_mutate(data: &mut [u8], size: usize) -> usize { + assert!(size <= data.len()); + let new_size = unsafe { LLVMFuzzerMutate(data.as_mut_ptr(), size, data.len()) }; + assert!(new_size <= data.len()); + new_size +} From b5dfd80a21e36d48af3c927aa046a53a64f0a6f7 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 12 May 2021 17:59:24 -0700 Subject: [PATCH 6/8] Also run doc tests in CI --- ci/script.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/script.sh b/ci/script.sh index de649a8..b94de7b 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -5,6 +5,8 @@ cd $(dirname $0)/.. export CARGO_TARGET_DIR=$(pwd)/target +cargo test --doc + pushd ./example cargo rustc \ --release \ From 9a13a0c31e9748a0debb5c1bd26d394d862baac7 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 May 2021 09:44:47 -0700 Subject: [PATCH 7/8] Add a `max_size` parameter to `fuzzer_mutate` This allows users to force libfuzzer to only perform shrinking mutations. --- example_mutator/src/main.rs | 2 +- src/lib.rs | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/example_mutator/src/main.rs b/example_mutator/src/main.rs index 41914ab..c0ca0b4 100755 --- a/example_mutator/src/main.rs +++ b/example_mutator/src/main.rs @@ -24,7 +24,7 @@ fuzz_mutator!( let len = decompressed.len(); let cap = decompressed.capacity(); decompressed.resize(cap, 0); - let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len); + let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len, cap); // Recompress the mutated data. let compressed = compress(&decompressed[..new_decompressed_size]); diff --git a/src/lib.rs b/src/lib.rs index 7e516f3..9262eca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -285,7 +285,7 @@ macro_rules! fuzz_target { /// let len = decompressed.len(); /// let cap = decompressed.capacity(); /// decompressed.resize(cap, 0); -/// let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len); +/// let new_decompressed_size = libfuzzer_sys::fuzzer_mutate(&mut decompressed, len, cap); /// /// // Recompress the mutated data. /// let compressed = compress(&decompressed[..new_decompressed_size]); @@ -374,25 +374,37 @@ macro_rules! fuzz_mutator { /// You generally don't have to use this at all unless you're defining a /// custom mutator with [the `fuzz_mutator!` macro][crate::fuzz_mutator]. /// -/// Mutates `data[..size]` in place and returns the new size of the mutated -/// data. +/// Mutates `data[..size]` in place such that the mutated data is no larger than +/// `max_size` and returns the new size of the mutated data. +/// +/// To only allow shrinking mutations, make `max_size < size`. +/// +/// To additionally allow mutations that grow the size of the data, make +/// `max_size > size`. +/// +/// Both `size` and `max_size` must be less than or equal to `data.len()`. /// /// # Example /// /// ```no_run /// // Create some data in a buffer. /// let mut data = vec![0; 128]; -/// data[..5].copy_from_slice(b"hello"); +/// data[..b"hello".len()].copy_from_slice(b"hello"); /// -/// // Ask `libFuzzer` to mutate the data. -/// let new_size = libfuzzer_sys::fuzzer_mutate(&mut data, 5); +/// // Ask `libFuzzer` to mutate the data. By setting `max_size` to our buffer's +/// // full length, we are allowing `libFuzzer` to perform mutations that grow +/// // the size of the data, such as insertions. +/// let size = b"hello".len(); +/// let max_size = data.len(); +/// let new_size = libfuzzer_sys::fuzzer_mutate(&mut data, size, max_size); /// /// // Get the mutated data out of the buffer. /// let mutated_data = &data[..new_size]; /// ``` -pub fn fuzzer_mutate(data: &mut [u8], size: usize) -> usize { +pub fn fuzzer_mutate(data: &mut [u8], size: usize, max_size: usize) -> usize { assert!(size <= data.len()); - let new_size = unsafe { LLVMFuzzerMutate(data.as_mut_ptr(), size, data.len()) }; + assert!(max_size <= data.len()); + let new_size = unsafe { LLVMFuzzerMutate(data.as_mut_ptr(), size, max_size) }; assert!(new_size <= data.len()); new_size } From e80623eb76bf163e7b4970a8e5803a6fdbd071be Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 May 2021 09:45:26 -0700 Subject: [PATCH 8/8] Ignore crash files inside `./example_mutator` --- example_mutator/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 example_mutator/.gitignore diff --git a/example_mutator/.gitignore b/example_mutator/.gitignore new file mode 100644 index 0000000..d8d1df6 --- /dev/null +++ b/example_mutator/.gitignore @@ -0,0 +1 @@ +crash-*