yara: update to v4.0.1, fix #758, fix #283.

Yaracpp needed some work, because YARA interface changed. Also, I simplified and refactored Yaracpp a bit.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 # dev
 
+* Enhancement: YARA updated to version 4.0.1 ([#758](https://github.com/avast/retdec/issues/758)), fixed Mach-O parsing issue ([#283](https://github.com/avast/retdec/issues/283)).
 * Enhancement: Improved detection of many packers/installers/compilers in `retdec-fileinfo`, including Armadillo ([#733](https://github.com/avast/retdec/pull/733)), VMProtect ([#734](https://github.com/avast/retdec/pull/734)), Petite ([#735](https://github.com/avast/retdec/pull/735)), Enigma ([#741](https://github.com/avast/retdec/pull/741)), ASPack ([#743](https://github.com/avast/retdec/pull/743)), Eziriz ([#746](https://github.com/avast/retdec/pull/746)), PyInstaller ([#748](https://github.com/avast/retdec/pull/748)), Astrum InstallWizard ([#753](https://github.com/avast/retdec/pull/753)), AutoHotKey ([#756](https://github.com/avast/retdec/pull/756)), AutoIt ([#757](https://github.com/avast/retdec/pull/757)), BAT to PE-EXE script compilers ([#761](https://github.com/avast/retdec/pull/761)), Bero ([#764](https://github.com/avast/retdec/pull/764)).
 * Enhancement: Enable .NET module in RetDec's YARA ([#747](https://github.com/avast/retdec/issues/747)).
 * Fix: Fixed build on some systems by adding missing includes of `<limits>` into `retdec-fileinfo` ([#745](https://github.com/avast/retdec/pull/745)).

cmake/deps.cmake

@@ -45,13 +45,12 @@ set(OPENSSL_ARCHIVE_SHA256
     CACHE INTERNAL ""
 )
 
-# 3.11.0
 set(YARA_URL
-    "https://github.com/VirusTotal/yara/archive/b9f925bb4e2b998bd6bb2f2e3cc2087c62fdd5b9.zip"
+    "https://github.com/VirusTotal/yara/archive/v4.0.1.zip"
     CACHE INTERNAL "URL of Yara archive to use."
 )
 set(YARA_ARCHIVE_SHA256
-    "8ebec236ede6f20f27ad597452a02dcc54ac3c73f09fea8c93c3a6bf10fbc890"
+    "4dcc6907b8537b67b52a61aa76f01196a4cc8e8e9f5fb6e4dd835692c2370e83"
     CACHE INTERNAL ""
 )
 

include/retdec/yaracpp/yara_detector.h

@@ -0,0 +1,140 @@
+/**
+ * @file include/retdec/yaracpp/yara_detector.h
+ * @brief Interpret of YARA rules.
+ * @copyright (c) 2017 Avast Software, licensed under the MIT license
+ */
+
+#ifndef RETDEC_YARACPP_YARA_DETECTOR_H
+#define RETDEC_YARACPP_YARA_DETECTOR_H
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "retdec/yaracpp/yara_rule.h"
+
+typedef struct _YR_COMPILER YR_COMPILER;
+typedef struct YR_RULES YR_RULES;
+typedef struct YR_SCAN_CONTEXT YR_SCAN_CONTEXT;
+
+namespace retdec {
+namespace yaracpp {
+
+/**
+ * Interpret of YARA rules
+ */
+class YaraDetector
+{
+	public:
+		/**
+		 * Structure for callback function
+		 */
+		class CallbackSettings
+		{
+			private:
+				/// set to @c true if you want store all rules (not only detected)
+				bool storeAll;
+				/// link to detected rules
+				std::vector<YaraRule> &storedDetected;
+				/// link to undetected rules
+				std::vector<YaraRule> &storedUndetected;
+			public:
+				CallbackSettings(
+						bool cStoreAll,
+						std::vector<YaraRule> &cDetected,
+						std::vector<YaraRule> &cUndetected
+				);
+
+				/// @name Other methods
+				/// @{
+				void addDetected(YaraRule &rule);
+				void addUndetected(YaraRule &rule);
+				bool storeAllRules() const;
+				/// @}
+		};
+
+		struct RuleFile
+		{
+			RuleFile(
+					const std::string& pathToFile_,
+					bool precompiled_,
+					FILE* handle_)
+					: pathToFile(pathToFile_)
+					, precompiled(precompiled_)
+					, handle(handle_)
+			{}
+
+			std::string pathToFile;
+			bool precompiled;
+			FILE* handle;
+		};
+
+	private:
+		/// compiler or text rules
+		YR_COMPILER *compiler = nullptr;
+		/// representation of files with rules
+		std::vector<FILE*> files;
+		/// representation of detected rules
+		std::vector<YaraRule> detectedRules;
+		/// representation of undetected rules
+		std::vector<YaraRule> undetectedRules;
+		/// rules from input text files
+		YR_RULES* textFilesRules = nullptr;
+		/// rules from precompiled files
+		std::vector<YR_RULES*> precompiledRules;
+		/// internal state of instance
+		bool stateIsValid = true;
+		/// indicates whether text files need recompilation
+		bool needsRecompilation = true;
+
+		/// @name Static auxiliary methods
+		/// @{
+		static int yaraCallback(
+				YR_SCAN_CONTEXT* context,
+				int message,
+				void *messageData,
+				void *userData
+		);
+		/// @}
+
+		/// @name Auxiliary detection methods
+		/// @{
+		template <typename T> bool analyzeWithScan(
+				T&& value,
+				bool storeAllRules = false
+		);
+		YR_RULES* getCompiledRules();
+		/// @}
+	public:
+		YaraDetector();
+		~YaraDetector();
+
+		/// @name Other methods
+		/// @{
+		bool addRules(const char *string);
+		bool addRuleFile(
+				const std::string &pathToFile,
+				const std::string &nameSpace = std::string()
+		);
+		bool isInValidState() const;
+		/// @}
+
+		/// @name Detection methods
+		/// @{
+		bool analyze(
+				const std::string &pathToInputFile,
+				bool storeAllRules = false
+		);
+		bool analyze(
+				std::vector<std::uint8_t> &bytes,
+				bool storeAllRules = false
+		);
+		const std::vector<YaraRule>& getDetectedRules() const;
+		const std::vector<YaraRule>& getUndetectedRules() const;
+		/// @}
+};
+
+} // namespace yaracpp
+} // namespace retdec
+
+#endif

include/retdec/yaracpp/types/yara_match.h → include/retdec/yaracpp/yara_match.h

@@ -1,10 +1,11 @@
 /**
- * @file include/retdec/yaracpp/types/yara_match.h
+ * @file include/retdec/yaracpp/yara_match.h
  * @brief Library representation of one YARA match.
  * @copyright (c) 2017 Avast Software, licensed under the MIT license
  */
 
-#pragma once
+#ifndef RETDEC_YARACPP_YARA_MATCH_H
+#define RETDEC_YARACPP_YARA_MATCH_H
 
 #include <cstdint>
 #include <vector>
@@ -18,8 +19,8 @@ namespace yaracpp {
 class YaraMatch
 {
 	private:
-		std::size_t offset;             ///< offset of match detection
-		std::vector<std::uint8_t> data; ///< data
+		std::size_t offset;
+		std::vector<std::uint8_t> data;
 	public:
 		/// @name Getters
 		/// @{
@@ -42,3 +43,5 @@ class YaraMatch
 
 } // namespace yaracpp
 } // namespace retdec
+
+#endif

include/retdec/yaracpp/types/yara_meta.h → include/retdec/yaracpp/yara_meta.h

@@ -1,10 +1,11 @@
 /**
- * @file include/retdec/yaracpp/types/yara_meta.h
+ * @file include/retdec/yaracpp/yara_meta.h
  * @brief Library representation of one YARA meta.
  * @copyright (c) 2017 Avast Software, licensed under the MIT license
  */
 
-#pragma once
+#ifndef RETDEC_YARACPP_YARA_META_H
+#define RETDEC_YARACPP_YARA_META_H
 
 #include <string>
 
@@ -23,10 +24,11 @@ class YaraMeta
 			Int
 		};
 	private:
-		std::string id;         ///< name of meta
-		Type type;              ///< type of meta
-		std::string strValue;   ///< string value of meta
-		std::uint64_t intValue; ///< int value of meta
+	    /// name of meta
+		std::string id;
+		Type type;
+		std::string strValue;
+		std::uint64_t intValue;
 	public:
 		/// @name Const getters
 		/// @{
@@ -53,3 +55,5 @@ class YaraMeta
 
 } // namespace yaracpp
 } // namespace retdec
+
+#endif

include/retdec/yaracpp/types/yara_rule.h → include/retdec/yaracpp/yara_rule.h

@@ -4,13 +4,14 @@
  * @copyright (c) 2017 Avast Software, licensed under the MIT license
  */
 
-#pragma once
+#ifndef RETDEC_YARACPP_YARA_RULE_H
+#define RETDEC_YARACPP_YARA_RULE_H
 
 #include <iosfwd>
 #include <vector>
 
-#include "retdec/yaracpp/types/yara_match.h"
-#include "retdec/yaracpp/types/yara_meta.h"
+#include "retdec/yaracpp/yara_match.h"
+#include "retdec/yaracpp/yara_meta.h"
 
 namespace retdec {
 namespace yaracpp {
@@ -21,9 +22,9 @@ namespace yaracpp {
 class YaraRule
 {
 	private:
-		std::string name;               ///< name of rule
-		std::vector<YaraMeta> metas;    ///< all meta-data related to rule
-		std::vector<YaraMatch> matches; ///< all matches of rule
+		std::string name;
+		std::vector<YaraMeta> metas;
+		std::vector<YaraMatch> matches;
 	public:
 		/// @name Const getters
 		/// @{
@@ -63,3 +64,5 @@ class YaraRule
 
 } // namespace yaracpp
 } // namespace retdec
+
+#endif

src/cpdetect/cpdetect.cpp

@@ -15,7 +15,7 @@
 #include "retdec/cpdetect/heuristics/macho_heuristics.h"
 #include "retdec/cpdetect/heuristics/pe_heuristics.h"
 #include "retdec/cpdetect/settings.h"
-#include "retdec/yaracpp/yara_detector/yara_detector.h"
+#include "retdec/yaracpp/yara_detector.h"
 
 using namespace retdec::fileformat;
 using namespace retdec::utils;

src/fileinfo/pattern_detector/pattern_detector.cpp

@@ -10,7 +10,7 @@
 #include "retdec/utils/filesystem_path.h"
 #include "retdec/utils/string.h"
 #include "fileinfo/pattern_detector/pattern_detector.h"
-#include "retdec/yaracpp/yara_detector/yara_detector.h"
+#include "retdec/yaracpp/yara_detector.h"
 
 using namespace retdec::utils;
 using namespace retdec::yaracpp;