From 4683b2d45cbb2d7c81790c5bc50f5092dec4bd3c Mon Sep 17 00:00:00 2001 From: michaelschratt Date: Sat, 24 Nov 2018 19:18:17 +0100 Subject: [PATCH 1/4] fix requirements, remove pickle as it is a python standard package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d1be5ad..fcd431a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ scandir pefile -pickle +lxml naiveBayesClassifier \ No newline at end of file From dbcd72f2e50d972a24240d99ef07e2fe6b9328db Mon Sep 17 00:00:00 2001 From: michaelschratt Date: Sat, 24 Nov 2018 19:19:14 +0100 Subject: [PATCH 2/4] add gitignore, makes working with IDEs more convenient --- .gitignore | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1eb6d35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,224 @@ +# Created by .ignore support plugin (hsz.mobi) +### Vagrant template +# General +.vagrant/ + +# Log files (if you are creating logs in debug mode, uncomment this) +# *.logs +### CVS template +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/dictionaries +.idea/**/shelf + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests +### Eclipse template + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders +dbs/ + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk +### VirtualEnv template +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +# [Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json +### SVN template +.svn/ +### Vim template +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* +### Dropbox template +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache +### JEnv template +# JEnv local Java version configuration file +.java-version + +# Used by previous versions of JEnv +.jenv-version +### JDeveloper template +# default application storage directory used by the IDE Performance Cache feature +.data/ + +# used for ADF styles caching +temp/ + +# default output directories +classes/ +deploy/ +javadoc/ + +# lock file, a part of Oracle Credential Store Framework +cwallet.sso.lck +.idea/encodings.xml +.idea/inspectionProfiles/ +.idea/misc.xml +.idea/modules.xml +.idea/vcs.xml +.idea/yarGen.iml From 3e67e52656c76975aa2fcd106f56f70fc50accac Mon Sep 17 00:00:00 2001 From: michaelschratt Date: Sat, 24 Nov 2018 19:20:07 +0100 Subject: [PATCH 3/4] add docker support --- .dockerignore | 2 ++ Dockerfile | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a4707ba --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +dbs \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5a63c49 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:2.7-alpine + +RUN apk add --update --no-cache g++ gcc libxslt-dev + +COPY . /app +WORKDIR /app + +RUN pip install -r requirements.txt + +ENTRYPOINT ["python", "yarGen.py"] \ No newline at end of file From e5bcf1e115cda12a72d93ac31353d56cc0e78986 Mon Sep 17 00:00:00 2001 From: michaelschratt Date: Sat, 24 Nov 2018 19:27:54 +0100 Subject: [PATCH 4/4] add docker section --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 60d7ad6..6f3b198 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,20 @@ The rule generation process also tries to identify similarities between the file 4. Run `python yarGen.py --update` to automatically download the built-in databases. The are saved into the './dbs' sub folder. (Download: 913 MB) 5. See help with `python yarGen.py --help` for more information on the command line parameters +### Docker + +- Build the image + +`docker build -t yargen .` + +- Update Datebases (make the dbs directory persistent) + +`docker run --rm -v /ABSPATH/TO/dbs:/app/dbs yargen --update` + +- Run yarGen + +`docker run --rm -v /ABSPATH/TO/dbs:/app/dbs yargen -a "Florian Roth" -r "Internal Research" -m /opt/mal/apt_case_32` + ### Memory Requirements Warning: yarGen pulls the whole goodstring database to memory and uses at least 3 GB of memory for a few seconds - 6 GB if opcodes evaluation is activated (--opcodes).