Server optimisations and docker smaller (#50)

* Optimalisations in server and searchcode

* Reduce docker image size (#47)

This reduces the image size from 1.44GB to 264MB. This is a decrease of more than 80%. 
We achieve this by using Alpine Linux with JDK 8 as our base image.

* Explicitly use python3 (#49)

* Fix for #46

README.md

@@ -39,7 +39,13 @@ Wordlists are in the following format:
 API_KEY|||80||| This contains an API key reference
 (https|http):\/\/.*api.*|||60||| This regex matches any URL containing 'api'
 ```
-__Note that these wordlists also support [regex](https://www.regular-expressions.info/examples.html) entries.__
+__Note that these wordlists support [regex](https://www.regular-expressions.info/examples.html) entries.__
+
+In the `exclusion_list.txt` you can define exclusions (if you have for some reason to much findings):
+```
+(https|http):\/\/.*api.*|||"res","layout"||| Like previously, note that "res","layout" resembles the path
+(https|http):\/\/.*api.*|||||| To exclude everywhere
+```
 
 ### Filetypes
 Any source file will be processed. This contains '.java', '.js', '.html', '.xml',... files.
@@ -120,6 +126,7 @@ PyInstaller can't handle subfolders with code, therefore we need to put the code
 sed -i 's/from helpers./from /g' helpers/*
 sed -i 's/from helpers./from /g' stacoan.py
 sed -i 's/os.path.join(parentdir, "config.ini")/"config.ini"/g' helpers/logger.py
+cp helpers/* ./ || :;
 ```
 Build stacoan:
 ```
@@ -134,6 +141,7 @@ PyInstaller can't handle subfolders with code, therefore we need to put the code
 sed -i '' 's/from helpers./from /g' helpers/*
 sed -i '' 's/from helpers./from /g' stacoan.py
 sed -i '' 's/os.path.join(parentdir, "config.ini")/"config.ini"/g' helpers/logger.py
+cp helpers/* ./ || :;
 ```
 Build stacoan:
 ```
@@ -147,6 +155,7 @@ PyInstaller can't handle subfolders with code, therefore we need to put the code
 sed -i 's/from helpers./from /g' helpers/*
 sed -i 's/from helpers./from /g' stacoan.py
 sed -i 's/os.path.join(parentdir, "config.ini")/"config.ini"/g' helpers/logger.py
+cp helpers/* ./ || :;
 ```
 Build stacoan:
 ```

docker/Dockerfile

@@ -1,27 +1,39 @@
-FROM python:3
+# -----------------------------------------------------------------------------------------
+# Start from Alpine OS with Oracle JDK8
+# -----------------------------------------------------------------------------------------
 
-RUN apt-get update && apt-get install upx software-properties-common -y
+FROM anapsix/alpine-java
 
-# https://github.com/re6exp/debian-jessie-oracle-jdk-8
-RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list  && \
-    echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list  && \
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886  && \
-    apt-get update
 
-RUN echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections  && \
-    echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections  && \
-    DEBIAN_FRONTEND=noninteractive  apt-get install -y --force-yes oracle-java8-installer oracle-java8-set-default
+# -----------------------------------------------------------------------------------------
+# Install Python3
+# -----------------------------------------------------------------------------------------
 
-RUN rm -rf /var/cache/oracle-jdk8-installer  && \
-    apt-get clean  && \
-    rm -rf /var/lib/apt/lists/*
+RUN apk add --no-cache python3
+RUN python3 -m ensurepip
+RUN rm -r /usr/lib/python*/ensurepip
+RUN pip3 install --upgrade pip setuptools
+RUN if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi
+RUN if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi
+RUN rm -r /root/.cache
 
+
+# -----------------------------------------------------------------------------------------
+# Install StaCoAn
+# -----------------------------------------------------------------------------------------
+
+RUN apk add --no-cache git
 RUN git clone https://github.com/vincentcox/StaCoAn/
 WORKDIR /StaCoAn/src
 RUN pip3 install -r requirements.txt && chmod u+rwx /StaCoAn/src/jadx/bin/jadx
 
 COPY stacoan.sh /stacoan.sh
 
+
+# -----------------------------------------------------------------------------------------
+# Expose us
+# -----------------------------------------------------------------------------------------
+
 EXPOSE 8000
 EXPOSE 8080
 ENTRYPOINT ["/bin/bash", "/stacoan.sh"]

docker/stacoan.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-python /StaCoAn/src/stacoan.py --disable-browser --enable-server
+python3 /StaCoAn/src/stacoan.py --disable-browser --enable-server
 
 # https://stackoverflow.com/questions/90418/exit-shell-script-based-on-process-exit-code
 rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi

src/config/exclusion_list.txt

@@ -1,2 +1,3 @@
 http:|||"res","layout"||| Suggested by Adi
-(https|http):\/\/.*api.*|||"res","layout"||| Suggested by Adi
+(https|http):\/\/.*api.*|||"res","layout"||| Suggested by Adi
+http:\/\/schemas\.android\.com\/apk\/res\/android||||||

src/helpers/file.py

@@ -41,15 +41,19 @@ def find_matches_in_db_file(self):
             for row in cursor.fetchall():
                 line += 1
                 for matchword in Searchwords.db_search_words:
-                    if matchword in str(row):
+                    exclude = False
+                    if re.match(File.non_regex_indicator, str(row)):
+                        Searchwords.db_search_words[matchword].regex = True
+                    if re.search(matchword, str(row), re.IGNORECASE):
                         for item in Searchwords.exclusion_list:
                             if item[0] == matchword and item[1] in self.file_path:
-                                Logger("Exclusion found: %s in file %s" % (str(item[0]), self.file_path))
-                            else:
-                                importance = Searchwords.db_search_words[matchword]
-                                db_match = MatchDatabase(matchword, line, str(table_name), str(row), importance)
-                                self.db_matches.append(db_match)
-                                self.all_matches.append(db_match)
+                                Logger("Database xclusion found: %s in file %s" % (str(item[0]), self.file_path), Logger.INFO)
+                                exclude = True
+                        if exclude == False:
+                            importance = Searchwords.db_search_words[matchword]
+                            db_match = MatchDatabase(matchword, line, str(table_name), str(row), importance)
+                            self.db_matches.append(db_match)
+                            self.all_matches.append(db_match)
         self.orden_matches()
 
     def find_matches_in_src_file(self, CODE_OFFSET, QUERY_IMPORTANCE):
@@ -64,31 +68,22 @@ def find_matches_in_src_file(self, CODE_OFFSET, QUERY_IMPORTANCE):
             for query in Searchwords.src_search_words.keys():
                 if int(Searchwords.src_search_words[query]) > QUERY_IMPORTANCE:
                     if re.match(File.non_regex_indicator, query):
-                        if query.lower() in line.lower():
-                            for item in Searchwords.exclusion_list:
-                                if item[0] == query and item[1] in self.file_path:
-                                    Logger("Exclusion found: %s in file %s" % (str(item[0]), self.file_path))
-                                else:
-                                    upper_range = min(line_index + CODE_OFFSET, len(lines_in_file)+1)
-                                    lower_range = max(line_index - CODE_OFFSET-1, 1)
-                                    src_match = MatchSource(query, line_index, lines_in_file[lower_range:upper_range],
-                                                            Searchwords.src_search_words[query], len(lines_in_file))
-                                    self.all_matches.append(src_match)
-                                    self.src_matches.append(src_match)
-
-                    else:
-                        if re.search(query, line.lower(), re.IGNORECASE):
-                            exclude = False
-                            for item in Searchwords.exclusion_list:
-                                if item[0] == query and item[1] in self.file_path:
+                        Searchwords.src_search_words[query].regex = True
+                    if re.search(query, line.lower(), re.IGNORECASE):
+                        exclude = False
+                        for item in Searchwords.exclusion_list:
+                            if re.search(item[0], line, re.IGNORECASE):
+                                if (item[1] in self.file_path or (item[1] == "" or item[1] is None)):
+                                    Logger("Exclusion found: %s in file %s" % (str(item[0]), self.file_path),
+                                           Logger.INFO)
                                     exclude = True
-                            if exclude == False:
-                                upper_range = min(line_index + CODE_OFFSET, len(lines_in_file)+1)
-                                lower_range = max(line_index - CODE_OFFSET-1, 1)
-                                src_match = MatchSource(query, line_index, lines_in_file[lower_range:upper_range],
-                                                        Searchwords.src_search_words[query], len(lines_in_file))
-                                self.all_matches.append(src_match)
-                                self.src_matches.append(src_match)
+                        if exclude == False:
+                            upper_range = min(line_index + CODE_OFFSET, len(lines_in_file)+1)
+                            lower_range = max(line_index - CODE_OFFSET-1, 1)
+                            src_match = MatchSource(query, line_index, lines_in_file[lower_range:upper_range],
+                                                    Searchwords.src_search_words[query], len(lines_in_file))
+                            self.all_matches.append(src_match)
+                            self.src_matches.append(src_match)
             line_index = line_index + 1
         self.orden_matches()
 

src/helpers/match.py

@@ -3,6 +3,7 @@ class Match:
     def __init__(self, matchword, importance):
         self.matchword = matchword
         self.importance = importance
+        self.regex = False
 
 
 class MatchSource(Match):

src/helpers/report_html.py

@@ -107,7 +107,7 @@ def get_source_code_from_file(self, file_path, project):
                                               Searchwords.all_searchwords[match.matchword])):
                                 self.text("report")
                             with self.tag('h5'):
-                                if re.match(Report_html.non_regex_indicator, match.matchword):
+                                if not match.regex:
                                     self.text(match.matchword)
                                     # print(match.matchword)
                                 else:
@@ -206,7 +206,7 @@ def get_source_code_from_file(self, file_path, project):
                                               Searchwords.all_searchwords[match.matchword])):
                                     self.text("report")
                                 with self.tag('h5'):
-                                    if re.match(Report_html.non_regex_indicator, match.matchword):
+                                    if not match.regex:
                                         self.text(match.matchword)
                                     else:
                                         self.text("regex: " + match.matchword)
@@ -346,7 +346,7 @@ def findings_overview(self, project):
                                               Searchwords.all_searchwords[match.matchword])):
                                             self.text("beenhere")
                                         with self.tag('h5'):
-                                            if re.match(Report_html.non_regex_indicator, match.matchword):
+                                            if not match.regex:
                                                 self.text(match.matchword)
                                                 # print(match.matchword)
                                             else:

src/helpers/server.py

@@ -40,10 +40,11 @@ def log_request(self, code='-', size='-'):
                 Logger(self.requestline + " " + str(code) + " " + str(size), Logger.INFO)
 
         def log_error(self, format, *args):
-            Logger(("%s - - [%s] %s\n" %
-                    (self.address_string(),
-                     self.log_date_time_string(),
-                     format % args)), Logger.WARNING)
+            if not "robots.txt" in self.requestline:
+                Logger(("%s - - [%s] %s\n" %
+                        (self.address_string(),
+                         self.log_date_time_string(),
+                         format % args)), Logger.WARNING)
 
         def log_message(self, format, *args):
             Logger(("%s - - [%s] %s\n" %
@@ -59,10 +60,11 @@ def log_request(self, code='-', size='-'):
             Logger(self.requestline+ " " + str(code) + " " + str(size), Logger.INFO)
 
         def log_error(self, format, *args):
-            Logger(("%s - - [%s] %s\n" %
-                    (self.address_string(),
-                     self.log_date_time_string(),
-                     format % args)), Logger.WARNING)
+            if not "robots.txt" in self.requestline:
+                Logger(("%s - - [%s] %s - %s\n" %
+                        (self.address_string(),
+                         self.log_date_time_string(),
+                         format % args, str(self.requestline))), Logger.WARNING)
 
         def log_message(self, format, *args):
             Logger(("%s - - [%s] %s\n" %
@@ -530,7 +532,9 @@ def list_directory(self, path):
                         <button type="submit" class="box__button">Upload</button>
                     </div>
                     <div class="box__uploading">Uploading&hellip;</div>
-                    <div class="box__success">Done! <a href="/" class="box__restart2" id="done_link" onclick="javascript:event.target.port=8080" role="button" target="_blank">Open report!</a></div>
+                    <div class="box__success">Done! <a href="/" class="box__restart2" id="done_link" onclick="javascript:event.target.port=""")
+            f.write(bytearray(ServerWrapper.REPORT_SERVER_PORT))
+            f.write(b"""" role="button" target="_blank">Open report!</a></div>
                     <div class="box__error">Error! <span></span>. <a href="/?" class="box__restart" role="button">Try again!</a></div>
                 </form>
             </div>

src/stacoan.py

@@ -47,11 +47,7 @@ def parse_args():
 
 # Note that this server(args) function CANNOT be placed in the server.py file. It calls "program()", which cannot be
 # called from the server.py file
-def server(args):
-    config = configparser.ConfigParser()
-    config.read("config.ini")
-    # Drag and drop server
-    server_enabled = config.getboolean("ProgramConfig", 'server_enabled')
+def server(args, server_enabled, DRAG_DROP_SERVER_PORT):
     # Windows multithreading is different on Linux and windows (fork <-> new instance without parent context and args)
     child=False
     if os.name == 'nt':
@@ -108,10 +104,12 @@ def serverlistener(in_q):
         drag_drop_server_thread.start()
 
         if not args.disable_browser:
-            Logger("Now automatically opening the HTML report.")
-            DRAG_DROP_SERVER_PORT = json.loads(config.get("Server", 'drag_drop_server_port'))
             # Open the webbrowser to the generated start page.
             report_folder_start = "http:///127.0.0.1:" + str(DRAG_DROP_SERVER_PORT)
+            if sys.platform == "darwin":  # check if on OSX
+                # strip off http:///
+                report_folder_start = str(report_folder_start).strip("http:///")
+                report_folder_start = "file:///" + report_folder_start
             webbrowser.open(report_folder_start)
 
         # Keep waiting until q is gone.
@@ -134,6 +132,8 @@ def program(args):
 
     config = configparser.ConfigParser()
     config.read("config.ini")
+    server_enabled = config.getboolean("ProgramConfig", 'server_enabled')
+    DRAG_DROP_SERVER_PORT = json.loads(config.get("Server", 'drag_drop_server_port'))
 
     # Update log level
     if not (args.log_warnings or args.log_errors):
@@ -148,12 +148,18 @@ def program(args):
     Searchwords.searchwords_import(Searchwords())
 
     # Server(args) checks if the server should be run and handles the spawning of the server and control of it
-    server(args)
+    server(args, server_enabled, DRAG_DROP_SERVER_PORT)
 
     # For each project (read .ipa or .apk file), run the scripts.
     all_project_paths = args.project
+
+    if not all_project_paths:
+        sys.exit(0)
     for project_path in all_project_paths:
-        Project.projects[project_path] = Project(project_path)
+        try:
+            Project.projects[project_path] = Project(project_path)
+        except:
+            sys.exit(0)
 
         report_folder = os.path.join(Project.projects[project_path].name, config.get("ProgramConfig", 'report_folder'))
         report_folder_start = os.path.join(os.getcwd(), report_folder, "start.html")
@@ -240,9 +246,10 @@ def program(args):
         Logger("HTML report is available at: %s" % report_folder_start)
         if (not args.disable_browser) and not (args.enable_server or server_enabled):
             Logger("Now automatically opening the HTML report.")
-
             # Open the webbrowser to the generated start page.
             if sys.platform == "darwin":  # check if on OSX
+                # strip off http:///
+                report_folder_start = str(report_folder_start).strip("http:///")
                 report_folder_start = "file:///" + report_folder_start
             webbrowser.open(report_folder_start)
     # Exit program