Ingress

.github/workflows/unit_tests.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
 
     steps:
     - uses: actions/checkout@v2
@@ -28,22 +28,17 @@ jobs:
         git clone -b categorical_features https://github.com/equalitie/spark-iforest.git
         cd spark-iforest/python
         python setup.py sdist
-        pip install dist/pyspark-iforest-2.4.0.tar.gz
+        pip install dist/pyspark-iforest-2.4.0.99.tar.gz
         cd ../../
-        git clone https://github.com/equalitie/esretriever.git
-        cd esretriever
-        sudo pip install -e .
-        cd ..
         mkdir ./src/baskerville/logs/
     - name: Lint with flake8
       run: |
-        cd ./src
-        flake8 . --count --ignore=C901,W503,W504,E226 --max-line-length=127 --statistics
+        ./linting.sh
 
     - name: Test with pytest
       run: |
         export PYTHONPATH="./src:./esretriever/src"
         pytest ./tests/unit
     - name: License check
       run: |
-        python ./src/baskerville/util/licensing.py
+        python ./src/baskerville/util/licensing.py

Dockerfile

@@ -15,7 +15,7 @@ RUN rm -r $JAVA_HOME/*
 RUN mv jdk262/openlogic-openjdk-8u262-b10-linux-64/* $JAVA_HOME/
 
 COPY ./src /usr/local/baskerville/src
-COPY ./data/jars /usr/local/baskerville/data/jars
+COPY ./data /usr/local/baskerville/data
 COPY ./requirements.txt /usr/local/baskerville
 
 WORKDIR /usr/local/baskerville

data/samples/ats_log_schema.json

@@ -5,7 +5,7 @@
          "type": "string",
          "format": "date",
          "pattern": "(\\d\\d\\d\\d-([0-2])?\\d-([0-3])?\\dT?([0-2])?\\d:([0-5])?\\d:([0-5])?\\d\\.\\d?\\d?\\d?Z?)",
-         "required": true
+         "required": false
       },
       "ISP": {
          "type": "string"

data/samples/weblog_schema.json

@@ -0,0 +1,64 @@
+{
+   "name": "Weblogs",
+   "properties": {
+      "datestamp": {
+         "type": "string",
+         "format": "date",
+         "pattern": "(\\d\\d\\d\\d-([0-2])?\\d-([0-3])?\\dT?([0-2])?\\d:([0-5])?\\d:([0-5])?\\d\\.\\d?\\d?\\d?Z?)",
+         "required": true
+      },
+      "cache_result": {
+         "type": "string"
+      },
+      "client_ip": {
+         "type": "string",
+         "pattern": "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
+      },
+      "client_request_host": {
+         "type": "string"
+      },
+      "client_request_method": {
+         "type": "string",
+         "default": ""
+      },
+      "client_ua": {
+         "type": "string"
+      },
+      "client_url": {
+         "type": "string"
+      },
+      "content_type": {
+        "type": "string"
+      },
+      "http_request_scheme": {
+         "type": "string"
+      },
+      "http_response_code": {
+         "type": "string",
+         "pattern": "[1-5][0-9][0-9]"
+      },
+      "querystring": {
+         "type": "string"
+      },
+      "reply_length_bytes": {
+         "type": "string"
+      },
+      "geoip": {
+	  "location": {
+              "lon": {
+		  "type": "string",
+		  "format": "number"
+	      },
+	      "lat": {
+		  "type": "string",
+		  "format": "number"
+	      }
+	  },
+	  "country_name":{
+              "type": "string"
+	  }
+      }
+   },
+   "required": ["datestamp", "client_ip", "client_request_host", "client_ua", "client_url", "content_type", "http_response_code", "querystring", "reply_length_bytes", "geoip"],
+   "additionalProperties": false
+}