Merge pull request #26 from obsrvbl-oss/update_public_repository

Update to 5.1.2

.gitignore

@@ -13,6 +13,10 @@ __pycache__/
 .installed.cfg
 *.egg
 
+# PyCharm
+.idea
+*.iml
+
 index.html
 .coverage
 

Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ARCH ?= amd64
-VERSION := 5.1.0
+VERSION := 5.1.2
 
 SCRIPTS_DIR := src/scripts
 uPNA_DIR := src/uPNA

README.md

@@ -2,22 +2,23 @@
 
 This repository is where the development of the Observable Networks Appliance (ONA) takes place. The ONA software is used to collect input data for Observable Networks' network security service. It can run on a variety of platforms, including embedded computers, physical servers, virtual machines, cloud servers, and Docker containers.
 
-See [observable.net](https://observable.net) for more information about Observable Networks' network security service.
-
 ## Supported platforms
 
 The following platforms are officially supported:
 
-* [Ubuntu 18.04 and later](https://onstatic.s3.amazonaws.com/ona/master/ona-service_UbuntuXenial_amd64.deb)
-* [RHEL 7 and compatible](https://onstatic.s3.amazonaws.com/ona/master/ona-service_RHEL_7_x86_64.rpm)
-* [RHEL 8 and compatible](https://onstatic.s3.amazonaws.com/ona/master/ona-service_RHEL_8_x86_64.rpm)
-* [Raspberry Pi with Raspbian](https://onstatic.s3.amazonaws.com/ona/master/ona-service_RaspbianJessie_armhf.deb)
+* [Ubuntu 18.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb)
+* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_7_x86_64.rpm)
+* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_8_x86_64.rpm)
+* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb)
+  ([installation guide](raspberry_pi_guide.md))
+* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_aarch64.deb)
+  ([installation guide](raspberry_pi_guide.md))
 * [Docker](https://github.com/obsrvbl/ona/blob/master/images/docker/Dockerfile)
 
 To install the latest version on 20.04 (recommended for physical and virtual machine installations):
 
 ```
-$ wget https://onstatic.s3.amazonaws.com/ona/master/ona-service_UbuntuXenial_amd64.deb
+$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb
 $ sudo apt install ./ona-service_UbuntuXenial_amd64.deb
 ```
 

images/docker/Dockerfile

@@ -1,17 +1,65 @@
-FROM ubuntu:18.04
-LABEL maintainer="Observable Networks <support@obsrvbl.com>"
+FROM debian:11 AS ona
 
-# Install dependencies.
-RUN apt-get update && apt-get install -y curl
+ARG APP_NAME=ona
 
-RUN curl -L -O https://github.com/mjschultz/netsa-pkg/releases/download/v0.1.27/netsa-pkg.deb
-RUN curl -L -O https://s3.amazonaws.com/onstatic/ona/master/ona-service_UbuntuXenialContainer_amd64.deb
-RUN apt install -y --fix-missing ./netsa-pkg.deb ./ona-service_UbuntuXenialContainer_amd64.deb
+ENV VIRTUAL_ENV "/root/.virtualenvs/$APP_NAME"
+ENV PATH "$VIRTUAL_ENV/bin:$PATH"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /obsrvbl
+
+# Set locales - https://hub.docker.com/_/debian
+RUN --mount=type=cache,target=/var/cache/apt apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --quiet --assume-yes \
+    locales \
+    && rm -rf /var/lib/apt/lists/* \
+	&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+ENV LANG en_US.utf8
+
+# Install python3 and pip
+RUN --mount=type=cache,target=/var/cache/apt apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --quiet --assume-yes \
+    build-essential \
+    python3 \
+    python3-pip \
+    python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN --mount=type=cache,target=/var/cache/apt apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --quiet --assume-yes \
+    wget curl \
+    nano \
+    vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set python venv
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN python -m venv $VIRTUAL_ENV
+
+# Install python packages
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --upgrade pip setuptools wheel ipython
+
+# Install netsa-pkg
+RUN curl -L -O https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb \
+    && apt-get update && apt-get install --assume-yes --fix-missing ./netsa-pkg.deb \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf netsa-pkg.deb
+
+# Use local copy of ONA service package if needed
+# COPY ona-service_UbuntuXenialContainer_amd64.deb ./
+
+# Install ONA service
+RUN if [ ! -f ./ona-service_UbuntuXenialContainer_amd64.deb ] ;\
+    then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenialContainer_amd64.deb ;\
+    else echo "Use cached package" ;fi \
+    && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuXenialContainer_amd64.deb \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf ona-service_UbuntuXenialContainer_amd64.deb
 
 # Switch to the unprivileged user, set some local configuration, and start.
 COPY run.sh /opt/obsrvbl-ona/run.sh
 RUN chmod +x /opt/obsrvbl-ona/run.sh
 
-USER obsrvbl_ona
-
-CMD ["/opt/obsrvbl-ona/run.sh"]
+CMD ["/opt/obsrvbl-ona/run.sh"]

images/iso/build_iso.sh

@@ -50,16 +50,18 @@ ubuntu_name="ubuntu-${RELEASE}-server-${ARCH}.iso"
 ona_name="ona-${RELEASE}-server-${ARCH}.iso"
 ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
 test -n "$ubuntu_url" || fatal "failed getting Ubuntu ISO download URL"
-ona_service_url="https://s3.amazonaws.com/onstatic/ona/master/ona-service_UbuntuXenial_amd64.deb"
-netsa_pkg_url="http://onstatic.s3.amazonaws.com/netsa-pkg.deb"
+ona_service_url="https://s3.amazonaws.com/onstatic/ona-service/master/ona-service_UbuntuXenial_amd64.deb"
+netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
 
 shift $(($OPTIND-1))
 
 test $EUID -ne 0 && sudo="sudo"
 which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
 which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
 
-mkdir "$DIR"/working || fatal
+[[ -d "$DIR" ]] || fatal  # invalid directory
+[[ -d "$DIR"/working && $(ls -A "$DIR"/working) ]] && fatal  # working directory exists and is not empty
+[[ -d "$DIR"/working ]] || mkdir "$DIR"/working # working directory does not exist, so create it
 (
   set -e
   cd "$DIR"/working

images/iso/isolinux/grub.cfg

@@ -11,13 +11,13 @@ set menu_color_normal=white/black
 set menu_color_highlight=black/light-gray
 
 set timeout=30
-menuentry "Install ONA (DHCP)" {
+menuentry "Install ONA (Static IP)" {
 	set gfxpayload=keep
-	linux	/install/vmlinuz  file=/cdrom/preseed/dhcp.seed quiet ---
+	linux	/install/vmlinuz  file=/cdrom/preseed/nodhcp.seed quiet ---
 	initrd	/install/initrd.gz
 }
-menuentry "Install ONA (Static IP)" {
+menuentry "Install ONA (DHCP)" {
 	set gfxpayload=keep
-	linux	/install/vmlinuz  file=/cdrom/preseed/nodhcp.seed quiet ---
+	linux	/install/vmlinuz  file=/cdrom/preseed/dhcp.seed quiet ---
 	initrd	/install/initrd.gz
 }

images/iso/isolinux/txt.cfg

@@ -1,9 +1,9 @@
 default observable
-label observable
-  menu label ^Install ONA (DHCP)
-  kernel /install/vmlinuz
-  append  file=/cdrom/preseed/dhcp.seed vga=788 initrd=/install/initrd.gz quiet --
 label observable
   menu label ^Install ONA (Static IP)
   kernel /install/vmlinuz
   append  file=/cdrom/preseed/nodhcp.seed vga=788 initrd=/install/initrd.gz quiet --
+label observable
+  menu label ^Install ONA (DHCP)
+  kernel /install/vmlinuz
+  append  file=/cdrom/preseed/dhcp.seed vga=788 initrd=/install/initrd.gz quiet --

pyproject.toml

@@ -0,0 +1,5 @@
+[tool.black]
+exclude = '/(\.git|\.local)/'
+skip-string-normalization = true
+target-version = ["py38"]
+line-length = 88

raspberry_pi_guide.md

@@ -0,0 +1,59 @@
+<!---
+title: Cisco Secure Cloud Analytics (SCA) - ONA Sensor with Raspberry PI
+author: 
+- Bruno Fagioli (bgimenez@cisco)
+- Iuri Mieiras (iuri@mieras.com)
+revision: 0
+--->
+##### Requirements before start 
+- RaspberryPI OS (32 or 64 bits); 
+- Make sure you have a monitor, mouse and keyboard connected in the device, at least until you get SSH access to it. 
+
+##### Required softwares
+- Raspberry PI ARM ONA image
+- CERT NetSA Security Suite (Silk and YAF)
+
+##### Step by step installation
+1. Download the .deb for ONA and tar.gz's for SILK and YAF:
+```bash
+cd /tmp
+wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb
+wget https://tools.netsa.cert.org/releases/silk-3.19.2.tar.gz
+wget https://tools.netsa.cert.org/releases/yaf-2.12.1.tar.gz
+```
+
+2. Install dependencies for compiling SILK and YAF:
+```bash
+sudo apt-get install build-essential libglib2.0-dev libfixbuf-dev libpcap-dev
+```
+
+3. Create ONA dirs for SILK and YAF binaries:
+```bash
+sudo mkdir /opt/silk /opt/yaf
+```
+
+4. Compile and install SILK:
+```bash
+tar -xvzf silk-3.19.2.tar.gz
+cd silk-3.19.2
+./configure --prefix=/opt/silk --with-libfixbuf
+make && sudo make install 
+cd ..
+```
+
+5. Compile and install YAF:
+```shell
+tar -xvzf yaf-2.12.1.tar.gz
+cd yaf-2.12.1 
+./configure --prefix=/opt/yaf 
+make && sudo make install 
+```
+
+6. Install ONA services: 
+```bash
+sudo apt install ./ona-service_RaspbianJessie_armhf.deb
+```
+
+7. After this last step, if you followed the SCA sensor guide, your sensor should be appearing within your SCA dashboard. Wait around 10-20min before the console start to show the netflows
+
+