Merge pull request #23 from CESNET/devel

Merge devel to master and bump version to 0.3.0
CESNET · Mar 11, 2022 · 936d3cb · 936d3cb
2 parents 0302f9c + b74793d
commit 936d3cb
Show file tree

Hide file tree

Showing 107 changed files with 11,840 additions and 91 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -8,12 +8,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image: ['ubuntu:18.04', 'ubuntu:20.04', 'debian:stretch', 'debian:buster', 'debian:bullseye', 'centos:7', 'centos:8']
+        image:
+          - 'ubuntu:18.04'
+          - 'ubuntu:20.04'
+          - 'debian:stretch'
+          - 'debian:buster'
+          - 'debian:bullseye'
+          - 'centos:7'
+          - 'quay.io/centos/centos:stream8'
+          - 'oraclelinux:8'
 
     name: Build on ${{ matrix.image }}
     container: ${{ matrix.image }}
     steps:
-    - uses: actions/checkout@v1
 
     # Dependencies ---------------------------------------------------------------------------
     - name: Install dependencies (Ubuntu/Debian)
@@ -23,14 +30,21 @@ jobs:
         apt-get -y install git gcc g++ cmake make libxml2-dev liblz4-dev libzstd-dev
       env:
         DEBIAN_FRONTEND: noninteractive
-    - name: Install dependencies (CentOS)
-      if: startsWith(matrix.image, 'centos')
+    - name: Enable EPEL (CentOS)
+      if: contains(matrix.image, 'centos')
       run: |
         yum -y install epel-release
+    - name: Enable EPEL (Oracle Linxu)
+      if: contains(matrix.image, 'oraclelinux')
+      run: |
+        dnf -y install oracle-epel-release-el8
+    - name: Install dependencies (CentOS, Oracle Linux)
+      if: contains(matrix.image, 'centos') || contains(matrix.image, 'oraclelinux')
+      run: |
         yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
-    - name: Install depedencies (Fedora)
-      if: startsWith(matrix.image, 'fedora')
-      run: dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
+
+    # Checkout repository --------------------------------------------------------------------
+    - uses: actions/checkout@v2
 
     # Build ----------------------------------------------------------------------------------
     # Note: Unit tests are disabled on CentOS7 due to outdated GCC version

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -15,13 +15,16 @@ jobs:
     container: ${{ matrix.image }}
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Define global variables
-      run: echo "::set-output name=zip_file::libfds-${IMAGE//:/}-$GITHUB_SHA.zip"
-      shell: bash
-      env:
-        IMAGE: ${{ matrix.image }}
-      id: vars
+    - uses: actions/checkout@v2
+    - name: Define variables
+      uses: actions/github-script@v5
+      with:
+        script: |
+          const sha = context.sha.substring(0, 8);
+          const image = `${{ matrix.image }}`
+          const distro = image.split('/').pop().replace(/:/g,'_');
+          const zip = `libfds-${distro}-${sha}`;
+          core.exportVariable('ZIP_FILE', zip);
 
     # Dependencies ---------------------------------------------------------------------------
     - name: Install dependencies (Ubuntu/Debian)
@@ -38,61 +41,70 @@ jobs:
         mkdir build && cd build
         cmake .. -DPACKAGE_BUILDER_DEB=On -DCPACK_PACKAGE_CONTACT="GitHub actions <no-reply@example.com>"
         make deb
-    - name: Pack DEB packages
-      working-directory: 'build/pkg/deb/debbuild/'
-      run: zip "$GITHUB_WORKSPACE/$ZIP_FILE" *.deb *.ddeb *.tar.gz *.dsc
-      env:
-        ZIP_FILE: ${{ steps.vars.outputs.zip_file }}
     - name: Archive DEB packages
       if: github.event_name == 'push'
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
-        name: ${{ steps.vars.outputs.zip_file }}
-        path: ${{ steps.vars.outputs.zip_file }}
+        name: ${{ env.ZIP_FILE }}
+        path: |
+          build/pkg/deb/debbuild/*.deb
+          build/pkg/deb/debbuild/*.ddeb
+          build/pkg/deb/debbuild/*.tar.gz
+          build/pkg/deb/debbuild/*.dsc
 
   rpm:
     # Try to build RPM packages
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        image: ['centos:7', 'centos:8']
+        image:
+          - 'centos:7'
+          - 'quay.io/centos/centos:stream8'
+          - 'oraclelinux:8'
 
     name: Build RPMs on ${{ matrix.image }}
     container: ${{ matrix.image }}
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Define global variables
-      run: echo "::set-output name=zip_file::libfds-${IMAGE//:/}-$GITHUB_SHA.zip"
-      env:
-        IMAGE: ${{ matrix.image }}
-      id: vars
+    - name: Define variables
+      uses: actions/github-script@v5
+      with:
+        script: |
+          const sha = context.sha.substring(0, 8);
+          const image = `${{ matrix.image }}`
+          const distro = image.split('/').pop().replace(/:/g,'_');
+          const zip = `libfds-${distro}-${sha}`;
+          core.exportVariable('ZIP_FILE', zip);
 
     # Dependencies ---------------------------------------------------------------------------
-    - name: Install dependencies (CentOS)
-      if: startsWith(matrix.image, 'centos')
+    - name: Enable EPEL (CentOS)
+      if: contains(matrix.image, 'centos')
       run: |
         yum -y install epel-release
+    - name: Enable EPEL (Oracle Linxu)
+      if: contains(matrix.image, 'oraclelinux')
+      run: |
+        dnf -y install oracle-epel-release-el8
+    - name: Install dependencies (CentOS, Oracle Linux)
+      if: contains(matrix.image, 'centos') || contains(matrix.image, 'oraclelinux')
+      run: |
         yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel rpm-build
-    - name: Install depedencies (Fedora)
-      if: startsWith(matrix.image, 'fedora')
-      run: dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel rpm-build
+
+    # Checkout repository --------------------------------------------------------------------
+    - uses: actions/checkout@v2
 
     # Build ----------------------------------------------------------------------------------
     - name: Build RPM packages
       run: |
         mkdir build && cd build
         cmake .. -DPACKAGE_BUILDER_RPM=On -DCPACK_PACKAGE_CONTACT="GitHub actions <no-reply@example.com>"
         make rpm
-    - name: Pack RPM packages
-      working-directory: 'build/pkg/rpm/rpmbuild'
-      run: zip -r "$GITHUB_WORKSPACE/$ZIP_FILE" RPMS SRPMS
-      env:
-        ZIP_FILE: ${{ steps.vars.outputs.zip_file }}
     - name: Archive RPM packages
       if: github.event_name == 'push'
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
-        name: ${{ steps.vars.outputs.zip_file }}
-        path: ${{ steps.vars.outputs.zip_file }}
+        name: ${{ env.ZIP_FILE }}
+        path: |
+          build/pkg/rpm/rpmbuild/RPMS/
+          build/pkg/rpm/rpmbuild/SRPMS/
diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,10 @@ cmake-build-release/
 # JetBrains files
 .idea/
 
+# VS Code files
+.vscode/
+
+
 # Prerequisites
 *.d
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,8 +13,8 @@ endif()
 
 # Versions and other informations
 set(LIBFDS_VERSION_MAJOR 0)
-set(LIBFDS_VERSION_MINOR 2)
-set(LIBFDS_VERSION_PATCH 1)
+set(LIBFDS_VERSION_MINOR 3)
+set(LIBFDS_VERSION_PATCH 0)
 set(LIBFDS_VERSION
 	${LIBFDS_VERSION_MAJOR}.${LIBFDS_VERSION_MINOR}.${LIBFDS_VERSION_PATCH})
 

diff --git a/config/system/aliases.xml b/config/system/aliases.xml
@@ -20,14 +20,31 @@
 		</aggregation>
 	</element>
 
+	<!-- Number of bytes -->
+	<element>
+		<name>Bytes</name>
+		<alias>bytes</alias>
+
+		<source>
+			<id>iana:octetDeltaCount</id>
+		</source>
+
+		<aggregation>
+			<method>sum</method>
+			<order>descendent</order>
+		</aggregation>
+	</element>
+
 	<!-- Source IP address (IPv4 or IPv6) -->
 	<element>
 		<name>SRC IP</name>
+		<alias>src ip</alias>
 		<alias>srcip</alias>
+		<alias>src host</alias>
 		<alias>srchost</alias>
 
 		<!-- possible data types: ipv4Address/ipv6Address -->
-		<source mode="anyOf">
+		<source mode="firstOf">
 			<id>iana:sourceIPv4Address</id>
 			<id>iana:sourceIPv6Address</id>
 		</source>
@@ -36,11 +53,13 @@
 	<!-- Destination IP address (IPv4 or IPv6) -->
 	<element>
 		<name>DST IP</name>
+		<alias>dst ip</alias>
 		<alias>dstip</alias>
+		<alias>dst host</alias>
 		<alias>dsthost</alias>
 
 		<!-- possible data types: ipv4Address/ipv6Address -->
-		<source mode="anyOf">
+		<source mode="firstOf">
 			<id>iana:destinationIPv4Address</id>
 			<id>iana:destinationIPv6Address</id>
 		</source>
@@ -54,8 +73,10 @@
 
 		<!-- possible data types: ipv4Address/ipv6Address -->
 		<source mode="anyOf">
-			<id>@srcip</id>
-			<id>@dstip</id>
+			<id>iana:sourceIPv4Address</id>
+			<id>iana:sourceIPv6Address</id>
+            <id>iana:destinationIPv4Address</id>
+			<id>iana:destinationIPv6Address</id>
 		</source>
 	</element>
 
@@ -64,7 +85,6 @@
 		<name>Flow Start</name>
 		<alias>flowstart</alias>
 
-		<dataType>dateTimeMilliseconds</dataType>
 		<aggregation>
 			<method>min</method>
 			<order>ascendent</order>
@@ -83,7 +103,6 @@
 		<name>Flow End</name>
 		<alias>flowend</alias>
 
-		<dataType>dateTimeMilliseconds</dataType>
 		<aggregation>
 			<method>max</method>
 			<order>ascendent</order>
@@ -97,47 +116,90 @@
 		</source>
 	</element>
 
-	<!-- Flow duration (calculation example) -->
 	<element>
-		<name>Duration</name>
-		<alias>duration</alias>
+		<name>Protocol</name>
 
-		<dataType>dateTimeMilliseconds</dataType>
-		<aggregation>
-			<method>sum</method>
-			<order>descendent</order>
-		</aggregation>
+		<alias>protocol</alias>
+		<alias>proto</alias>
 
-		<source mode="firstOf">
-			<!-- First, try to calculate the duration -->
-			<calculated expr="$1 - $2">
-				<arg id="1">@flowEnd</arg>
-				<arg id="2">@flowStart</arg>
-			</calculated>
-			<!-- If calculation failed, try to find other elements -->
-			<id>iana:flowDurationMilliseconds</id>
-			<id>iana:flowDurationMicroseconds</id>
+		<source>
+			<id>iana:protocolIdentifier</id>
 		</source>
 	</element>
 
-	<!-- Bytes per second (another calculated value) -->
 	<element>
-		<name>Bytes per second</name>
-		<alias>bps</alias>
+		<name>IPv4 Address</name>
 
-		<dataType>float64</dataType>
-		<aggregation>
-			<method>sum</method>
-			<order>descendent</order>
-		</aggregation>
+		<alias>ip4</alias>
+		<alias>ipv4</alias>
+		<alias>ipv4addr</alias>
+		<alias>ipv4address</alias>
+
+		<source>
+			<id>iana:sourceIPv4Address</id>
+			<id>iana:destinationIPv4Address</id>
+		</source>
+	</element>
+
+	<element>
+		<name>IPv6 Address</name>
+
+		<alias>ip6</alias>
+		<alias>ipv6</alias>
+		<alias>ipv6addr</alias>
+		<alias>ipv6address</alias>
+
+		<source>
+			<id>iana:sourceIPv6Address</id>
+			<id>iana:destinationIPv6Address</id>
+		</source>
+	</element>
+
+	<element>
+		<name>Port</name>
+
+		<alias>port</alias>
+
+		<source>
+			<id>iana:sourceTransportPort</id>
+			<id>iana:destinationTransportPort</id>
+		</source>
+	</element>
+
+	<element>
+		<name>Source Port</name>
+
+		<alias>srcport</alias>
+		<alias>sourceport</alias>
+		<alias>src port</alias>
+
+		<source>
+			<id>iana:sourceTransportPort</id>
+		</source>
+	</element>
+
+	<element>
+		<name>Destination Port</name>
+
+		<alias>dstport</alias>
+		<alias>destport</alias>
+		<alias>destinationport</alias>
+		<alias>dst port</alias>
 
 		<source>
-			<calculated expr="($1 * 1000) / $2">
-				<arg id="1">iana:octetDeltaCount</arg>
-				<arg id="2">@duration</arg>
-			</calculated>
+			<id>iana:destinationTransportPort</id>
 		</source>
 	</element>
+
+	<element>
+		<name>TCP Flags</name>
+
+		<alias>tcpflags</alias>
+
+		<source>
+			<id>iana:tcpControlBits</id>
+		</source>
+	</element>
+
 </ipfix-aliases>
 
-<!-- TODO: How to express "net a.b.c.d/XX"? -->