Add CI test

.github/workflows/build_steps.yml

@@ -69,6 +69,17 @@ jobs:
     defaults:
       run: {shell: bash}
     steps:
+      - name: Extra envs
+        # This has to come after msvc-dev-cmd to overwrite the bad VCPKG_ROOT it sets
+        run: |
+          echo ${{fromJSON(inputs.matrix)[0]}}
+          echo ${{fromJSON(inputs.matrix)[0]}}.os
+          echo ${{fromJSON(inputs.matrix)}}
+          echo ${{fromJSON(inputs.matrix)[0].os}}
+          echo ${{matrix.os}}
+          echo ${{inputs.job_type}}
+          echo ${{inputs}}
+
       - name: Checkout
         uses: actions/checkout@v3.3.0
         with:
@@ -254,6 +265,38 @@ jobs:
       test_dirs: ${{env.test_dirs}}
       cibw_build: ${{env.CIBW_BUILD}}
 
+  proxy_test:
+    if: |
+      always() &&
+      !failure() &&
+      !cancelled() &&
+      inputs.job_type == 'build-python-wheels'
+    needs: [compile]
+    uses: ./.github/workflows/run_proxy_tests.yml
+    secrets: inherit
+    strategy:
+      matrix:
+        # Declaring the dummy fields here to aid the Github Actions linting in VSCode and to provide documentation
+        os: [0] # Decouples the steps from any distro version changes
+        cmake_preset_prefix: [0]
+        cibw_build_suffix: [0]
+        envs: [0]
+        build_dir: [0] # Must be an absolute path
+        vcpkg_installed_dir: [0]
+        vcpkg_packages_dir: [0]
+        symbols: [0] # Glob for symbol symbol files. Used for including in build-python-wheels builds and exclusion on others.
+        do_not_archive: [0]
+        test_services: [0] # Github service containers to spin up for the pytest run
+        container: [0]
+        exclude:
+            - os: 0
+        include:
+            - ${{fromJSON(inputs.matrix)[0]}} # The items after 0 are for tests only
+    with:
+      python3: ${{inputs.python3}}
+      wheel: wheel-${{needs.compile.outputs.cibw_build}}
+      os: ${{matrix.os}}
+
   stop-ec2-runner:
     needs: [start_ec2_runner, compile]
     if: |

.github/workflows/run_proxy_tests.yml

@@ -0,0 +1,94 @@
+name: 'Run proxy tests'
+on:
+  workflow_call:
+    inputs:
+      wheel:             {required: true, type: string, description: The name of the wheel artifact to test }
+      os:                {required: true, type: string, description: The operating system to run the tests on }
+      python3:           {default: '7', type: string, description: Specifies the version of python for testing }
+jobs:
+  run_proxy_tests:   
+    # Only run on Linux
+    # select python support python > 3.6. Since we are dropping 3.6 support anyway, we can skip it
+    if: |
+      inputs.os == 'linux' &&
+      inputs.python3 != '6' 
+    runs-on: ubuntu-22.04
+    container:
+      image: ubuntu:22.04
+      options: --cap-add=NET_ADMIN
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.3.0
+
+      - name: Set persistent storage variables
+        uses: ./.github/actions/set_persistent_storage_env_vars
+        with:
+          aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}"
+          aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}"
+
+      - name: Get wheel artifact
+        uses: actions/download-artifact@v3.0.2
+        with:
+          name: ${{inputs.wheel}}
+
+      - name: Select Python
+        uses: actions/setup-python@v4.7.1
+        with:
+          python-version: "3.${{inputs.python3}}"
+
+      - name: Install deps
+        shell: bash
+        run: |
+          ls arcticdb*.whl
+          ls -al
+          apt update
+          apt install -y software-properties-common iptables squid
+          apt install -y curl
+          echo "whl"
+          python -m pip install --force-reinstall $(ls arcticdb*.whl)[Testing]
+          echo "grpcio"
+          python -m pip install grpcio grpcio-tools
+
+          echo "
+          acl localnet src 127.0.0.1/32
+          http_access allow localnet
+          http_access deny all
+
+          http_port 3128
+          " >> /etc/squid/squid.conf
+
+          echo "squid.conf"
+          service squid restart
+          iptables-legacy -A OUTPUT -p tcp --dport 80 -m owner ! --uid-owner proxy -j REJECT
+          iptables-legacy -A OUTPUT -p tcp --dport 443 -m owner ! --uid-owner proxy -j REJECT
+
+          python -m grpc_tools.protoc -Icpp/proto --python_out=./python cpp/proto/arcticc/pb2/*.proto
+          
+          echo "http_proxy=http://127.0.0.1:3128" >> $GITHUB_ENV
+          echo "https_proxy=http://127.0.0.1:3128" >> $GITHUB_ENV
+
+
+      - name: Setup softlink for SSL
+        shell: bash -el {0}
+        run: |
+            mkdir -p /etc/pki/tls
+            ln -s /usr/lib/ssl/certs /etc/pki/tls/certs
+            ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
+
+      - name: Run proxy test
+        shell: bash
+        run: |
+          echo ${http_proxy}
+          echo ${https_proxy}
+          curl google.com
+          python -m pytest --timeout=3600 python/tests/integration/arcticdb/test_arctic.py::test_s3_sts_auth
+
+      # Fallback if the clean up at test fixutre tear down fails due to crash or etc
+      - name: Remove AWS testing account and credentials
+        if: always()
+        run: |
+          python -c "
+          from arcticdb.storage_fixtures.s3 import real_s3_sts_clean_up
+          real_s3_sts_clean_up('${ARCTICDB_REAL_S3_STS_TEST_USERNAME}', '${ARCTICDB_REAL_S3_STS_TEST_ROLE}', '${ARCTICDB_REAL_S3_STS_TEST_POLICY_NAME}')
+          "
+        continue-on-error: true

cpp/arcticdb/storage/s3/apt update.sh

@@ -0,0 +1,40 @@
+docker run --cap-add=NET_ADMIN --cap-add=NET_RAW --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v/tmp:/tmp/host -it ubuntu:22.04 /bin/bash
+
+apt update
+apt install -y software-properties-common
+add-apt-repository ppa:deadsnakes/ppa -y
+apt install -y python3.8 python3.8-venv python3-pip
+python3.8 -m venv /root/pyenvs/venv38
+cd /tmp/host
+
+source /root/pyenvs/venv38/bin/activate
+apt install -y iptables
+apt install -y squid vim
+pip install /tmp/host/arcticdb-4.1.0.dev0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+pip install ipython
+
+sh -c 'echo "
+# Allow access from localhost
+acl localnet src 127.0.0.1/32
+http_access allow localnet
+
+# Deny all other access
+http_access deny all
+
+# Set the port for the proxy server
+http_port 3128
+" >> /etc/squid/squid.conf'
+
+
+service squid restart
+iptables-nft -A OUTPUT -p tcp --dport 80 -m owner ! --uid-owner proxy -j REJECT
+iptables-nft -A OUTPUT -p tcp --dport 443 -m owner ! --uid-owner proxy -j REJECT
+
+export http_proxy="http://127.0.0.1:3128"
+export https_proxy="http://127.0.0.1:3128"
+
+unset http_proxy https_proxy
+
+import arcticdb as adb
+ac = adb.Arctic('s3://s3.eu-north-1.amazonaws.com:phoebus-test-bucket-1?aws_auth=sts&aws_profile=sts_test_profile&path_prefix=144_2024-12-18T10_36_20_161572')
+ac.list_libraries()