Merge branch 'main' into jackhe/1es-azure-pools

microsoft · Jun 20, 2024 · 3565cf3 · 3565cf3
2 parents aae2a05 + f5bec53
commit 3565cf3
Show file tree

Hide file tree

Showing 58 changed files with 777 additions and 242 deletions.
diff --git a/.azure/OneBranch.Official.yml b/.azure/OneBranch.Official.yml
@@ -76,19 +76,13 @@ extends:
         image: 'cdpxb7b51c2f738e43e48f7605d9a8e5f6d700.azurecr.io/b7b51c2f-738e-43e4-8f76-05d9a8e5f6d7/official/msquicbuild:${{ parameters.WindowsContainerImage2DockerTag }}'
         type: Windows
       - container: linux_build_container # Default container
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-x86_64'
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-cross'
         type: Linux
-      - container: ubuntu_2004_x86_64
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-x86_64'
+      - container: ubuntu_2204_xdp
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-xdp'
         type: Linux
-      - container: ubuntu_2004_arm
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-arm'
-        type: Linux
-      - container: ubuntu_2204_x86_64
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-x86_64'
-        type: Linux
-      - container: ubuntu_2204_arm
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-arm'
+      - container: ubuntu_2204_cross
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-cross'
         type: Linux
 
     stages:
@@ -186,10 +180,12 @@ extends:
         parameters:
           config: Release
           tls: openssl3
+          xdp: "-UseXdp"
       - template: .azure/obtemplates/build-linux.yml@self
         parameters:
           config: Debug
           tls: openssl3
+          xdp: "-UseXdp"
 
     - stage: package_linux
       displayName: Package Linux

diff --git a/.azure/OneBranch.PullRequest.yml b/.azure/OneBranch.PullRequest.yml
@@ -72,19 +72,13 @@ extends:
         image: 'cdpxb7b51c2f738e43e48f7605d9a8e5f6d700.azurecr.io/b7b51c2f-738e-43e4-8f76-05d9a8e5f6d7/official/msquicbuild:${{ parameters.WindowsContainerImage2DockerTag }}'
         type: Windows
       - container: linux_build_container # Default container
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-x86_64'
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-cross'
         type: Linux
-      - container: ubuntu_2004_x86_64
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-x86_64'
+      - container: ubuntu_2204_xdp
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-xdp'
         type: Linux
-      - container: ubuntu_2004_arm
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-arm'
-        type: Linux
-      - container: ubuntu_2204_x86_64
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-x86_64'
-        type: Linux
-      - container: ubuntu_2204_arm
-        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-arm'
+      - container: ubuntu_2204_cross
+        image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-cross'
         type: Linux
 
     stages:
@@ -182,10 +176,12 @@ extends:
         parameters:
           config: Release
           tls: openssl3
+          xdp: "-UseXdp"
       - template: .azure/obtemplates/build-linux.yml@self
         parameters:
           config: Debug
           tls: openssl3
+          xdp: "-UseXdp"
 
     - stage: package_linux
       displayName: Package Linux

diff --git a/.azure/obtemplates/build-linux.yml b/.azure/obtemplates/build-linux.yml
@@ -2,6 +2,7 @@ parameters:
   config: ''
   tls: 'openssl'
   platform: 'linux'
+  xdp: ''
 
 jobs:
 - job: build_${{ parameters.platform }}_${{ parameters.tls }}_${{ parameters.config }}
@@ -16,40 +17,39 @@ jobs:
   - task: PowerShell@2
     displayName: Prepare Build Machine
     ${{ if eq(parameters.tls, 'openssl') }}:
-      target: ubuntu_2004_x86_64
+      target: linux_build_container
     ${{ else }}:
-      target: ubuntu_2204_x86_64
+      target: ubuntu_2204_xdp
     inputs:
       pwsh: true
       filePath: scripts/prepare-machine.ps1
       arguments: -Tls ${{ parameters.tls }} -ForContainerBuild
   - task: PowerShell@2
     displayName: x64
     ${{ if eq(parameters.tls, 'openssl') }}:
-      target: ubuntu_2004_x86_64
+      target: linux_build_container
     ${{ else }}:
-      target: ubuntu_2204_x86_64
-      xdp: true
+      target: ubuntu_2204_xdp
     inputs:
       pwsh: true
       filePath: scripts/build.ps1
-      arguments: -Tls ${{ parameters.tls }} -Config ${{ parameters.config }} -Platform ${{ parameters.platform }} -EnableLinuxXDP ${{ xdp }} -Arch x64 -CI -UseSystemOpenSSLCrypto -OneBranch -OfficialRelease
+      arguments: -Tls ${{ parameters.tls }} -Config ${{ parameters.config }} -Platform ${{ parameters.platform }} ${{ parameters.xdp }} -Arch x64 -CI -UseSystemOpenSSLCrypto -OneBranch -OfficialRelease
   - task: PowerShell@2
     displayName: arm64
     ${{ if eq(parameters.tls, 'openssl') }}:
-      target: ubuntu_2004_arm
+      target: linux_build_container
     ${{ else }}:
-      target: ubuntu_2204_arm
+      target: ubuntu_2204_cross
     inputs:
       pwsh: true
       filePath: scripts/build.ps1
       arguments: -Tls ${{ parameters.tls }} -Config ${{ parameters.config }} -Platform ${{ parameters.platform }} -Arch arm64 -CI -UseSystemOpenSSLCrypto -OneBranch -OfficialRelease
   - task: PowerShell@2
     displayName: arm
     ${{ if eq(parameters.tls, 'openssl') }}:
-      target: ubuntu_2004_arm
+      target: linux_build_container
     ${{ else }}:
-      target: ubuntu_2204_arm
+      target: ubuntu_2204_cross
     inputs:
       pwsh: true
       filePath: scripts/build.ps1

diff --git a/.docker/ubuntu-20.04/Dockerfile b/.docker/ubuntu-20.04/Dockerfile
@@ -32,6 +32,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     cppcheck \
     clang-tidy \
     gdb \
+    liblttng-ust-dev \
+    libssl-dev \
+    libnuma-dev \
     && rm -rf /var/lib/apt/lists/*
 
 RUN gem install --version 2.8.1 dotenv
@@ -47,17 +50,8 @@ RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsof
 
 RUN git config --global safe.directory '*'
 
-# Stage 2: x86_64 build
-FROM base AS x86_64-build
-
-RUN apt-get update -y && apt-get install -y \
-    liblttng-ust-dev \
-    libssl-dev \
-    libnuma-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Stage 3: ARM build
-FROM base AS arm-build
+# Stage 2-2: dependencies for cross-build
+FROM base AS cross-build
 
 RUN apt-get update && apt-get install -y \
     gcc-arm-linux-gnueabihf \

diff --git a/.docker/ubuntu-22.04/Dockerfile b/.docker/ubuntu-22.04/Dockerfile
@@ -33,6 +33,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     cppcheck \
     clang-tidy \
     gdb \
+    liblttng-ust-dev \
+    libssl-dev \
+    libnuma-dev \
     && rm -rf /var/lib/apt/lists/*
 
 RUN gem install fpm
@@ -47,13 +50,10 @@ RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsof
 
 RUN git config --global safe.directory '*'
 
-# Stage 2: x86_64 build
-FROM base AS x86_64-build
+# Stage 2-1: dependencies for xdp-build
+FROM base AS xdp-build
 
 RUN apt-get update -y && apt-get install -y \
-    liblttng-ust-dev \
-    libssl-dev \
-    libnuma-dev \
     && echo "deb [arch=amd64] http://cz.archive.ubuntu.com/ubuntu noble main" > /etc/apt/sources.list.d/xdp.list \
     && apt-get update -y && apt-get install --no-install-recommends -y -t noble \
     libnl-3-dev \
@@ -70,8 +70,8 @@ RUN apt-get update -y && apt-get install -y \
     libbpf-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Stage 3: ARM build
-FROM base AS arm-build
+# Stage 2-2: dependencies for cross-build
+FROM base AS cross-build
 
 RUN apt-get update && apt-get install -y \
     gcc-arm-linux-gnueabihf \

diff --git a/.docker/ubuntu-24.04/Dockerfile b/.docker/ubuntu-24.04/Dockerfile
@@ -0,0 +1,106 @@
+# Stage 1: Base image
+FROM ubuntu:24.04@sha256:e3f92abc0967a6c19d0dfa2d55838833e947b9d74edbcb0113e48535ad4be12a AS base
+
+LABEL org.opencontainers.image.source https://github.com/microsoft/msquic
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    apt-transport-https \
+    ca-certificates \
+    gnupg \
+    software-properties-common \
+    wget && \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
+    apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main' && \
+    apt-add-repository ppa:lttng/stable-2.13 && \
+    apt-get update && \
+    apt-get install -y tzdata \
+    build-essential \
+    cmake \
+    clang \
+    git \
+    make \
+    sudo \
+    lttng-tools \
+    perl \
+    nasm \
+    ruby \
+    ruby-dev \
+    rpm \
+    cppcheck \
+    clang-tidy \
+    gdb \
+    liblttng-ust-dev \
+    libssl-dev \
+    libnuma-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN gem install fpm
+RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+files/libicu72_72.1-3ubuntu3_amd64.deb \
+    && dpkg -i ./libicu72_72.1-3ubuntu3_amd64.deb \
+    && rm ./libicu72_72.1-3ubuntu3_amd64.deb \
+    && wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
+    sudo dpkg -i packages-microsoft-prod.deb && \
+    sudo add-apt-repository universe && \
+    sudo apt-get update -y && \
+    sudo apt-get install -y \
+        # INSTALL POWERSHELl
+        powershell \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git config --global safe.directory '*'
+
+# Stage 2: dependencies for cross-build.
+#          ubuntu 24.04 can install both xdp and arm dependencies
+FROM base AS cross-build
+
+RUN apt-get update -y && apt-get install --no-install-recommends -y \
+    libnl-3-dev \
+    libnl-genl-3-dev \
+    libnl-route-3-dev \
+    zlib1g-dev \
+    zlib1g \
+    pkg-config \
+    m4 \
+    libpcap-dev \
+    libelf-dev \
+    libc6-dev-i386 \
+    libxdp-dev \
+    libbpf-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y \
+    gcc-arm-linux-gnueabihf \
+    g++-arm-linux-gnueabihf \
+    binutils-arm-linux-gnueabihf \
+    gcc-aarch64-linux-gnu \
+    g++-aarch64-linux-gnu \
+    binutils-aarch64-linux-gnu \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN dpkg --add-architecture arm64 && \
+    dpkg --add-architecture armhf
+
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.int && \
+    cat /etc/apt/sources.list.int | grep "^deb" | sed 's/deb /deb [arch=amd64] /g' > /etc/apt/sources.list && \
+    rm /etc/apt/sources.list.int && \
+    rm /etc/apt/sources.list.d/ubuntu.sources
+
+RUN echo $' \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble main restricted universe multiverse \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble-updates main restricted universe multiverse \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble-backports main restricted universe multiverse \n\
+' > /etc/apt/sources.list.d/arm-cross-compile-sources.list
+
+RUN apt-get update \
+    && apt-get --no-install-recommends install -y \
+    liblttng-ust-dev:arm64 \
+    liblttng-ust-dev:armhf \
+    libssl-dev:arm64 \
+    libssl-dev:armhf \
+    libnuma-dev:arm64 \
+    libnuma-dev:armhf \
+    && rm -rf /var/lib/apt/lists/*
diff --git a/.github/workflows/build-reuse-darwin-framework.yml b/.github/workflows/build-reuse-darwin-framework.yml
@@ -53,7 +53,7 @@ jobs:
     runs-on: macos-12
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}
@@ -90,7 +90,7 @@ jobs:
     runs-on: macos-12
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}
@@ -114,7 +114,7 @@ jobs:
     runs-on: macos-12
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}

diff --git a/.github/workflows/build-reuse-unix.yml b/.github/workflows/build-reuse-unix.yml
@@ -85,10 +85,10 @@ jobs:
     name: Build
     runs-on: ${{ inputs.os }}
     container:
-      image: ${{ (inputs.plat == 'linux' && format('ghcr.io/microsoft/msquic/linux-build-xcomp:{0}-{1}', inputs.os, (startsWith(inputs.arch, 'x') && 'x86_64') || 'arm')) || '' }}
+      image: ${{ (inputs.plat == 'linux' && format('ghcr.io/microsoft/msquic/linux-build-xcomp:{0}-{1}', inputs.os, ((inputs.xdp == '-UseXdp' && inputs.os == 'ubuntu-22.04') && 'xdp') || 'cross')) || '' }}
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}
@@ -99,21 +99,21 @@ jobs:
         chown -R $(id -u):$(id -g) $PWD
     - name: Prepare Machine
       shell: pwsh
-      run: scripts/prepare-machine.ps1 ${{ inputs.plat == 'linux' && '-ForContainerBuild' || '-ForBuild' }} -Tls ${{ inputs.tls }} ${{ inputs.xdp }}
+      run: scripts/prepare-machine.ps1 ${{ inputs.plat == 'linux' && '-ForContainerBuild' || '-ForBuild' }} -Tls ${{ inputs.tls }}
     - name: Build For Test
       if: inputs.build == '-Test'
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} -OneBranch
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} -OneBranch
     - name: Build For Perf
       if: inputs.build == '-Perf'
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisableTools -DisableTest ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }}
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisableTools -DisableTest ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }}
     - name: Build
       if: inputs.build == ''
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} -OneBranch
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} -OneBranch
     - name: Upload build artifacts
       uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
       with:
-        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.static }}${{ inputs.clang }}${{ inputs.systemcrypto }}${{ inputs.codecheck }}${{ inputs.sanitize }}${{ inputs.build }}
+        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.static }}${{ inputs.clang }}${{ inputs.systemcrypto }}${{ inputs.codecheck }}${{ inputs.sanitize }}${{ inputs.xdp }}${{ inputs.build }}
         path: artifacts
diff --git a/.github/workflows/build-reuse-win.yml b/.github/workflows/build-reuse-win.yml
@@ -69,7 +69,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}

diff --git a/.github/workflows/build-reuse-winkernel.yml b/.github/workflows/build-reuse-winkernel.yml
@@ -58,7 +58,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     steps:
     - name: Checkout repository
-      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         repository: microsoft/msquic
         ref: ${{ inputs.ref }}