From ee5dd4440d0afd2bc08bfa72dd3fad6e292325d4 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Tue, 1 Mar 2022 14:58:51 -0800 Subject: [PATCH 1/2] [C#] Write range to object log should cover max read ranges --- cs/src/core/Allocator/GenericAllocator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cs/src/core/Allocator/GenericAllocator.cs b/cs/src/core/Allocator/GenericAllocator.cs index 967080ead..077a18f3d 100644 --- a/cs/src/core/Allocator/GenericAllocator.cs +++ b/cs/src/core/Allocator/GenericAllocator.cs @@ -441,7 +441,7 @@ private void WriteAsync(long flushPage, ulong alignedDestinationAddres if (endPosition > ObjectBlockSize || i == (end / recordSize) - 1) { - var memoryStreamLength = (int)ms.Position; + var memoryStreamLength = (int)endPosition; var _objBuffer = bufferPool.Get(memoryStreamLength); From e0f2ee591d7afa49b015d269f10dfb1a8106935d Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Tue, 1 Mar 2022 15:55:24 -0800 Subject: [PATCH 2/2] Update ADO pipeline --- azure-pipelines-full.yml | 69 ++++++++++---------- azure-pipelines.yml | 137 ++++++--------------------------------- 2 files changed, 54 insertions(+), 152 deletions(-) diff --git a/azure-pipelines-full.yml b/azure-pipelines-full.yml index 8877af88b..2d0b6fd7a 100644 --- a/azure-pipelines-full.yml +++ b/azure-pipelines-full.yml @@ -6,7 +6,7 @@ variables: jobs: - job: 'csharpWindows' pool: - vmImage: windows-latest + vmImage: windows-2022 displayName: 'C# (Windows)' timeoutInMinutes: 75 @@ -27,17 +27,15 @@ jobs: buildConfiguration: 'Release' steps: - - powershell: 'Invoke-WebRequest -OutFile azure-storage-emulator.msi -Uri "https://go.microsoft.com/fwlink/?LinkId=717179&clcid=0x409"' - displayName: 'Download Azure Storage Emulator' - - - powershell: 'msiexec /passive /lvx installation.log /a azure-storage-emulator.msi TARGETDIR="C:\storage-emulator"' - displayName: 'Install Azure Storage Emulator' + - task: NodeTool@0 + inputs: + versionSpec: 13.x - - script: '"C:\Program Files\Microsoft SQL Server\130\Tools\Binn\SqlLocalDB.exe" create "v13.0" 13.0 -s' - displayName: 'Init Test Db' + - script : npm install -g azurite + displayName: Install Azurite - - script: '"C:\storage-emulator\root\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe" start' - displayName: 'Start Storage Emulator' + - script : start /B azurite + displayName: Start Azurite - task: DotNetCoreCLI@2 displayName: 'dotnet build $(buildConfiguration)' @@ -62,7 +60,7 @@ jobs: - job: 'cppWindows' pool: - vmImage: vs2017-win2016 + vmImage: windows-2022 displayName: 'C++ (Windows)' strategy: @@ -77,10 +75,10 @@ jobs: steps: - task: CMake@1 - displayName: 'CMake .. -G"Visual Studio 15 2017 Win64"' + displayName: 'CMake .. -G"Visual Studio 17 2022"' inputs: workingDirectory: 'cc/build' - cmakeArgs: '.. -G"Visual Studio 15 2017 Win64"' + cmakeArgs: '.. -G"Visual Studio 17 2022"' - task: MSBuild@1 displayName: 'Build solution cc/build/FASTER.sln' @@ -94,33 +92,40 @@ jobs: - job: 'cppLinux' pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 displayName: 'C++ (Linux)' steps: - script: | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt update - sudo apt install -y g++-7 libaio-dev uuid-dev libtbb-dev + sudo apt install -y g++ libaio-dev uuid-dev libtbb-dev displayName: 'Install depdendencies' - script: | - export CXX='g++-7' + git clone https://git.kernel.dk/liburing + cd liburing + git checkout liburing-0.7 + ./configure + sudo make install + displayName: Install Liburing + + - script: | cd cc mkdir -p build/Debug build/Release cd build/Debug - cmake -DCMAKE_BUILD_TYPE=Debug ../.. + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_URING=ON ../.. make -j cd ../../build/Release - cmake -DCMAKE_BUILD_TYPE=Release ../.. + cmake -DCMAKE_BUILD_TYPE=Release -DUSE_URING=ON ../.. make -j displayName: 'Compile' - script: | CTEST_OUTPUT_ON_FAILURE=1 make test workingDirectory: 'cc/build/Debug' displayName: 'Run Tests (Debug)' + - job: 'csharpLinux' pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 displayName: 'C# (Linux)' strategy: @@ -134,6 +139,12 @@ jobs: buildConfiguration: 'Release' steps: + - bash: | + sudo npm install -g azurite + sudo mkdir azurite + sudo azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + - task: DotNetCoreCLI@2 displayName: 'dotnet build $(buildConfiguration)' inputs: @@ -157,7 +168,7 @@ jobs: # - job: 'cppBlobsWindows' # pool: -# vmImage: vs2017-win2016 +# vmImage: windows-2022 # displayName: 'C++ Blobs (Windows)' # strategy: @@ -172,10 +183,10 @@ jobs: # steps: # - task: CMake@1 -# displayName: 'CMake .. -G"Visual Studio 15 2017 Win64" -DUSE_BLOBS=ON' +# displayName: 'CMake .. -G"Visual Studio 16 2019" -DUSE_BLOBS=ON' # inputs: # workingDirectory: 'cc/build' -# cmakeArgs: '.. -G"Visual Studio 15 2017 Win64" -DUSE_BLOBS=ON' +# cmakeArgs: '.. -G"Visual Studio 16 2019" -DUSE_BLOBS=ON' # - script: 'git clone https://github.com/microsoft/vcpkg' # workingDirectory: 'cc/build' @@ -199,18 +210,6 @@ jobs: # solution: 'cc/build/FASTER.sln' # msbuildArguments: '/m /p:Configuration=$(buildConfiguration) /p:Platform=$(buildPlatform)' -# - powershell: 'Invoke-WebRequest -OutFile azure-storage-emulator.msi -Uri "https://go.microsoft.com/fwlink/?LinkId=717179&clcid=0x409"' -# displayName: 'Download Azure Storage Emulator' - -# - powershell: 'msiexec /passive /lvx installation.log /a azure-storage-emulator.msi TARGETDIR="C:\storage-emulator"' -# displayName: 'Install Azure Storage Emulator' - -# - script: '"C:\Program Files\Microsoft SQL Server\130\Tools\Binn\SqlLocalDB.exe" create "v13.0" 13.0 -s' -# displayName: 'Init Test Db' - -# - script: '"C:\storage-emulator\root\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe" start' -# displayName: 'Start Storage Emulator' - # - script: | # ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "azure_test" # ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "storage_test" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 90c5d83f2..ddbd9c9fb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,7 +6,7 @@ variables: jobs: - job: 'csharpWindows' pool: - vmImage: windows-latest + vmImage: windows-2022 displayName: 'C# (Windows)' timeoutInMinutes: 75 @@ -27,17 +27,15 @@ jobs: buildConfiguration: 'Release' steps: - - powershell: 'Invoke-WebRequest -OutFile azure-storage-emulator.msi -Uri "https://go.microsoft.com/fwlink/?LinkId=717179&clcid=0x409"' - displayName: 'Download Azure Storage Emulator' - - - powershell: 'msiexec /passive /lvx installation.log /a azure-storage-emulator.msi TARGETDIR="C:\storage-emulator"' - displayName: 'Install Azure Storage Emulator' + - task: NodeTool@0 + inputs: + versionSpec: 13.x - - script: '"C:\Program Files\Microsoft SQL Server\130\Tools\Binn\SqlLocalDB.exe" create "v13.0" 13.0 -s' - displayName: 'Init Test Db' + - script : npm install -g azurite + displayName: Install Azurite - - script: '"C:\storage-emulator\root\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe" start' - displayName: 'Start Storage Emulator' + - script : start /B azurite + displayName: Start Azurite - task: DotNetCoreCLI@2 displayName: 'dotnet build $(buildConfiguration)' @@ -62,7 +60,7 @@ jobs: - job: 'cppWindows' pool: - vmImage: vs2017-win2016 + vmImage: windows-2022 displayName: 'C++ (Windows)' strategy: @@ -77,10 +75,10 @@ jobs: steps: - task: CMake@1 - displayName: 'CMake .. -G"Visual Studio 15 2017 Win64"' + displayName: 'CMake .. -G"Visual Studio 17 2022"' inputs: workingDirectory: 'cc/build' - cmakeArgs: '.. -G"Visual Studio 15 2017 Win64"' + cmakeArgs: '.. -G"Visual Studio 17 2022"' - task: MSBuild@1 displayName: 'Build solution cc/build/FASTER.sln' @@ -124,9 +122,10 @@ jobs: CTEST_OUTPUT_ON_FAILURE=1 make test workingDirectory: 'cc/build/Debug' displayName: 'Run Tests (Debug)' + - job: 'csharpLinux' pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 displayName: 'C# (Linux)' strategy: @@ -140,6 +139,12 @@ jobs: buildConfiguration: 'Release' steps: + - bash: | + sudo npm install -g azurite + sudo mkdir azurite + sudo azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + - task: DotNetCoreCLI@2 displayName: 'dotnet build $(buildConfiguration)' inputs: @@ -159,106 +164,4 @@ jobs: inputs: testResultsFormat: 'VSTest' testResultsFiles: '*.trx' - searchFolder: '$(Agent.TempDirectory)' - -# - job: 'cppBlobsWindows' -# pool: -# vmImage: vs2017-win2016 -# displayName: 'C++ Blobs (Windows)' - -# strategy: -# maxParallel: 2 -# matrix: -# x64-Debug: -# buildPlatform: 'x64' -# buildConfiguration: 'Debug' -# x64-Release: -# buildPlatform: 'x64' -# buildConfiguration: 'Release' - -# steps: -# - task: CMake@1 -# displayName: 'CMake .. -G"Visual Studio 15 2017 Win64" -DUSE_BLOBS=ON' -# inputs: -# workingDirectory: 'cc/build' -# cmakeArgs: '.. -G"Visual Studio 15 2017 Win64" -DUSE_BLOBS=ON' - -# - script: 'git clone https://github.com/microsoft/vcpkg' -# workingDirectory: 'cc/build' -# displayName: 'Download Vcpkg' - -# - script: '.\vcpkg\bootstrap-vcpkg.bat' -# workingDirectory: 'cc/build' -# displayName: 'Install Vcpkg' - -# - script: '.\vcpkg\vcpkg.exe install azure-storage-cpp:x64-windows' -# workingDirectory: 'cc/build' -# displayName: 'Install Azure dependencies' - -# - script: '.\vcpkg\vcpkg.exe integrate install' -# workingDirectory: 'cc/build' -# displayName: 'Integrate vcpkg with msbuild' - -# - task: MSBuild@1 -# displayName: 'Build solution cc/build/FASTER.sln' -# inputs: -# solution: 'cc/build/FASTER.sln' -# msbuildArguments: '/m /p:Configuration=$(buildConfiguration) /p:Platform=$(buildPlatform)' - -# - powershell: 'Invoke-WebRequest -OutFile azure-storage-emulator.msi -Uri "https://go.microsoft.com/fwlink/?LinkId=717179&clcid=0x409"' -# displayName: 'Download Azure Storage Emulator' - -# - powershell: 'msiexec /passive /lvx installation.log /a azure-storage-emulator.msi TARGETDIR="C:\storage-emulator"' -# displayName: 'Install Azure Storage Emulator' - -# - script: '"C:\Program Files\Microsoft SQL Server\130\Tools\Binn\SqlLocalDB.exe" create "v13.0" 13.0 -s' -# displayName: 'Init Test Db' - -# - script: '"C:\storage-emulator\root\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe" start' -# displayName: 'Start Storage Emulator' - -# - script: | -# ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "azure_test" -# ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "storage_test" -# ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "faster_blobs_example" -# workingDirectory: 'cc/build' -# displayName: 'Run Ctest' - -# - job: 'cppBlobsLinux' -# pool: -# vmImage: ubuntu-18.04 -# displayName: 'C++ Blobs (Linux)' - -# steps: -# - script: | -# sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -# sudo apt update -# sudo apt install -y libaio-dev uuid-dev libtbb-dev npm -# displayName: 'Install dependencies' - -# - script: | -# sudo ./scripts/linux/azure/blob.sh -# workingDirectory: 'cc' -# displayName: 'Install Azure dependencies' - -# - script: | -# cd cc -# mkdir -p build/Debug build/Release -# cd build/Debug -# cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_BLOBS=ON ../.. -# make -j -# cd ../../build/Release -# cmake -DCMAKE_BUILD_TYPE=Release -DUSE_BLOBS=ON ../.. -# make -j -# displayName: 'Compile' - -# - script: | -# sudo npm install -g azurite -# azurite -s & -# displayName: 'Install and launch azurite (linux storage emulator)' - -# - script: | -# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib -# CTEST_OUTPUT_ON_FAILURE=1 make test -# workingDirectory: 'cc/build/Debug' -# displayName: 'Run Tests (Debug)' \ No newline at end of file + searchFolder: '$(Agent.TempDirectory)' \ No newline at end of file