Skip to content

Commit

Permalink
700 switch to GitHub shared runner mac and win x86 (#705)
Browse files Browse the repository at this point in the history
* switched selfhosted to GH shared runner

* switched selfhosted to GH shared runner

* changed ci_windows workflow to work with github hosted runners and updated thirdparty.inc with new versions of prerequisite packages

* removed obsolete parts from ci_windows.yml

* Update ci_windows.yml

fixed typos

* removed commented and obsplete options

* restored thirdparty

* restored thirdparty

* restored thirdparty

* restored thirdparty

* changed thirtparty.inc to work with prerequisite packages insstalled on the runner

* moved updated thirdparty.inco to tools folder

* dummy change

* moved tools/thirdparty.inc to tools/.thirdparty.inc

* changed to thirdparty.txt

* updated ci_windows to use thirdparty.txt

* minor fixes

* minor fixes

* minor fixes

* minor fixes

* minor fixes

* removed comments and prepared for merging
  • Loading branch information
maxb-io authored and udi-speedb committed Dec 6, 2023
1 parent 13ffdec commit 797029e
Show file tree
Hide file tree
Showing 4 changed files with 288 additions and 23 deletions.
12 changes: 9 additions & 3 deletions .github/workflows/build_and_publish_jar.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ jobs:
uses: ./.github/workflows/ci_macos.yml
with:
verSion: ${{ needs.pre_build.outputs.out1 }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_BUCKET: ${{ secrets.AWS_BUCKET }}

Mac_ARM:
needs: pre_build
Expand Down Expand Up @@ -100,9 +104,11 @@ jobs:
mvn versions:set -DnewVersion=$VERSION-SNAPSHOT
mvn deploy -X -e -DskipTests
mvn versions:set -DnewVersion=$the_version
#mvn versions:set -DnewVersion=$the_version
#mvn clean deploy -P release -X -e -DskipTests
- name: show path
- name: show next step
run: |
echo "versions:set -DnewVersion=$VERSION-SNAPSHOT"
echo "Make sure the SNAPSHOT is fine and run these to publish the artifact"
echo "run: mvn versions:set -DnewVersion=$the_version"
echo "run: mvn clean deploy -P release -X -e -DskipTests"
13 changes: 8 additions & 5 deletions .github/workflows/ci_macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ on:
verSion:
required: true
type: string
secrets:
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_BUCKET:
required: true
workflow_dispatch:

jobs:
Expand All @@ -27,11 +34,7 @@ jobs:
export CXXFLAGS="-I$JAVA_HOME/include"
brew install zlib
brew install bzip2 lz4 snappy
ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 2 rocksdbjavastatic
# - name: 'upload artifacts' #This step executed only when this workflow is called by another and a version is provided
# if: inputs.verSion != ' '
# run: aws s3 cp java/target/libspeedbjni-osx-x86_64.jnilib s3://spdb-builder/jar_test/v${{ inputs.verSion }}/libspeedbjni-osx-x86_64.jnilib
ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 2 rocksdbjavastatic #more CPU cores makes shared github runner to overload cpu and stop
- name: Upload artifacts to S3
if: inputs.verSion != ' '
Expand Down
18 changes: 3 additions & 15 deletions .github/workflows/ci_windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,28 +136,16 @@ jobs:
- name: Build Speedb
run: |
copy C:\Users\runneradmin\thirdparty\snappy-1.1.9\build\snappy-stubs-public.h C:\Users\runneradmin\thirdparty\snappy-1.1.9\
copy C:\Users\runneradmin\thirdparty\snappy-1.1.9\build\snappy-stubs-public.h C:\Users\runneradmin\thirdparty\snappy-1.1.9\
copy tools\thirdparty.txt thirdparty.inc # copy the thirdparty.inc that reflects the env on the runner machine
mkdir build
cd build
#& $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
& $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -A x64 -DJNI=1 -DGFLAGS=1 -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 -DSNAPPY=1 -DLZ4=1 -DZLIB=1 -DZSTD=1 -DXPRESS=1 -DFAIL_ON_WARNINGS=0 ..
cd ..
echo "Building with VS version: $Env:CMAKE_GENERATOR"
msbuild build/speedb.sln /p:Configuration=Release /t:speedbjni-shared
#msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
#msbuild.exe build/speedb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
#- name: 'build and present'
# shell: powershell # For Linux, use bash
# run: |
# $env:THIRDPARTY_HOME='C:/Users/builder/code'
# cp $env:THIRDPARTY_HOME\thirdparty.inc . # copy the thirdparty.inc that reflects the env on the runner machine
# mkdir runner_bld
# cd runner_bld
# cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 -DJNI=1 -DGFLAGS=1 -DSNAPPY=1 -DLZ4=1 -DZLIB=1 -DZSTD=1 -DXPRESS=1 -DFAIL_ON_WARNINGS=0 ..
# msbuild speedb.sln /p:Configuration=Release /t:speedbjni-shared

- name: Upload artifacts to S3
if: inputs.verSion != ' '
uses: NotCoffee418/s3-zip-upload@v1
Expand All @@ -168,5 +156,5 @@ jobs:
BUCKET_NAME: spdb-builder
AWS_REGION: us-east-1
SOURCE_MODE: FILE
SOURCE_PATH: runner_bld\java\Release\speedbjni-shared.dll
SOURCE_PATH: build\java\Release\speedbjni-shared.dll
DEST_FILE: jar_test/v${{ inputs.verSion }}/libspeedbjni-win64.dll
268 changes: 268 additions & 0 deletions tools/thirdparty.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# Edit definitions below to specify paths to include files and libraries of all 3rd party libraries

# TODO: Make this work with find_package and/or get rid of it
#
# This example assumes all the libraries locate in directories under THIRDPARTY_HOME environment variable
# Set environment variable THIRDPARTY_HOME to point to your third party libraries home (Unix style dir separators)
# or change the paths below to reflect where the libraries actually reside
#
set (THIRDPARTY_LIBS "") # Initialization, don't touch

#
# Defaults
#
set(GFLAGS_HOME $ENV{THIRDPARTY_HOME}/gflags-2.2.2)
set(GFLAGS_INCLUDE ${GFLAGS_HOME}/target/include)
set(GFLAGS_LIB_DEBUG ${GFLAGS_HOME}/target/lib/Debug/gflags_static.lib)
set(GFLAGS_LIB_RELEASE ${GFLAGS_HOME}/target/lib/Release/gflags_static.lib)

# ================================================== GFLAGS ==================================================
# For compatibility
if (GFLAGS)
set(WITH_GFLAGS ON)
endif ()

if (WITH_GFLAGS)
message(STATUS "GFLAGS library is enabled")

if(DEFINED ENV{GFLAGS_INCLUDE})
set(GFLAGS_INCLUDE $ENV{GFLAGS_INCLUDE})
endif()

if(DEFINED ENV{GFLAGS_LIB_DEBUG})
set(GFLAGS_LIB_DEBUG $ENV{GFLAGS_LIB_DEBUG})
endif()

if(DEFINED ENV{GFLAGS_LIB_RELEASE})
set(GFLAGS_LIB_RELEASE $ENV{GFLAGS_LIB_RELEASE})
endif()

set(GFLAGS_CXX_FLAGS -DGFLAGS=gflags)
set(GFLAGS_LIBS debug ${GFLAGS_LIB_DEBUG} optimized ${GFLAGS_LIB_RELEASE})

add_definitions(${GFLAGS_CXX_FLAGS})
include_directories(${GFLAGS_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${GFLAGS_LIBS})
else ()
message(STATUS "GFLAGS library is disabled")
endif ()

# ================================================== SNAPPY ==================================================
#
# Edit these 4 lines to define paths to Snappy
#
set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/snappy-1.1.9)
set(SNAPPY_INCLUDE ${SNAPPY_HOME} ${SNAPPY_HOME}/build)
set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/build/Debug/snappy.lib)
set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/build/Release/snappy.lib)

# For compatibility
if(SNAPPY)
set(WITH_SNAPPY ON)
endif ()

if (WITH_SNAPPY)
message(STATUS "SNAPPY library is enabled")

if(DEFINED ENV{SNAPPY_INCLUDE})
set(SNAPPY_INCLUDE $ENV{SNAPPY_INCLUDE})
endif()

if(DEFINED ENV{SNAPPY_LIB_DEBUG})
set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG})
endif()

if(DEFINED ENV{SNAPPY_LIB_RELEASE})
set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE})
endif()

set(SNAPPY_CXX_FLAGS -DSNAPPY)
set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE})

add_definitions(${SNAPPY_CXX_FLAGS})
include_directories(${SNAPPY_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${SNAPPY_LIBS})
else ()
message(STATUS "SNAPPY library is disabled")
endif ()

# ================================================== LZ4 ==================================================
#
# Edit these 4 lines to define paths to LZ4
#
set(LZ4_HOME $ENV{THIRDPARTY_HOME}/lz4-1.9.2)
set(LZ4_INCLUDE ${LZ4_HOME}/lib)
set(LZ4_LIB_DEBUG ${LZ4_HOME}/visual/VS2017/bin/x64_Debug/liblz4_static.lib)
set(LZ4_LIB_RELEASE ${LZ4_HOME}/visual/VS2017/bin/x64_Release/liblz4_static.lib)

# For compatibility
if (LZ4)
set(WITH_LZ4 ON)
endif ()

if (WITH_LZ4)
message(STATUS "LZ4 library is enabled")

if(DEFINED ENV{LZ4_INCLUDE})
set(LZ4_INCLUDE $ENV{LZ4_INCLUDE})
endif()

if(DEFINED ENV{LZ4_LIB_DEBUG})
set(LZ4_LIB_DEBUG $ENV{LZ4_LIB_DEBUG})
endif()

if(DEFINED ENV{LZ4_LIB_RELEASE})
set(LZ4_LIB_RELEASE $ENV{LZ4_LIB_RELEASE})
endif()

set(LZ4_CXX_FLAGS -DLZ4)
set(LZ4_LIBS debug ${LZ4_LIB_DEBUG} optimized ${LZ4_LIB_RELEASE})

add_definitions(${LZ4_CXX_FLAGS})
include_directories(${LZ4_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${LZ4_LIBS})
else ()
message(STATUS "LZ4 library is disabled")
endif ()

# ================================================== ZLIB ==================================================
#
# Edit these 4 lines to define paths to ZLIB
#
set(ZLIB_HOME $ENV{THIRDPARTY_HOME}/zlib-1.3)
set(ZLIB_INCLUDE ${ZLIB_HOME})
set(ZLIB_LIB_DEBUG ${ZLIB_HOME}/contrib/vstudio/vc14/x64/ZlibStatDebug/zlibstat.lib)
set(ZLIB_LIB_RELEASE ${ZLIB_HOME}/contrib/vstudio/vc14/x64/ZlibStatRelease/zlibstat.lib)

# For compatibilty
if (ZLIB)
set(WITH_ZLIB ON)
endif ()

if (WITH_ZLIB)
message(STATUS "ZLIB library is enabled")

if(DEFINED ENV{ZLIB_INCLUDE})
set(ZLIB_INCLUDE $ENV{ZLIB_INCLUDE})
endif()

if(DEFINED ENV{ZLIB_LIB_DEBUG})
set(ZLIB_LIB_DEBUG $ENV{ZLIB_LIB_DEBUG})
endif()

if(DEFINED ENV{ZLIB_LIB_RELEASE})
set(ZLIB_LIB_RELEASE $ENV{ZLIB_LIB_RELEASE})
endif()

set(ZLIB_CXX_FLAGS -DZLIB)
set(ZLIB_LIBS debug ${ZLIB_LIB_DEBUG} optimized ${ZLIB_LIB_RELEASE})

add_definitions(${ZLIB_CXX_FLAGS})
include_directories(${ZLIB_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZLIB_LIBS})
else ()
message(STATUS "ZLIB library is disabled")
endif ()

# ================================================== XPRESS ==================================================
# This makes use of built-in Windows API, no additional includes, links to a system lib

# For compatibilty
if (XPRESS)
set(WITH_XPRESS ON)
endif ()

if (WITH_XPRESS)
message(STATUS "XPRESS is enabled")

add_definitions(-DXPRESS)

# We are using the implementation provided by the system
set (SYSTEM_LIBS ${SYSTEM_LIBS} Cabinet.lib)
else ()
message(STATUS "XPRESS is disabled")
endif ()


# ================================================== ZSTD ==================================================
#
# Edit these 4 lines to define paths to ZSTD
#

set(ZSTD_HOME $ENV{THIRDPARTY_HOME}/zstd-1.5.2)
set(ZSTD_INCLUDE ${ZSTD_HOME}/lib ${ZSTD_HOME}/lib/dictBuilder)
set(ZSTD_LIB_DEBUG ${ZSTD_HOME}/build/VS2010/bin/x64_Debug/libzstd_static.lib)
set(ZSTD_LIB_RELEASE ${ZSTD_HOME}/build/VS2010/bin/x64_Release/libzstd_static.lib)

# For compatibility
if (ZSTD)
set(WITH_ZSTD ON)
endif ()

if (WITH_ZSTD)
message(STATUS "ZSTD library is enabled")

if(DEFINED ENV{ZSTD_INCLUDE})
set(ZSTD_INCLUDE $ENV{ZSTD_INCLUDE})
endif()

if(DEFINED ENV{ZSTD_LIB_DEBUG})
set(ZSTD_LIB_DEBUG $ENV{ZSTD_LIB_DEBUG})
endif()

if(DEFINED ENV{ZSTD_LIB_RELEASE})
set(ZSTD_LIB_RELEASE $ENV{ZSTD_LIB_RELEASE})
endif()

# ZSTD_STATIC_LINKING_ONLY only allows us to create an allocation functions override
# When jemalloc is in use
set(ZSTD_LIBS debug ${ZSTD_LIB_DEBUG} optimized ${ZSTD_LIB_RELEASE})

add_definitions(-DZSTD -DZSTD_STATIC_LINKING_ONLY)
include_directories(${ZSTD_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZSTD_LIBS})
else ()
message(STATUS "ZSTD library is disabled")
endif ()

#
# Edit these 4 lines to define paths to Jemalloc
#
set(JEMALLOC_HOME $ENV{THIRDPARTY_HOME}/Jemalloc.Library)
set(JEMALLOC_INCLUDE ${JEMALLOC_HOME}/build/native/inc)
set(JEMALLOC_LIB_DEBUG ${JEMALLOC_HOME}/lib/native/debug/amd64/jemalloc.lib)
set(JEMALLOC_LIB_RELEASE ${JEMALLOC_HOME}/lib/native/retail/amd64/jemalloc.lib)

# ================================================== JEMALLOC ==================================================
if(JEMALLOC)
set(WITH_JEMALLOC ON)
endif()

if (WITH_JEMALLOC)
message(STATUS "JEMALLOC library is enabled")
set(JEMALLOC_CXX_FLAGS "-DROCKSDB_JEMALLOC -DJEMALLOC_EXPORT= -DJEMALLOC_NO_RENAME")

if(DEFINED ENV{JEMALLOC_INCLUDE})
set(JEMALLOC_INCLUDE $ENV{JEMALLOC_INCLUDE})
endif()

if(DEFINED ENV{JEMALLOC_LIB_DEBUG})
set(JEMALLOC_LIB_DEBUG $ENV{JEMALLOC_LIB_DEBUG})
endif()

if(DEFINED ENV{JEMALLOC_LIB_RELEASE})
set(JEMALLOC_LIB_RELEASE $ENV{JEMALLOC_LIB_RELEASE})
endif()

set(JEMALLOC_LIBS debug ${JEMALLOC_LIB_DEBUG} optimized ${JEMALLOC_LIB_RELEASE})

add_definitions(${JEMALLOC_CXX_FLAGS})
include_directories(${JEMALLOC_INCLUDE})
set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${JEMALLOC_LIBS})
set (ARTIFACT_SUFFIX "_je")

else ()
set (ARTIFACT_SUFFIX "")
message(STATUS "JEMALLOC library is disabled")
endif ()

0 comments on commit 797029e

Please sign in to comment.