Protobuf 22.5 cpp

COMPILING.md

@@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code
 generation. Most users only need to use `skipCodegen=true` as discussed above.
 
 ### Build Protobuf
-The codegen plugin is C++ code and requires protobuf 21.7 or later.
+The codegen plugin is C++ code and requires protobuf 22.5 or later.
 
 For Linux, Mac and MinGW:
 ```
-$ PROTOBUF_VERSION=21.7
+$ PROTOBUF_VERSION=22.5
 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz
 $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz
 $ cd protobuf-$PROTOBUF_VERSION

buildscripts/grpc-java-artifacts/Dockerfile

@@ -27,7 +27,11 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \
     mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \
     yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses
 
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local    
+
 # Install Maven
 RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \
     tar xz -C /var/local
-ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH
+

buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base

@@ -10,5 +10,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         g++-aarch64-linux-gnu \
         g++-powerpc64le-linux-gnu \
         openjdk-8-jdk \
+        pkg-config \
         && \
     rm -rf /var/lib/apt/lists/*
+
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:$PATH
+

buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base

@@ -9,5 +9,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         curl \
         g++-s390x-linux-gnu \
         openjdk-8-jdk \
+        pkg-config \
         && \
     rm -rf /var/lib/apt/lists/*
+
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:$PATH
+

buildscripts/kokoro/android-interop.sh

@@ -7,8 +7,8 @@ set -exu -o pipefail
 
 cd github/grpc-java
 
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS=-I/tmp/protobuf/include
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
 export LD_LIBRARY_PATH=/tmp/protobuf/lib
 export OS_NAME=$(uname)
 

buildscripts/kokoro/android.sh

@@ -9,9 +9,6 @@ BASE_DIR="$(pwd)"
 
 cd "$BASE_DIR/github/grpc-java"
 
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS=-I/tmp/protobuf/include
-export LD_LIBRARY_PATH=/tmp/protobuf/lib
 export OS_NAME=$(uname)
 
 cat <<EOF >> gradle.properties
@@ -30,10 +27,18 @@ unzip -qd "${ANDROID_HOME}/cmdline-tools" cmdline.zip
 rm cmdline.zip
 mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest"
 (yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses
-
+curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /tmp
+export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH
+
 # Proto deps
 buildscripts/make_dependencies.sh
 
+sudo apt-get update && sudo apt-get install pkg-config
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
+export LD_LIBRARY_PATH=/tmp/protobuf/lib
+
 # Build Android with Java 11, this adds it to the PATH
 sudo update-java-alternatives --set java-1.11.0-openjdk-amd64
 # Unset any existing JAVA_HOME env var to stop Gradle from using it
@@ -98,6 +103,7 @@ cd $BASE_DIR/github/grpc-java
 ./gradlew clean
 git checkout HEAD^
 ./gradlew --stop  # use a new daemon to build the previous commit
+GRADLE_FLAGS="${GRADLE_FLAGS} -PskipCodegen=true" # skip codegen for build from previous commit since it wasn't built with --std=c++14 when making this change
 ./gradlew publishToMavenLocal $GRADLE_FLAGS
 cd examples/android/helloworld/
 ../../gradlew build $GRADLE_FLAGS

buildscripts/kokoro/unix.sh

@@ -51,9 +51,9 @@ fi
 export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
 
 # Make protobuf discoverable by :grpc-compiler
-export LD_LIBRARY_PATH=/tmp/protobuf/lib
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS="-I/tmp/protobuf/include"
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
+export LIBRARY_PATH=/tmp/protobuf/lib
 
 ./gradlew grpc-compiler:clean $GRADLE_FLAGS
 

buildscripts/kokoro/windows.cfg

@@ -2,7 +2,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc-java/buildscripts/kokoro/windows.bat"
-timeout_mins: 45
+timeout_mins: 90 
 
 # We always build mvn artifacts.
 action {

buildscripts/kokoro/windows32.bat

@@ -15,19 +15,21 @@ set ESCWORKSPACE=%WORKSPACE:\=\\%
 
 @rem Clear JAVA_HOME to prevent a different Java version from being used
 set JAVA_HOME=
-set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
 
 mkdir grpc-java-helper32
 cd grpc-java-helper32
-call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1
+call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1
 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
 
 cd "%WORKSPACE%"
 
 SET TARGET_ARCH=x86_32
 SET FAIL_ON_WARNINGS=true
-SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release
-SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include
+SET PROTOBUF_VER=22.5
+SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
+SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
+SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
+call :Get_Libs
 SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
 SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
 
@@ -50,3 +52,32 @@ IF NOT %GRADLEEXIT% == 0 (
 cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
 
 cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
+
+goto :eof
+:Get_Libs
+SetLocal EnableDelayedExpansion
+set "libs_list="
+for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
+  for %%b in (%%a) do (
+    set lib=%%b
+    set libfirst2char=!lib:~0,2!
+    if !libfirst2char!==-l (
+      @rem remove the leading -l
+      set lib=!lib:~2!
+      @rem remove spaces
+      set lib=!lib: =!
+      @rem Because protobuf is specified as libprotobuf and elsewhere
+      if !lib! NEQ protobuf (
+        set lib=!lib!.lib
+        if "!libs_list!"=="" (
+          set libs_list=!lib!
+        ) else (
+          set libs_list=!libs_list!,!lib!
+        )
+      )
+    )
+  )
+)
+EndLocal & set "VC_PROTOBUF_LIBS=%VC_PROTOBUF_LIBS%,%libs_list%" 
+exit /b 0
+

buildscripts/kokoro/windows64.bat

@@ -14,23 +14,54 @@ set ESCWORKSPACE=%WORKSPACE:\=\\%
 
 @rem Clear JAVA_HOME to prevent a different Java version from being used
 set JAVA_HOME=
-set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
 
 mkdir grpc-java-helper64
 cd grpc-java-helper64
-call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" || exit /b 1
+call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1
 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
 
 cd "%WORKSPACE%"
 
 SET TARGET_ARCH=x86_64
 SET FAIL_ON_WARNINGS=true
-SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release
-SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include
+SET PROTOBUF_VER=22.5
+SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
+SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
+SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
+call :Get_Libs
 SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
 SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
 
 @rem make sure no daemons have any files open
 cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
 
 cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
+
+goto :eof
+:Get_Libs
+SetLocal EnableDelayedExpansion
+set "libs_list="
+for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
+  for %%b in (%%a) do (
+    set lib=%%b
+    set libfirst2char=!lib:~0,2!
+    if !libfirst2char!==-l (
+      @rem remove the leading -l
+      set lib=!lib:~2!
+      @rem remove spaces
+      set lib=!lib: =!
+      @rem Because protobuf is specified as libprotobuf and elsewhere
+      if !lib! NEQ protobuf (
+        set lib=!lib!.lib
+        if "!libs_list!"=="" (
+          set libs_list=!lib!
+        ) else (
+          set libs_list=!libs_list!,!lib!
+        )
+      )
+    )
+  )
+)
+EndLocal & set "VC_PROTOBUF_LIBS=%VC_PROTOBUF_LIBS%,%libs_list%" 
+exit /b 0
+

buildscripts/make_dependencies.bat

@@ -1,12 +1,16 @@
-set PROTOBUF_VER=21.7
-set CMAKE_NAME=cmake-3.3.2-win32-x86
+choco install -y pkgconfiglite
+choco install -y openjdk --version=17.0
+set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-17\bin"
+set PROTOBUF_VER=22.5
+set ABSL_VERSION=20230125.4
+set CMAKE_NAME=cmake-3.26.3-windows-x86_64
 
 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" (
   call :installProto || exit /b 1
 )
 
 echo Compile gRPC-Java with something like:
-echo -PtargetArch=x86_32 -PvcProtobufLibs=%cd%\protobuf-%PROTOBUF_VER%\build\Release -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\include
+echo -PtargetArch=x86_32 -PvcProtobufLibPath=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\lib -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\include -PvcProtobufLibs=insert-list-of-libs-from-pkg-config-output-here
 goto :eof
 
 
@@ -20,25 +24,35 @@ if not exist "%CMAKE_NAME%" (
 set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin
 :hasCmake
 @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled
-powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
+powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1
 del protobuf.zip
+powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
+powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
+del absl.zip
+rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
+move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
 mkdir protobuf-%PROTOBUF_VER%\build
 pushd protobuf-%PROTOBUF_VER%\build
 
-@rem Workaround https://github.com/protocolbuffers/protobuf/issues/10174
-powershell -command "(Get-Content ..\cmake\extract_includes.bat.in) -replace '\.\.\\', '' | Out-File -encoding ascii ..\cmake\extract_includes.bat.in"
 @rem cmake does not detect x86_64 from the vcvars64.bat variables.
-@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS
-if "%PLATFORM%" == "X64" (
-  @rem Note the space
-  SET CMAKE_VSARCH= Win64
+@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS, likewise for x32
+if "%PLATFORM%" == "x64" (
+  SET CMAKE_VSARCH=-A x64
+) else if "%PLATFORM%" == "x86" (
+  @rem -A x86 doesn't work: https://github.com/microsoft/vcpkg/issues/15465
+  SET CMAKE_VSARCH=-DCMAKE_GENERATOR_PLATFORM=WIN32
 ) else (
   SET CMAKE_VSARCH=
 )
-cmake -Dprotobuf_BUILD_TESTS=OFF -G "Visual Studio %VisualStudioVersion:~0,2%%CMAKE_VSARCH%" .. || exit /b 1
-msbuild /maxcpucount /p:Configuration=Release /verbosity:minimal libprotoc.vcxproj || exit /b 1
-call extract_includes.bat || exit /b 1
+for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do (
+  SET VC_YEAR=%%a
+)
+for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do (
+  SET visual_studio_major_version=%%a
+)
+cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. || exit /b 1
+cmake --build . --config Release --target install || exit /b 1
 popd
 goto :eof
 
@@ -49,3 +63,4 @@ powershell -command "$ErrorActionPreference = 'stop'; & { iwr https://cmake.org/
 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('cmake.zip', '.') }" || exit /b 1
 del cmake.zip
 goto :eof
+