Fixed issues in Cache initialization routes with async libpq modes

Server should start listening to requests only when both cache/database init routines are complete [docker_compare][skip_build]
sumeetchhetri · Apr 13, 2021 · 75bba20 · 75bba20
1 parent 65d9cca
commit 75bba20
Show file tree

Hide file tree

Showing 55 changed files with 429 additions and 370 deletions.
diff --git a/.cproject b/.cproject
@@ -42,6 +42,7 @@
 							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.macosx.base.342787389" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.macosx.base">
 								<option IS_BUILTIN_EMPTY="false" IS_VALUE_EMPTY="false" id="gnu.cpp.compiler.option.include.paths.1062586156" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
 									<listOptionValue builtIn="false" value="/usr/local/opt/libpq/include"/>
+									<listOptionValue builtIn="false" value="/usr/local/Cellar/vcpkg/2020.11-1/libexec/installed/x64-osx/include/"/>
 									<listOptionValue builtIn="false" value="/usr/local/opt/emsdk/upstream/emscripten/system/include"/>
 									<listOptionValue builtIn="false" value="/Users/sumeetc/Projects/GitHub/fmt/include/"/>
 									<listOptionValue builtIn="false" value="/Users/sumeetc/Projects/GitHub/seastar/include/"/>
@@ -59,6 +60,7 @@
 								</option>
 								<option IS_BUILTIN_EMPTY="false" IS_VALUE_EMPTY="false" id="gnu.cpp.compiler.option.preprocessor.def.228093075" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
 									<listOptionValue builtIn="false" value="__cplusplus=201103L"/>
+									<listOptionValue builtIn="false" value="RAPIDJSON_HAS_STDSTRING=1"/>
 								</option>
 								<option id="gnu.cpp.compiler.option.optimization.level.392374030" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
 								<option defaultValue="gnu.cpp.compiler.debugging.level.max" id="gnu.cpp.compiler.option.debugging.level.1405958163" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" valueType="enumerated"/>
@@ -167,5 +169,12 @@
 		</scannerConfigBuildInfo>
 	</storageModule>
 	<storageModule moduleId="org.eclipse.cdt.core.pathentry"/>
-	<storageModule moduleId="refreshScope"/>
+	<storageModule moduleId="refreshScope" versionNumber="2">
+		<configuration configurationName="Debug">
+			<resource resourceType="PROJECT" workspacePath="/ffead-cpp"/>
+		</configuration>
+		<configuration configurationName="Release">
+			<resource resourceType="PROJECT" workspacePath="/ffead-cpp"/>
+		</configuration>
+	</storageModule>
 </cproject>
diff --git a/.github/workflows/ci-docker-tfb-test.yml b/.github/workflows/ci-docker-tfb-test.yml
@@ -19,4 +19,4 @@ jobs:
         cp -rf docker/techempower-config/* FrameworkBenchmarks/frameworks/C++/ffead-cpp/
         cd FrameworkBenchmarks
         #sed -i'' -e 's|apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql postgresql-contrib|apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql-12 postgresql-contrib-12|g' toolset/databases/postgres/postgres.dockerfile
-        ./tfb --mode verify --test ffead-cpp
+        ./tfb --mode verify --test-dir C++/ffead-cpp
diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
@@ -57,18 +57,6 @@ jobs:
         cd hiredis-1.0.0/
         cmake . && sudo make install
         rm -rf hiredis-1.0.0
-        git clone https://github.com/sumeetchhetri/cinatra.git
-        cd cinatra
-        git checkout sum_master
-        cd /tmp
-        git clone --recurse-submodules https://github.com/sumeetchhetri/drogon
-        cd  drogon
-        mkdir build
-        cd build
-        cmake -DCMAKE_BUILD_TYPE=Release ..
-        make && sudo make install
-        cd /tmp
-        rm -rf drogon
         popd
 
     - name: Install ffead-cpp
@@ -107,6 +95,8 @@ jobs:
         done
         echo "ffead-cpp start successful"
         sleep 5
-        cd tests && chmod +x *.sh && ./runTests.sh
+        cd tests
+        sudo chmod +x *.sh
+        sudo ./runTests.sh
         echo "ffead-cpp normal shutdown"
         sudo pkill ffead-cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -85,6 +85,7 @@ if(DEBUG)
 	#if (APPLE)
 	#	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 	#endif()
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_DEBUG")
 else()
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 endif(DEBUG)

diff --git a/docker/benchmark_compare.json b/docker/benchmark_compare.json
@@ -5,10 +5,7 @@
 			"json_url": "/te-benchmark-um-pq/json",
 			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq/db",
-			"query_url": "/te-benchmark-um-pq/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq/fortunes",
-			"update_url": "/te-benchmark-um-pq/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",
@@ -26,13 +23,10 @@
 			"tags": []
 		},
 		"postgresql-raw52": {
-			"json_url": "/te-benchmark-um-pq/json",
-			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq/db",
 			"query_url": "/te-benchmark-um-pq/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq/fortunes",
 			"update_url": "/te-benchmark-um-pq/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",
@@ -92,13 +86,10 @@
 			"tags": []
 		},
 		"postgresql-raw-async52": {
-			"json_url": "/te-benchmark-um-pq-async/json",
-			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq-async/db",
 			"query_url": "/te-benchmark-um-pq-async/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq-async/fortunes",
 			"update_url": "/te-benchmark-um-pq-async/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq-async/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",
@@ -161,10 +152,7 @@
 			"json_url": "/te-benchmark-um-pq/json",
 			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq/db",
-			"query_url": "/te-benchmark-um-pq/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq/fortunes",
-			"update_url": "/te-benchmark-um-pq/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",
@@ -182,13 +170,10 @@
 			"tags": []
 		},
 		"postgresql-raw53": {
-			"json_url": "/te-benchmark-um-pq/json",
-			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq/db",
 			"query_url": "/te-benchmark-um-pq/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq/fortunes",
 			"update_url": "/te-benchmark-um-pq/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",
@@ -248,13 +233,10 @@
 			"tags": []
 		},
 		"postgresql-raw-async53": {
-			"json_url": "/te-benchmark-um-pq-async/json",
-			"plaintext_url": "/plaintext",
 			"db_url": "/te-benchmark-um-pq-async/db",
 			"query_url": "/te-benchmark-um-pq-async/queries?queries=",
 			"fortune_url": "/te-benchmark-um-pq-async/fortunes",
 			"update_url": "/te-benchmark-um-pq-async/updates?queries=",
-			"cached_query_url": "/te-benchmark-um-pq-async/cached-worlds?count=",
 			"port": 8080,
 			"approach": "Realistic",
 			"classification": "Fullstack",

diff --git a/docker/techempower-config/ffead-cpp-sql-raw-async-clibpqb-profiled-base.dockerfile b/docker/techempower-config/ffead-cpp-sql-raw-async-clibpqb-profiled-base.dockerfile
@@ -79,7 +79,15 @@ ENV accept accept
 
 WORKDIR ${IROOT}
 
+COPY sql-profiled-util.sh ${IROOT}/
+RUN chmod 755 ${IROOT}/sql-profiled-util.sh
+RUN ./sql-profiled-util.sh batch clang async
+
+#COPY TeBkUmLpqAsync.cpp ${IROOT}/ffead-cpp-src/web/te-benchmark-um-pq-async/src/
+#COPY TeBkUmLpqAsync.h ${IROOT}/ffead-cpp-src/web/te-benchmark-um-pq-async/include/
+#COPY LibpqDataSourceImpl.cpp ${IROOT}/ffead-cpp-src/src/modules/sdorm/sql/libpq/
+#COPY LibpqDataSourceImpl.h ${IROOT}/ffead-cpp-src/src/modules/sdorm/sql/libpq/
+
 COPY sql-async-profiled-install-clang.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
 RUN chmod 755 ${IROOT}/sql-async-profiled-install-clang.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh
-
 RUN ./sql-async-profiled-install-clang.sh batch
diff --git a/docker/techempower-config/ffead-cpp-sql-raw-async-profiled-base.dockerfile b/docker/techempower-config/ffead-cpp-sql-raw-async-profiled-base.dockerfile
@@ -79,7 +79,10 @@ ENV accept accept
 
 WORKDIR ${IROOT}
 
+COPY sql-profiled-util.sh ${IROOT}/
+RUN chmod 755 ${IROOT}/sql-profiled-util.sh
+RUN ./sql-profiled-util.sh nobatch noclang async
+
 COPY sql-async-profiled-install.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
 RUN chmod 755 ${IROOT}/sql-async-profiled-install.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh
-
 RUN ./sql-async-profiled-install.sh
diff --git a/docker/techempower-config/ffead-cpp-sql-raw-clibpqb-profiled-base.dockerfile b/docker/techempower-config/ffead-cpp-sql-raw-clibpqb-profiled-base.dockerfile
@@ -79,7 +79,10 @@ ENV accept accept
 
 WORKDIR ${IROOT}
 
+COPY sql-profiled-util.sh ${IROOT}/
+RUN chmod 755 ${IROOT}/sql-profiled-util.sh
+RUN ./sql-profiled-util.sh batch clang noasync
+
 COPY sql-profiled-install-clang.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
 RUN chmod 755 ${IROOT}/sql-profiled-install-clang.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh
-
 RUN ./sql-profiled-install-clang.sh batch
diff --git a/docker/techempower-config/ffead-cpp-sql-raw-profiled-base.dockerfile b/docker/techempower-config/ffead-cpp-sql-raw-profiled-base.dockerfile
@@ -79,7 +79,10 @@ ENV accept accept
 
 WORKDIR ${IROOT}
 
+COPY sql-profiled-util.sh ${IROOT}/
+RUN chmod 755 ${IROOT}/sql-profiled-util.sh
+RUN ./sql-profiled-util.sh nobatch noclang noasync
+
 COPY sql-profiled-install.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
 RUN chmod 755 ${IROOT}/sql-profiled-install.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh
-
 RUN ./sql-profiled-install.sh
diff --git a/docker/techempower-config/run_ffead.sh b/docker/techempower-config/run_ffead.sh
@@ -107,10 +107,6 @@ then
 	for i in $(seq 0 $(($(nproc --all)-1))); do
 		taskset -c $i ./ffead-cpp $FFEAD_CPP_PATH &
 	done
-	if [ "$3" = "postgresql-raw-async" ]
-	then
-		sleep 20
-	fi
 elif [ "$2" = "lithium" ]
 then
 	./ffead-cpp-lithium $FFEAD_CPP_PATH &

diff --git a/docker/techempower-config/sql-async-profiled-install-clang-dbg.sh b/docker/techempower-config/sql-async-profiled-install-clang-dbg.sh
@@ -1,44 +1,4 @@
-mkdir /tmp/profile-data
-
-rm -rf $IROOT/ffead-cpp-5.0-sql
-
-if [ "$1" = "batch" ]
-then
-	apt remove -yqq libpq-dev
-	apt autoremove -yqq
-	rm -f /usr/lib/x86_64-linux-gnu/libpq.*
-	apt update && apt install -y bison flex libreadline-dev
-	cd /tmp
-	#wget -q https://github.com/an-tao/postgres/archive/batch_mode_ubuntu.tar.gz
-	#tar -xzf batch_mode_ubuntu.tar.gz
-	#cd postgres-batch_mode_ubuntu
-	#./configure --prefix=/usr CFLAGS='-O2 -pipe -march=native'
-	#make && make install
-	wget -nv https://github.com/postgres/postgres/archive/b787d4ce6d910080065025bcd5f968544997271f.zip
-	unzip -q b787d4ce6d910080065025bcd5f968544997271f.zip
-	cd postgres-b787d4ce6d910080065025bcd5f968544997271f
-	wget -nv https://www.postgresql.org/message-id/attachment/115223/v22-0001-libpq-batch.patch
-	git apply ./v22-0001-libpq-batch.patch
-	./configure --prefix=/usr CFLAGS='-O3 -march=native -flto'
-	cd src/interfaces/libpq
-	make all install -j4
-	cp ../../../src/include/postgres_ext.h ../../../src/include/pg_config_ext.h libpq-fe.h /usr/include
-fi
-
-apt update -yqq && apt install -yqq clang
-
 cd $IROOT/ffead-cpp-src/
-rm -rf CMakeCache.txt CMakeFiles
-rm -rf web/te-benchmark-um web/te-benchmark-um-mgr web/te-benchmark-um-pq
-
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-mgr)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-pq)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um/libte_benchmark_um${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-mgr/libte_benchmark_um_mgr${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-pq/libte_benchmark_um_pq${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-
-sed -i 's|tfb-database|localhost|g' $IROOT/ffead-cpp-src/web/te-benchmark-um-pq-async/config/sdorm.xml
 
 rm -rf build
 mkdir build
@@ -53,7 +13,7 @@ service postgresql stop
 cd $IROOT/
 sed -i 's|cmake |CC=/usr/bin/clang CXX=/usr/bin/clang++ cmake |g' $IROOT/ffead-cpp-sql-raw/resources/rundyn-automake.sh
 #sed -i 's|-fprofile-instr-generate=/tmp/cprof.prof|-fprofile-instr-generate=/tmp/cprofdi.prof|g' $IROOT/ffead-cpp-sql-raw/rtdcf/CMakeLists.txt.template
-apt update -yqq && apt install -yqq vim gdb
+apt update -yqq && apt install -yqq vim gdb net-tools telnet iputils-ping
 ./install_ffead-cpp-sql-raw-profiled.sh async
 
 #mv $IROOT/ffead-cpp-sql-raw $IROOT/ffead-cpp-5.0-sql

diff --git a/docker/techempower-config/sql-async-profiled-install-clang.sh b/docker/techempower-config/sql-async-profiled-install-clang.sh
@@ -1,45 +1,4 @@
-mkdir /tmp/profile-data
-
-rm -rf $IROOT/ffead-cpp-5.0-sql
-
-if [ "$1" = "batch" ]
-then
-	apt remove -yqq libpq-dev
-	apt autoremove -yqq
-	rm -f /usr/lib/x86_64-linux-gnu/libpq.*
-	apt update && apt install -y bison flex libreadline-dev
-	cd /tmp
-	#wget -q https://github.com/an-tao/postgres/archive/batch_mode_ubuntu.tar.gz
-	#tar -xzf batch_mode_ubuntu.tar.gz
-	#cd postgres-batch_mode_ubuntu
-	#./configure --prefix=/usr CFLAGS='-O2 -pipe -march=native'
-	#make && make install
-	wget -nv https://github.com/postgres/postgres/archive/b787d4ce6d910080065025bcd5f968544997271f.zip
-	unzip -q b787d4ce6d910080065025bcd5f968544997271f.zip
-	cd postgres-b787d4ce6d910080065025bcd5f968544997271f
-	wget -nv https://www.postgresql.org/message-id/attachment/115223/v22-0001-libpq-batch.patch
-	git apply ./v22-0001-libpq-batch.patch
-	./configure --prefix=/usr CFLAGS='-O3 -march=native -flto'
-	cd src/interfaces/libpq
-	make all install -j4
-	cp ../../../src/include/postgres_ext.h ../../../src/include/pg_config_ext.h libpq-fe.h /usr/include
-fi
-
-apt update -yqq && apt install -yqq clang
-
 cd $IROOT/ffead-cpp-src/
-rm -rf $IROOT/ffead-cpp-sql-raw
-rm -rf CMakeCache.txt CMakeFiles
-rm -rf web/te-benchmark-um web/te-benchmark-um-mgr web/te-benchmark-um-pq
-
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-mgr)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-pq)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um/libte_benchmark_um${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-mgr/libte_benchmark_um_mgr${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-pq/libte_benchmark_um_pq${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-
-sed -i 's|tfb-database|localhost|g' $IROOT/ffead-cpp-src/web/te-benchmark-um-pq-async/config/sdorm.xml
 
 rm -rf build
 mkdir build

diff --git a/docker/techempower-config/sql-async-profiled-install.sh b/docker/techempower-config/sql-async-profiled-install.sh
@@ -1,42 +1,4 @@
-mkdir /tmp/profile-data
-
-rm -rf $IROOT/ffead-cpp-5.0-sql
-
-if [ "$1" = "batch" ]
-then
-	apt remove -yqq libpq-dev
-	apt autoremove -yqq
-	rm -f /usr/lib/x86_64-linux-gnu/libpq.*
-	apt update && apt install -y bison flex libreadline-dev
-	cd /tmp
-	#wget -q https://github.com/an-tao/postgres/archive/batch_mode_ubuntu.tar.gz
-	#tar -xzf batch_mode_ubuntu.tar.gz
-	#cd postgres-batch_mode_ubuntu
-	#./configure --prefix=/usr CFLAGS='-O2 -pipe -march=native'
-	#make && make install
-	wget -nv https://github.com/postgres/postgres/archive/b787d4ce6d910080065025bcd5f968544997271f.zip
-	unzip -q b787d4ce6d910080065025bcd5f968544997271f.zip
-	cd postgres-b787d4ce6d910080065025bcd5f968544997271f
-	wget -nv https://www.postgresql.org/message-id/attachment/115223/v22-0001-libpq-batch.patch
-	git apply ./v22-0001-libpq-batch.patch
-	./configure --prefix=/usr CFLAGS='-O3 -march=native -flto'
-	cd src/interfaces/libpq
-	make all install -j4
-	cp ../../../src/include/postgres_ext.h ../../../src/include/pg_config_ext.h libpq-fe.h /usr/include
-fi
-
 cd $IROOT/ffead-cpp-src/
-rm -rf CMakeCache.txt CMakeFiles
-rm -rf web/te-benchmark-um web/te-benchmark-um-mgr web/te-benchmark-um-pq
-
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-mgr)||g' CMakeLists.txt
-sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark-um-pq)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um/libte_benchmark_um${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-mgr/libte_benchmark_um_mgr${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark-um-pq/libte_benchmark_um_pq${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
-
-sed -i 's|tfb-database|localhost|g' $IROOT/ffead-cpp-src/web/te-benchmark-um-pq-async/config/sdorm.xml
 
 rm -rf build
 mkdir build