From e75100f5e12e4337c8063bf5d80f03da816056cd Mon Sep 17 00:00:00 2001 From: Sumeet Chhetri Date: Sun, 8 May 2022 15:32:40 +0530 Subject: [PATCH 1/3] Added support for scons builds Added support for shellb builds Fixed several compilation issues Added support for redis clsuter Adding support for SOLR/Elasticsearch (WIP) [skip_build] --- CMakeLists.txt | 60 ++- .../install_ffead-cpp-dependencies.sh | 10 + .../os-based/DockerFile-Alpine-x64-ffead-cpp | 10 + .../DockerFile-Alpine-x64-ffead-cpp_autoconf | 10 + .../DockerFile-ArchLinux-x64-ffead-cpp | 10 + ...ockerFile-ArchLinux-x64-ffead-cpp_autoconf | 10 + .../os-based/DockerFile-Gentoo-x64-ffead-cpp | 10 + .../DockerFile-Gentoo-x64-ffead-cpp_autoconf | 10 + .../DockerFile-OpenSuse-x64-ffead-cpp | 10 + ...DockerFile-OpenSuse-x64-ffead-cpp_autoconf | 10 + .../os-based/DockerFile-Ubuntu-x64-ffead-cpp | 10 + .../DockerFile-Ubuntu-x64-ffead-cpp-meson | 10 + .../DockerFile-Ubuntu-x64-ffead-cpp-scons | 64 +++ .../DockerFile-Ubuntu-x64-ffead-cpp-shellb | 64 +++ .../DockerFile-Ubuntu-x64-ffead-cpp-xmake | 10 + .../DockerFile-Ubuntu-x64-ffead-cpp_autoconf | 10 + ...ckerFile-Ubuntu-x64-ffead-cpp_with_iouring | 10 + .../install_ffead-cpp-dependencies.sh | 10 + ...File-UbuntuBionic-x64-ffead-cpp-forprofile | 10 + ...kerFile-UbuntuBionic-x64-ffead-cpp-fortest | 10 + ...File-UbuntuBionic-x64-ffead-cpp-fortest-nb | 10 + ...untuBionic-x64-ffead-cpp-fortest_localhost | 10 + docker/test/install_deps.sh | 10 + .../DockerFile-UbuntuBionic-x64-ffead-cpp | 10 + ...erFile-UbuntuBionic-x64-ffead-cpp_autoconf | 10 + ffead-cpp-shellb.sh | 149 +++++++ meson.build | 45 ++- meson_options.txt | 2 + resources/rundyn-automake.sh | 13 +- rtdcf/inter-shellb.sh.tem | 28 ++ rtdcf/sconstruct.in | 36 ++ scons_options.py | 16 + sconstruct | 376 ++++++++++++++++++ src/framework/ConfigurationData.h | 2 + src/framework/ConfigurationHandler.cpp | 134 +++++++ src/framework/ConfigurationHandler.h | 12 +- src/framework/ConnectionSettings.cpp | 40 -- src/framework/ConnectionSettings.h | 60 --- src/framework/ServerInitUtil.cpp | 14 +- src/framework/meson.build | 4 +- src/framework/sconscript | 28 ++ src/modules/CMakeLists.txt | 30 +- src/modules/cache/CacheInterface.h | 4 + src/modules/cache/CacheManager.cpp | 29 +- src/modules/cache/CacheManager.h | 6 +- .../cache/redis/RedisClusterCacheImpl.cpp | 264 ++++++++++++ .../cache/redis/RedisClusterCacheImpl.h | 82 ++++ src/modules/client-util/HttpClient.cpp | 4 +- src/modules/client-util/HttpClient.h | 4 + src/modules/common/AppDefines.bazel.h.in | 31 ++ ...{AppDefines.h.in.cm => AppDefines.h.in.mc} | 4 + src/modules/common/AppDefines.meson.h.in | 6 + src/modules/common/AppDefines.scons.h.in | 31 ++ src/modules/common/AppDefines.xmake.h.in | 4 + src/modules/common/CommonUtils.cpp | 20 + src/modules/common/CommonUtils.h | 3 + src/modules/component/BeanContext.cpp | 6 +- src/modules/component/ComponentHandler.cpp | 6 +- src/modules/meson.build | 74 ++-- src/modules/methinvoker/MethodInvoc.cpp | 10 +- src/modules/sconscript | 49 +++ src/modules/search/DocumentQuery.cpp | 24 +- src/modules/search/DocumentQuery.h | 14 +- src/modules/search/IndexQuery.cpp | 27 +- src/modules/search/IndexQuery.h | 17 +- src/modules/search/SearchEngineInterface.h | 54 ++- src/modules/search/SearchEngineManager.cpp | 78 ++-- src/modules/search/SearchEngineManager.h | 15 +- src/modules/search/SearchQuery.cpp | 33 +- src/modules/search/SearchQuery.h | 15 + .../search/elasticsearch/ElasticSearch.cpp | 5 +- .../search/elasticsearch/ElasticSearch.h | 5 +- src/modules/search/solr/SolrSearch.cpp | 173 ++++---- src/modules/search/solr/SolrSearch.h | 7 +- src/modules/server-util/ServiceHandler.cpp | 21 +- src/modules/server-util/SocketInterface.h | 5 +- src/modules/xmake.lua | 7 + src/server/embedded/CHServer.cpp | 19 +- src/server/sconscript | 26 ++ tests/sconscript | 26 ++ web/default/sconscript | 29 ++ web/flexApp/sconscript | 29 ++ web/markers/sconscript | 29 ++ web/oauthApp/sconscript | 29 ++ web/peer-server/sconscript | 29 ++ web/t1/sconscript | 29 ++ web/t2/sconscript | 29 ++ web/t3/sconscript | 29 ++ web/t4/sconscript | 29 ++ web/t5/sconscript | 29 ++ web/te-benchmark/sconscript | 29 ++ xmake.lua | 42 +- 92 files changed, 2586 insertions(+), 380 deletions(-) create mode 100644 docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-scons create mode 100644 docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-shellb create mode 100644 ffead-cpp-shellb.sh create mode 100644 rtdcf/inter-shellb.sh.tem create mode 100644 rtdcf/sconstruct.in create mode 100644 scons_options.py create mode 100644 sconstruct delete mode 100644 src/framework/ConnectionSettings.cpp delete mode 100644 src/framework/ConnectionSettings.h create mode 100644 src/framework/sconscript create mode 100644 src/modules/cache/redis/RedisClusterCacheImpl.cpp create mode 100644 src/modules/cache/redis/RedisClusterCacheImpl.h create mode 100644 src/modules/common/AppDefines.bazel.h.in rename src/modules/common/{AppDefines.h.in.cm => AppDefines.h.in.mc} (94%) create mode 100644 src/modules/common/AppDefines.scons.h.in create mode 100644 src/modules/sconscript create mode 100644 src/server/sconscript create mode 100644 tests/sconscript create mode 100644 web/default/sconscript create mode 100644 web/flexApp/sconscript create mode 100644 web/markers/sconscript create mode 100644 web/oauthApp/sconscript create mode 100644 web/peer-server/sconscript create mode 100644 web/t1/sconscript create mode 100644 web/t2/sconscript create mode 100644 web/t3/sconscript create mode 100644 web/t4/sconscript create mode 100644 web/t5/sconscript create mode 100644 web/te-benchmark/sconscript diff --git a/CMakeLists.txt b/CMakeLists.txt index 121275e5b..044cd40a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,8 @@ option(MOD_REDIS "Enable Redis Cache module" OFF) option(MOD_SDORM_SQL "Enable SQL Sdorm module" ON) option(MOD_SDORM_MONGO "Enable Mongo Sdorm module" OFF) option(MOD_SDORM_SCYLLA "Enable Scylladb Sdorm module" OFF) +option(MOD_SOLR "Enable SOLR Search module" OFF) +option(MOD_ELASTIC "Enable Elasticsearch Search module" OFF) option(MOD_SER_BIN "Enable Binary Serialization module" OFF) option(MOD_JOBS "Enable Jobs module" ON) option(MOD_APACHE "Enable Apache module" OFF) @@ -107,6 +109,23 @@ include_directories("${CMAKE_SOURCE_DIR}/modules/http" "${CMAKE_SOURCE_DIR}/modu include_directories("${CMAKE_SOURCE_DIR}/modules/jobs" "${CMAKE_SOURCE_DIR}/modules/reflection" "${CMAKE_SOURCE_DIR}/modules/sdorm" "${CMAKE_SOURCE_DIR}/modules/search") include_directories("${CMAKE_SOURCE_DIR}/modules/serialization" "${CMAKE_SOURCE_DIR}/modules/serialization/xml" "${CMAKE_SOURCE_DIR}/modules/serialization/json") include_directories("${CMAKE_SOURCE_DIR}/modules/server-util" "${CMAKE_SOURCE_DIR}/modules/ssl" "${CMAKE_SOURCE_DIR}/modules/threads") + +include_directories("${CMAKE_SOURCE_DIR}/modules/search") +if(MOD_SOLR) + set(HAVE_SOLR "") + include_directories("${CMAKE_SOURCE_DIR}/modules/search/solr") +endif() +if(MOD_ELASTIC) + CHECK_INCLUDE_FILE("elasticlient/client.h" HAVE_ELASTIC) + if(HAVE_ELASTIC) + include_directories("${CMAKE_SOURCE_DIR}/modules/search/elasticsearch") + FIND_LIBRARY(HAVE_ELASTICLIB elasticlient) + if(NOT HAVE_ELASTICLIB) + message(FATAL_ERROR "elasticlient library not found") + endif() + endif() +endif() + if(CMAKE_INC_PATH) include_directories("${CMAKE_SOURCE_DIR}/framework" "${CMAKE_INC_PATH}/include") else() @@ -428,6 +447,13 @@ if(MOD_REDIS) message(FATAL_ERROR "hiredis library not found") endif() include_directories("${CMAKE_SOURCE_DIR}/modules/cache/redis") + CHECK_INCLUDE_FILE_CXX("sw/redis++/redis++.h" HAVE_REDIS_CLUSTERINC) + if(HAVE_REDIS_CLUSTERINC) + FIND_LIBRARY(HAVE_REDIS_CLUSTERLIB redis++) + if(NOT HAVE_REDIS_CLUSTERLIB) + message(FATAL_ERROR "redis++ library not found") + endif() + endif() endif(MOD_REDIS) if(MOD_MEMCACHED) set(INC_MEMCACHED 1) @@ -742,7 +768,7 @@ elseif(SRV_CPPSRV) endif() configure_file ( - "${PROJECT_SOURCE_DIR}/src/modules/common/AppDefines.h.in.cm" + "${PROJECT_SOURCE_DIR}/src/modules/common/AppDefines.h.in.mc" "${PROJECT_SOURCE_DIR}/src/modules/common/AppDefines.h" ) configure_file ( @@ -781,75 +807,75 @@ endif() if(SRV_ALL) add_executable(ffead-cpp src/server/embedded/CHServer.cpp) set_property(TARGET ffead-cpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp ffead-modules ffead-framework ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp ffead-modules ffead-framework ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) include_directories("${CINATRA_INCLUDES}") add_executable(ffead-cpp-cinatra src/server/cinatra/CinatraServer.cpp) find_package(Boost 1.65 REQUIRED COMPONENTS system) set_property(TARGET ffead-cpp-cinatra PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-cinatra ffead-modules ffead-framework stdc++fs ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-cinatra ffead-modules ffead-framework stdc++fs ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) add_executable(ffead-cpp-lithium src/server/lithium/LithiumServer.cpp) set_property(TARGET ffead-cpp-lithium PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-lithium ffead-modules ffead-framework boost_context ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-lithium ffead-modules ffead-framework boost_context ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) include_directories("${CMAKE_JSONCPP_INC}") add_executable(ffead-cpp-drogon src/server/drogon/DrogonServer.cpp) set_property(TARGET ffead-cpp-drogon PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-drogon ffead-modules ffead-framework ${HAVE_DROGONLIB} ${HAVE_TRANTORLIB} ${HAVE_JSONCPP} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-drogon ffead-modules ffead-framework ${HAVE_DROGONLIB} ${HAVE_TRANTORLIB} ${HAVE_JSONCPP} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) add_executable(ffead-cpp-nghttp2 src/server/nghttp2/Nghttp2Server.cpp) find_package(Boost 1.65 REQUIRED COMPONENTS system thread) set_property(TARGET ffead-cpp-nghttp2 PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) add_executable(ffead-cpp-mongols src/server/mongols/MongolsServer.cpp) set_property(TARGET ffead-cpp-mongols PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-mongols ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_MONGOLSLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-mongols ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_MONGOLSLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) add_executable(ffead-cpp-uvcpp src/server/uv-cpp/UvCppServer.cpp) set_property(TARGET ffead-cpp-uvcpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-uvcpp ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_UVCPPLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-uvcpp ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_UVCPPLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) add_executable(ffead-cpp-cppsrv src/server/cppserver/CppServerServer.cpp) set_property(TARGET ffead-cpp-cppsrv PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-cppsrv ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_CPPSRV} ${HAVE_CPPSRVASIO} ${HAVE_CPPSRVCPPCMN} ${HAVE_CPPSRVFMT} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-cppsrv ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_CPPSRV} ${HAVE_CPPSRVASIO} ${HAVE_CPPSRVCPPCMN} ${HAVE_CPPSRVFMT} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_EMB) add_executable(ffead-cpp src/server/embedded/CHServer.cpp) set_property(TARGET ffead-cpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp ffead-modules ffead-framework ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp ffead-modules ffead-framework ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_CINATRA) include_directories("${CINATRA_INCLUDES}") add_executable(ffead-cpp src/server/cinatra/CinatraServer.cpp) find_package(Boost 1.65 REQUIRED COMPONENTS system) set_property(TARGET ffead-cpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp ffead-modules ffead-framework stdc++fs ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp ffead-modules ffead-framework stdc++fs ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_LITHIUM) add_executable(ffead-cpp src/server/lithium/LithiumServer.cpp) set_property(TARGET ffead-cpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp ffead-modules ffead-framework boost_context ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp ffead-modules ffead-framework boost_context ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_DROGON) include_directories("${CMAKE_JSONCPP_INC}") add_executable(ffead-cpp src/server/drogon/DrogonServer.cpp) set_property(TARGET ffead-cpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp ffead-modules ffead-framework ${HAVE_DROGONLIB} ${HAVE_TRANTORLIB} ${HAVE_JSONCPP} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp ffead-modules ffead-framework ${HAVE_DROGONLIB} ${HAVE_TRANTORLIB} ${HAVE_JSONCPP} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_NGHTTP2) add_executable(ffead-cpp-nghttp2 src/server/nghttp2/Nghttp2Server.cpp) find_package(Boost 1.65 REQUIRED COMPONENTS system thread) set_property(TARGET ffead-cpp-nghttp2 PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_MONGOLS) add_executable(ffead-cpp-mongols src/server/mongols/MongolsServer.cpp) set_property(TARGET ffead-cpp-mongols PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-mongols ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_MONGOLSLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-mongols ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_MONGOLSLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_UVCPP) add_executable(ffead-cpp-uvcpp src/server/uv-cpp/UvCppServer.cpp) set_property(TARGET ffead-cpp-uvcpp PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-uvcpp ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_UVCPPLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-uvcpp ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_UVCPPLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) elseif(SRV_CPPSRV) add_executable(ffead-cpp-cppsrv src/server/cppserver/CppServerServer.cpp) set_property(TARGET ffead-cpp-cppsrv PROPERTY POSITION_INDEPENDENT_CODE ON) - target_link_libraries(ffead-cpp-cppsrv ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_CPPSRV} ${HAVE_CPPSRVASIO} ${HAVE_CPPSRVCPPCMN} ${HAVE_CPPSRVFMT} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) + target_link_libraries(ffead-cpp-cppsrv ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_CPPSRV} ${HAVE_CPPSRVASIO} ${HAVE_CPPSRVCPPCMN} ${HAVE_CPPSRVFMT} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) endif() MESSAGE( STATUS "CMAKE_REQUIRED_INCLUDES: " ${CMAKE_REQUIRED_INCLUDES} ) diff --git a/docker/compose-techempower-tests/ffead-cpp/install_ffead-cpp-dependencies.sh b/docker/compose-techempower-tests/ffead-cpp/install_ffead-cpp-dependencies.sh index bc965f903..082dc1592 100644 --- a/docker/compose-techempower-tests/ffead-cpp/install_ffead-cpp-dependencies.sh +++ b/docker/compose-techempower-tests/ffead-cpp/install_ffead-cpp-dependencies.sh @@ -29,6 +29,16 @@ cmake . && make install cd $IROOT rm -rf hiredis-1.0.0 +wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +tar xf 1.3.3.tar.gz +rm -f 1.3.3.tar.gz +cd redis-plus-plus-1.3.3/ +mkdir build +cd build +cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +cd $IROOT +rm -rf redis-plus-plus-1.3.3 + mkdir -p /usr/lib/x86_64-linux-gnu/odbc wget -q https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.19-linux-ubuntu18.04-x86-64bit.tar.gz tar xf mysql-connector-odbc-8.0.19-linux-ubuntu18.04-x86-64bit.tar.gz diff --git a/docker/os-based/DockerFile-Alpine-x64-ffead-cpp b/docker/os-based/DockerFile-Alpine-x64-ffead-cpp index 801933f0e..e7ebac021 100644 --- a/docker/os-based/DockerFile-Alpine-x64-ffead-cpp +++ b/docker/os-based/DockerFile-Alpine-x64-ffead-cpp @@ -29,6 +29,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install ffead-cpp WORKDIR /tmp RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip diff --git a/docker/os-based/DockerFile-Alpine-x64-ffead-cpp_autoconf b/docker/os-based/DockerFile-Alpine-x64-ffead-cpp_autoconf index 9827c6c67..7b45baea0 100644 --- a/docker/os-based/DockerFile-Alpine-x64-ffead-cpp_autoconf +++ b/docker/os-based/DockerFile-Alpine-x64-ffead-cpp_autoconf @@ -29,6 +29,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install ffead-cpp WORKDIR /tmp RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip diff --git a/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp b/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp index 693021129..d91884cf4 100644 --- a/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp +++ b/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp @@ -25,6 +25,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp_autoconf b/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp_autoconf index cc2fc1f00..b291de3b4 100644 --- a/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp_autoconf +++ b/docker/os-based/DockerFile-ArchLinux-x64-ffead-cpp_autoconf @@ -25,6 +25,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp b/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp index 12df9a47e..4e22dfeec 100644 --- a/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp +++ b/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp @@ -18,6 +18,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp_autoconf b/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp_autoconf index 5db3cfa3a..926d609f8 100644 --- a/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp_autoconf +++ b/docker/os-based/DockerFile-Gentoo-x64-ffead-cpp_autoconf @@ -18,6 +18,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp b/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp index ee4f41983..aed49b28b 100644 --- a/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp +++ b/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp @@ -20,6 +20,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp_autoconf b/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp_autoconf index 4b6f01619..b0bc4b361 100644 --- a/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp_autoconf +++ b/docker/os-based/DockerFile-OpenSuse-x64-ffead-cpp_autoconf @@ -20,6 +20,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp index 6f72061ea..6fb332282 100644 --- a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp @@ -22,6 +22,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-meson b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-meson index f68491108..eae57389c 100644 --- a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-meson +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-meson @@ -25,6 +25,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-scons b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-scons new file mode 100644 index 000000000..6aaf6d241 --- /dev/null +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-scons @@ -0,0 +1,64 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND noninteractive +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +RUN apt update -yqq && apt install -yqq gcc g++ software-properties-common cmake unzip libpcre3-dev zlib1g-dev libpq-dev \ + libssl-dev uuid-dev odbc-postgresql unixodbc unixodbc-dev libcurl4-openssl-dev libmemcached-dev wget netcat \ + python3 python3-pip python3-setuptools python3-wheel ninja-build rapidjson-dev libpugixml-dev && rm -rf /var/lib/apt/lists/* +RUN pip3 install scons + +#Install libcuckoo headers +WORKDIR /tmp +RUN wget -q https://github.com/efficient/libcuckoo/archive/master.zip +RUN unzip -qq master.zip +RUN rm -f master.zip +WORKDIR /tmp/libcuckoo-master +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ . +RUN make install +WORKDIR /tmp +RUN rm -rf /tmp/libcuckoo-master + +RUN wget -q https://github.com/redis/hiredis/archive/v1.0.0.tar.gz +RUN tar xf v1.0.0.tar.gz +RUN rm -f v1.0.0.tar.gz +RUN cd hiredis-1.0.0/ && cmake . && make install +WORKDIR /tmp +RUN rm -rf hiredis-1.0.0 + +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + +#Install mongodb c driver +RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz +RUN tar xf mongo-c-driver-1.4.2.tar.gz +RUN rm -f mongo-c-driver-1.4.2.tar.gz +RUN cd mongo-c-driver-1.4.2/ && ./configure --disable-automatic-init-and-cleanup --disable-tests --disable-ssl --disable-sasl && make && make install +WORKDIR /tmp +RUN rm -rf mongo-c-driver-1.4.2 + +#Install ffead-cpp +WORKDIR /tmp +RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip +RUN unzip -qq master.zip +RUN mv ffead-cpp-master ffead-cpp-src +RUN rm -f master.zip +WORKDIR /tmp/ffead-cpp-src + +RUN scons -Q +RUN mv /tmp/ffead-cpp-src/ffead-cpp-6.0-bin /tmp/ +WORKDIR /tmp +RUN rm -rf /tmp/ffead-cpp-src + +COPY *.sh /opt/ + +WORKDIR /opt + +RUN chmod +x install_ffead-cpp.sh +RUN ./install_ffead-cpp.sh diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-shellb b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-shellb new file mode 100644 index 000000000..18dba25ca --- /dev/null +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-shellb @@ -0,0 +1,64 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND noninteractive +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +RUN apt update -yqq && apt install -yqq gcc g++ software-properties-common cmake unzip libpcre3-dev zlib1g-dev libpq-dev \ + libssl-dev uuid-dev odbc-postgresql unixodbc unixodbc-dev libcurl4-openssl-dev libmemcached-dev wget netcat \ + rapidjson-dev libpugixml-dev && rm -rf /var/lib/apt/lists/* + +#Install libcuckoo headers +WORKDIR /tmp +RUN wget -q https://github.com/efficient/libcuckoo/archive/master.zip +RUN unzip -qq master.zip +RUN rm -f master.zip +WORKDIR /tmp/libcuckoo-master +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ . +RUN make install +WORKDIR /tmp +RUN rm -rf /tmp/libcuckoo-master + +RUN wget -q https://github.com/redis/hiredis/archive/v1.0.0.tar.gz +RUN tar xf v1.0.0.tar.gz +RUN rm -f v1.0.0.tar.gz +RUN cd hiredis-1.0.0/ && cmake . && make install +WORKDIR /tmp +RUN rm -rf hiredis-1.0.0 + +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + +#Install mongodb c driver +RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz +RUN tar xf mongo-c-driver-1.4.2.tar.gz +RUN rm -f mongo-c-driver-1.4.2.tar.gz +RUN cd mongo-c-driver-1.4.2/ && ./configure --disable-automatic-init-and-cleanup --disable-tests --disable-ssl --disable-sasl && make && make install +WORKDIR /tmp +RUN rm -rf mongo-c-driver-1.4.2 + +#Install ffead-cpp +WORKDIR /tmp +RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip +RUN unzip -qq master.zip +RUN mv ffead-cpp-master ffead-cpp-src +RUN rm -f master.zip +WORKDIR /tmp/ffead-cpp-src + +RUN wget -q https://github.com/sumeetchhetri/shellb/releases/download/1.0.0/shellb -P /usr/local/bin && chmod +x /usr/local/bin/shellb +RUN shellb ffead-cpp-shellb +RUN mv /tmp/ffead-cpp-src/shellb_out/ffead-cpp-6.0-bin /tmp/ +WORKDIR /tmp +RUN rm -rf /tmp/ffead-cpp-src + +COPY *.sh /opt/ + +WORKDIR /opt + +RUN chmod +x install_ffead-cpp.sh +RUN ./install_ffead-cpp.sh diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-xmake b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-xmake index 8e60ef70c..2c98dd5bc 100644 --- a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-xmake +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp-xmake @@ -27,6 +27,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_autoconf b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_autoconf index 267bcf39f..8730990c5 100644 --- a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_autoconf +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_autoconf @@ -22,6 +22,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_with_iouring b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_with_iouring index 0363c502b..55cf7c761 100644 --- a/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_with_iouring +++ b/docker/os-based/DockerFile-Ubuntu-x64-ffead-cpp_with_iouring @@ -22,6 +22,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/techempower-config/install_ffead-cpp-dependencies.sh b/docker/techempower-config/install_ffead-cpp-dependencies.sh index 0febbd332..01a107df1 100644 --- a/docker/techempower-config/install_ffead-cpp-dependencies.sh +++ b/docker/techempower-config/install_ffead-cpp-dependencies.sh @@ -56,6 +56,16 @@ cmake . && make install cd $IROOT rm -rf hiredis-1.0.0 +wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +tar xf 1.3.3.tar.gz +rm -f 1.3.3.tar.gz +cd redis-plus-plus-1.3.3/ +mkdir build +cd build +cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +cd $IROOT +rm -rf redis-plus-plus-1.3.3 + cd $IROOT wget -q https://github.com/microsoft/mimalloc/archive/v2.0.2.tar.gz tar xf v2.0.2.tar.gz diff --git a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-forprofile b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-forprofile index 9395b63ae..7b4ba3441 100644 --- a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-forprofile +++ b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-forprofile @@ -85,6 +85,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest index f98d88376..2484c1698 100644 --- a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest +++ b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest @@ -126,6 +126,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest-nb b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest-nb index 93f00037c..22467b440 100644 --- a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest-nb +++ b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest-nb @@ -126,6 +126,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest_localhost b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest_localhost index 4eeb35fec..9bc19021c 100644 --- a/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest_localhost +++ b/docker/test/DockerFile-UbuntuBionic-x64-ffead-cpp-fortest_localhost @@ -26,6 +26,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver RUN wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz RUN tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/test/install_deps.sh b/docker/test/install_deps.sh index 1d95026bb..937489056 100644 --- a/docker/test/install_deps.sh +++ b/docker/test/install_deps.sh @@ -99,6 +99,16 @@ cd hiredis-1.0.0/ && cmake . && make install cd /tmp rm -rf hiredis-1.0.0 +wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +tar xf 1.3.3.tar.gz +rm -f 1.3.3.tar.gz +cd redis-plus-plus-1.3.3/ +mkdir build +cd build +cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +cd /tmp +rm -rf redis-plus-plus-1.3.3 + #Install mongodb c driver wget -q https://github.com/mongodb/mongo-c-driver/releases/download/1.4.2/mongo-c-driver-1.4.2.tar.gz tar xf mongo-c-driver-1.4.2.tar.gz diff --git a/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp b/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp index 63ca56076..0355bdba1 100644 --- a/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp +++ b/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp @@ -20,6 +20,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install ffead-cpp WORKDIR /tmp RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip diff --git a/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp_autoconf b/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp_autoconf index 2ab8e1706..f01c4e8ce 100644 --- a/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp_autoconf +++ b/docker/webrtc-peerjs/DockerFile-UbuntuBionic-x64-ffead-cpp_autoconf @@ -20,6 +20,16 @@ RUN cd hiredis-1.0.0/ && cmake . && make install WORKDIR /tmp RUN rm -rf hiredis-1.0.0 +RUN wget -q https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz +RUN tar xf 1.3.3.tar.gz +RUN rm -f 1.3.3.tar.gz +WORKDIR redis-plus-plus-1.3.3/ +RUN mkdir build +WORKDIR build +RUN cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=17 .. && make && make install +WORKDIR /tmp +RUN rm -rf redis-plus-plus-1.3.3 + #Install ffead-cpp WORKDIR /tmp RUN wget -q https://github.com/sumeetchhetri/ffead-cpp/archive/master.zip diff --git a/ffead-cpp-shellb.sh b/ffead-cpp-shellb.sh new file mode 100644 index 000000000..b76556f55 --- /dev/null +++ b/ffead-cpp-shellb.sh @@ -0,0 +1,149 @@ +function do_setup() { + LOG_MODE=1 + BUILD_SYS=emb + BUILD_PLATFORM=c_cpp + DEFS_FILE="src/modules/common/AppDefines.h" +} +function do_config() { + configs+=$'MOD_MEMORY|Enable Memory Cache module|1\n' + configs+=$'MOD_MEMCACHED|Enable Memcached Cache module|1\n' + configs+=$'MOD_REDIS|Enable Redis Cache module|1\n' + configs+=$'MOD_SDORM_SQL|Enable SQL Sdorm module|1\n' + configs+=$'MOD_SDORM_MONGO|Enable Mongo Sdorm module|1\n' + configs+=$'MOD_SDORM_SCYLLA|Enable Scylla Sdorm module|0\n' + configs+=$'MOD_SOLR|Enable SOLR Search module|1\n' + configs+=$'MOD_ELASTIC|Enable Elasticsearch Search module|1\n' + configs+=$'MOD_SER_BIN|Enable Binary Serialization module|1\n' + configs+=$'MOD_JOBS|Enable Jobs module|1\n' + configs+=$'SRV_ALL|Build All inbuilt HTTP Server Engines|1\n' + configs+=$'SRV_EMB|Embedded Http Server Engine|1\n' + configs+=$'WITH_RAPIDJSON|Use rapidjson for json handling|1\n' + configs+=$'WITH_PUGIXML|Use pugixml for xml handling|1\n' + configs+=$'WITH_PICOEV|Enable picoev engine|0\n' + echo "$configs" +} +function do_start() { + set_out "shellb_out" + set_install "ffead-cpp-6.0-bin" + finc_cpp_compiler "Please install a c++ compiler to proceed" + cpp_flags "-std=c++17 -Wall -g" + l_flags "" + add_lib_path "/usr/local/opt/openssl/lib" "/usr/local/lib" + add_inc_path "/usr/local/opt/openssl/include" "/usr/local/include" "/usr/include/libmongoc-1.0" + add_inc_path "/usr/include/libbson-1.0" "/usr/local/include/libmongoc-1.0" "/usr/local/include/libbson-1.0" + add_def "OS_${OS_NAME}" "BUILD_SHELLB" "INC_WEBSVC" "INC_TPE" "INC_DVIEW" "INC_DCP" "INC_XMLSER" + if [ "$OS_DARWIN" = "1" ]; then add_def "APPLE"; fi + c_hdr "execinfo.h" "HAVE_EXECINFOINC" + c_hdr "sys/sendfile.h" "IS_SENDFILE" + if c_hdr "liburing" "HAVE_LIBURING"; then + add_def "USE_IO_URING" + elif c_hdr "sys/epoll.h" "HAVE_EPOLL"; then + add_def "USE_EPOLL" + elif c_hdr "sys/event.h" "HAVE_KQUEUE"; then + add_def "USE_KQUEUE" + elif c_hdr "port.h" "HAVE_EVPORT"; then + add_def "USE_EVPORT" + elif c_hdr "sys/devpoll.h" "HAVE_DEVPOLL"; then + add_def "USE_DEVPOLL" + elif c_hdr "sys/poll.h" "HAVE_POLL"; then + add_def "USE_POLL" + elif c_hdr "sys/select.h" "HAVE_SELECT"; then + add_def "USE_SELECT" + fi + c_code "#include \n#include \n#include \nint test() { return TCP_QUICKACK; }" "HAVE_TCP_QUICKACK" + c_code "#include \n#include \n#include \nint test() { return TCP_DEFER_ACCEPT; }" "HAVE_TCP_DEFER_ACCEPT" + c_code "#include \n#include \n#include \nint test() { return TCP_FASTOPEN; }" "HAVE_TCP_FASTOPEN" + c_code "#include \n#include \n#include \n#include \n#include \n#include \nint main() { return SO_ATTACH_REUSEPORT_CBPF; }" "HAVE_SO_ATTACH_REUSEPORT_CBPF" + if ! c_hdr "regex.h" "HAVE_REGEX"; then + c_hdr_lib "onigposix.h" "onig" "HAVE_ONIG_REGEX,HAVE_ONIG_REGEX_LIB" "regex devel not found" + fi + c_lib "z" "" "zlib lib not found" + c_func "accept4" "HAVE_ACCEPT4" + c_hdr_lib "uuid/uuid.h" "uuid" "HAVE_UUIDINC" "uuid devel not found" + c_hdr_lib "curl/curl.h" "curl" "HAVE_CURLINC,HAVE_CURLLIB" "curl devel not found" + if ! c_hdr_lib "openssl/ssl.h" "ssl" "HAVE_SSLINC,HAVE_SSLLIB" || ! c_lib "crypto" "HAVE_CRYPTOLIB"; then + set_exclude_src "src/modules/ssl" "src/modules/client-util/ssl" + fi + c_hdr_lib "libpq-fe.h" "pq" "HAVE_PQHDR,HAVE_LIBPQ" "libpq devel not found" + c_hdr_lib "sql.h" "odbc" "HAVE_SQLINC,HAVE_ODBCLIB,HAVE_LIBODBC,INC_SDORM,INC_SDORM_SQL" "odbc devel not found" + c_hdr_lib "mongoc.h" "mongoc-1.0" "HAVE_MONGOINC,HAVE_MONGOCLIB,INC_SDORM_MONGO" "libmongoc devel not found" + c_hdr_lib "bson.h" "bson-1.0" "HAVE_BSONINC,HAVE_BSONLIB" "libbson devel not found" + if ! is_config "MOD_SER_BIN"; then + set_exclude_src "src/modules/serialization/binary" + fi + if ! is_config "MOD_MEMORY"; then + set_exclude_src "src/modules/cache/memory" + fi + if ! is_config "MOD_REDIS" || ! c_hdr_lib "hiredis/hiredis.h" "hiredis" "HAVE_REDISINC,HAVE_REDISLIB,INC_REDISCACHE"; then + set_exclude_src "src/modules/cache/redis" + fi + if defined "HAVE_REDISINC"; then + cpp_hdr_lib "sw/redis++/redis++.h" "redis++" "HAVE_REDIS_CLUSTERINC,HAVE_REDIS_CLUSTERLIB,INC_REDISCACHE" "" + fi + if ! is_config "MOD_MEMCACHED" || ! c_hdr_lib "libmemcached/memcached.h" "memcached" "HAVE_MEMCACHEDINC,HAVE_MEMCACHEDLIB,INC_MEMCACHED"; then + set_exclude_src "src/modules/cache/memcached" + fi + if defined "HAVE_MEMCACHEDINC"; then + c_lib "memcachedutil" + fi + is_config "WITH_RAPIDJSON" && cpp_hdr "rapidjson/document.h" "HAVE_RAPID_JSON" + is_config "WITH_PUGIXML" && cpp_hdr_lib "pugixml.hpp" "pugixml" "HAVE_PUGI_XML" + if ! is_config "MOD_SDORM_SQL"; then + set_exclude_src "src/modules/sdorm/sql" + fi + if ! is_config "MOD_SDORM_MONGO"; then + set_exclude_src "src/modules/sdorm/mongo" + fi + if ! is_config "MOD_SDORM_SCYLLA" || ! cpp_hdr_lib "cassandra.h" "scylla-cpp-driver" "HAVE_SCYLLAINC,HAVE_SCYLLALIB"; then + set_exclude_src "src/modules/sdorm/scylla" + fi + if ! is_config "MOD_ELASTIC" || ! cpp_hdr_lib "elasticlient/client.h" "elasticlient" "HAVE_ELASTIC"; then + set_exclude_src "src/modules/search/elasticsearch" + fi + if ! is_config "MOD_SOLR"; then + set_exclude_src "src/modules/search/solr" + else + add_def "HAVE_SOLR" + fi + if ! is_config "MOD_JOBS"; then + set_exclude_src "src/modules/jobs" + fi + if ! is_config "WITH_PICOEV"; then + set_exclude_src "src/modules/common/picoev" + else + add_def "USE_PICOEV" + fi + set_exclude_src "src/modules/sdorm/gtm" + if ! defined "OS_MINGW"; then + set_exclude_src "src/modules/wepoll" + fi + add_inc_path "src/framework" + set_src "src/modules" "stared:ffead-modules" + set_src "src/framework" "stared:ffead-framework" "ffead-modules" + set_src "tests" "binary:tests" "ffead-framework,ffead-modules" + set_src "src/server/embedded" "binary:ffead-cpp" "ffead-framework,ffead-modules" + + set_inc_src "web/default/include" "web/default/src" "stared:default" "ffead-framework,ffead-modules" + set_inc_src "web/flexApp/include" "web/flexApp/src" "stared:flexApp" "ffead-framework,ffead-modules" + set_inc_src "web/markers/include" "web/markers/src" "stared:markers" "ffead-framework,ffead-modules" + set_inc_src "web/oauthApp/include" "web/oauthApp/src" "stared:oauthApp" "ffead-framework,ffead-modules" + set_inc_src "web/peer-server/include" "web/peer-server/src" "stared:peer-server" "ffead-framework,ffead-modules" + set_inc_src "web/te-benchmark/include" "web/te-benchmark/src" "stared:te-benchmark" "ffead-framework,ffead-modules" + set_inc_src "web/t1/include" "web/t1/src" "stared:t1" "ffead-framework,ffead-modules" + if is_config "MOD_SDORM_MONGO"; then + set_inc_src "web/t2/include" "web/t2/src" "stared:t2" "ffead-framework,ffead-modules" + fi + if is_config "MOD_SDORM_SQL"; then + set_inc_src "web/t3/include" "web/t3/src" "stared:t3" "ffead-framework,ffead-modules" + set_inc_src "web/t4/include" "web/t4/src" "stared:t4" "ffead-framework,ffead-modules" + set_inc_src "web/t5/include" "web/t5/src" "stared:t5" "ffead-framework,ffead-modules" + fi + templatize "rtdcf/inter-shellb.sh.tem" "rtdcf/inter-shellb.sh" "CPPFLAGS,LFLAGS,LIBS" +} +function do_install() { + install_here "." "ffead-cpp" "script@*.sh,*.key,*.pem,*.crt" "resources" "public" "rtdcf" "web" + install_here "lib" "lib*.${SHLIB_EXT}" + install_here "include" "src/framework@*.h" "src/modules@*.h" + install_here "logs" + install_here "tests" "tests" "tests/@*.sh,*.pem,*.csv,*.prop" +} diff --git a/meson.build b/meson.build index 02c7c9b75..fd0eb8c2b 100644 --- a/meson.build +++ b/meson.build @@ -110,6 +110,15 @@ if get_option('MOD_REDIS') == true endif global_libs += [libhiredis] endif + librredispp = cc.find_library('redis++', dirs: ['/usr/local/lib']) + if librredispp.found() + conf_data.set('HAVE_REDIS_CLUSTERLIB', 1) + if cc.has_header('sw/redis++/redis++.h') + conf_data.set('HAVE_REDIS_CLUSTERINC', 1) + conf_data.set('INC_REDISCACHE', 1) + endif + global_libs += [librredispp] + endif endif global_includes = ['/usr/local/include'] @@ -231,6 +240,24 @@ if get_option('MOD_JOBS') == true conf_data.set('INC_JOBS', 1) endif +if get_option('MOD_SOLR') == true + conf_data.set('HAVE_SOLR', 1) +endif + +if get_option('MOD_ELASTIC') == true + if cc.has_header('elasticlient/client.h', args : '-I/usr/local/include/') + conf_data.set('HAVE_ELASTIC', 1) + else + error('elasticlient headers not found') + endif + libelastic = cc.find_library('elasticlient') + if libelastic.found() + global_libs += [libelastic] + else + error('elastic lib not found') + endif +endif + if not cc.has_header('libcuckoo/cuckoohash_map.hh', args : '-I/usr/local/include/') error('libcuckoo headers not found') endif @@ -253,6 +280,10 @@ if get_option('WITH_PUGIXML') == true endif endif +if get_option('MOD_MEMORY') == true + conf_data.set('INC_MEMORYCACHE', 1) +endif + if get_option('WITH_PICOEV') == true conf_data.set('USE_PICOEV', 1) endif @@ -333,16 +364,16 @@ endif configure_file(input : 'src/modules/common/AppDefines.meson.h.in', output : 'AppDefines.h', configuration : conf_data) conf_file_path = meson.current_build_dir() + '/AppDefines.h' -run_command('mv', conf_file_path, 'src/modules/common/') -run_command('chmod', '+x', 'meson_grabber.sh') -run_command('chmod', '+x', 'meson_postinstall.sh') +run_command('mv', conf_file_path, 'src/modules/common/', check: true) +run_command('chmod', '+x', 'meson_grabber.sh', check: true) +run_command('chmod', '+x', 'meson_postinstall.sh', check: true) meson_grabber = meson.source_root() + '/meson_grabber.sh' meson_post_install = meson.source_root() + '/meson_postinstall.sh' bin_dir = meson.current_source_dir()+'/'+meson.project_name()+'-'+meson.project_version()+'-bin/' -run_command('rm', '-rf', bin_dir) -run_command('mkdir', bin_dir) +run_command('rm', '-rf', bin_dir, check: true) +run_command('mkdir', bin_dir, check: true) subdir('src/modules') subdir('src/framework') @@ -370,10 +401,10 @@ emb_includes = ['src/modules/common','src/modules/cache','src/modules/cache/memo 'src/modules/ioc','src/modules/jobs','src/modules/reflection','src/modules/sdorm','src/modules/sdorm/sql','src/modules/sdorm/sql/libpq', 'src/modules/client-util/ssl','src/modules/sdorm/mongo','src/modules/sdorm/mongo/raw','src/modules/search','src/modules/serialization', 'src/modules/serialization/xml','src/modules/serialization/json','src/modules/server-util','src/modules/ssl','src/modules/threads', - 'src/framework'] + 'src/framework','src/modules/search/solr','src/modules/search/elasticsearch'] foreach inc: emb_includes - c = run_command(meson_grabber, inc, 'all', '*.h') + c = run_command(meson_grabber, inc, 'all', '*.h', check: true) incs = c.stdout().strip().split('\n') install_headers(incs, install_dir: bin_dir+'include') endforeach diff --git a/meson_options.txt b/meson_options.txt index 7aec0f2ab..1ccede6c5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -4,6 +4,8 @@ option('MOD_REDIS', type: 'boolean', description: 'Enable Redis Cache module', v option('MOD_SDORM_SQL', type: 'boolean', description: 'Enable SQL Sdorm module', value: true) option('MOD_SDORM_MONGO', type: 'boolean', description: 'Enable Mongo Sdorm module', value: true) option('MOD_SDORM_SCYLLA', type: 'boolean', description: 'Enable Scylla Sdorm module', value: false) +option('MOD_SOLR', type: 'boolean', description: 'Enable SOLR Search module', value: true) +option('MOD_ELASTIC', type: 'boolean', description: 'Enable Elasticsearch Search module', value: true) option('MOD_SER_BIN', type: 'boolean', description: 'Enable Binary Serialization module', value: true) option('MOD_JOBS', type: 'boolean', description: 'Enable Jobs module', value: true) option('SRV_ALL', type: 'boolean', description: 'Build All inbuilt HTTP Server Engines', value: true) diff --git a/resources/rundyn-automake.sh b/resources/rundyn-automake.sh index 2416fedf1..6b8d1074c 100644 --- a/resources/rundyn-automake.sh +++ b/resources/rundyn-automake.sh @@ -55,8 +55,19 @@ then rm -rf build_meson || true meson setup build_meson && cd build_meson ninja install +elif [ "$TYPE" = "scons" ] +then + rm -rf .scons_temp + scons +elif [ "$TYPE" = "bazel" ] +then + : +elif [ "$TYPE" = "shellb" ] +then + rm -rf .bin || true + ./shellb.sh inter-shellb else - echo "Invalid Build Type specified, only cmake, xmake and meson supported..." + echo "Invalid Build Type specified, only cmake, xmake, meson, scons, bazel and shellb supported..." fi cp -f *inter* $FFEAD_CPP_PATH/lib/ diff --git a/rtdcf/inter-shellb.sh.tem b/rtdcf/inter-shellb.sh.tem new file mode 100644 index 000000000..bd014eb25 --- /dev/null +++ b/rtdcf/inter-shellb.sh.tem @@ -0,0 +1,28 @@ +function do_setup() { + LOG_MODE=1 + BUILD_SYS=emb + BUILD_PLATFORM=c_cpp +} +function do_start() { + set_out "." + finc_cpp_compiler "Please install a c++ compiler to proceed" + cpp_flags "@CPPFLAGS@" + l_flags "@LFLAGS@" + add_lib_path "/usr/local/opt/openssl/lib" "/usr/local/lib" + add_inc_path "/usr/local/opt/openssl/include" "/usr/local/include" "/usr/include/libmongoc-1.0" + add_inc_path "/usr/include/libbson-1.0" "/usr/local/include/libmongoc-1.0" "/usr/local/include/libbson-1.0" + add_lib "@LIBS@" + add_inc_path "../include" + add_lib_path "../lib" + for wedbdir in ../web/* + do + wedbdir=${wedbdir%*/} + wedbdir="${wedbdir##*/}" + add_inc_path "../web/$wedbdir/include" + done + set_src_files "ReflectorInterface.cpp,SerializeInterface.cpp,AjaxInterface.cpp,WsInterface.cpp" "shared:inter" "ffead-framework,ffead-modules" + set_src_files "DCPInterface.cpp,TemplateInterface.cpp" "shared:dinter" "ffead-framework,ffead-modules" +} +function do_install() { + mv ".bin/libinter.${SHLIB_EXT}" ".bin/libdinter.${SHLIB_EXT}" ../lib/ +} \ No newline at end of file diff --git a/rtdcf/sconstruct.in b/rtdcf/sconstruct.in new file mode 100644 index 000000000..f74d1b4a8 --- /dev/null +++ b/rtdcf/sconstruct.in @@ -0,0 +1,36 @@ +import os +import shutil + +# Set our required libraries +libraries = @LIBS@ +library_paths = @LIBPATH@ +cppDefines = @CPPDEFINES@ +cppFlags = @CPPFLAGS@ +cxxFlags = @CXXFLAGS@ +cppPath = @CPPPATH@ + +library_paths.append("../lib") +cppPath.append("../include") +libraries.append('-lffead-modules') +libraries.append('-lffead-framework') + +dirnames=next(os.walk('../web'))[1] +dirnames[:] = [d for d in dirnames if not d[0] == '.'] +for dirname in dirnames: + dirPath = os.path.join('../web', dirname, 'include') + cppPath.append(dirPath) + libraries.append('-l'+dirname) + +env = Environment() +env.Append(CPPPATH = cppPath) +env.Append(LIBPATH = library_paths) +env.Append(CXXFLAGS = cxxFlags) +env.Append(LIBS = libraries) +env.Append(CPPDEFINES = cppDefines) +env.Append(CPPFLAGS = cppFlags) + +inter_sources = ['ReflectorInterface.cpp' ,'SerializeInterface.cpp', 'AjaxInterface.cpp' ,'WsInterface.cpp'] +env.SharedLibrary('inter', inter_sources) + +dinter_sources = ['DCPInterface.cpp' ,'TemplateInterface.cpp'] +env.SharedLibrary('dinter', dinter_sources) \ No newline at end of file diff --git a/scons_options.py b/scons_options.py new file mode 100644 index 000000000..1b5f358eb --- /dev/null +++ b/scons_options.py @@ -0,0 +1,16 @@ +MOD_DEBUG = False +MOD_MEMORY = True +MOD_MEMCACHED = True +MOD_REDIS = True +MOD_SDORM_SQL = True +MOD_SDORM_MONGO = True +MOD_SDORM_SCYLLA = False +MOD_SER_BIN = True +MOD_JOBS = True +#SRV_ALL = True +SRV_EMB = True +WITH_RAPIDJSON = True +WITH_PUGIXML = True +WITH_PICOEV = False +MOD_SOLR = True +MOD_ELASTIC = False \ No newline at end of file diff --git a/sconstruct b/sconstruct new file mode 100644 index 000000000..db93d21ad --- /dev/null +++ b/sconstruct @@ -0,0 +1,376 @@ +import os +import shutil +import json + +vars = Variables('scons_options.py') +vars.Add(BoolVariable('MOD_DEBUG', 'Enable Debug', False)) +vars.Add(BoolVariable('MOD_MEMORY', 'Enable Memory Cache module', True)) +vars.Add(BoolVariable('MOD_MEMCACHED', 'Enable Memcached Cache module', True)) +vars.Add(BoolVariable('MOD_REDIS', 'Enable Redis Cache module', True)) +vars.Add(BoolVariable('MOD_SDORM_SQL', 'Enable SQL Sdorm module', True)) +vars.Add(BoolVariable('MOD_SDORM_MONGO', 'Enable Mongo Sdorm module', True)) +vars.Add(BoolVariable('MOD_SDORM_SCYLLA', 'Enable Scylla Sdorm module', False)) +vars.Add(BoolVariable('MOD_SER_BIN', 'Enable Binary Serialization module', True)) +vars.Add(BoolVariable('MOD_JOBS', 'Enable Jobs module', True)) +vars.Add(BoolVariable('MOD_SOLR', 'Enable SOLR Search module', True)) +vars.Add(BoolVariable('MOD_ELASTIC', 'Enable Elasticsearch Search module', False)) +vars.Add(BoolVariable('SRV_EMB', 'Embedded Http Server Engine', True)) +vars.Add(BoolVariable('WITH_RAPIDJSON', 'Use rapidjson for json handling', True)) +vars.Add(BoolVariable('WITH_PUGIXML', 'Use pugixml for xml handling', True)) +vars.Add(BoolVariable('WITH_PICOEV', 'Enable picoev engine', False)) + +#print(vars) +# Set our required libraries +libraries = ['z'] +library_paths = [] +cppDefines = {} +cppFlags = ['-Wall'] +cxxFlags = ['-std=c++17'] +cppPath = [] + +shutil.copy2('src/modules/common/AppDefines.scons.h.in', 'src/modules/common/AppDefines.h') + +# define the attributes of the build environment shared between +# both the debug and release builds +common_env = Environment(variables = vars) +Help(vars.GenerateHelpText(common_env)) + +platform = common_env['PLATFORM'] +if platform == 'darwin': + if os.path.isdir('/usr/local/opt/openssl/lib/'): + library_paths.append("/usr/local/opt/openssl/lib/") + cppPath.append("/usr/local/opt/openssl/include") + +library_paths.append("/usr/local/lib") +cppPath.append("/usr/local/include") +#print "library_paths = %s" % library_paths + +common_env.Append(CPPPATH = cppPath) +common_env.Append(LIBPATH = library_paths) +common_env.Append(CXXFLAGS = cxxFlags) + +conf = Configure(common_env) + +platform = common_env['PLATFORM'] +if platform == 'darwin': + conf.env.Append(CPPDEFINES=['OS_DARWIN', 'APPLE']) +elif platform == 'mingw': + conf.env.Append(CPPDEFINES=['MINGW', 'OS_MINGW', 'USE_WIN_IOCP']) +elif platform == 'cygwin': + conf.env.Append(CPPDEFINES=['CYGWIN']) +elif platform == 'posix': + conf.env.Append(CPPDEFINES=['OS_LINUX']) +else: + print 'Incompatible platform, exiting!' + Exit(1) + +if conf.CheckLib('uuid'): + conf.env.Append(CPPDEFINES=['HAVE_UUIDLIB']) +elif conf.CheckLib('ossp-uuid'): + conf.env.Append(CPPDEFINES=['HAVE_UUIDLIB']) +else: + print 'Did not find uuid library!' + Exit(1) + +if conf.CheckCHeader('uuid/uuid.h'): + conf.env.Append(CPPDEFINES=['HAVE_UUIDINC']) +elif conf.CheckCHeader('ossp/uuid.h'): + conf.env.Append(CPPDEFINES=['HAVE_OSSPUUIDINC']) +elif conf.CheckCHeader('ossp/uuid.h'): + conf.env.Append(CPPDEFINES=['HAVE_OSSPUUIDINC_2']) +else: + print 'Did not find uuid headers!' + Exit(1) + +if not conf.CheckLibWithHeader('curl', 'curl/curl.h', 'c'): + print 'Did not find curl library!' +else: + conf.env.Append(CPPDEFINES=['HAVE_CURLLIB', 'HAVE_CURLINC']) + +if conf.CheckCHeader('regex.h'): + conf.env.Append(CPPDEFINES=['HAVE_REGEX']) +else: + if not conf.CheckLibWithHeader('onig', 'onigposix.h', 'c'): + print 'Did not find oniguruma library!' + else: + conf.env.Append(CPPDEFINES=['HAVE_ONIG_REGEX', 'HAVE_ONIG_REGEX_LIB']) + +mod_scylla = ARGUMENTS.get('MOD_SDORM_SCYLLA', False) +if bool(mod_scylla): + if not conf.CheckLibWithHeader('scylla-cpp-driver', 'cassandra.h', 'cxx'): + print 'Did not find scylla development files!' + Exit(1) + +mod_sql = ARGUMENTS.get('MOD_SDORM_SQL', True) +if bool(mod_sql): + if conf.CheckLibWithHeader('odbc', 'sql.h', 'c'): + conf.env.Append(CPPDEFINES=['HAVE_SQLINC', 'INC_SDORM', 'INC_SDORM_SQL', 'HAVE_ODBCLIB']) + if conf.CheckLib('pq'): + pqhdr = False + if conf.CheckCHeader('libpq-fe.h'): + pqhdr = True + if FindFile('libpq-fe.h', '/usr/include/postgresql'): + pqhdr = True + conf.env.Append(CPPPATH=['/usr/include/postgresql']) + if FindFile('libpq-fe.h', '/usr/include/pgsql'): + pqhdr = True + conf.env.Append(CPPPATH=['/usr/include/pgsql']) + if FindFile('libpq-fe.h', '/usr/local/include/postgresql'): + pqhdr = True + conf.env.Append(CPPPATH=['/usr/local/include/postgresql']) + if FindFile('libpq-fe.h', '/usr/local/include/pgsql'): + pqhdr = True + conf.env.Append(CPPPATH=['/usr/local/include/pgsql']) + if pqhdr: + conf.env.Append(CPPDEFINES=['HAVE_PQHDR', 'HAVE_LIBPQ']) + if conf.CheckFunc('PQenterBatchMode', '#include '): + conf.env.Append(CPPDEFINES=['HAVE_LIBPQ_BATCH']) + + +mod_mongo = ARGUMENTS.get('MOD_SDORM_MONGO', True) +if bool(mod_mongo): + if conf.CheckLib('bson-1.0') and conf.CheckLib('mongoc-1.0'): + oi = FindFile('bson.h', '/usr/include/libbson-1.0') + li = FindFile('bson.h', '/usr/local/include/libbson-1.0') + if oi: + conf.env.Append(CPPPATH=['/usr/include/libbson-1.0']) + conf.env.Append(CPPPATH=['/usr/include/libmongoc-1.0']) + elif li: + conf.env.Append(CPPPATH=['/usr/local/include/libbson-1.0']) + conf.env.Append(CPPPATH=['/usr/local/include/libmongoc-1.0']) + + if oi or li: + conf.env.Append(CPPDEFINES=['HAVE_BSONLIB', 'HAVE_MONGOCLIB', 'HAVE_BSONINC', 'HAVE_MONGOINC', 'INC_SDORM', 'INC_SDORM_MONGO']) + + else: + print 'Did not find mongo development files!' + Exit(1) + +if conf.CheckLib('ssl') and conf.CheckLib('crypto') and conf.CheckCHeader('openssl/ssl.h'): + conf.env.Append(CPPDEFINES=['HAVE_SSLINC', 'HAVE_CRYPTOLIB', 'HAVE_SSLLIB']) + +mod_memcached = ARGUMENTS.get('MOD_MEMCACHED', True) +if bool(mod_memcached): + if conf.CheckLib('memcachedutil') and conf.CheckLibWithHeader('memcached', 'libmemcached/memcached.h', 'c'): + conf.env.Append(CPPDEFINES=['HAVE_MEMCACHEDINC', 'INC_MEMCACHED', 'HAVE_MEMCACHEDLIB']) + else: + mod_memcached = False + +mod_redis = ARGUMENTS.get('MOD_REDIS', True) +if bool(mod_redis): + if conf.CheckLibWithHeader('hiredis', 'hiredis/hiredis.h', 'c'): + conf.env.Append(CPPDEFINES=['HAVE_REDISINC', 'INC_REDISCACHE', 'HAVE_REDISLIB']) + if conf.CheckLibWithHeader('redis++', 'sw/redis++/redis++.h', 'cxx'): + conf.env.Append(CPPDEFINES=['HAVE_REDIS_CLUSTERINC', 'HAVE_REDIS_CLUSTERLIB']) + else: + mod_redis = False + +mod_solr = ARGUMENTS.get('MOD_SOLR', True) +if bool(mod_solr): + conf.env.Append(CPPDEFINES=['HAVE_SOLR']) + +mod_elastic = ARGUMENTS.get('MOD_ELASTIC', False) +if bool(mod_elastic): + if conf.CheckLibWithHeader('elasticlient', 'elasticlient/client.h', 'cxx'): + conf.env.Append(CPPDEFINES=['HAVE_ELASTIC']) + +if conf.CheckLib('dl'): + pass + +if conf.CheckLib('pthread'): + pass + +if not conf.CheckCXXHeader('libcuckoo/cuckoohash_map.hh'): + if not FindFile('libcuckoo/cuckoohash_map.hh', '/usr/include') and not FindFile('libcuckoo/cuckoohash_map.hh', '/usr/local/include'): + print 'Did not find libcuckoo header files!' + Exit(1) + +mod_ser_bin = ARGUMENTS.get('MOD_SER_BIN', True) +if bool(mod_ser_bin): + conf.env.Append(CPPDEFINES=['INC_BINSER']) + +mod_jobs = ARGUMENTS.get('MOD_JOBS', True) +if bool(mod_jobs): + conf.env.Append(CPPDEFINES=['INC_JOBS']) + +mod_withpicoev = ARGUMENTS.get('WITH_PICOEV', False) +if bool(mod_withpicoev): + conf.env.Append(CPPDEFINES=['USE_PICOEV']) + +mod_memory = ARGUMENTS.get('MOD_MEMORY', True) +if bool(mod_memory): + conf.env.Append(CPPDEFINES=['INC_MEMORYCACHE']) + +mod_srvenb = ARGUMENTS.get('SRV_EMB', True) +if bool(mod_srvenb): + conf.env.Append(CPPDEFINES=['SRV_EMB']) + +mod_withrj = ARGUMENTS.get('WITH_RAPIDJSON', True) +if bool(mod_withrj) and conf.CheckCXXHeader('rapidjson/document.h'): + conf.env.Append(CPPDEFINES=['HAVE_RAPID_JSON']) + +mod_withpx = ARGUMENTS.get('WITH_PUGIXML', True) +if bool(mod_withrj) and conf.CheckLibWithHeader('pugixml', 'pugixml.hpp', 'cxx'): + conf.env.Append(CPPDEFINES=['HAVE_PUGI_XML']) + +if conf.CheckCHeader('liburing'): + conf.env.Append(CPPDEFINES=['USE_IO_URING']) +elif conf.CheckCHeader('sys/epoll.h'): + conf.env.Append(CPPDEFINES=['USE_EPOLL']) +elif conf.CheckCHeader('sys/event.h'): + conf.env.Append(CPPDEFINES=['USE_KQUEUE']) +elif conf.CheckCHeader('port.h'): + conf.env.Append(CPPDEFINES=['USE_EVPORT']) +elif conf.CheckCHeader('sys/devpoll.h'): + conf.env.Append(CPPDEFINES=['USE_DEVPOLL']) +elif conf.CheckCHeader('sys/poll.h'): + conf.env.Append(CPPDEFINES=['USE_POLL']) +elif conf.CheckCHeader('sys/select.h'): + conf.env.Append(CPPDEFINES=['USE_SELECT']) + +if conf.CheckCHeader('execinfo.h'): + conf.env.Append(CPPDEFINES=['HAVE_EXECINFOINC']) + +if conf.CheckCHeader('sys/sendfile.h'): + conf.env.Append(CPPDEFINES=['IS_SENDFILE']) + +if not conf.CheckFunc('accept4'): + print 'Did not find accept4(), using local version' + +common_env = conf.Finish() + +mod_debug = ARGUMENTS.get('MOD_DEBUG', False) +if bool(mod_debug): + cppFlags.Append('-g'); + +common_env.Append(CPPPATH = cppPath) +common_env.Append(LIBPATH = library_paths) +common_env.Append(CXXFLAGS = cxxFlags) +common_env.Append(LIBS = libraries) +common_env.Append(CPPDEFINES = cppDefines) +common_env.Append(CPPFLAGS = cppFlags) + +# uncomment to force g++ for c code also--creates warnings but +# avoids need for extern "C" +# env['CC'] = 'g++' + +all_libs=[] +libs=common_env.get('LIBS') +for lb in libs: + all_libs.append('-l'+lb) + +#print all_libs + +Delete("rtdcf/sconstruct") +subst = Environment(tools = ['textfile']) +substitutions = {'@CPPPATH@': json.dumps(common_env.get('CPPPATH')), '@LIBPATH@': json.dumps(common_env.get('LIBPATH')), '@CPPFLAGS@': json.dumps(cppFlags), + '@CXXFLAGS@': json.dumps(cxxFlags), '@LIBS@': json.dumps(all_libs), '@CPPDEFINES@': json.dumps(common_env.get('CPPDEFINES'))} +subst.Substfile('rtdcf/sconstruct.in', SUBST_DICT = substitutions) + + +# Our release build is derived from the common build environment... +release_env = common_env.Clone() +release_env.Append(CPPDEFINES=['RELEASE']) +release_env.VariantDir('build/release/src/modules', 'src/modules', duplicate=0) +release_env.VariantDir('build/release/src/framework', 'src/framework', duplicate=0) +release_env.VariantDir('build/release/tests', 'tests', duplicate=0) +release_env.VariantDir('build/release/src/server', 'src/server', duplicate=0) +release_env.VariantDir('build/release/web/default', 'web/default', duplicate=0) +release_env.VariantDir('build/release/web/oauthApp', 'web/oauthApp', duplicate=0) +release_env.VariantDir('build/release/web/flexApp', 'web/flexApp', duplicate=0) +release_env.VariantDir('build/release/web/peer-server', 'web/peer-server', duplicate=0) +release_env.VariantDir('build/release/web/markers', 'web/markers', duplicate=0) +release_env.VariantDir('build/release/web/te-benchmark', 'web/te-benchmark', duplicate=0) +release_env.VariantDir('build/release/web/t1', 'web/t1', duplicate=0) +release_env.VariantDir('build/release/web/t2', 'web/t2', duplicate=0) +release_env.VariantDir('build/release/web/t3', 'web/t3', duplicate=0) +release_env.VariantDir('build/release/web/t4', 'web/t4', duplicate=0) +release_env.VariantDir('build/release/web/t5', 'web/t5', duplicate=0) + +# We define our debug build environment in a similar fashion... +debug_env = common_env.Clone() +debug_env.Append(CPPDEFINES=['DEBUG']) +debug_env.VariantDir('build/debug/src/modules', 'src/modules', duplicate=0) +debug_env.VariantDir('build/debug/src/framework', 'src/framework', duplicate=0) +debug_env.VariantDir('build/debug/tests', 'tests', duplicate=0) +debug_env.VariantDir('build/debug/src/server', 'src/server', duplicate=0) +debug_env.VariantDir('build/debug/web/default', 'web/default', duplicate=0) +debug_env.VariantDir('build/debug/web/oauthApp', 'web/oauthApp', duplicate=0) +debug_env.VariantDir('build/debug/web/flexApp', 'web/flexApp', duplicate=0) +debug_env.VariantDir('build/debug/web/peer-server', 'web/peer-server', duplicate=0) +debug_env.VariantDir('build/debug/web/markers', 'web/markers', duplicate=0) +debug_env.VariantDir('build/debug/web/te-benchmark', 'web/te-benchmark', duplicate=0) +debug_env.VariantDir('build/debug/web/t1', 'web/t1', duplicate=0) +debug_env.VariantDir('build/debug/web/t2', 'web/t2', duplicate=0) +debug_env.VariantDir('build/debug/web/t3', 'web/t3', duplicate=0) +debug_env.VariantDir('build/debug/web/t4', 'web/t4', duplicate=0) +debug_env.VariantDir('build/debug/web/t5', 'web/t5', duplicate=0) + +build_env=release_env +mode='release' +if bool(mod_debug): + mode='debug' + build_env=debug_env + +modeDir = 'build/%s' % mode +build_env.SConscript('%s/src/modules/sconscript' % modeDir, {'env': build_env, 'mod_mongo': bool(mod_mongo), 'mod_scylla': bool(mod_scylla), 'mod_solr': bool(mod_solr), 'mod_elastic': bool(mod_elastic)}) +build_env.SConscript('%s/src/framework/sconscript' % modeDir, {'env': build_env, 'mod_mongo': bool(mod_mongo), 'mod_scylla': bool(mod_scylla), 'mod_solr': bool(mod_solr), 'mod_elastic': bool(mod_elastic)}) +build_env.SConscript('%s/src/server/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/tests/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/default/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/oauthApp/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/flexApp/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/peer-server/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/markers/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/te-benchmark/sconscript' % modeDir, {'env': build_env}) +build_env.SConscript('%s/web/t1/sconscript' % modeDir, {'env': build_env}) +if bool(mod_mongo): + build_env.SConscript('%s/web/t2/sconscript' % modeDir, {'env': build_env}) +if bool(mod_sql): + build_env.SConscript('%s/web/t3/sconscript' % modeDir, {'env': build_env}) + build_env.SConscript('%s/web/t4/sconscript' % modeDir, {'env': build_env}) + build_env.SConscript('%s/web/t5/sconscript' % modeDir, {'env': build_env}) + +Delete("ffead-cpp-6.0-bin") +Mkdir("ffead-cpp-6.0-bin") +Mkdir("ffead-cpp-6.0-bin/include") +Mkdir("ffead-cpp-6.0-bin/lib") +Mkdir("ffead-cpp-6.0-bin/tests") +Mkdir("ffead-cpp-6.0-bin/logs") +build_env.Install('ffead-cpp-6.0-bin', source = ['resources', 'public', 'web', 'rtdcf', modeDir+'/bin/ffead-cpp']) +build_env.Install('ffead-cpp-6.0-bin', source = Glob('script/*.sh') + Glob('script/*.key') + Glob('script/*.pem') + Glob('script/*.crt')) +build_env.Install('ffead-cpp-6.0-bin/include', source = Glob('src/framework/*.h') + Glob('src/modules/*.h')) +for root, dirnames, filenames in os.walk('src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if build_env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + if 'gtm' in dirPath: + continue + if 'mongo' in dirPath: + if not mod_mongo: + continue + if 'scylla' in dirPath: + if not mod_scylla: + continue + if 'solr' in dirPath: + if not mod_solr: + continue + if 'elasticsearch' in dirPath: + if not mod_elastic: + continue + build_env.Install('ffead-cpp-6.0-bin/include', source = Glob("%s/*.h" % dirPath)) +for root, dirnames, filenames in os.walk('web'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root, dirname, 'include') + if bool(mod_mongo) and 't1' in dirPath: + continue + if bool(mod_sql) and ('t3' in dirPath or 't4' in dirPath or 't5' in dirPath): + continue + #delete src and other build files from web directories +build_env.Install('ffead-cpp-6.0-bin/lib', source = Glob(modeDir+'/bin/lib*')) +build_env.Install('ffead-cpp-6.0-bin/tests', source = Glob('tests/*.sh') + Glob('tests/*.csv') + Glob('tests/*.prop') + Glob('tests/*.pem') + [modeDir+'/bin/tests']) \ No newline at end of file diff --git a/src/framework/ConfigurationData.h b/src/framework/ConfigurationData.h index 61cae4f74..86a23bf26 100644 --- a/src/framework/ConfigurationData.h +++ b/src/framework/ConfigurationData.h @@ -276,6 +276,7 @@ class ConfigurationData { std::map >, std::less<> > sdormConnProperties; std::map >, std::less<> > sdormEntityMappings; std::map >, std::less<> > cacheConnProperties; + std::map >, std::less<> > searchConnProperties; std::map, std::less<> > webserviceDetailMap; std::map >, std::less<> > classStructureMap; std::map > appPropertiesMap; @@ -327,6 +328,7 @@ class ConfigurationData { friend class JobScheduler; friend class DataSourceManager; friend class CacheManager; + friend class SearchEngineManager; friend class ServerInitUtil; public: static void enableFeatures(bool enableCors, bool enableSecurity, bool enableFilters, bool enableControllers, diff --git a/src/framework/ConfigurationHandler.cpp b/src/framework/ConfigurationHandler.cpp index b0f4ffc34..6abde4ce6 100644 --- a/src/framework/ConfigurationHandler.cpp +++ b/src/framework/ConfigurationHandler.cpp @@ -967,6 +967,7 @@ void ConfigurationHandler::handle(strVec webdirs, const strVec& webdirs1, const configureDataSources(name, defpath+"config/sdorm.xml", allclsmap); configureCaches(name, defpath+"config/cache.xml"); + configureSearches(name, defpath+"config/search.xml"); logger << "started reading fviews.xml " << std::endl; Document doc1; @@ -2606,6 +2607,139 @@ void ConfigurationHandler::initializeDataSources() } } +void ConfigurationHandler::configureSearches(const std::string &name, const std::string &configFile) { + Logger logger = LoggerFactory::getLogger("ConfigurationHandler"); + SimpleXmlParser parser("Parser"); + logger << ("started reading search config file " + configFile) << std::endl; + + Document doc; + parser.readDocument(configFile, doc); + const Element& dbroot = doc.getRootElement(); + + if(dbroot.getTagName()=="searches") + { + ElementList datasrcs = dbroot.getChildElements(); + bool found_default = false; + for (unsigned int dsnu = 0; dsnu < datasrcs.size(); dsnu++) + { + if(datasrcs.at(dsnu).getTagName()=="search") + { + ConnectionProperties cprops; + if(!found_default && (datasrcs.at(dsnu).getAttribute("default")=="true" || datasrcs.size()==1)) { + cprops.addProperty("_isdefault_", "true"); + found_default = true; + } + int psize = 1; + ElementList confs = datasrcs.at(dsnu).getChildElements(); + for (unsigned int cns = 0; cns < confs.size(); cns++) + { + if(confs.at(cns).getTagName()=="nodes") + { + ElementList nodec = confs.at(cns).getChildElements(); + for (unsigned int nn = 0; nn < nodec.size(); nn++) + { + if(nodec.at(nn).getTagName()!="node")continue; + ConnectionNode cnode; + ElementList nodes = nodec.at(nn).getChildElements(); + for (unsigned int ncc = 0; ncc < nodes.size(); ncc++) + { + if(nodes.at(ncc).getTagName()=="url") + { + cnode.url = nodes.at(ncc).getText(); + } + else if(nodes.at(ncc).getTagName()=="username") + { + cnode.username = nodes.at(ncc).getText(); + } + else if(nodes.at(ncc).getTagName()=="password") + { + cnode.password = nodes.at(ncc).getText(); + } + else if(nodes.at(ncc).getTagName()=="host") + { + cnode.host = nodes.at(ncc).getText(); + } + else if(nodes.at(ncc).getTagName()=="port") + { + try { + cnode.port = CastUtil::toInt(nodes.at(ncc).getText()); + } catch(const std::exception& e) { + } + } + else if(nodes.at(ncc).getTagName()=="readTimeout") + { + try { + cnode.readTimeout = CastUtil::toFloat(nodes.at(ncc).getText()); + } catch(const std::exception& e) { + } + } + else if(nodes.at(ncc).getTagName()=="connectionTimeout") + { + try { + cnode.connectionTimeout = CastUtil::toFloat(nodes.at(ncc).getText()); + } catch(const std::exception& e) { + } + } + } + cprops.addNode(cnode); + } + } + else if(confs.at(cns).getTagName()=="pool-size") + { + if(confs.at(cns).getText()!="") + { + try { + psize = CastUtil::toInt(confs.at(cns).getText()); + } catch(const std::exception& e) { + } + } + cprops.poolWriteSize = psize; + } + else if(confs.at(cns).getTagName()=="name") + { + cprops.name = confs.at(cns).getText(); + } + else if(confs.at(cns).getTagName()=="type") + { + cprops.type = confs.at(cns).getText(); + } + else + { + cprops.addProperty(confs.at(cns).getTagName(), confs.at(cns).getText()); + } + } + + if(cprops.getName()!="") + { + ConfigurationData::getInstance()->searchConnProperties[name][cprops.getName()] = cprops; + } + } + } + } + + logger << "done reading search config file " + configFile << std::endl; +} + +void ConfigurationHandler::initializeSearches() { + std::map > mycntxts = ConfigurationData::getInstance()->servingContexts; + std::map >, std::less<> > searchConnProperties = ConfigurationData::getInstance()->searchConnProperties; + std::map >::iterator mit; + for (mit=mycntxts.begin();mit!=mycntxts.end();++mit) { + if(searchConnProperties.find(mit->first)!=searchConnProperties.end()) + { + std::map > allProps = searchConnProperties[mit->first]; + std::map>::iterator it; + for (it=allProps.begin();it!=allProps.end();++it) { + SearchEngineManager::initSearch(allProps[it->first], mit->first, &ConfigurationHandler::populateClassBeanInstanceCb); + } + } + } +} + +void ConfigurationHandler::destroySearches() { + SearchEngineManager::destroy(); +} + void ConfigurationHandler::populateClassBeanInstanceCb(std::string& clsn, std::string appn, ClassBeanIns* cbi) { cbi->clas = ConfigurationData::getReflector()->getClassInfo(clsn, appn); if(cbi->clas->getClassName()!="") { diff --git a/src/framework/ConfigurationHandler.h b/src/framework/ConfigurationHandler.h index 4abcf6a3c..6787e63b0 100644 --- a/src/framework/ConfigurationHandler.h +++ b/src/framework/ConfigurationHandler.h @@ -55,6 +55,7 @@ #include "iostream" #include "DataSourceManager.h" #include "CacheManager.h" +#include "SearchEngineManager.h" #include "CommonUtils.h" class ConfigurationHandler { @@ -68,12 +69,19 @@ class ConfigurationHandler { ConfigurationHandler(); virtual ~ConfigurationHandler(); static void handle(strVec webdirs, const strVec& webdirs1, const std::string& incpath, const std::string& rtdcfpath, const std::string& serverRootDirectory, const std::string& respath); + static void configureDataSources(const std::string& name, const std::string& configFile, std::map >& allclsmap); + static void initializeDataSources(); static void destroyDataSources(); + static void configureCaches(const std::string& name, const std::string& configFile); - static void destroyCaches(); - static void initializeDataSources(); static void initializeCaches(); + static void destroyCaches(); + + static void configureSearches(const std::string& name, const std::string& configFile); + static void initializeSearches(); + static void destroySearches(); + static void initializeWsdls(); static void populateClassBeanInstanceCb(std::string& clsn, std::string appn, ClassBeanIns* cbi); }; diff --git a/src/framework/ConnectionSettings.cpp b/src/framework/ConnectionSettings.cpp deleted file mode 100644 index cd6334e76..000000000 --- a/src/framework/ConnectionSettings.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2009-2020, Sumeet Chhetri - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -/* - * ConnectionSettings.cpp - * - * Created on: Jan 30, 2010 - * Author: sumeet - */ - -#include "ConnectionSettings.h" - -ConnectionSettings::ConnectionSettings() { - - -} - -ConnectionSettings::~ConnectionSettings() { - -} - -ConnectionSettings::ConnectionSettings(std::string mod,std::string src,std::string usn,std::string pas,std::string add) { - this->mode = mod; - this-> source = src; - this->username = usn; - this->passwd = pas; - this->address = add; -} diff --git a/src/framework/ConnectionSettings.h b/src/framework/ConnectionSettings.h deleted file mode 100644 index ae4c6b2c2..000000000 --- a/src/framework/ConnectionSettings.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright 2009-2020, Sumeet Chhetri - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -/* - * ConnectionSettings.h - * - * Created on: Jan 30, 2010 - * Author: sumeet - */ - -#ifndef CONNECTIONSETTINGS_H_ -#define CONNECTIONSETTINGS_H_ -#include "string" - - -class ConnectionSettings { - std::string mode;//ldap|database|filesystem - std::string source; - std::string username; - std::string passwd; - std::string address; -public: - ConnectionSettings(); - ConnectionSettings(std::string,std::string,std::string,std::string,std::string); - virtual ~ConnectionSettings(); - friend class ComponentGen; -}; - -#endif /* CONNECTIONSETTINGS_H_ */ -/* -mode=ldap -source=host:port -username=user -passwd=user -address=ldap://example.com:9765 - -mode=database -source=dsn-name -username=user -passwd=user -address=oracle://host:port - -mode=filesystem -source=host:port -username=user //ftp -passwd=user //ftp -address=ftp://example.com:9765 -*/ diff --git a/src/framework/ServerInitUtil.cpp b/src/framework/ServerInitUtil.cpp index e05e74db5..d3ae8e79f 100644 --- a/src/framework/ServerInitUtil.cpp +++ b/src/framework/ServerInitUtil.cpp @@ -220,8 +220,14 @@ void ServerInitUtil::bootstrap(std::string serverRootDirectory, Logger& logger, std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " xmake"; #elif defined(BUILD_MESON) std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " meson"; +#elif defined(BUILD_SCONS) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " scons"; +#elif defined(BUILD_BAZEL) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " bazel"; +#elif defined(BUILD_SHELLB) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " shellb"; #else - logger << "Invalid Build Type specified, only cmake, xmake and meson supported...\n" << std::endl; + logger << "Invalid Build Type specified, only autotools, cmake, xmake, meson, scons and bazel supported...\n" << std::endl; #endif std::string output = ScriptHandler::execute(compres, true); logger << "Intermediate code generation task\n\n" << std::endl; @@ -401,6 +407,10 @@ void ServerInitUtil::init(Logger& logger) { ConfigurationHandler::initializeCaches(); logger << ("Initializing Caches done....") << std::endl; + logger << ("Initializing Searches....") << std::endl; + ConfigurationHandler::initializeSearches(); + logger << ("Initializing Searches done....") << std::endl; + #ifdef INC_JOBS if(ConfigurationData::getInstance()->isJobsEnabled()) { JobScheduler::start(); @@ -423,6 +433,8 @@ void ServerInitUtil::cleanUp() { ConfigurationHandler::destroyCaches(); + ConfigurationHandler::destroySearches(); + ConfigurationData::getInstance()->clearAllSingletonBeans(); #ifdef INC_JOBS diff --git a/src/framework/meson.build b/src/framework/meson.build index b9a420401..124c05be6 100644 --- a/src/framework/meson.build +++ b/src/framework/meson.build @@ -4,11 +4,11 @@ module_includes = ['/usr/local/include','../modules/common','../modules/cache',' '../modules/ioc','../modules/jobs','../modules/reflection','../modules/sdorm','../modules/sdorm/sql','../modules/sdorm/sql/libpq', '../modules/client-util/ssl','../modules/sdorm/mongo','../modules/sdorm/mongo/raw','../modules/search','../modules/serialization', '../modules/serialization/xml','../modules/serialization/json','../modules/server-util','../modules/ssl','../modules/threads', - '../modules/sdorm/scylla/raw'] + '../modules/sdorm/scylla/raw','../modules/search/solr','../modules/search/elasticsearch'] module_includes += global_includes module_libs = [global_libs] -c = run_command(meson_grabber, '.') +c = run_command(meson_grabber, '.', check: true) module_sources = c.stdout().strip().split('\n') libffeadframework = shared_library('ffead-framework', module_sources, include_directories: module_includes, dependencies: module_libs, diff --git a/src/framework/sconscript b/src/framework/sconscript new file mode 100644 index 000000000..3e4c451da --- /dev/null +++ b/src/framework/sconscript @@ -0,0 +1,28 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +for root, dirnames, filenames in os.walk('../modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['.']) +matches.append(Glob("./*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/ffead-framework', matches, LIBS=['ffead-modules'], LIBPATH='../../bin/') diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt index fa621f3f5..c2f49ec65 100644 --- a/src/modules/CMakeLists.txt +++ b/src/modules/CMakeLists.txt @@ -149,11 +149,29 @@ file(GLOB serialization_module_sources install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/serialization/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/serialization/xml/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/serialization/json/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") -#file(GLOB search_module_sources -# "search/*.h" -# "search/*.cpp" -#) -#install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/search/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") + +file (GLOB search_module_sources + "search/*.h" + "search/*.cpp" +) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/search/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") +if(MOD_SOLR) + file (GLOB search_module_sources_solr + "search/solr/*.h" + "search/solr/*.cpp" + ) + list (APPEND search_module_sources ${search_module_sources_solr}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/search/solr/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") +endif() +if(MOD_ELASTIC) + file (GLOB search_module_sources_elas + "search/elasticsearch/*.h" + "search/elasticsearch/*.cpp" + ) + list (APPEND search_module_sources ${search_module_sources_elas}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/modules/search/elasticsearch/ DESTINATION ${PROJECT_NAME}-bin/include FILES_MATCHING PATTERN "*.h") +endif() + file(GLOB serv_module_sources "server-util/*.h" "server-util/*.cpp" @@ -178,4 +196,4 @@ if(BUILD_STATIC_LIBS OR EMSCRIPTEN) else() add_library(ffead-modules ${sql_pq_module_sources} ${mongo_raw_module_sources} ${scylla_raw_module_sources} ${wepoll_sources} ${comm_module_sources} ${comm_module_picoev_sources} ${ssl_client_module_sources} ${client_module_sources} ${cache_module_sources} ${sdorm_module_sources} ${http_module_sources} ${ioc_module_sources} ${reflection_module_sources} ${serialization_module_sources} ${search_module_sources} ${serv_module_sources} ${ssl_module_sources} ${threads_module_sources} ${mem_module_sources} ${red_module_sources} ${memc_module_sources} ${sql_module_sources} ${mongo_module_sources} ${serb_module_sources} ${job_module_sources}) endif() -target_link_libraries(ffead-modules ${HAVE_LIBPQ} ${HAVE_LIBURING} ${HAVE_ONIG_REGEX_LIB} ${HAVE_WINREGEX} ${HAVE_WSOCK32} ${HAVE_WS232} ${HAVE_DLLIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) +target_link_libraries(ffead-modules ${HAVE_LIBPQ} ${HAVE_LIBURING} ${HAVE_ONIG_REGEX_LIB} ${HAVE_WINREGEX} ${HAVE_WSOCK32} ${HAVE_WS232} ${HAVE_DLLIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB}) diff --git a/src/modules/cache/CacheInterface.h b/src/modules/cache/CacheInterface.h index 4ead17fd8..8bd04ae4b 100644 --- a/src/modules/cache/CacheInterface.h +++ b/src/modules/cache/CacheInterface.h @@ -38,6 +38,10 @@ class CacheInterface { virtual ~CacheInterface(); virtual void init()=0; + ConnectionPooler* getPool() { + return pool; + } + bool set(const std::string& key, const char* value, int expireSeconds = -1); bool add(const std::string& key, const char* value, int expireSeconds = -1); bool replace(const std::string& key, const char* value, int expireSeconds = -1); diff --git a/src/modules/cache/CacheManager.cpp b/src/modules/cache/CacheManager.cpp index 9cf2b59cd..c1eb6d784 100644 --- a/src/modules/cache/CacheManager.cpp +++ b/src/modules/cache/CacheManager.cpp @@ -153,8 +153,16 @@ void CacheManager::cleanImpl(CacheInterface* ccImpl) { } else if(StringUtil::toLowerCopy(ccImpl->pool->getProperties().getType())=="redis") { -#ifdef INC_REDISCACHE - delete (RedisCacheImpl*)ccImpl; +#if defined(HAVE_REDISINC) + if(StringUtil::toLowerCopy(ccImpl->pool->getProperties().getProperty("cluster"))=="true") { +#if defined(HAVE_REDIS_CLUSTERINC) + delete (RedisClusterCacheImpl*)ccImpl; +#else + delete (RedisCacheImpl*)ccImpl; +#endif + } else { + delete (RedisCacheImpl*)ccImpl; + } #endif } } @@ -196,7 +204,13 @@ CacheInterface* CacheManager::getImpl(std::string name, std::string appName) { else if(StringUtil::toLowerCopy(cchMgr->props.getType())=="redis") { #ifdef INC_REDISCACHE - t = new RedisCacheImpl(cchMgr->pool); + if(StringUtil::toLowerCopy(cchMgr->props.getProperty("cluster"))=="true") { +#ifdef HAVE_REDIS_CLUSTERINC + t = new RedisClusterCacheImpl(cchMgr->pool); +#endif + } else { + t = new RedisCacheImpl(cchMgr->pool); + } #endif } t->init(); @@ -209,7 +223,6 @@ CacheInterface* CacheManager::getImpl(std::string name, std::string appName) { CacheManager::CacheManager(const ConnectionProperties& props) { logger = LoggerFactory::getLogger("CacheManager"); - this->reflector = NULL; this->pool = NULL; this->props = props; if(StringUtil::toLowerCopy(props.getType()) == "memory") { @@ -220,7 +233,13 @@ CacheManager::CacheManager(const ConnectionProperties& props) { #endif } else if(StringUtil::toLowerCopy(props.getType()) == "redis") { #ifdef INC_REDISCACHE - this->pool = new RedisCacheConnectionPool(props); + if(StringUtil::toLowerCopy(props.getProperty("cluster"))=="true") { +#ifdef HAVE_REDIS_CLUSTERINC + this->pool = new RedisClusterCacheConnectionPool(props); +#endif + } else { + this->pool = new RedisCacheConnectionPool(props); + } #endif } } diff --git a/src/modules/cache/CacheManager.h b/src/modules/cache/CacheManager.h index e860b44ab..ab41291ce 100644 --- a/src/modules/cache/CacheManager.h +++ b/src/modules/cache/CacheManager.h @@ -26,7 +26,10 @@ #ifdef INC_MEMCACHED #include"MemcachedImpl.h" #endif -#ifdef INC_REDISCACHE +#ifdef HAVE_REDIS_CLUSTERINC +#include "RedisClusterCacheImpl.h" +#endif +#if HAVE_REDISINC #include "RedisCacheImpl.h" #endif #include "GenericObject.h" @@ -40,7 +43,6 @@ class CacheManager { static std::map appInitCompletionStatus; ConnectionProperties props; ConnectionPooler* pool; - Reflector* reflector; static void initCache(const ConnectionProperties& props, const std::string& appName, GetClassBeanIns f); static void destroy(); CacheManager(const ConnectionProperties& props); diff --git a/src/modules/cache/redis/RedisClusterCacheImpl.cpp b/src/modules/cache/redis/RedisClusterCacheImpl.cpp new file mode 100644 index 000000000..8be9b163b --- /dev/null +++ b/src/modules/cache/redis/RedisClusterCacheImpl.cpp @@ -0,0 +1,264 @@ +/* + Copyright 2009-2020, Sumeet Chhetri + + Licensed under the Apache License, Version 2.0 (const the& "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +/* + * RedisClusterCacheImpl.cpp + * + * Created on: 07-May-2014 + * Author: sumeetc + */ + +#include "RedisClusterCacheImpl.h" + +#ifdef HAVE_REDIS_CLUSTERINC + +RedisClusterCacheImpl::RedisClusterCacheImpl(ConnectionPooler* pool) { + this->pool = pool; + this->properties = pool->getProperties(); + this->defaultExpireSeconds = -1; + if(properties.getProperty("expiryTime")!="") { + try { + this->defaultExpireSeconds = CastUtil::toInt(properties.getProperty("expiryTime")); + } catch(const std::exception& e) { + } + } + if(this->defaultExpireSeconds<=0) { + this->defaultExpireSeconds = 3600; + } +} + +RedisClusterCacheImpl::~RedisClusterCacheImpl() { +} + +bool RedisClusterCacheImpl::remove(const std::string& key) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + return r->del(key)>0; +} + +long long RedisClusterCacheImpl::increment(const std::string& key, const int& number) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + return r->incrby(key, number); +} + +long long RedisClusterCacheImpl::decrement(const std::string& key, const int& number) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + return r->decrby(key, number); +} + +long double RedisClusterCacheImpl::incrementFloat(const std::string& key, const double& number) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + return r->incrbyfloat(key, number); +} + +long double RedisClusterCacheImpl::decrementFloat(const std::string& key, const double& number) { + return incrementFloat(key, -number); +} + +std::map RedisClusterCacheImpl::statistics() { + std::map stats; + return stats; +} + +bool RedisClusterCacheImpl::flushAll() { + return false; +} + +bool RedisClusterCacheImpl::set(const std::string& key, GenericObject& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::string valueStr = value.getSerilaizedState(); + r->set(key, valueStr, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::ALWAYS); + return true; +} + +bool RedisClusterCacheImpl::add(const std::string& key, GenericObject& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::string valueStr = value.getSerilaizedState(); + r->set(key, valueStr, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::NOT_EXIST); + return true; +} + +bool RedisClusterCacheImpl::replace(const std::string& key, GenericObject& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::string valueStr = value.getSerilaizedState(); + r->set(key, valueStr, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::EXIST); + return true; +} + +bool RedisClusterCacheImpl::setRaw(const std::string& key, const char* value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(key, value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::ALWAYS); + return true; +} + +bool RedisClusterCacheImpl::addRaw(const std::string& key, const char* value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(key, value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::NOT_EXIST); + return true; +} + +bool RedisClusterCacheImpl::replaceRaw(const std::string& key, const char* value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(key, value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::EXIST); + return true; +} + +std::string RedisClusterCacheImpl::getValue(const std::string& key) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::optional os = r->get(key); + return os.has_value()?os.value():""; +} + +std::vector RedisClusterCacheImpl::getValues(const std::vector& keys) { + std::vector rv; + mgetRaw(keys, rv); + return rv; +} + +bool RedisClusterCacheImpl::setRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(std::to_string(key), value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::ALWAYS); + return true; +} +bool RedisClusterCacheImpl::addRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(std::to_string(key), value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::NOT_EXIST); + return true; +} +bool RedisClusterCacheImpl::replaceRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->set(std::to_string(key), value, std::chrono::milliseconds((expireSeconds<=0?defaultExpireSeconds:expireSeconds)*1000), sw::redis::UpdateType::EXIST); + return true; +} +std::string RedisClusterCacheImpl::getValue(const unsigned long long& key) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::optional os = r->get(std::to_string(key)); + return os.has_value()?os.value():""; +} +void RedisClusterCacheImpl::getValues(const std::vector& keys, std::vector& rvalues) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + + std::vector skeys; + for(int i=0;i<(int)keys.size();++i) { + skeys.push_back(std::to_string(keys.at(i))); + } + std::vector> values; + r->mget(skeys.begin(), skeys.end(), std::back_inserter(values)); + + for (const auto &val : values) { + if (val) { + rvalues.push_back(val.value()); + } else { + rvalues.push_back(""); + } + } +} +bool RedisClusterCacheImpl::remove(const unsigned long long& key) { + return remove(std::to_string(key)); +} + +void RedisClusterCacheImpl::mgetRaw(const std::vector& skeys, std::vector& rvalues) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + + std::vector> values; + r->mget(skeys.begin(), skeys.end(), std::back_inserter(values)); + + for (const auto &val : values) { + if (val) { + rvalues.push_back(val.value()); + } else { + rvalues.push_back(""); + } + } +} + +void* RedisClusterCacheImpl::executeCommand(const std::string command, ...) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + r->command>(command, ""); + return NULL; +} + +bool RedisClusterCacheImpl::addToQ(const std::string& qname, const std::string& value) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + return r->lpush(qname, value)>0; +} + +std::string RedisClusterCacheImpl::getFromQ(const std::string& qname) { + Connection* connection = pool->checkout(); + sw::redis::RedisCluster* r = (sw::redis::RedisCluster*)connection->getConn(); + std::optional os = r->rpop(qname); + return os.has_value()?os.value():""; +} + +void RedisClusterCacheImpl::init() { +} + +RedisClusterCacheConnectionPool::RedisClusterCacheConnectionPool(const ConnectionProperties& props) { + r = NULL; +} + +RedisClusterCacheConnectionPool::~RedisClusterCacheConnectionPool() { + destroyPool(); +} + +void* RedisClusterCacheConnectionPool::newConnection(const bool& isWrite, const ConnectionNode& node) { + return r; +} +void RedisClusterCacheConnectionPool::closeConnection(void* conn) { +} + +void RedisClusterCacheConnectionPool::initEnv() { + getProps().setNewConnectionStrategy(true); + sw::redis::ConnectionOptions connection_options; + connection_options.host = getProperties().getNodes().at(0).getHost(); // Required. + connection_options.port = getProperties().getNodes().at(0).getPort(); // Optional. The default port is 6379. + //connection_options.password = "auth"; // Optional. No password by default. + //connection_options.db = 1; // Optional. Use the 0th database by default. + + // Optional. Timeout before we successfully send request to or receive response from redis. + // By default, the timeout is 0ms, i.e. never timeout and block until we send or receive successfuly. + // NOTE: if any command is timed out, we throw a TimeoutError exception. + //connection_options.socket_timeout = getProperties().getNodes().at(0).getConnectionTimeout(); + connection_options.connect_timeout = std::chrono::milliseconds((int)getProperties().getNodes().at(0).getConnectionTimeout()); + + int numConns = getProperties().getPoolReadSize() + getProperties().getPoolWriteSize(); + sw::redis::ConnectionPoolOptions pool_options; + pool_options.size = numConns; + + r = new sw::redis::RedisCluster(connection_options, pool_options); +} + +void RedisClusterCacheConnectionPool::destroy() { + delete r; +} +#endif diff --git a/src/modules/cache/redis/RedisClusterCacheImpl.h b/src/modules/cache/redis/RedisClusterCacheImpl.h new file mode 100644 index 000000000..025a5ffca --- /dev/null +++ b/src/modules/cache/redis/RedisClusterCacheImpl.h @@ -0,0 +1,82 @@ +/* + Copyright 2009-2020, Sumeet Chhetri + + Licensed under the Apache License, Version 2.0 (const the& "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +/* + * RedisClusterCacheImpl.h + * + * Created on: 07-May-2014 + * Author: sumeetc + */ + +#ifndef REDISCLUSTERCACHEIMPL_H_ +#define REDISCLUSTERCACHEIMPL_H_ +#include "CacheInterface.h" +#include "ConnectionPooler.h" +#include +#include +#ifdef HAVE_REDIS_CLUSTERINC +#include + +class RedisClusterCacheConnectionPool: public ConnectionPooler { + void initEnv(); + void* newConnection(const bool& isWrite, const ConnectionNode& node); + void closeConnection(void* conn); + void destroy(); + sw::redis::RedisCluster* r; +public: + RedisClusterCacheConnectionPool(const ConnectionProperties& props); + virtual ~RedisClusterCacheConnectionPool(); +}; + +class RedisClusterCacheImpl: public CacheInterface { + ConnectionProperties properties; +public: + RedisClusterCacheImpl(ConnectionPooler* pool); + ~RedisClusterCacheImpl(); + void init(); + + bool setRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds = -1); + bool addRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds = -1); + bool replaceRaw(const unsigned long long& key, const std::string_view& value, int expireSeconds = -1); + std::string getValue(const unsigned long long& key); + void getValues(const std::vector& keys, std::vector& values); + bool remove(const unsigned long long& key); + + bool set(const std::string& key, GenericObject& value, int expireSeconds); + bool add(const std::string& key, GenericObject& value, int expireSeconds); + bool replace(const std::string& key, GenericObject& value, int expireSeconds); + + bool setRaw(const std::string& key, const char* value, int expireSeconds = -1); + bool addRaw(const std::string& key, const char* value, int expireSeconds = -1); + bool replaceRaw(const std::string& key, const char* value, int expireSeconds = -1); + void mgetRaw(const std::vector& keys, std::vector& values); + + std::string getValue(const std::string& key); + std::vector getValues(const std::vector& keys); + + bool remove(const std::string& key); + long long increment(const std::string& key, const int& number= 1); + long long decrement(const std::string& key, const int& number= 1); + long double incrementFloat(const std::string& key, const double& number = 1.0); + long double decrementFloat(const std::string& key, const double& number = 1.0); + std::map statistics(); + bool flushAll(); + + void* executeCommand(const std::string command, ...); + bool addToQ(const std::string& qname, const std::string& value); + std::string getFromQ(const std::string& qname); +}; +#endif +#endif /* REDISCLUSTERCACHEIMPL_H_ */ diff --git a/src/modules/client-util/HttpClient.cpp b/src/modules/client-util/HttpClient.cpp index c0daf8a55..d0f0c75b9 100644 --- a/src/modules/client-util/HttpClient.cpp +++ b/src/modules/client-util/HttpClient.cpp @@ -199,13 +199,13 @@ void HttpClient::execute(HttpRequest* request, HttpResponse* response, propMap& size_t HttpClient::onContent(void *res, size_t len, size_t mb, void *data) { std::string* _bd = (std::string*) data; - _bd->append(reinterpret_cast(data), len * mb); + _bd->append(reinterpret_cast(res), len * mb); return len * mb; } size_t HttpClient::onHeaders(void *res, size_t len, size_t mb, void *data) { std::string* _hd = (std::string*) data; - _hd->append(reinterpret_cast(data), len * mb); + _hd->append(reinterpret_cast(res), len * mb); return len * mb; } diff --git a/src/modules/client-util/HttpClient.h b/src/modules/client-util/HttpClient.h index 66a582739..4b4f6c9a4 100644 --- a/src/modules/client-util/HttpClient.h +++ b/src/modules/client-util/HttpClient.h @@ -20,6 +20,10 @@ * Author: sumeet */ +//https://github.com/libcpr/cpr +//https://github.com/seznam/elasticlient +//https://github.com/sewenew/redis-plus-plus#install-redis-plus-plus + #ifndef SRC_MODULES_CLIENT_UTIL_HTTP_HTTPCLIENT_H_ #define SRC_MODULES_CLIENT_UTIL_HTTP_HTTPCLIENT_H_ diff --git a/src/modules/common/AppDefines.bazel.h.in b/src/modules/common/AppDefines.bazel.h.in new file mode 100644 index 000000000..3ceaaf894 --- /dev/null +++ b/src/modules/common/AppDefines.bazel.h.in @@ -0,0 +1,31 @@ +/* + Copyright 2009-2020, Sumeet Chhetri + + Licensed under the Apache License, Version 2.0 (const the& "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define BUILD_BAZEL 1 + +#ifdef HAVE_ODBCLIB +#define HAVE_LIBODBC 1 +#endif + +#define INC_WEBSVC 1 +#define INC_TPE 1 +#define INC_DVIEW 1 +#define INC_DCP 1 +#define INC_XMLSER 1 + +#ifdef APPLE +#define OS_DARWIN 1 +#endif diff --git a/src/modules/common/AppDefines.h.in.cm b/src/modules/common/AppDefines.h.in.mc similarity index 94% rename from src/modules/common/AppDefines.h.in.cm rename to src/modules/common/AppDefines.h.in.mc index 5a9080339..e5d28f0c1 100644 --- a/src/modules/common/AppDefines.h.in.cm +++ b/src/modules/common/AppDefines.h.in.mc @@ -82,6 +82,10 @@ #cmakedefine HAVE_RAPID_JSON @HAVE_RAPID_JSON@ #cmakedefine HAVE_PUGI_XML @HAVE_PUGI_XML@ #cmakedefine HAVE_SYSINFO @HAVE_SYSINFO@ +#cmakedefine HAVE_SOLR @HAVE_SOLR@ +#cmakedefine HAVE_ELASTIC @HAVE_ELASTIC@ +#cmakedefine HAVE_REDIS_CLUSTERINC @HAVE_REDIS_CLUSTERINC@ +#cmakedefine HAVE_REDIS_CLUSTERLIB @HAVE_REDIS_CLUSTERLIB@ #ifdef HAVE_ODBCLIB #define HAVE_LIBODBC 1 diff --git a/src/modules/common/AppDefines.meson.h.in b/src/modules/common/AppDefines.meson.h.in index 84b8ba5dc..577e020bd 100644 --- a/src/modules/common/AppDefines.meson.h.in +++ b/src/modules/common/AppDefines.meson.h.in @@ -82,6 +82,12 @@ #mesondefine HAVE_RAPID_JSON #mesondefine HAVE_PUGI_XML #mesondefine HAVE_SYSINFO +#mesondefine HAVE_PUGI_XML +#mesondefine HAVE_SYSINFO +#mesondefine HAVE_SOLR +#mesondefine HAVE_ELASTIC +#mesondefine HAVE_REDIS_CLUSTERINC +#mesondefine HAVE_REDIS_CLUSTERLIB #ifdef HAVE_ODBCLIB #define HAVE_LIBODBC 1 diff --git a/src/modules/common/AppDefines.scons.h.in b/src/modules/common/AppDefines.scons.h.in new file mode 100644 index 000000000..da4484d89 --- /dev/null +++ b/src/modules/common/AppDefines.scons.h.in @@ -0,0 +1,31 @@ +/* + Copyright 2009-2020, Sumeet Chhetri + + Licensed under the Apache License, Version 2.0 (const the& "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define BUILD_SCONS 1 + +#ifdef HAVE_ODBCLIB +#define HAVE_LIBODBC 1 +#endif + +#define INC_WEBSVC 1 +#define INC_TPE 1 +#define INC_DVIEW 1 +#define INC_DCP 1 +#define INC_XMLSER 1 + +#ifdef APPLE +#define OS_DARWIN 1 +#endif diff --git a/src/modules/common/AppDefines.xmake.h.in b/src/modules/common/AppDefines.xmake.h.in index 208d3e50c..60cfe82c3 100644 --- a/src/modules/common/AppDefines.xmake.h.in +++ b/src/modules/common/AppDefines.xmake.h.in @@ -82,6 +82,10 @@ ${define IS_SENDFILE} ${define HAVE_RAPID_JSON} ${define HAVE_PUGI_XML} ${define HAVE_SYSINFO} +${define HAVE_SOLR} +${define HAVE_ELASTIC} +${define HAVE_REDIS_CLUSTERINC} +${define HAVE_REDIS_CLUSTERLIB} #ifdef HAVE_ODBCLIB #define HAVE_LIBODBC 1 diff --git a/src/modules/common/CommonUtils.cpp b/src/modules/common/CommonUtils.cpp index af5d46dac..e9caf4263 100644 --- a/src/modules/common/CommonUtils.cpp +++ b/src/modules/common/CommonUtils.cpp @@ -58,6 +58,26 @@ int CommonUtils::g_seed = 0; const char* CommonUtils::dateStr; +static const char* get_date() { + time_t t; + struct tm tm; + static const char *days[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; + static const char *months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + static __thread char date[59] = "Date: Thu, 01 Jan 1970 00:00:00 GMT\r\nServer: FFEAD 2.0\r\n"; + + time(&t); + gmtime_r(&t, &tm); + strftime(date, 58, "Date: ---, %d --- %Y %H:%M:%S GMT\r\nServer: FFEAD 2.0\r\n", &tm); + memcpy(date + 6, days[tm.tm_wday], 3); + memcpy(date + 14, months[tm.tm_mon], 3); + + return date; +} + +void CommonUtils::setDate() { + dateStr = get_date(); +} + CommonUtils::CommonUtils() { } diff --git a/src/modules/common/CommonUtils.h b/src/modules/common/CommonUtils.h index de7dd28ae..171ff1339 100644 --- a/src/modules/common/CommonUtils.h +++ b/src/modules/common/CommonUtils.h @@ -73,6 +73,9 @@ class CommonUtils { friend class ConfigurationHandler; friend class CHServer; friend class ServiceHandler; + friend class RequestHandler2; + friend class RequestReaderHandler; + static void setDate(); static void addContext(std::string appName); static int g_seed; public: diff --git a/src/modules/component/BeanContext.cpp b/src/modules/component/BeanContext.cpp index d665225f4..2318dbfa2 100644 --- a/src/modules/component/BeanContext.cpp +++ b/src/modules/component/BeanContext.cpp @@ -105,7 +105,8 @@ void* BeanContext::invoke(const std::string& name, std::vector ar { for (unsigned int var = 0; var < args.size(); ++var) { - argus += ""+ser.serializeUnknown(args.at(var).getPointer(),rettyp)+""; + int serOpt = SerializeBase::identifySerOption(rettyp); + argus += ""+ser.serializeUnknown(args.at(var).getPointer(),serOpt,rettyp,NULL)+""; } } std::string call = ""+argus+""; @@ -126,7 +127,8 @@ void* BeanContext::invoke(const std::string& name, std::vector ar message.setTagName(tag); call = message.render(); //logger << call << std::flush; - retval = ser.unSerializeUnknown(call,tag); + int serOpt = SerializeBase::identifySerOption(tag); + retval = ser.unSerializeUnknown(call,serOpt,tag); } else { diff --git a/src/modules/component/ComponentHandler.cpp b/src/modules/component/ComponentHandler.cpp index fba6856ba..6b1fae665 100644 --- a/src/modules/component/ComponentHandler.cpp +++ b/src/modules/component/ComponentHandler.cpp @@ -119,8 +119,8 @@ void* ComponentHandler::service(void* arg) } if(argts.at(var).getAttribute("type")!="") { - std::string objxml = argts.at(var).getChildElements().at(0).render(); // @suppress("Invalid arguments") - std::string objClassName = argts.at(var).getChildElements().at(0).getTagName(); + std::string objxml = argts.at(var).render(); // @suppress("Invalid arguments") + std::string objClassName = argts.at(var).getTagName(); int serOpt = SerializeBase::identifySerOption(argts.at(var).getAttribute("type")); value = ser.unSerializeUnknown(objxml, serOpt, argts.at(var).getAttribute("type")); } @@ -200,7 +200,7 @@ void* ComponentHandler::service(void* arg) { void* retobj = reflector.invokeMethodUnknownReturn(_temp,meth,valus,true); int serOpt = SerializeBase::identifySerOption(returnType); - std::string oxml = ser.serializeUnknown(retobj,serOpt,returnType); + std::string oxml = ser.serializeUnknown(retobj,serOpt,returnType,NULL); retValue = (""+oxml+""); } } diff --git a/src/modules/meson.build b/src/modules/meson.build index 01108ca59..912c9ec53 100644 --- a/src/modules/meson.build +++ b/src/modules/meson.build @@ -2,118 +2,136 @@ module_sources = [] module_includes = ['/usr/local/include','common','cache','cache/memory','cache/redis', 'cache/memcached','client-util','http','http/http11','http/http20', 'common/picoev', - 'ioc','jobs','reflection','sdorm','sdorm/sql','sdorm/sql/libpq','../framework', - 'client-util/ssl','sdorm/mongo','sdorm/mongo/raw','search','serialization', + 'ioc','jobs','reflection','sdorm','sdorm/sql','sdorm/sql/libpq','../framework', 'search/elasticsearch', + 'client-util/ssl','sdorm/mongo','sdorm/mongo/raw','search','serialization', 'search/solr', 'serialization/xml','serialization/json','server-util','ssl','threads', 'sdorm/scylla/raw'] module_includes += global_includes -c = run_command(meson_grabber, 'common') +c = run_command(meson_grabber, 'common', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'client-util') +c = run_command(meson_grabber, 'client-util', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'server-util') +c = run_command(meson_grabber, 'server-util', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'http', 'all') +c = run_command(meson_grabber, 'http', 'all', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'ioc') +c = run_command(meson_grabber, 'ioc', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'reflection') +c = run_command(meson_grabber, 'reflection', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'threads') +c = run_command(meson_grabber, 'threads', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -if get_option('MOD_JOBS') == true - c = run_command(meson_grabber, 'ssl') +if libssl.found() + c = run_command(meson_grabber, 'ssl', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp - c = run_command(meson_grabber, 'client-util/ssl') + c = run_command(meson_grabber, 'client-util/ssl', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif -c = run_command(meson_grabber, 'cache') +c = run_command(meson_grabber, 'cache', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'cache/memory') -tmp = c.stdout().strip().split('\n') -module_sources += tmp +if get_option('MOD_MEMORY') == true + c = run_command(meson_grabber, 'cache/memory', check: true) + tmp = c.stdout().strip().split('\n') + module_sources += tmp +endif if get_option('MOD_REDIS') == true - c = run_command(meson_grabber, 'cache/redis') + c = run_command(meson_grabber, 'cache/redis', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif if get_option('MOD_MEMCACHED') == true - c = run_command(meson_grabber, 'cache/memcached') + c = run_command(meson_grabber, 'cache/memcached', check: true) + tmp = c.stdout().strip().split('\n') + module_sources += tmp +endif + +c = run_command(meson_grabber, 'search', check: true) +tmp = c.stdout().strip().split('\n') +module_sources += tmp + +if get_option('MOD_SOLR') == true + c = run_command(meson_grabber, 'search/solr', check: true) + tmp = c.stdout().strip().split('\n') + module_sources += tmp +endif + +if get_option('MOD_ELASTIC') == true + c = run_command(meson_grabber, 'search/elasticsearch', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif -c = run_command(meson_grabber, 'sdorm') +c = run_command(meson_grabber, 'sdorm', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp if get_option('MOD_SDORM_SQL') == true - c = run_command(meson_grabber, 'sdorm/sql', 'all') + c = run_command(meson_grabber, 'sdorm/sql', 'all', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif if get_option('MOD_SDORM_MONGO') == true - c = run_command(meson_grabber, 'sdorm/mongo', 'all') + c = run_command(meson_grabber, 'sdorm/mongo', 'all', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif if get_option('MOD_SDORM_SCYLLA') == true - c = run_command(meson_grabber, 'sdorm/scylla', 'all') + c = run_command(meson_grabber, 'sdorm/scylla', 'all', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif -c = run_command(meson_grabber, 'serialization') +c = run_command(meson_grabber, 'serialization', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'serialization/xml') +c = run_command(meson_grabber, 'serialization/xml', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp -c = run_command(meson_grabber, 'serialization/json') +c = run_command(meson_grabber, 'serialization/json', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp if get_option('MOD_SER_BIN') == true - c = run_command(meson_grabber, 'serialization/binary') + c = run_command(meson_grabber, 'serialization/binary', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp module_includes += ['serialization/binary'] endif if get_option('MOD_JOBS') == true - c = run_command(meson_grabber, 'jobs') + c = run_command(meson_grabber, 'jobs', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif if get_option('WITH_PICOEV') == true - c = run_command(meson_grabber, 'common/picoev', 'all') + c = run_command(meson_grabber, 'common/picoev', 'all', check: true) tmp = c.stdout().strip().split('\n') module_sources += tmp endif diff --git a/src/modules/methinvoker/MethodInvoc.cpp b/src/modules/methinvoker/MethodInvoc.cpp index f8684e772..2669be591 100644 --- a/src/modules/methinvoker/MethodInvoc.cpp +++ b/src/modules/methinvoker/MethodInvoc.cpp @@ -92,7 +92,7 @@ void* MethodInvoc::service(void* arg) for (unsigned var = 0; var < argts.size(); var++) { void *value = NULL; - Element* arg = argts.at(var); + Element* arg = &argts.at(var); if(arg->getTagName()!="argument" || arg->getAttribute("type")=="") throw MethodInvokerException("every argument tag should have a name and type attribute\n",retValue); if(arg->getText()=="" && arg->getChildElements().size()==0) @@ -135,9 +135,9 @@ void* MethodInvoc::service(void* arg) } else if(arg->getAttribute("type")!="") { - Element* obj = arg->getChildElements().at(0); - std::string objxml = obj->render(); - std::string objClassName = obj->getTagName(); + Element obj = arg->getChildElements().at(0); + std::string objxml = obj.render(); + std::string objClassName = obj.getTagName(); int serOpt = SerializeBase::identifySerOption(arg->getAttribute("type")); value = ser.unSerializeUnknown(objxml,serOpt,arg->getAttribute("type")); } @@ -210,7 +210,7 @@ void* MethodInvoc::service(void* arg) { void* retobj = reflector.invokeMethodUnknownReturn(_temp,meth,valus,true); int serOpt = SerializeBase::identifySerOption(returnType); - std::string oxml = ser.serializeUnknown(retobj,serOpt,returnType); + std::string oxml = ser.serializeUnknown(retobj,serOpt,returnType,NULL); retValue = (""+oxml+""); } } diff --git a/src/modules/sconscript b/src/modules/sconscript new file mode 100644 index 000000000..6b9952cc3 --- /dev/null +++ b/src/modules/sconscript @@ -0,0 +1,49 @@ +import os + +Import('env') +Import('mod_mongo') +Import('mod_scylla') +Import('mod_solr') +Import('mod_elastic') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +env.Append(CPPPATH=['../framework']) + +# add source directories (at all levels) to the include path +# and add their contents are build targets +matches = [] +for root, dirnames, filenames in os.walk('./'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + if 'gtm' in dirPath: + continue + if 'mongo' in dirPath: + if not mod_mongo: + continue + if 'scylla' in dirPath: + if not mod_scylla: + continue + if 'solr' in dirPath: + if not mod_solr: + continue + if 'elasticsearch' in dirPath: + if not mod_elastic: + continue + + #print('adding dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + globStr = "%s/*.cpp" % dirPath + matches.append(Glob(globStr)) + +# debug output +#print('modules env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/ffead-modules', matches) diff --git a/src/modules/search/DocumentQuery.cpp b/src/modules/search/DocumentQuery.cpp index 5edf51bd2..6ea1ca3a5 100644 --- a/src/modules/search/DocumentQuery.cpp +++ b/src/modules/search/DocumentQuery.cpp @@ -29,18 +29,10 @@ DocumentQuery::DocumentQuery() { DocumentQuery::~DocumentQuery() { } -const std::string& DocumentQuery::getData() const { +std::string& DocumentQuery::getData() { return data; } -const std::map& DocumentQuery::getFields() const { - return fields; -} - -void DocumentQuery::setFields(const std::map& fields) { - this->fields = fields; -} - const std::string& DocumentQuery::getId() const { return id; } @@ -57,10 +49,22 @@ void DocumentQuery::setIndexName(const std::string& indexName) { this->indexName = indexName; } -const std::map& DocumentQuery::getProperties() const { +std::map& DocumentQuery::getProperties() { return properties; } void DocumentQuery::setProperties(const std::map& properties) { this->properties = properties; } + +void DocumentQuery::setData(const std::string &data) { + this->data = data; +} + +const std::string& DocumentQuery::getOpType() const { + return opType; +} + +void DocumentQuery::setOpType(const std::string &opType) { + this->opType = opType; +} diff --git a/src/modules/search/DocumentQuery.h b/src/modules/search/DocumentQuery.h index bf7d56c5f..3eae5b650 100644 --- a/src/modules/search/DocumentQuery.h +++ b/src/modules/search/DocumentQuery.h @@ -32,22 +32,26 @@ class DocumentQuery { private: std::string id; std::string indexName; + std::string opType; std::map properties; - std::map fields; std::string data; + friend class SolrSearch; public: DocumentQuery(); virtual ~DocumentQuery(); - const std::string& getData() const; - const std::map& getFields() const; + void setData(const std::string &data); + std::string& getData(); + std::map& getFields(); void setFields(const std::map& fields); const std::string& getId() const; void setId(const std::string& id); const std::string& getIndexName() const; void setIndexName(const std::string& indexName); - const std::map& getProperties() const; + std::map& getProperties(); void setProperties(const std::map& properties); + const std::string& getOpType() const; + void setOpType(const std::string &opType); template bool add(T& t) { if(data=="") { @@ -65,7 +69,7 @@ class DocumentQuery { } return false; } - template std::map addMulti(std::vector& vecT) { + template bool addMulti(std::vector& vecT) { if(data=="") { std::string cn = CastUtil::getClassName(vecT); int serOpt = SerializeBase::identifySerOption(cn); diff --git a/src/modules/search/IndexQuery.cpp b/src/modules/search/IndexQuery.cpp index 8a33b577e..aece764d7 100644 --- a/src/modules/search/IndexQuery.cpp +++ b/src/modules/search/IndexQuery.cpp @@ -35,12 +35,11 @@ IndexQuery& IndexQuery::setError(const std::string& error) { return *this; } -const std::map& IndexQuery::getMappings() const { +std::map& IndexQuery::getMappings() { return mappings; } -IndexQuery& IndexQuery::setMappings( - const std::map& mappings) { +IndexQuery& IndexQuery::setMappings(const std::map& mappings) { this->mappings = mappings; return *this; } @@ -54,16 +53,24 @@ IndexQuery& IndexQuery::setName(const std::string& name) { return *this; } -const std::map& IndexQuery::getProperties() const { +std::map& IndexQuery::getProperties() { return properties; } -IndexQuery& IndexQuery::setProperties( - const std::map& properties) { +IndexQuery& IndexQuery::setProperties(const std::map& properties) { this->properties = properties; return *this; } +std::map& IndexQuery::getExtras() { + return extras; +} + +IndexQuery& IndexQuery::setExtras(const std::map& extras) { + this->extras = extras; + return *this; +} + int IndexQuery::getStatus() const { return status; } @@ -73,6 +80,14 @@ IndexQuery& IndexQuery::setStatus(int status) { return *this; } +const std::string& IndexQuery::getOpType() const { + return opType; +} + +void IndexQuery::setOpType(const std::string &opType) { + this->opType = opType; +} + IndexQuery::~IndexQuery() { } diff --git a/src/modules/search/IndexQuery.h b/src/modules/search/IndexQuery.h index 860659b7f..dc849eea0 100644 --- a/src/modules/search/IndexQuery.h +++ b/src/modules/search/IndexQuery.h @@ -23,7 +23,7 @@ #ifndef INDEXQUERY_H_ #define INDEXQUERY_H_ -#include "string" +#include "CastUtil.h" #include "map" class IndexQuery { @@ -31,23 +31,30 @@ class IndexQuery { int status; std::string error; std::string name; - std::map properties; - std::map mappings; + std::string opType; + std::map properties;//settings + std::map mappings;//field mappings + std::map extras;//field mappings + friend class SolrSearch; public: IndexQuery(); virtual ~IndexQuery(); const std::string& getError() const; IndexQuery& setError(const std::string& error); - const std::map& getMappings() const; + std::map& getMappings(); IndexQuery& setMappings(const std::map& mappings); const std::string& getName() const; IndexQuery& setName(const std::string& name); - const std::map& getProperties() const; + std::map& getProperties(); IndexQuery& setProperties(const std::map& properties); + std::map& getExtras(); + IndexQuery& setExtras(const std::map& extras); int getStatus() const; IndexQuery& setStatus(int status); IndexQuery& addProperty(std::string& prop, std::string& value); IndexQuery& addMapping(std::string& prop, std::string& value); + const std::string& getOpType() const; + void setOpType(const std::string &opType); }; #endif /* INDEXQUERY_H_ */ diff --git a/src/modules/search/SearchEngineInterface.h b/src/modules/search/SearchEngineInterface.h index 9067913a7..a0a3bf445 100644 --- a/src/modules/search/SearchEngineInterface.h +++ b/src/modules/search/SearchEngineInterface.h @@ -22,20 +22,22 @@ #ifndef SEARCHENGINEINTERFACE_H_ #define SEARCHENGINEINTERFACE_H_ -#include "IndexQuery.h" -#include "DocumentQuery.h" -#include "SearchQuery.h" #include "GenericObject.h" #include "ConnectionPooler.h" #include "HttpRequest.h" #include "HttpResponse.h" #include "JSONUtil.h" #include "HttpClient.h" +#include "IndexQuery.h" +#include "DocumentQuery.h" +#include "SearchQuery.h" class SearchEngineInterface { +protected: ConnectionPooler* pool; std::string appName; Reflector* reflector; + friend class SearchEngineManager; public: SearchEngineInterface(); virtual ~SearchEngineInterface(); @@ -43,27 +45,59 @@ class SearchEngineInterface { virtual void createIndex(IndexQuery& iq)=0; virtual void updateIndex(IndexQuery& iq)=0; virtual void removeIndex(IndexQuery& iq)=0; + virtual void indexOp(IndexQuery& iq)=0; virtual void add(DocumentQuery& iq)=0; virtual void update(DocumentQuery& iq)=0; virtual void remove(DocumentQuery& iq)=0; + virtual void docOp(DocumentQuery& iq)=0; virtual std::string query(SearchQuery& q)=0; - - template T get(GenericObject& id) { - +#ifdef HAVE_RAPID_JSON + std::string query(SearchQuery& q, rapidjson::Value& doc) { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc.Accept(w); + q.setData(std::string(b.GetString(), b.GetSize())); + return query(q); + } +#endif + + template T get(std::string indxName, std::string& id) { + T t; + std::string cn = CastUtil::getClassName(t); + int serOpt = SerializeBase::identifySerOption(cn); + SearchQuery q; + q.setIndexName(indxName); + q.setId(id); + std::string resp = query(q); + return JSONSerialize::unserialize(resp, serOpt); } - template std::vector getAll() { - + template std::vector getAll(std::string indxName) { + T t; + std::string cn = CastUtil::getClassName(t); + int serOpt = SerializeBase::identifySerOption(cn); + SearchQuery q; + q.setIndexName(indxName); + std::string resp = query(q); + return JSONSerialize::unserialize>(resp, serOpt); } template std::vector getList(SearchQuery& q) { - + T t; + std::string cn = CastUtil::getClassName(t); + int serOpt = SerializeBase::identifySerOption(cn); + std::string resp = query(q); + return JSONSerialize::unserialize>(resp, serOpt); } template T get(SearchQuery& q) { - + T t; + std::string cn = CastUtil::getClassName(t); + int serOpt = SerializeBase::identifySerOption(cn); + std::string resp = query(q); + return JSONSerialize::unserialize(resp, serOpt); } }; diff --git a/src/modules/search/SearchEngineManager.cpp b/src/modules/search/SearchEngineManager.cpp index 0183b99ba..bb4108315 100644 --- a/src/modules/search/SearchEngineManager.cpp +++ b/src/modules/search/SearchEngineManager.cpp @@ -23,9 +23,35 @@ #include "SearchEngineManager.h" std::map SearchEngineManager::engines; -std::string SearchEngineManager::defEngineName; +std::map SearchEngineManager::defEngineNames; +std::map SearchEngineManager::appInitCompletionStatus; + +void SearchEngineManager::triggerAppInitCompletion(std::string appNameN) { + std::string appName = appNameN; + if(appName=="") { + appName = CommonUtils::getAppName(); + } else { + StringUtil::replaceAll(appName, "-", "_"); + RegexUtil::replace(appName, "[^a-zA-Z0-9_]+", ""); + } + if(appInitCompletionStatus.find(appName)!=appInitCompletionStatus.end()) { + appInitCompletionStatus[appName] = true; + } +} -void SearchEngineManager::initCache(const ConnectionProperties& props, const std::string& appName) { +bool SearchEngineManager::isInitCompleted() { + bool flag = true; + if(appInitCompletionStatus.size()>0) { + std::map::iterator it = appInitCompletionStatus.begin(); + for(;it!=appInitCompletionStatus.end();++it) { + flag &= it->second; + } + } + return flag; +} + +void SearchEngineManager::initSearch(const ConnectionProperties& props, const std::string& appName, GetClassBeanIns f) { + Logger logger = LoggerFactory::getLogger("SearchEngineManager"); std::string name = StringUtil::trimCopy(props.getName()); if(name=="") { @@ -38,7 +64,7 @@ void SearchEngineManager::initCache(const ConnectionProperties& props, const std throw std::runtime_error("Search Engine Already exists"); } if(props.getProperty("_isdefault_")=="true") { - defEngineName = StringUtil::trimCopy(props.getName()); + defEngineNames[appName] = StringUtil::trimCopy(props.getName()); } SearchEngineManager* mgr = new SearchEngineManager(props); engines[name] = mgr; @@ -49,26 +75,25 @@ void SearchEngineManager::initCache(const ConnectionProperties& props, const std std::vector v; StringUtil::split(v, meth, "."); if(v.size()==2) { - std::string scappName = appName; - StringUtil::replaceAll(scappName, "-", "_"); - RegexUtil::replace(scappName, "[^a-zA-Z0-9_]+", ""); - CommonUtils::setAppName(scappName); - ClassInfo* clas = ref->getClassInfo(v.at(0), appName); - if(clas->getClassName()!="") { - args argus; - vals valus; - const Constructor& ctor = clas->getConstructor(argus); - void* _temp = ref->newInstanceGVP(ctor); - try { - if(_temp!=NULL) { - const Method& meth = clas->getMethod(v.at(1), argus); - if(meth.getMethodName()!="") - { - ref->invokeMethodGVP(_temp, meth, valus); - } + CommonUtils::setAppName(appName); + ClassBeanIns cbi; + f(v.at(0), appNameN, &cbi); + void* _temp = cbi.instance; + try { + if(_temp!=NULL) { + args argus; + vals valus; + const Method& meth = cbi.clas->getMethod(v.at(1), argus); + if(meth.getMethodName()!="") + { + appInitCompletionStatus[appName] = false; + ref->invokeMethodGVP(_temp, meth, valus); } - } catch(const std::exception& e) { } + } catch(const std::exception& e) { + logger.info("Error during init call for Seacrh " + appNameN + "@" + props.getName() + " " + std::string(e.what())); + } + if(cbi.cleanUp) { ref->destroy(_temp, v.at(0), appName); } } @@ -93,7 +118,7 @@ SearchEngineInterface* SearchEngineManager::getImpl(std::string name) { std::string appName = CommonUtils::getAppName(); StringUtil::trim(name); if(name=="") { - name = defEngineName; + name = defEngineNames[appName]; } name = appName + name; if(engines.find(name)==engines.end()) @@ -104,14 +129,14 @@ SearchEngineInterface* SearchEngineManager::getImpl(std::string name) { SearchEngineInterface* t = NULL; if(StringUtil::toLowerCopy(engine->props.getType())=="elasticsearch") { -#ifdef INC_MEMCACHED +#ifdef HAVE_ELASTIC t = NULL; #endif } else if(StringUtil::toLowerCopy(engine->props.getType())=="solr") { -#ifdef INC_REDISCACHE - t = NULL; +#ifdef HAVE_SOLR + t = new SolrSearch(engine->pool); #endif } if(t == NULL) @@ -125,7 +150,6 @@ SearchEngineInterface* SearchEngineManager::getImpl(std::string name) { SearchEngineManager::SearchEngineManager(const ConnectionProperties& props) { this->props = props; - reflector = NULL; this->pool = new SearchEngineConnectionPool(props); } @@ -133,7 +157,6 @@ SearchEngineManager::~SearchEngineManager() { } void SearchEngineConnectionPool::initEnv() { - HttpClient::init(); } void* SearchEngineConnectionPool::newConnection(const bool& isWrite, const ConnectionNode& node) { @@ -148,7 +171,6 @@ void SearchEngineConnectionPool::destroy() { } SearchEngineConnectionPool::SearchEngineConnectionPool(const ConnectionProperties& props) { - logger = LoggerFactory::getLogger("SearchEngineConnectionPool"); createPool(props); } diff --git a/src/modules/search/SearchEngineManager.h b/src/modules/search/SearchEngineManager.h index cce3ee1ef..169e79216 100644 --- a/src/modules/search/SearchEngineManager.h +++ b/src/modules/search/SearchEngineManager.h @@ -29,9 +29,14 @@ #include "Mutex.h" #include "SearchEngineInterface.h" #include "HttpClient.h" +#ifdef HAVE_SOLR +#include "SolrSearch.h" +#endif +#ifdef HAVE_ELASTIC +#include "ElasticSearch.h" +#endif class SearchEngineConnectionPool: public ConnectionPooler { - Logger logger; void initEnv(); void* newConnection(const bool& isWrite, const ConnectionNode& node); void closeConnection(void* conn); @@ -43,17 +48,19 @@ class SearchEngineConnectionPool: public ConnectionPooler { class SearchEngineManager { static std::map engines; - static std::string defEngineName; + static std::map defEngineNames; + static std::map appInitCompletionStatus; ConnectionProperties props; ConnectionPooler* pool; - Reflector* reflector; - static void initCache(const ConnectionProperties& props, const std::string& appName); + static void initSearch(const ConnectionProperties& props, const std::string& appName, GetClassBeanIns f); static void destroy(); SearchEngineManager(const ConnectionProperties& props); friend class ConfigurationHandler; public: virtual ~SearchEngineManager(); static SearchEngineInterface* getImpl(std::string name = ""); + static void triggerAppInitCompletion(std::string appName = ""); + static bool isInitCompleted(); }; #endif /* SEARCHENGINEMANAGER_H_ */ diff --git a/src/modules/search/SearchQuery.cpp b/src/modules/search/SearchQuery.cpp index 0ee772ae5..66fbd6221 100644 --- a/src/modules/search/SearchQuery.cpp +++ b/src/modules/search/SearchQuery.cpp @@ -23,11 +23,38 @@ #include "SearchQuery.h" SearchQuery::SearchQuery() { - - } SearchQuery::~SearchQuery() { - } +const std::string& SearchQuery::getIndexName() const { + return indexName; +} + +void SearchQuery::setIndexName(const std::string &indexName) { + this->indexName = indexName; +} + +std::map& SearchQuery::getProperties() { + return properties; +} + +const std::string& SearchQuery::getId() const { + return id; +} + +void SearchQuery::setData(const std::string &data) { +} + +std::string& SearchQuery::getData() { + return data; +} + +void SearchQuery::setId(const std::string &id) { + this->id = id; +} + +void SearchQuery::setProperties(const std::map &properties) { + this->properties = properties; +} diff --git a/src/modules/search/SearchQuery.h b/src/modules/search/SearchQuery.h index 4d1d9f626..122525cdf 100644 --- a/src/modules/search/SearchQuery.h +++ b/src/modules/search/SearchQuery.h @@ -22,11 +22,26 @@ #ifndef SEARCHQUERY_H_ #define SEARCHQUERY_H_ +#include "CastUtil.h" class SearchQuery { + std::string id; + std::string indexName; + std::map properties; + std::string data; + friend class SolrSearch; public: SearchQuery(); virtual ~SearchQuery(); + + void setData(const std::string &data); + std::string& getData(); + const std::string& getIndexName() const; + void setIndexName(const std::string &indexName); + std::map& getProperties(); + void setProperties(const std::map &properties); + const std::string& getId() const; + void setId(const std::string &id); }; #endif /* SEARCHQUERY_H_ */ diff --git a/src/modules/search/elasticsearch/ElasticSearch.cpp b/src/modules/search/elasticsearch/ElasticSearch.cpp index 478b43f0f..76e2b21ae 100644 --- a/src/modules/search/elasticsearch/ElasticSearch.cpp +++ b/src/modules/search/elasticsearch/ElasticSearch.cpp @@ -22,9 +22,11 @@ #include "ElasticSearch.h" +#ifdef HAVE_ELASTIC std::string ElasticSearch::COLL_URL = "/admin/collections"; -ElasticSearch::ElasticSearch() { +ElasticSearch::ElasticSearch(ConnectionPooler* pool) { + this->pool = pool; } ElasticSearch::~ElasticSearch() { @@ -68,3 +70,4 @@ void ElasticSearch::remove(DocumentQuery& iq) { std::string ElasticSearch::query(SearchQuery& q) { return std::string(); } +#endif diff --git a/src/modules/search/elasticsearch/ElasticSearch.h b/src/modules/search/elasticsearch/ElasticSearch.h index ee1abda8a..ca5d57c8c 100644 --- a/src/modules/search/elasticsearch/ElasticSearch.h +++ b/src/modules/search/elasticsearch/ElasticSearch.h @@ -23,11 +23,13 @@ #ifndef ELASTICSEARCH_H_ #define ELASTICSEARCH_H_ #include "SearchEngineInterface.h" +#ifdef HAVE_ELASTIC +#include class ElasticSearch: public SearchEngineInterface { static std::string COLL_URL; public: - ElasticSearch(); + ElasticSearch(ConnectionPooler* pool); virtual ~ElasticSearch(); void createIndex(IndexQuery& iq); @@ -42,3 +44,4 @@ class ElasticSearch: public SearchEngineInterface { }; #endif /* ELASTICSEARCH_H_ */ +#endif diff --git a/src/modules/search/solr/SolrSearch.cpp b/src/modules/search/solr/SolrSearch.cpp index 27d481f92..8684b1562 100644 --- a/src/modules/search/solr/SolrSearch.cpp +++ b/src/modules/search/solr/SolrSearch.cpp @@ -26,17 +26,15 @@ std::string SolrSearch::COLL_C_PARAMS = ",router.name,numShards,shards,replicati "maxShardsPerNode,createNodeSet,createNodeSet.shuffle,collection.configName,router.field,autoAddReplicas,async,rule,snitch,policy,waitForFinalState,"; std::string SolrSearch::COLL_M_PARAMS = ",maxShardsPerNode,replicationFactor,autoAddReplicas,collection.configName,rule,snitch,policy,"; -SolrSearch::SolrSearch() { - isCloud = StringUtil::toLowerCopy(pool->getProperties().getProperty("isCloud"))=="true"; +SolrSearch::SolrSearch(ConnectionPooler* pool) { + this->pool = pool; + v2 = StringUtil::toLowerCopy(pool->getProperties().getProperty("v2"))=="true"; } SolrSearch::~SolrSearch() { } -void SolrSearch::createIndex(IndexQuery& iq) { - if(!isCloud) { - return; - } +void SolrSearch::indexOp(IndexQuery& iq) { Connection* connection = pool->checkout(); HttpClient* hc = (HttpClient*)connection->getConn(); HttpRequest rq; @@ -46,97 +44,47 @@ void SolrSearch::createIndex(IndexQuery& iq) { std::string uri; uri.append("/admin/collections"); - uri.append("?action=CREATE"); - uri.append("&name=" + iq.getName()); + uri.append("?wt=json&action="); + uri.append(iq.getOpType()); for(it=iq.properties.begin();it!=iq.properties.end();++it) { - if(COLL_C_PARAMS.find(","+it->first+",")!=std::string::npos) { + //if(COLL_C_PARAMS.find(","+it->first+",")!=std::string::npos) { + if(it->first.find("____")==0) continue; uri.append("&" + it->first + "=" + it->second); - } + //} } rq.setUrl(uri); rq.setMethod("GET"); hc->execute(&rq, &rs, p); + pool->release(connection); if(rs.getStatusCode()!="200") { if(rs.getContent()!="") { - throw std::runtime_error("Unable to create index " + rs.getContent()); + throw std::runtime_error("Error during index op " + rs.getContent()); } - throw std::runtime_error("Unable to create index"); + throw std::runtime_error("Error during index op "); } } -void SolrSearch::updateIndex(IndexQuery& iq) { - if(!isCloud) { - return; - } - Connection* connection = pool->checkout(); - HttpClient* hc = (HttpClient*)connection->getConn(); - HttpRequest rq; - HttpResponse rs; - propMap p; - std::map::iterator it; - - std::string uri; - uri.append("/admin/collections/" + iq.getName() + "/update/json?split=/"); - uri.append("?action=MODIFYCOLLECTION"); - uri.append("&collection=" + iq.getName()); - for(it=iq.properties.begin();it!=iq.properties.end();++it) { - if(COLL_M_PARAMS.find(","+it->first+",")!=std::string::npos) { - uri.append("&" + it->first + "=" + it->second); - } - } - - rq.setUrl(uri); - rq.setMethod("GET"); +void SolrSearch::createIndex(IndexQuery& iq) { + iq.getProperties().emplace("action", "CREATE"); + iq.getProperties().emplace("name", iq.getName()); + return indexOp(iq); +} - hc->execute(&rq, &rs, p); - if(rs.getStatusCode()!="200") { - if(rs.getContent()!="") { - throw std::runtime_error("Unable to update index " + rs.getContent()); - } - throw std::runtime_error("Unable to update index"); - } +void SolrSearch::updateIndex(IndexQuery& iq) { + iq.getProperties().emplace("action", "MODIFYCOLLECTION"); + iq.getProperties().emplace("name", iq.getName()); + return indexOp(iq); } void SolrSearch::removeIndex(IndexQuery& iq) { - Connection* connection = pool->checkout(); - HttpClient* hc = (HttpClient*)connection->getConn(); - HttpRequest rq; - HttpResponse rs; - propMap p; - std::map::iterator it; - - std::string uri; - if(!isCloud) { - uri.append("/admin/cores"); - uri.append("?action=UNLOAD"); - uri.append("&core=" + iq.getName()); - if(iq.properties.find("deleteInstanceDir")!=iq.properties.end()) { - uri.append("&deleteInstanceDir=" + iq.properties["deleteInstanceDir"]); - } - } else { - uri.append("/admin/collections"); - uri.append("?action=DELETE"); - uri.append("&name=" + iq.getName()); - if(iq.properties.find("async")!=iq.properties.end()) { - uri.append("&async=" + iq.properties["async"]); - } - } - - rq.setUrl(uri); - rq.setMethod("GET"); - - hc->execute(&rq, &rs, p); - if(rs.getStatusCode()!="200") { - if(rs.getContent()!="") { - throw std::runtime_error("Unable to delete index " + rs.getContent()); - } - throw std::runtime_error("Unable to delete index"); - } + iq.getProperties().emplace("action", "DELETE"); + iq.getProperties().emplace("name", iq.getName()); + return indexOp(iq); } -void SolrSearch::add(DocumentQuery& iq) { +void SolrSearch::docOp(DocumentQuery& iq) { Connection* connection = pool->checkout(); HttpClient* hc = (HttpClient*)connection->getConn(); HttpRequest rq; @@ -145,35 +93,62 @@ void SolrSearch::add(DocumentQuery& iq) { std::map::iterator it; std::string uri; - if(!isCloud) { - uri.append("/api/cores/" + iq.getIndexName() + "/update/json?split=/"); + if(!v2) { + uri.append("/" + iq.getIndexName() + "/update" + iq.getProperties()["____urprefix"] + "?"); } else { - uri.append("/api/collections/" + iq.getIndexName() + "/update/json?split=/"); + uri.append("/api/c/" + iq.getIndexName() + "/update" + iq.getProperties()["____urprefix"] + "?"); } - if(iq.properties.find("commit")!=iq.properties.end()) { - uri.append("&commit=" + iq.properties["commit"]); + for(it=iq.properties.begin();it!=iq.properties.end();++it) { + //if(COLL_C_PARAMS.find(","+it->first+",")!=std::string::npos) { + if(it->first.find("____")==0) continue; + uri.append("&" + it->first + "=" + it->second); + //} } rq.setUrl(uri); rq.setMethod("POST"); - rq.setContent(iq.getData()); + rq.setContent(std::move(iq.getData())); rq.addHeader(HttpRequest::ContentType, "application/json"); hc->execute(&rq, &rs, p); + pool->release(connection); if(rs.getStatusCode()!="200") { if(rs.getContent()!="") { - throw std::runtime_error("Unable to add document " + rs.getContent()); + throw std::runtime_error("Error during document op " + rs.getContent()); } - throw std::runtime_error("Unable to add document"); + throw std::runtime_error("Error during document op "); + } +} + +void SolrSearch::add(DocumentQuery& iq) { + if(!v2) { + iq.getProperties().emplace("____urprefix", "/json/docs"); + } else { + iq.getProperties().emplace("____urprefix", "/json"); } + docOp(iq); } void SolrSearch::update(DocumentQuery& iq) { - add(iq); + if(!v2) { + iq.getProperties().emplace("____urprefix", "/json/docs"); + } else { + iq.getProperties().emplace("____urprefix", "/json"); + } + docOp(iq); } void SolrSearch::remove(DocumentQuery& iq) { + if(iq.getId().length()>0) { + iq.setData("{\"delete\":{\"query\":\"id:"+iq.getId()+"\"}}"); + } else { + iq.setData("{\"delete\":{\"query\":\""+iq.getData()+"\"}}"); + } + docOp(iq); +} + +std::string SolrSearch::query(SearchQuery& iq) { Connection* connection = pool->checkout(); HttpClient* hc = (HttpClient*)connection->getConn(); HttpRequest rq; @@ -182,30 +157,34 @@ void SolrSearch::remove(DocumentQuery& iq) { std::map::iterator it; std::string uri; - if(!isCloud) { - uri.append("/api/cores/" + iq.getIndexName() + "/update/json?"); + if(!v2) { + uri.append("/" + iq.getIndexName() + "/query?"); } else { - uri.append("/api/collections/" + iq.getIndexName() + "/update/json?"); + uri.append("/api/c/" + iq.getIndexName() + "/query?"); } - if(iq.properties.find("commit")!=iq.properties.end()) { - uri.append("&commit=" + iq.properties["commit"]); + for(it=iq.properties.begin();it!=iq.properties.end();++it) { + //if(COLL_C_PARAMS.find(","+it->first+",")!=std::string::npos) { + if(it->first.find("____")==0) continue; + uri.append("&" + it->first + "=" + it->second); + //} } rq.setUrl(uri); rq.setMethod("POST"); - rq.setContent("{\"delete\": {\"id\":\""+iq.getId()+"\"}}"); + if(iq.getData().length()>0) { + rq.setContent(std::move(iq.getData())); + } rq.addHeader(HttpRequest::ContentType, "application/json"); hc->execute(&rq, &rs, p); + pool->release(connection); if(rs.getStatusCode()!="200") { if(rs.getContent()!="") { - throw std::runtime_error("Unable to add document " + rs.getContent()); + throw std::runtime_error("Error during search op " + rs.getContent()); } - throw std::runtime_error("Unable to add document"); + throw std::runtime_error("Error during search op "); } -} -std::string SolrSearch::query(SearchQuery& q) { - return std::string(); + return rs.getContent(); } diff --git a/src/modules/search/solr/SolrSearch.h b/src/modules/search/solr/SolrSearch.h index 39144e81d..a32c1e929 100644 --- a/src/modules/search/solr/SolrSearch.h +++ b/src/modules/search/solr/SolrSearch.h @@ -26,18 +26,21 @@ class SolrSearch: public SearchEngineInterface { static std::string COLL_C_PARAMS, COLL_M_PARAMS; - bool isCloud; + bool v2; + public: - SolrSearch(); + SolrSearch(ConnectionPooler* pool); virtual ~SolrSearch(); void createIndex(IndexQuery& iq); void updateIndex(IndexQuery& iq); void removeIndex(IndexQuery& iq); + void indexOp(IndexQuery& iq); void add(DocumentQuery& iq); void update(DocumentQuery& iq); void remove(DocumentQuery& iq); + void docOp(DocumentQuery& iq); std::string query(SearchQuery& q); }; diff --git a/src/modules/server-util/ServiceHandler.cpp b/src/modules/server-util/ServiceHandler.cpp index f61590cc7..423fc6ab4 100644 --- a/src/modules/server-util/ServiceHandler.cpp +++ b/src/modules/server-util/ServiceHandler.cpp @@ -58,26 +58,11 @@ void* ServiceHandler::closeConnections(void *arg) { return NULL; } -static const char* get_date() { - time_t t; - struct tm tm; - static const char *days[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; - static const char *months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - static __thread char date[59] = "Date: Thu, 01 Jan 1970 00:00:00 GMT\r\nServer: FFEAD 2.0\r\n"; - - time(&t); - gmtime_r(&t, &tm); - strftime(date, 58, "Date: ---, %d --- %Y %H:%M:%S GMT\r\nServer: FFEAD 2.0\r\n", &tm); - memcpy(date + 6, days[tm.tm_wday], 3); - memcpy(date + 14, months[tm.tm_mon], 3); - - return date; -} void* ServiceHandler::timer(void *arg) { ServiceHandler* ths = (ServiceHandler*)arg; while(ths->run) { - CommonUtils::dateStr = get_date(); + CommonUtils::setDate(); Thread::sSleep(1); } return NULL; @@ -128,8 +113,8 @@ void ServiceHandler::start() { run = true; Thread* mthread = new Thread(&closeConnections, this); mthread->execute(-1); - Thread* tthread = new Thread(&timer, this); - tthread->execute(); + //Thread* tthread = new Thread(&timer, this); + //tthread->execute(); } } diff --git a/src/modules/server-util/SocketInterface.h b/src/modules/server-util/SocketInterface.h index d3ff84e29..88d7f56d7 100644 --- a/src/modules/server-util/SocketInterface.h +++ b/src/modules/server-util/SocketInterface.h @@ -30,13 +30,11 @@ #include "string" #include "Mutex.h" #include -#include "LoggerFactory.h" #ifdef HAVE_SSLINC #include #include #include "SSLHandler.h" #endif -#include "Timer.h" #include #include #include "vector" @@ -51,6 +49,7 @@ #include #endif #include "blockingconcurrentqueue.h" +#include "StringUtil.h" class BaseSocket; @@ -77,7 +76,7 @@ class ResponseData { class EventHandler { moodycamel::BlockingConcurrentQueue* wQ; template void queueWrite(Func1 f, BaseSocket* bs, void* arg) { - wQ->enqueue(std::move(SockWriteRequest(f, bs, arg))); + wQ->enqueue(SockWriteRequest(f, bs, arg)); } friend class BaseSocket; friend class RequestHandler2; diff --git a/src/modules/xmake.lua b/src/modules/xmake.lua index f9c2c07ca..e2e9e567f 100644 --- a/src/modules/xmake.lua +++ b/src/modules/xmake.lua @@ -36,6 +36,13 @@ target("ffead-modules") add_files("$(projectdir)/src/modules/cache/memcached/*.cpp") add_links("memcached","memcachedutil") end + add_files("$(projectdir)/src/modules/search/*.cpp") + if has_config("MOD_SOLR") then + add_files("$(projectdir)/src/modules/search/solr/*.cpp") + end + if has_config("MOD_ELASTIC") then + add_files("$(projectdir)/src/modules/search/elasticsearch/*.cpp") + end add_files("$(projectdir)/src/modules/sdorm/*.cpp") if has_config("MOD_SDORM_SQL") then add_files("$(projectdir)/src/modules/sdorm/sql/*.cpp") diff --git a/src/server/embedded/CHServer.cpp b/src/server/embedded/CHServer.cpp index cdd962c29..003bd7bfd 100644 --- a/src/server/embedded/CHServer.cpp +++ b/src/server/embedded/CHServer.cpp @@ -837,8 +837,14 @@ int CHServer::entryPoint(int vhostNum, bool isMain, std::string serverRootDirect std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " xmake"; #elif defined(BUILD_MESON) std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " meson"; +#elif defined(BUILD_SCONS) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " scons"; +#elif defined(BUILD_BAZEL) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " bazel"; +#elif defined(BUILD_SHELLB) + std::string compres = respath+"rundyn-automake.sh "+serverRootDirectory + " shellb"; #else - logger << "Invalid Build Type specified, only cmake, xmake and meson supported...\n" << std::endl; + logger << "Invalid Build Type specified, only cmake, xmake, meson, scons, bazel and shellb supported...\n" << std::endl; #endif std::string output = ScriptHandler::execute(compres, true); logger << "Intermediate code generation task\n\n" << std::endl; @@ -1286,6 +1292,10 @@ void CHServer::serve(std::string port, std::string ipaddr, int thrdpsiz, std::st ConfigurationHandler::initializeCaches(); logger << ("Initializing Caches done....") << std::endl; + logger << ("Initializing Searches....") << std::endl; + ConfigurationHandler::initializeSearches(); + logger << ("Initializing Searches done....") << std::endl; + //struct sockaddr_storage their_addr; // connector's address information //socklen_t sin_size; SOCKET sockfd = Server::createListener(ipaddr, CastUtil::toInt(port), true, isSinglEVH); @@ -1342,7 +1352,8 @@ void CHServer::serve(std::string port, std::string ipaddr, int thrdpsiz, std::st struct stat buffer; while(stat (serverCntrlFileNm.c_str(), &buffer) == 0) { - Thread::sSleep(10); + CommonUtils::setDate(); + Thread::sSleep(1); //CommonUtils::printStats(); } @@ -1369,6 +1380,8 @@ void CHServer::serve(std::string port, std::string ipaddr, int thrdpsiz, std::st #endif ConfigurationHandler::destroyCaches(); + ConfigurationHandler::destroySearches(); + ConfigurationData::getInstance()->clearAllSingletonBeans(); ConfigurationData::clearInstance(); @@ -1450,5 +1463,5 @@ Http11Socket* CHServer::createSocketInterface2(SOCKET fd) { bool CHServer::doRegisterListenerFunc() { struct stat buffer; - return stat (serverCntrlFileNm.c_str(), &buffer)==0 && CacheManager::isInitCompleted() && DataSourceManager::isInitCompleted(); + return stat (serverCntrlFileNm.c_str(), &buffer)==0 && CacheManager::isInitCompleted() && DataSourceManager::isInitCompleted() && SearchEngineManager::isInitCompleted(); } diff --git a/src/server/sconscript b/src/server/sconscript new file mode 100644 index 000000000..0771354f4 --- /dev/null +++ b/src/server/sconscript @@ -0,0 +1,26 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +env.Append(CPPPATH=['../framework']) +for root, dirnames, filenames in os.walk('../modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = ['embedded/CHServer.cpp'] +env.Append(CPPPATH=['.']) + +#print('server env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.Program('../../bin/ffead-cpp', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/tests/sconscript b/tests/sconscript new file mode 100644 index 000000000..564195987 --- /dev/null +++ b/tests/sconscript @@ -0,0 +1,26 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +env.Append(CPPPATH=['../src/framework']) +for root, dirnames, filenames in os.walk('../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = ['FfeadServerTestSuite.cpp'] +env.Append(CPPPATH=['.']) + +#print('tests env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.Program('../bin/tests', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../bin') diff --git a/web/default/sconscript b/web/default/sconscript new file mode 100644 index 000000000..1f172772d --- /dev/null +++ b/web/default/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/default', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/flexApp/sconscript b/web/flexApp/sconscript new file mode 100644 index 000000000..1ca89ae69 --- /dev/null +++ b/web/flexApp/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/flexApp', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/markers/sconscript b/web/markers/sconscript new file mode 100644 index 000000000..ec24be43f --- /dev/null +++ b/web/markers/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/markers', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/oauthApp/sconscript b/web/oauthApp/sconscript new file mode 100644 index 000000000..d3125cecc --- /dev/null +++ b/web/oauthApp/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/oauthApp', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/peer-server/sconscript b/web/peer-server/sconscript new file mode 100644 index 000000000..24b782f5b --- /dev/null +++ b/web/peer-server/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/peer-server', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/t1/sconscript b/web/t1/sconscript new file mode 100644 index 000000000..28babce0f --- /dev/null +++ b/web/t1/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/t1', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/web/t2/sconscript b/web/t2/sconscript new file mode 100644 index 000000000..63951d815 --- /dev/null +++ b/web/t2/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/t2', matches, LIBS=['ffead-modules', 'ffead-framework', 'mongoc-1.0', 'bson-1.0'], LIBPATH='../../bin/') diff --git a/web/t3/sconscript b/web/t3/sconscript new file mode 100644 index 000000000..f72a6e52a --- /dev/null +++ b/web/t3/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/t3', matches, LIBS=['ffead-modules', 'ffead-framework', 'pq'], LIBPATH='../../bin/') diff --git a/web/t4/sconscript b/web/t4/sconscript new file mode 100644 index 000000000..b57e97073 --- /dev/null +++ b/web/t4/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/t4', matches, LIBS=['ffead-modules', 'ffead-framework', 'pq'], LIBPATH='../../bin/') diff --git a/web/t5/sconscript b/web/t5/sconscript new file mode 100644 index 000000000..49b458497 --- /dev/null +++ b/web/t5/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/t5', matches, LIBS=['ffead-modules', 'ffead-framework', 'pq'], LIBPATH='../../bin/') diff --git a/web/te-benchmark/sconscript b/web/te-benchmark/sconscript new file mode 100644 index 000000000..56513a43c --- /dev/null +++ b/web/te-benchmark/sconscript @@ -0,0 +1,29 @@ +import os + +Import('env') + +# copy parent environment so our alterations don't affect it +env = env.Clone() + +# add src dirs to include path +env.Append(CPPPATH=['../../src/framework']) +for root, dirnames, filenames in os.walk('../../src/modules'): + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for dirname in dirnames: + dirPath = os.path.join(root,dirname) + if env['PLATFORM'] != 'mingw' and 'wepoll' in dirPath: + continue + #print('adding src dir to path: %s' % dirPath) + env.Append(CPPPATH=[dirPath]) + +matches = [] +env.Append(CPPPATH=['include']) +matches.append(Glob("src/*.cpp")) + +# debug output +#print('framework env CPPPATH:') +#print(env['CPPPATH']) + +# build a shared library +env.SharedLibrary('../../bin/te-benchmark', matches, LIBS=['ffead-modules', 'ffead-framework'], LIBPATH='../../bin/') diff --git a/xmake.lua b/xmake.lua index a5d7c40a7..b4662e712 100644 --- a/xmake.lua +++ b/xmake.lua @@ -76,6 +76,12 @@ option("MOD_REDIS") raise('hiredis library not found') end option:add(l) + l = find_package("redis++") + if l then + option:add(l) + option:set("configvar", "HAVE_REDIS_CLUSTERINC", 1) + option:set("configvar", "HAVE_REDIS_CLUSTERLIB", 1) + end option:set("configvar", "HAVE_REDISINC", 1) option:set("configvar", "HAVE_REDISLIB", 1) option:set("configvar", "INC_REDISCACHE", 1) @@ -83,6 +89,40 @@ option("MOD_REDIS") end) option_end() +option("MOD_SOLR") + set_default(false) + set_showmenu(true) + set_description("Enable SOLR Search module") + after_check(function (option) + if option:enabled() then + option:set("configvar", "HAVE_SOLR", 1) + end + end) +option_end() + +option("MOD_ELASTIC") + set_default(false) + set_showmenu(true) + set_description("Enable Elasticsearch Search module") + after_check(function (option) + if option:enabled() then + import("lib.detect.find_package") + local l = find_package("elasticlient") + if not l then + raise('elasticlient library not found') + end + option:add(l) + import("lib.detect.has_cxxincludes") + local ok = has_cxxincludes({"elasticlient/client.h"}, {target = target}) + if ok then + option:set("configvar", "HAVE_ELASTIC", 1) + else + raise('elasticlient includes not found') + end + end + end) +option_end() + option("MOD_SER_BIN") set_default(false) set_showmenu(true) @@ -708,7 +748,7 @@ function getOptions() "SELECT", "POLL", "DEVPOLL", "IOURING", "EPOLL", "KQUEUE", "EVPORT", "ACCEPT4", "TCP_QUICKACK", "TCP_DEFER_ACCEPT", "TCP_FASTOPEN", "MOD_MEMORY","MOD_MEMCACHED", "MOD_REDIS", "MOD_SDORM_SQL", "MOD_SDORM_MONGO", "MOD_SDORM_SCYLLA", "MOD_SER_BIN", "MOD_JOBS", "SRV_EMB", "WITH_RAPIDJSON", - "WITH_PUGIXML", "WITH_PICOEV"} + "WITH_PUGIXML", "WITH_PICOEV", "MOD_ELASTIC", "MOD_SOLR"} end includes("src/modules") From 49eacceca9d8e2ca261f5599d627648a3b642637 Mon Sep 17 00:00:00 2001 From: Sumeet Chhetri Date: Sun, 8 May 2022 15:37:39 +0530 Subject: [PATCH 2/3] Added workflow for scons build [skip_build] --- .github/workflows/ci-scons.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/ci-scons.yml diff --git a/.github/workflows/ci-scons.yml b/.github/workflows/ci-scons.yml new file mode 100644 index 000000000..a1e1e780d --- /dev/null +++ b/.github/workflows/ci-scons.yml @@ -0,0 +1,16 @@ +name: Scons build + +on: + push: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, '[skip_build]')" + steps: + - uses: actions/checkout@v2 + - name: Build ffead-cpp with meson + run: | + cd docker/os-based + docker build -f DockerFile-Ubuntu-x64-ffead-cpp-scons . From 73f930cd0ec26b1649b0ed6ee65838a42ca4d5cf Mon Sep 17 00:00:00 2001 From: Sumeet Chhetri Date: Sun, 8 May 2022 15:39:19 +0530 Subject: [PATCH 3/3] Added workflow for shellb build [skip_build] --- .github/workflows/ci-shellb.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/ci-shellb.yml diff --git a/.github/workflows/ci-shellb.yml b/.github/workflows/ci-shellb.yml new file mode 100644 index 000000000..f1596afd8 --- /dev/null +++ b/.github/workflows/ci-shellb.yml @@ -0,0 +1,16 @@ +name: Shellb build + +on: + push: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, '[skip_build]')" + steps: + - uses: actions/checkout@v2 + - name: Build ffead-cpp with meson + run: | + cd docker/os-based + docker build -f DockerFile-Ubuntu-x64-ffead-cpp-shellb .