Topic/mstress hadoop 3

.github/workflows/build.yml

@@ -55,19 +55,19 @@ jobs:
             buser: qfsbuild
             runs_on: ubuntu-latest
           - distro: ubuntu
-            ver: 18.04
+            ver: 20.04
             codecov: yes
             btype: release
             buser: qfsbuild
             runs_on: ubuntu-latest
           - distro: ubuntu
-            ver: 18.04
+            ver: 20.04
             codecov: no
             btype: debug
             buser: qfsbuild
             runs_on: ubuntu-latest
           - distro: ubuntu
-            ver: 18.04
+            ver: 20.04
             codecov: no
             btype: release
             buser: root
@@ -102,12 +102,6 @@ jobs:
             btype: release
             buser: qfsbuild
             runs_on: ubuntu-latest
-          - distro: debian
-            ver: 10
-            codecov: no
-            btype: release
-            buser: qfsbuild
-            runs_on: ubuntu-latest
           - distro: centos
             ver: 6
             codecov: no
@@ -165,6 +159,7 @@ jobs:
       CODECOV: ${{ matrix.codecov }}
       BTYPE: ${{ matrix.btype }}
       BUSER: ${{ matrix.buser }}
+      UBUNTU_APT_OPTIONS: "-o Acquire:Queue-mode=host"
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

CMakeLists.txt

@@ -4,7 +4,7 @@
 # Created 2006/10/20
 # Author: Sriram Rao (Kosmix Corp)
 #
-# Copyright 2008-2017 Quantcast Corporation. All rights reserved.
+# Copyright 2008-2025 Quantcast Corporation. All rights reserved.
 # Copyright 2006 Kosmix Corp.
 #
 # This file is part of Quantcast File System.
@@ -22,25 +22,25 @@
 # permissions and limitations under the License.
 
 cmake_minimum_required(VERSION 2.8.4...10.0 FATAL_ERROR)
-cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION})
+cmake_policy(VERSION
+    ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION})
 
-IF (NOT DEFINED KFS_DIR_PREFIX)
-    set (KFS_DIR_PREFIX ${CMAKE_SOURCE_DIR})
-ENDIF (NOT DEFINED KFS_DIR_PREFIX)
+if(NOT DEFINED KFS_DIR_PREFIX)
+    set(KFS_DIR_PREFIX ${CMAKE_SOURCE_DIR})
+endif()
 
-project (QFS)
+project(QFS)
 set(CMAKE_MODULE_PATH ${KFS_DIR_PREFIX}/cmake/Modules)
 
-IF (NOT DEFINED Boost_NO_BOOST_CMAKE)
-    IF (${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 8 AND
-            5 LESS ${CMAKE_PATCH_VERSION})
+if(NOT DEFINED Boost_NO_BOOST_CMAKE)
+    if(${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 8 AND
+        5 LESS ${CMAKE_PATCH_VERSION})
         # Turn off by default with 2.8.5 < cmake < 2.9.0 to
         # work around cmake28 and boost libraries problem on centos 6.
         # Boost_NO_BOOST_CMAKE was introduced int cmake 2.8.6
         set(Boost_NO_BOOST_CMAKE ON)
-    ENDIF (${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 8 AND
-        5 LESS ${CMAKE_PATCH_VERSION})
-ENDIF (NOT DEFINED Boost_NO_BOOST_CMAKE)
+    endif()
+endif()
 
 if(NOT DEFINED QFS_USE_STATIC_LIB_LINKAGE)
     set(QFS_USE_STATIC_LIB_LINKAGE ON)
@@ -53,9 +53,8 @@ endif()
 set(Boost_USE_MULTITHREADED ON)
 
 # Require the packages we need to build
-if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR
-        CYGWIN OR
-        CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+if(CYGWIN OR
+    CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
     find_package(Boost COMPONENTS regex system REQUIRED)
 else()
     find_package(Boost COMPONENTS regex REQUIRED)
@@ -67,46 +66,64 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     set(CMAKE_INSTALL_RPATH @loader_path)
 endif()
 
-if (CYGWIN)
+if(CYGWIN)
     set(CMAKE_FIND_LIBRARY_PREFIXES ${CMAKE_FIND_LIBRARY_PREFIXES} "")
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES} ".dll.a" ".lib")
+    set(CMAKE_FIND_LIBRARY_SUFFIXES
+        ${CMAKE_FIND_LIBRARY_SUFFIXES} ".dll.a" ".lib")
+
     # For now only for cygwin to get strptime, and gnu specific glob flags.
     add_definitions(-D_GNU_SOURCE)
 endif()
 
-if (QFS_OMIT_JNI)
-    message (STATUS "Not building QFS JNI")
+if(QFS_OMIT_JNI)
+    message(STATUS "Not building QFS JNI")
 else()
     set(JAVA_AWT_LIBRARY NotNeeded)
     set(JAVA_JVM_LIBRARY NotNeeded)
     set(JAVA_AWT_INCLUDE_PATH NotNeeded)
     find_package(JNI REQUIRED)
 endif()
+
 find_package(Jerasure REQUIRED)
 find_package(Krb5 REQUIRED krb5)
 find_package(OpenSSL REQUIRED)
 find_package(FUSE)
 
 add_definitions(-DBOOST_BIND_GLOBAL_PLACEHOLDERS)
+
+# BOOST_SP_USE_QUICK_ALLOCATOR and BOOST_SP_USE_STD_ALLOCATOR are deprecated in
+# Boost 1.87 and later.
+if(Boost_MAJOR_VERSION LESS 1 OR
+    (Boost_MAJOR_VERSION EQUAL 1 AND Boost_MINOR_VERSION LESS 87))
+    add_definitions(-DBOOST_SP_USE_QUICK_ALLOCATOR)
+endif()
+
 # Define various options based on the library configuration we use
 if(KRB5_FLAVOR)
     add_definitions(-DKFS_KRB_USE_${KRB5_FLAVOR})
 endif()
+
 if(KRB5_USE_KRB5H)
     add_definitions(-DKFS_KRB_USE_KRB5H)
 endif()
-if("${KRB5_FLAVOR}" STREQUAL "HEIMDAL" OR KRB5_HAS_krb5_get_init_creds_opt_set_out_ccache)
+
+if("${KRB5_FLAVOR}" STREQUAL "HEIMDAL" OR
+    KRB5_HAS_krb5_get_init_creds_opt_set_out_ccache)
     add_definitions(-DKFS_KRB_USE_KRB5_GET_INIT_CREDS_OPT)
 endif()
+
 if(KRB5_HAS_krb5_unparse_name_flags_ext)
     add_definitions(-DKRB5_HAS_krb5_unparse_name_flags_ext)
 endif()
+
 if(KRB5_HAS_krb5_unparse_name_ext)
     add_definitions(-DKRB5_HAS_krb5_unparse_name_ext)
 endif()
+
 if(KRB5_HAS_krb5_free_keytab_entry_contents)
     add_definitions(-DKRB5_HAS_krb5_free_keytab_entry_contents)
 endif()
+
 if(KRB5_HAS_krb5_kt_free_entry)
     add_definitions(-DKRB5_HAS_krb5_kt_free_entry)
 endif()
@@ -125,38 +142,38 @@ endif()
 
 # Build with statically linked libraries; the value for this variable has to be
 # defined here overwriting whatever is in the cache.
-# When setto ON, we build with statically linked libraries; when off we
+# When set to ON, we build with statically linked libraries; when off we
 # link with dynamically linked libs
+if(NOT DEFINED QFS_USE_STATIC_LIB_LINKAGE)
+    set(QFS_USE_STATIC_LIB_LINKAGE TRUE)
+endif()
 
-IF (NOT DEFINED QFS_USE_STATIC_LIB_LINKAGE)
-    set (QFS_USE_STATIC_LIB_LINKAGE TRUE)
-ENDIF (NOT DEFINED QFS_USE_STATIC_LIB_LINKAGE)
+if(QFS_USE_STATIC_LIB_LINKAGE)
+    message(STATUS "Build binaries with statically linked QFS libraries")
+else()
+    message(STATUS "Build binaries with dynamically linked QFS libraries")
+endif()
 
-IF (QFS_USE_STATIC_LIB_LINKAGE)
-    message (STATUS "Build binaries with statically linked QFS libraries")
-ELSE (QFS_USE_STATIC_LIB_LINKAGE)
-    message (STATUS "Build binaries with dynamically linked QFS libraries")
-ENDIF (QFS_USE_STATIC_LIB_LINKAGE)
-set (USE_STATIC_LIB_LINKAGE ${QFS_USE_STATIC_LIB_LINKAGE} CACHE BOOL
+set(USE_STATIC_LIB_LINKAGE ${QFS_USE_STATIC_LIB_LINKAGE} CACHE BOOL
     "Build binaries with statically linked libraries" FORCE)
 
 if(ENABLE_COVERAGE)
     message(STATUS "Enabling code coverage with gcov")
-    set(CMAKE_CXX_FLAGS  "-coverage")
+    set(CMAKE_CXX_FLAGS "-coverage")
     set(CMAKE_C_FLAGS "-coverage")
 endif()
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-    set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=gnu++14")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14")
 endif()
 
 if(ENABLE_PROFILING)
     message(STATUS "Enabling profiling with gprof")
-    set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -pg")
-    set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -pg")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg")
     set(CMAKE_SHAREDBoost_USE_MULTITHREADED_LINKER_FLAGS
         "${CMAKE_SHARED_LINKER_FLAGS} -pg")
-    set(CMAKE_EXE_FLAGS  "${CMAKE_EXE_FLAGS} -pg")
+    set(CMAKE_EXE_FLAGS "${CMAKE_EXE_FLAGS} -pg")
 endif()
 
 # Change the line to Release to build release binaries
@@ -169,14 +186,14 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D NDEBUG -g3")
 endif()
 
-set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -Wall -DBOOST_SP_USE_QUICK_ALLOCATOR")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 string(TOUPPER KFS_OS_NAME_${CMAKE_SYSTEM_NAME} KFS_OS_NAME)
-add_definitions (-D${KFS_OS_NAME})
+add_definitions(-D${KFS_OS_NAME})
 
-IF (ENABLE_IO_BUFFER_DEBUG)
+if(ENABLE_IO_BUFFER_DEBUG)
     add_definitions(-DDEBUG_IOBuffer)
     message(STATUS "Enabled IO buffer debug")
-ENDIF (ENABLE_IO_BUFFER_DEBUG)
+endif()
 
 if(DEFINED QFS_EXTRA_CXX_OPTIONS)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${QFS_EXTRA_CXX_OPTIONS}")
@@ -195,11 +212,13 @@ if(FUSE_FOUND)
     message(STATUS "fuse definitions:  ${FUSE_DEFINITIONS}")
     message(STATUS "fuse version: ${FUSE_MAJOR_VERSION}.${FUSE_MINOR_VERSION}")
     add_definitions(${FUSE_DEFINITIONS})
+
     if(${FUSE_MAJOR_VERSION} LESS 3)
         add_definitions(-D FUSE_USE_VERSION=26)
     else()
         add_definitions(-D FUSE_USE_VERSION=30)
     endif()
+
     include_directories(${FUSE_INCLUDE_DIRS})
 else()
     message(STATUS "Not building qfs_fuse")
@@ -215,10 +234,11 @@ include_directories(
     ${JAVA_INCLUDE_PATH2}
 )
 
-# get the subdirs we want
-if (NOT QFS_OMIT_JNI)
+# Get the sub directories we want.
+if(NOT QFS_OMIT_JNI)
     add_subdirectory(${KFS_DIR_PREFIX}/src/cc/access src/cc/access)
 endif()
+
 add_subdirectory(${KFS_DIR_PREFIX}/src/cc/chunk src/cc/chunk)
 add_subdirectory(${KFS_DIR_PREFIX}/src/cc/common src/cc/common)
 add_subdirectory(${KFS_DIR_PREFIX}/src/cc/devtools src/cc/devtools)
@@ -242,7 +262,7 @@ if(FUSE_FOUND)
     add_subdirectory(${KFS_DIR_PREFIX}/src/cc/fuse src/cc/fuse)
 endif()
 
-add_custom_target (
+add_custom_target(
     rat ${KFS_DIR_PREFIX}/scripts/rat.sh ${KFS_DIR_PREFIX}
     COMMENT "Running license release audit tool (rat)"
     VERBATIM

Makefile

@@ -49,8 +49,8 @@ run-cmake: dir
 build: run-cmake
 	cd build/${BUILD_TYPE} && $(MAKE) ${MAKE_OPTIONS} install \
 	`${QFS_MSTRESS_ON} && \
-		echo ${QFSHADOOP_VERSIONS} | grep 2.10.2 >/dev/null 2>&1 && \
-		mvn --version >/dev/null 2>&1 && echo mstress-tarball`
+		echo ${QFSHADOOP_VERSIONS} | grep '3\.4\.1' >/dev/null 2>&1 && \
+		mvn --version >/dev/null 2>&1 && echo mstress-bootstrap mstress-tarball`
 
 .PHONY: java
 java: build