#
# Use "make help" to find out about configuration options.
#

EXE_NAME           ?= elbencho
EXE_VER_MAJOR      ?= 3
EXE_VER_MINOR      ?= 3
EXE_VER_PATCHLEVEL ?= 1
EXE_VERSION        ?= $(EXE_VER_MAJOR).$(EXE_VER_MINOR)-$(EXE_VER_PATCHLEVEL)
EXE                ?= $(BIN_PATH)/$(EXE_NAME)

SOURCE_PATH        ?= ./source
BIN_PATH           ?= ./bin
EXTERNAL_PATH      ?= ./external
PACKAGING_PATH     ?= ./packaging
BUILD_HELPERS_PATH ?= ./build_helpers
PLUGINS_PATH       ?= ./contrib/plugins

INST_PATH          ?= /usr/local/bin
PKG_INST_PATH      ?= /usr/bin

CXX                ?= g++
STRIP              ?= strip
CXX_FLAVOR         ?= c++17

# Recipes print a short tag per step and hide the commands, unless BUILD_VERBOSE=1 is given.
ifdef BUILD_VERBOSE
  QUIET :=
  MSG   := @:
else
  QUIET := @
  MSG   := @echo
endif

# "BOOST_STACKTRACE_USE_BACKTRACE" is to force libbacktrace backend, which is most reliable for
# static builds on Alpine with musl.
CXXFLAGS_BOOST     ?= -DBOOST_BIND_GLOBAL_PLACEHOLDERS -DBOOST_SPIRIT_THREADSAFE \
	-DBOOST_STACKTRACE_USE_BACKTRACE

LDFLAGS_AIO        ?= -laio
LDFLAGS_BOOST      ?= -lboost_program_options -lboost_thread
LDFLAGS_LINUX      ?= -l rt -l stdc++fs
LDFLAGS_NUMA       ?= -lnuma

COMMA              := ,

ALTHTTPSVC_SUPPORT ?= 0
BUILD_LIBFABRIC    ?= 0
COREBIND_SUPPORT   ?= 1
IBVERBS_SUPPORT    ?=
RDMACM_SUPPORT     ?=
LIBAIO_SUPPORT     ?= 1
LIBFABRIC_SUPPORT  ?= 0
LIBNUMA_SUPPORT    ?= 1
SYNCFS_SUPPORT     ?= 1
S3_AWSCRT          ?= 0
S3_SUPPORT         ?= 0
SYSCALLH_SUPPORT   ?= 1
THREADNAME_SUPPORT ?= 1

CXXFLAGS_COMMON   = -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 $(CXXFLAGS_BOOST) \
	-DEXE_NAME=\"$(EXE_NAME)\" -DEXE_VERSION=\"$(EXE_VERSION)\" \
	-I $(SOURCE_PATH) -I $(EXTERNAL_PATH)/Simple-Web-Server \
	-Wall -Wunused-variable -Woverloaded-virtual -Wextra -Wno-unused-parameter -fmessage-length=0 \
	-fno-strict-aliasing -pthread -ggdb -std=$(CXX_FLAVOR)
CXXFLAGS_RELEASE  = -O3 -Wuninitialized -g1 # "-g1" is for line numbers in backtraces
CXXFLAGS_DEBUG    = -O0 -D_FORTIFY_SOURCE=2 -DBUILD_DEBUG

LDFLAGS_COMMON       = -pthread $(LDFLAGS_BOOST) -gz=zlib
LDFLAGS_RELASE       = -O3
LDFLAGS_DEBUG        = -O0

SOURCES          := $(shell find $(SOURCE_PATH) -name '*.cpp')

# Release & debug flags for compiler and linker
ifeq ($(BUILD_DEBUG), 1)
CXXFLAGS = $(CXXFLAGS_COMMON) $(CXXFLAGS_DEBUG) $(CXXFLAGS_EXTRA)
LDFLAGS  = $(LDFLAGS_COMMON) $(LDFLAGS_DEBUG) $(LDFLAGS_EXTRA)
else
CXXFLAGS = $(CXXFLAGS_COMMON) $(CXXFLAGS_RELEASE) $(CXXFLAGS_EXTRA)
LDFLAGS  = $(LDFLAGS_COMMON) $(LDFLAGS_RELASE) $(LDFLAGS_EXTRA)

  ifeq ($(CXX), g++)
  # This is to enable gcc's automatic SIMD vectorization for RandAlgoXoshiro256ppSIMD.
  # ("-fopt-info-vec" can be used to confirm that all loops in nextInternalNway() get vectorized.)
  # For background see https://prng.di.unimi.it/ section "Vectorization".
  CXXFLAGS_SPECIAL_SIMD += -fdisable-tree-cunrolli
  endif
endif

# Dynamic or static linking
ifeq ($(BUILD_STATIC), 1)

  LDFLAGS += -static -no-pie

  ifeq ($(S3_AWSCRT), 1)
    LDFLAGS_S3_STATIC  += -l z
  else
    LDFLAGS_S3_STATIC  += -l curl -l ssl -l crypto -l tls -l z -l nghttp2 -l nghttp3 -l brotlidec \
      -l brotlicommon -l idn2 -l unistring -l psl -l cares -l zstd
  endif

else # dynamic linking

  LDFLAGS += -rdynamic -l dl

  ifeq ($(S3_AWSCRT), 1)
    LDFLAGS_S3_DYNAMIC += -l z
  else
    LDFLAGS_S3_DYNAMIC += -l crypto -l ssl -l curl -l z
  endif

endif

# S3 AWS CRT and SPDK are mutually exclusive because of conflicting openssl/crypto libraries.
ifeq ($(S3_AWSCRT)$(SPDK_SUPPORT),11)
  $(error Error: S3_AWSCRT and SPDK_SUPPORT are mutually exclusive due to conflicting dependencies.)
endif

# Compiler and linker flags for S3 support
# "-Wno-overloaded-virtual" because AWS SDK shows a lot of warnings about this otherwise
ifeq ($(S3_SUPPORT), 1)
  CXXFLAGS += -DS3_SUPPORT -Wno-overloaded-virtual
  LDFLAGS  += -L $(EXTERNAL_PATH)/aws-sdk-cpp_install/lib* -l aws-sdk-all \
	$(LDFLAGS_S3_DYNAMIC) $(LDFLAGS_S3_STATIC)

  ifeq ($(S3_AWSCRT), 1)
    CXXFLAGS += -DS3_AWSCRT
  endif

  # Apply user-provided AWS SDK include dir if given
  ifeq ($(AWS_INCLUDE_DIR),)
    CXXFLAGS += -I $(EXTERNAL_PATH)/aws-sdk-cpp_install/include
  else
    CXXFLAGS += -I $(AWS_INCLUDE_DIR)
  endif

endif

# Use Microsoft mimalloc for memory allocations.
# Note: This needs to come as very last in link order, thus we have a separate variable to ensure
# it's the trailing arg for the linker. (Can be confirmed e.g. via MIMALLOC_SHOW_STATS=1)
ifeq ($(USE_MIMALLOC), 1)
  CXXFLAGS += -DUSE_MIMALLOC
  LDFLAGS_MIMALLOC_TAIL := -L external/mimalloc/build -l:libmimalloc.a
endif

# Support for uWS (Micro Web Sockets) HTTP service.
ifeq ($(ALTHTTPSVC_SUPPORT), 1)
  CXXFLAGS += -DALTHTTPSVC_SUPPORT -I $(EXTERNAL_PATH)/uWebSockets/src \
	-I $(EXTERNAL_PATH)/uWebSockets/uSockets/src -DUWS_NO_ZLIB -DLIBUS_NO_SSL
  LDFLAGS  += -L $(EXTERNAL_PATH)/uWebSockets/uSockets -l:uSockets.a
endif

# Support for Hadoop HDFS
# (Note: "lib/amd64/server" is for OpenJDK 1.8 on RHEL8.)
ifeq ($(HDFS_SUPPORT), 1)
  CXXFLAGS += -DHDFS_SUPPORT -I $(HADOOP_HOME)/include
  LDFLAGS  += -L $(HADOOP_HOME)/lib/native -l:libhdfs.a -L $(JAVA_HOME)/lib/server \
	-L $(JAVA_HOME)/lib/amd64/server -l jvm
endif

# Support for ftxui
CXXFLAGS += -I $(EXTERNAL_PATH)/ftxui/include
LDFLAGS += -L $(EXTERNAL_PATH)/ftxui/build -l ftxui-component -l ftxui-dom -l ftxui-screen

# Support build in Cygwin environment
ifeq ($(CYGWIN_SUPPORT), 1)
  EXE                := $(EXE).exe

  BACKTRACE_SUPPORT  := 0
  COREBIND_SUPPORT   := 0
  LIBAIO_SUPPORT     := 0
  LIBNUMA_SUPPORT    := 0
  SYNCFS_SUPPORT     := 0
  SYSCALLH_SUPPORT   := 0
  THREADNAME_SUPPORT := 0

  CXX_FLAVOR         := gnu++17
  CXXFLAGS           += -DCYGWIN_SUPPORT
  LDFLAGS_COMMON     := $(filter-out -gz=zlib, $(LDFLAGS_COMMON))
endif

# Support build on macOS
ifeq ($(shell uname -s),Darwin)
  ifeq ($(S3_SUPPORT), 1)
    LDFLAGS += -framework CoreFoundation -framework Security -framework Network
  endif

  CXXFLAGS_BOOST += -DBOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED
  CXXFLAGS_BOOST := $(filter-out -DBOOST_STACKTRACE_USE_BACKTRACE, $(CXXFLAGS_BOOST))

  COREBIND_SUPPORT   := 0
  IBVERBS_SUPPORT    := 0
  RDMACM_SUPPORT     := 0
  LIBAIO_SUPPORT     := 0
  LIBNUMA_SUPPORT    := 0
  SYNCFS_SUPPORT     := 0
  THREADNAME_SUPPORT := 0

  CXXFLAGS         += -I/opt/homebrew/include
  LDFLAGS          += -L/opt/homebrew/lib
endif

# Extra flags for Linux & Cygwin, but not for macOS
ifneq ($(shell uname -s),Darwin)
  LDFLAGS += $(LDFLAGS_LINUX)
endif

# libaio support
# Note: Gets set by CYGWIN_SUPPORT=1, so needs to come after that
ifeq ($(LIBAIO_SUPPORT), 1)
  CXXFLAGS += -DLIBAIO_SUPPORT
  LDFLAGS  += $(LDFLAGS_AIO)
endif

# syncfs() call support
# Note: Gets set by CYGWIN_SUPPORT=1, so needs to come after that
ifeq ($(SYNCFS_SUPPORT), 1)
  CXXFLAGS += -DSYNCFS_SUPPORT
endif

# libnuma support
# Note: Gets set by CYGWIN_SUPPORT=1, so needs to come after that
ifeq ($(LIBNUMA_SUPPORT), 1)
  CXXFLAGS += -DLIBNUMA_SUPPORT
  LDFLAGS  += $(LDFLAGS_NUMA)
endif

# CPU core binding support
# Note: Gets set by CYGWIN_SUPPORT=1, so needs to come after that
ifeq ($(COREBIND_SUPPORT), 1)
  CXXFLAGS += -DCOREBIND_SUPPORT
endif

# sys/syscall.h include file support (for definition of SYS_..., e.g. SYS_gettid)
# Note: Gets set by CYGWIN_SUPPORT=1, so needs to come after that
ifeq ($(SYSCALLH_SUPPORT), 1)
  CXXFLAGS += -DSYSCALLH_SUPPORT
endif

# support for naming threads (e.g. in 'ps' or 'top' output)
# Note: Gets set by macOS/Darwin detection, so needs to come after that
ifeq ($(THREADNAME_SUPPORT), 1)
  CXXFLAGS += -DTHREADNAME_SUPPORT
endif

# Contributed plugins (see contrib/plugins/README.md). Each plugin.mk adds its sources and flags
# when its ELB_PLUGIN_<NAME>=1 option is set. Included before the auto detection, because a plugin
# may need to force e.g. CUDA_SUPPORT=1 before it gets detected.
include $(wildcard $(PLUGINS_PATH)/*/plugin.mk)

# Include build helpers for auto detection
include build_helpers/AutoDetection.mk

# Backtrace support
# Note: Gets auto-detected and gets set by CYGWIN_SUPPORT=1, so needs to come after those
ifeq ($(BACKTRACE_SUPPORT), 1)
  CXXFLAGS += -DBACKTRACE_SUPPORT
  LDFLAGS  += -l backtrace

  ifeq ($(LIB_BACKTRACE_DETECTED), )
    PREP_LIBBACKTRACE := 1
    CXXFLAGS += -I $(EXTERNAL_PATH)/libbacktrace/install/include
    LDFLAGS  += -L $(EXTERNAL_PATH)/libbacktrace/install/lib
  endif

endif

# Netbench engines. The tcp engine is always built and needs no library.
# The libfabric engine (LIBFABRIC_SUPPORT, off by default) uses a system-installed libfabric (found via
# pkg-config, min version 1.11). Without it, or with BUILD_LIBFABRIC=1, or for static binaries, our
# own copy gets built as static lib in external/ and its link flags come from its pkg-config file.
# That lookup is deferred (recursive variable) because it depends on the "externals" target having
# built the lib.
# The ibverbs engine (IBVERBS_SUPPORT) needs libibverbs with ibv_query_gid_ex() (rdma-core 30 or
# newer) and gets auto-detected when not set explicitly. Not available for static binaries. The
# rdmacm engine (RDMACM_SUPPORT) additionally needs librdmacm (same rdma-core) and is auto-detected
# the same way.
ifeq ($(LIBFABRIC_SUPPORT), 1)
  CXXFLAGS += -DLIBFABRIC_SUPPORT

  LIBFABRIC_DETECTED := $(shell pkg-config --atleast-version=1.11 libfabric 2>/dev/null && echo 1)

  ifeq ($(BUILD_LIBFABRIC)$(LIBFABRIC_DETECTED),01)
    ifneq ($(BUILD_STATIC),1)
      USE_SYSTEM_LIBFABRIC := 1
    endif
  endif

  ifeq ($(USE_SYSTEM_LIBFABRIC),1)
    CXXFLAGS += $(shell pkg-config --cflags libfabric 2>/dev/null)
    LDFLAGS  += -l fabric
  else
    PREP_LIBFABRIC := 1
    LIBFABRIC_PKG_CONFIG_PATH := $(EXTERNAL_PATH)/libfabric_install/lib/pkgconfig
    CXXFLAGS += -I $(EXTERNAL_PATH)/libfabric_install/include
    LIBFABRIC_PKG_LIBS = $(shell PKG_CONFIG_PATH=$(LIBFABRIC_PKG_CONFIG_PATH) pkg-config --static \
      --libs libfabric 2>/dev/null)
    LDFLAGS  += $(if $(LIBFABRIC_PKG_LIBS),$(LIBFABRIC_PKG_LIBS),\
      -L $(EXTERNAL_PATH)/libfabric_install/lib -l fabric)

    ifeq ($(BUILD_STATIC), 1)
      LIBFABRIC_TCP_ONLY := 1
    endif
  endif
endif

# (for the features info)
ifeq ($(PREP_LIBFABRIC),1)
  LIBFABRIC_INFO := libfabric (built in external/$(if $(LIBFABRIC_TCP_ONLY), with tcp provider only,))
else
  LIBFABRIC_INFO := libfabric (system$(if $(BUILD_VERBOSE), $(shell pkg-config --modversion libfabric),))
endif

ifeq ($(BUILD_STATIC), 1)
  IBVERBS_SUPPORT := 0
endif

ifeq ($(IBVERBS_SUPPORT),)
  IBVERBS_SUPPORT := $(shell printf '\043include <infiniband/verbs.h>\nint main() { struct ibv_gid_entry e; return ibv_query_gid_ex(0, 1, 0, &e, 0); }\n' | $(CXX) -x c++ -fsyntax-only - > /dev/null 2>&1 && echo 1 || echo 0)
endif

ifeq ($(IBVERBS_SUPPORT), 1)
  CXXFLAGS += -DIBVERBS_SUPPORT
  LDFLAGS  += -libverbs

  ifeq ($(RDMACM_SUPPORT),)
    RDMACM_SUPPORT := $(shell printf '\043include <rdma/rdma_cma.h>\nint main() { struct ibv_ece e; return rdma_get_remote_ece(0, &e); }\n' | $(CXX) -x c++ -fsyntax-only - > /dev/null 2>&1 && echo 1 || echo 0)
  endif
else
  RDMACM_SUPPORT := 0
endif

ifeq ($(RDMACM_SUPPORT), 1)
  CXXFLAGS += -DRDMACM_SUPPORT
  LDFLAGS  += -lrdmacm
endif



# (after the plugin includes, because plugins add to SOURCES)
OBJECTS          := $(SOURCES:.cpp=.o)
OBJECTS_CLEANUP  := $(shell find $(SOURCE_PATH) $(PLUGINS_PATH) -name '*.o') # for renamed sources
DEPENDENCY_FILES := $(shell find $(SOURCE_PATH) $(PLUGINS_PATH) -name '*.d')

# The resolved build configuration as a file that the objects and the externals depend on. It is
# only rewritten when the configuration changes, so changing a feature rebuilds exactly what
# depends on it (no "make clean-all" needed) and a build with the same features leaves everything
# alone. The externals get their own copy as stamp, so the (slow) preparation script only runs
# when the configuration changed since it last ran.
FEATURES_FILE    := $(BIN_PATH)/.build-features
EXTERNALS_STAMP  := $(EXTERNAL_PATH)/.prepared-features
FEATURES_STR     := ALTHTTPSVC_SUPPORT=$(ALTHTTPSVC_SUPPORT) BACKTRACE_SUPPORT=$(BACKTRACE_SUPPORT) \
	BUILD_DEBUG=$(BUILD_DEBUG) BUILD_LIBFABRIC=$(BUILD_LIBFABRIC) BUILD_STATIC=$(BUILD_STATIC) \
	COREBIND_SUPPORT=$(COREBIND_SUPPORT) CUDA_SUPPORT=$(CUDA_SUPPORT) \
	CUFILE_SUPPORT=$(CUFILE_SUPPORT) HDFS_SUPPORT=$(HDFS_SUPPORT) \
	IBVERBS_SUPPORT=$(IBVERBS_SUPPORT) LIBAIO_SUPPORT=$(LIBAIO_SUPPORT) \
	LIBFABRIC_SUPPORT=$(LIBFABRIC_SUPPORT) LIBNUMA_SUPPORT=$(LIBNUMA_SUPPORT) \
	PLUGINS_ENABLED=$(PLUGINS_ENABLED) \
	RDMACM_SUPPORT=$(RDMACM_SUPPORT) S3_AWSCRT=$(S3_AWSCRT) S3_SUPPORT=$(S3_SUPPORT) \
	SPDK_SUPPORT=$(SPDK_SUPPORT) SYNCFS_SUPPORT=$(SYNCFS_SUPPORT) \
	SYSCALLH_SUPPORT=$(SYSCALLH_SUPPORT) THREADNAME_SUPPORT=$(THREADNAME_SUPPORT) \
	USE_MIMALLOC=$(USE_MIMALLOC)

# "install" and the packaging targets take the executable as it is when it exists (whatever
# features it was built with) and only build it when it is missing.
EXE_IF_MISSING   := $(if $(wildcard $(EXE)),,$(EXE))


all: $(SOURCES) $(EXE)


$(EXE): $(OBJECTS)
	$(MSG) [LINK] $@
	$(QUIET)$(CXX) -o $(EXE) $(OBJECTS) $(LDFLAGS) $(LDFLAGS_MIMALLOC_TAIL)


.c.o:
	$(MSG) [DEP] $(@:.o=.d)
	$(QUIET)$(CXX) $(CXXFLAGS) $(@:.o=.c) -E -MMD -MF $(@:.o=.d) -MT $(@) -o /dev/null
	$(MSG) [CXX] $@
	$(QUIET)$(CXX) $(CXXFLAGS) $(@:.o=.c) -c -o $(@)


.cpp.o:
	$(MSG) [DEP] $(@:.o=.d)
	$(QUIET)$(CXX) $(CXXFLAGS) $(@:.o=.cpp) -E -MMD -MF $(@:.o=.d) -MT $(@) -o /dev/null
	$(MSG) [CXX] $@
	$(QUIET)$(CXX) $(CXXFLAGS) $(@:.o=.cpp) -c -o $(@)

# This is a special case because we need special compile options to get gcc to enable loop unrolling
# and SIMD conversion, hence the additional "CXXFLAGS_SPECIAL_SIMD".
source/toolkits/random/RandAlgoSelectorTk.o: source/toolkits/random/RandAlgoSelectorTk.cpp
	$(MSG) [DEP] $(@:.o=.d)
	$(QUIET)$(CXX) $(CXXFLAGS) $(@:.o=.cpp) -E -MMD -MF $(@:.o=.d) -MT $(@) -o /dev/null
	$(MSG) [CXX special SIMD] $@
	$(QUIET)$(CXX) $(CXXFLAGS) $(CXXFLAGS_SPECIAL_SIMD) $(@:.o=.cpp) -c -o $(@)


# "Makefile" and the build configuration as dependencies to rebuild all on their change.
# Dependency chain is: Makefile -> features-info -> features-detect -> externals -> FEATURES_FILE
$(OBJECTS): Makefile $(wildcard $(PLUGINS_PATH)/*/plugin.mk) $(FEATURES_FILE) | features-info

# (always checked, but only written when the configuration differs, so its timestamp only changes
# then)
$(FEATURES_FILE): FORCE
	$(QUIET)mkdir -p $(dir $@)
	$(QUIET)echo "$(FEATURES_STR)" | cmp -s - $@ 2>/dev/null || echo "$(FEATURES_STR)" > $@

FORCE:

externals: $(EXTERNALS_STAMP)

$(EXTERNALS_STAMP): $(FEATURES_FILE)
# Note: The "+" prefix is to let "make" know that it needs to increase the MAKELEVEL env var because
# there will be sub-make calls in this script.
	$(QUIET)+PREP_AWS_SDK=$(S3_SUPPORT) S3_AWSCRT=$(S3_AWSCRT) AWS_LIB_DIR=$(AWS_LIB_DIR) \
		AWS_INCLUDE_DIR=$(AWS_INCLUDE_DIR) PREP_MIMALLOC=$(USE_MIMALLOC) \
		PREP_UWS=$(ALTHTTPSVC_SUPPORT) PREP_LIBBACKTRACE=$(PREP_LIBBACKTRACE) \
		PREP_SPDK=$(SPDK_SUPPORT) PREP_LIBFABRIC=$(PREP_LIBFABRIC) \
		LIBFABRIC_TCP_ONLY=$(LIBFABRIC_TCP_ONLY) $(PLUGINS_EXTERNALS_ENV) \
		$(EXTERNAL_PATH)/prepare-external.sh
	$(QUIET)cp $(FEATURES_FILE) $@

# Compiler and linker flags for SPDK support.
# This is located under the "externals" make target because the "pkg-config" depends on the SPDK
# build from the prepare-external.sh script to have completed.
ifeq ($(SPDK_SUPPORT), 1)
  SPDK_PATH = "$(EXTERNAL_PATH)/spdk"

  CXXFLAGS += -DSPDK_SUPPORT

  SPDK_PKGS = spdk_bdev_nvme spdk_bdev spdk_nvme spdk_thread spdk_accel spdk_env_dpdk spdk_sock \
	spdk_sock_posix spdk_json

  CXXFLAGS += $(shell \
    PKG_CONFIG_PATH="$(SPDK_PATH)/isa-l:$(SPDK_PATH)/build/lib/pkgconfig:$(PKG_CONFIG_PATH)" \
	pkg-config --cflags $(SPDK_PKGS) )

  LDFLAGS += -L$(SPDK_PATH)/isa-l/.libs -L$(SPDK_PATH)/isa-l-crypto/.libs

  # extracts library search paths (--libs-only-L)
  LDFLAGS += $(shell \
    PKG_CONFIG_PATH="$(SPDK_PATH)/isa-l:$(SPDK_PATH)/isa-l-crypto:$(SPDK_PATH)/build/lib/pkgconfig:$(PKG_CONFIG_PATH)" \
	pkg-config --static --libs-only-L $(SPDK_PKGS) libisal libisal_crypto | \
	sed 's/-libverbs//g;s/-lrdmacm//g')

  # temporarily switch to static linking for spdk libs
  LDFLAGS += -Wl,-Bstatic

  # force inclusion of all symbols of some specific SPDK/DPDK libs
  # (otherwise e.g. tcp transport, bdev_nvme, or the "ring_mp_mc" mempool ops driver that
  # spdk_thread_lib_init() needs would get optimized out because they're only ever called
  # indirectly via registration macros; spdk_rdma_provider/spdk_rdma_utils are needed the same
  # way for the rdma transport when SPDK was configured with "--with-rdma")
  LDFLAGS += -Wl,--whole-archive
  LDFLAGS += -lspdk_nvme -lspdk_sock_posix -lspdk_bdev_nvme -lspdk_accel -lrte_mempool_ring \
	-lspdk_rdma_provider -lspdk_rdma_utils
  LDFLAGS += -Wl,--no-whole-archive

  # add remaining static libs via pkg-config (--libs-only-l); libisal_crypto is needed by
  # spdk_accel's software AES-XTS implementation, now pulled in via --whole-archive above
  LDFLAGS += $(shell \
	PKG_CONFIG_PATH="$(SPDK_PATH)/isa-l:$(SPDK_PATH)/isa-l-crypto:$(SPDK_PATH)/build/lib/pkgconfig:$(PKG_CONFIG_PATH)" \
	pkg-config --static --libs-only-l $(SPDK_PKGS) libisal libisal_crypto | \
	sed 's/-libverbs//g;s/-lrdmacm//g;s/-lssl//g;s/-lcrypto//g')

  # switch back to dynamic linking for system/standard libs
  LDFLAGS += -Wl,-Bdynamic
  LDFLAGS += -lboost_json -lstdc++fs -lpthread -ldl -lnuma -luuid -libverbs -lrdmacm -lssl -lcrypto

endif # SPDK_SUPPORT


features-info: features-detect
ifeq ($(BACKTRACE_SUPPORT),1)
	$(if $(BUILD_VERBOSE),$(info [OPT] Backtrace support enabled))
else
	$(info [OPT] Backtrace support disabled)
endif

ifeq ($(CUFILE_SUPPORT),1)
	$(info [OPT] CUFILE (GDS) support enabled$(if $(BUILD_VERBOSE), (CUFILE_INCLUDE_PATH: $(CUFILE_INCLUDE_PATH))))
else
	$(info [OPT] CUFILE (GDS) support disabled)
endif

ifeq ($(CUDA_SUPPORT),1)
	$(info [OPT] CUDA support enabled$(if $(BUILD_VERBOSE), (CUDA_INCLUDE_PATH: $(CUDA_INCLUDE_PATH))))
else
	$(info [OPT] CUDA support disabled)
endif

ifeq ($(S3_SUPPORT),1)
  ifeq ($(S3_AWSCRT),1)
	$(info [OPT] S3 support enabled (AWS CRT))
  else
	$(info [OPT] S3 support enabled)
  endif
else
	$(info [OPT] S3 support disabled. (Enable via S3_SUPPORT=1))
endif

ifeq ($(USE_MIMALLOC),1)
	$(info [OPT] mimalloc enabled)
endif

ifeq ($(ALTHTTPSVC_SUPPORT),1)
	$(info [OPT] Alternative HTTP service enabled)
endif

ifeq ($(HDFS_SUPPORT),1)
	$(info [OPT] HDFS support enabled. (HADOOP_HOME: $(HADOOP_HOME); JAVA_HOME: $(JAVA_HOME)))
endif

ifeq ($(SPDK_SUPPORT),1)
	$(info [OPT] SPDK support enabled)
endif

	$(info [OPT] Netbench engines enabled: tcp$(if $(filter 1,$(IBVERBS_SUPPORT)),$(COMMA) ibverbs)$(if $(filter 1,$(RDMACM_SUPPORT)),$(COMMA) rdmacm)$(if $(filter 1,$(LIBFABRIC_SUPPORT)),$(COMMA) $(LIBFABRIC_INFO)))

ifneq ($(PLUGINS_ENABLED),)
	$(info [OPT] Plugins enabled: $(PLUGINS_ENABLED))
else
	$(info [OPT] Plugins enabled: none)
endif


clean: clean-packaging clean-buildhelpers
	$(MSG) "[DELETE] OBJECTS, DEPENDENCY_FILES, EXECUTABLES"
	$(QUIET)rm -rf $(OBJECTS_CLEANUP) $(DEPENDENCY_FILES) $(EXE) $(EXE).exe


clean-externals:
	$(MSG) "[DELETE] EXTERNALS"
	$(QUIET)rm -f "$(EXTERNALS_STAMP)" "$(EXTERNAL_PATH)/alpine-chroot-install"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/aws-sdk-cpp" "$(EXTERNAL_PATH)/aws-sdk-cpp_install"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/ftxui"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/libbacktrace"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/libfabric" "$(EXTERNAL_PATH)/libfabric_install"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/mimalloc"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/Simple-Web-Server"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/spdk"
	$(QUIET)rm -rf "$(EXTERNAL_PATH)/uWebSockets"


clean-packaging:
	$(MSG) "[DELETE] PACKAGING_FILES"
	$(QUIET)rm -rf \
		$(EXE)-*-static-* \
		$(PACKAGING_PATH)/$(EXE_NAME)-*static-*.tar.*
	$(QUIET)rm -rf \
		$(PACKAGING_PATH)/BUILDROOT \
		$(PACKAGING_PATH)/RPMS/* $(PACKAGING_PATH)/SPECS/rpm.spec
	$(QUIET)bash -c "rm -rf $(PACKAGING_PATH)/$(EXE_NAME)*.{deb,ddeb,build,buildinfo,changes}"


clean-all: clean clean-externals clean-packaging clean-buildhelpers clean-s3rdma-tools

# The S3-over-RDMA test tools in tools/s3rdma have their own Makefiles. Without BUILD_VERBOSE,
# their output is reduced to the one tag here (errors go to stderr, so they are still shown).
clean-s3rdma-tools:
	$(MSG) "[DELETE] S3RDMA_TOOLS"
	$(QUIET)$(MAKE) -C tools/s3rdma clean-all $(if $(BUILD_VERBOSE),,-s --no-print-directory > /dev/null)


install: $(EXE_IF_MISSING)
	@echo "Installing elbencho..."

	install -p -m u=rwx,g=rx,o=rx $(EXE) $(INST_PATH)/
	install -p -m u=rwx,g=rx,o=rx dist/usr/bin/$(EXE_NAME)-chart $(INST_PATH)/
	install -p -m u=rwx,g=rx,o=rx dist/usr/bin/$(EXE_NAME)-scan-path $(INST_PATH)/
	install -p -m u=rwx,g=rx,o=rx dist/usr/bin/$(EXE_NAME)-summarize-json $(INST_PATH)/

	install -p -m u=rwx,g=rx,o=rx -D dist/etc/bash_completion.d/$(EXE_NAME) \
		/etc/bash_completion.d/$(EXE_NAME)
	install -p -m u=rwx,g=rx,o=rx -D dist/etc/bash_completion.d/$(EXE_NAME)-chart \
		/etc/bash_completion.d/$(EXE_NAME)-chart

	@echo
	@echo "Installing contributed tools..."

	install -p -m u=rwx,g=rx,o=rx contrib/storage_sweep/mtelbencho.sh $(INST_PATH)/
	install -p -m u=rwx,g=rx,o=rx contrib/storage_sweep/graph_sweep.sh $(INST_PATH)/

	@echo
	@echo "NOTE: The $(EXE_NAME) executable was installed to $(INST_PATH). The sudo"
	@echo "  command might drop $(INST_PATH) from PATH. In case sudo is needed, the"
	@echo "  absolute path can be used. Or alternatively use a rpm/deb package."


uninstall:
	rm -f $(INST_PATH)/$(EXE_NAME)
	rm -f $(INST_PATH)/$(EXE_NAME)-chart
	rm -f $(INST_PATH)/$(EXE_NAME)-scan-path
	rm -f $(INST_PATH)/$(EXE_NAME)-summarize-json
	rm -f $(INST_PATH)/mtelbencho.sh
	rm -f $(INST_PATH)/graph_sweep.sh
	rm -f /etc/bash_completion.d/$(EXE_NAME)
	rm -f /etc/bash_completion.d/$(EXE_NAME)-chart


# prepare generic part of build-root (not the .rpm or .deb specific part)
prepare-buildroot: | $(EXE_IF_MISSING) clean-packaging
	@echo "[PACKAGING] PREPARE BUILDROOT"

	mkdir -p $(PACKAGING_PATH)/BUILDROOT/$(PKG_INST_PATH)

	# copy main executable
	cp --preserve $(EXE) $(PACKAGING_PATH)/BUILDROOT/$(PKG_INST_PATH)

	# copy contents of dist subdir
	for dir in $(shell find dist/ -mindepth 1 -type d -printf "%P\n"); do \
		mkdir -p $(PACKAGING_PATH)/BUILDROOT/$$dir; \
	done

	for file in $(shell find dist/ -mindepth 1 -type f -printf "%P\n"); do \
		cp --preserve dist/$$file $(PACKAGING_PATH)/BUILDROOT/$$file; \
	done

	# copy contents of contrib subdir
	cp --preserve contrib/storage_sweep/mtelbencho.sh $(PACKAGING_PATH)/BUILDROOT/$(PKG_INST_PATH)
	cp --preserve contrib/storage_sweep/graph_sweep.sh $(PACKAGING_PATH)/BUILDROOT/$(PKG_INST_PATH)


rpm: | prepare-buildroot
	@echo "[PACKAGING] PREPARE RPM PACKAGE"

	cp $(PACKAGING_PATH)/SPECS/rpm.spec.template $(PACKAGING_PATH)/SPECS/rpm.spec
	sed -i "s/__NAME__/$(EXE_NAME)/" $(PACKAGING_PATH)/SPECS/rpm.spec
	sed -i "s/__VERSION__/$(EXE_VER_MAJOR).$(EXE_VER_MINOR).$(EXE_VER_PATCHLEVEL)/" \
		$(PACKAGING_PATH)/SPECS/rpm.spec

	rpmbuild $(PACKAGING_PATH)/SPECS/rpm.spec --bb --define "_topdir $(PWD)/$(PACKAGING_PATH)" \
		--define "__spec_install_pre /bin/true" --buildroot=$(PWD)/$(PACKAGING_PATH)/BUILDROOT

	@echo
	@echo "All done. Your package is here:"
	@find $(PACKAGING_PATH)/RPMS -name $(EXE_NAME)*.rpm


deb: | prepare-buildroot
	@echo "[PACKAGING] PREPARE DEB PACKAGE"

	cp -r $(PACKAGING_PATH)/debian $(PACKAGING_PATH)/BUILDROOT

	cp $(PACKAGING_PATH)/BUILDROOT/debian/control.template \
		$(PACKAGING_PATH)/BUILDROOT/debian/control

	sed -i "s/__NAME__/$(EXE_NAME)/" $(PACKAGING_PATH)/BUILDROOT/debian/control

	cd $(PACKAGING_PATH)/BUILDROOT && \
	EDITOR=/bin/true VISUAL=/bin/true DEBEMAIL=elbencho@localhost.localdomain debchange --create \
		--package elbencho --urgency low --noquery \
		--newversion "$(EXE_VER_MAJOR).$(EXE_VER_MINOR).$(EXE_VER_PATCHLEVEL)" \
		"Custom package build."

	cd $(PACKAGING_PATH)/BUILDROOT && \
		debuild -b -us -uc

	@echo
	@echo "All done. Your package is here:"
	@find $(PACKAGING_PATH) -name $(EXE_NAME)*.deb


version:
	@echo $(EXE_VERSION)


help:
	@echo 'Optional Build Features:'
	@echo '   ALTHTTPSVC_SUPPORT=0|1  - Build with support for alternative HTTP service.'
	@echo '                             (Default: 0)'
	@echo '   BACKTRACE_SUPPORT=0|1   - Build with backtrace support. (Default: detect)'
	@echo '   BUILD_LIBFABRIC=0|1     - Build our own libfabric (static lib) for the netbench'
	@echo '                             libfabric engine instead of using a system-installed'
	@echo '                             libfabric. Happens automatically if no system'
	@echo '                             libfabric is found or for BUILD_STATIC=1. (Default: 0)'
	@echo '   CUDA_SUPPORT=0|1        - Manually enable (=1) or disable (=0) support for'
	@echo '                             CUDA to work with GPU memory. By default, CUDA'
	@echo '                             support will be enabled when CUDA development files'
	@echo '                             are found. (cuda_runtime.h and libcudart.so)'
	@echo '   CUFILE_SUPPORT=0|1      - Manually enable (=1) or disable (=0) support for'
	@echo '                             GPUDirect Storage (GDS) through the cuFile API.'
	@echo '                             By default, GDS support will be enabled when GDS'
	@echo '                             development files are found. (cufile.h and'
	@echo '                             libcufile.so)'
	@echo '   CYGWIN_SUPPORT=0|1      - Reduce build features to enable build in Cygwin'
	@echo '                             environment. (Default: 0)'
	@echo '   HDFS_SUPPORT=0|1        - Build with support for Hadoop HDFS. HADOOP_HOME'
	@echo '                             and JAVA_HOME need to be set correctly so that'
	@echo '                             $$HADDOP_HOME/include/hdfs.h and'
	@echo '                             $$JAVA_HOME/lib/server/libjvm.so can be found.'
	@echo '                             (Default: 0)'
	@echo '   IBVERBS_SUPPORT=0|1     - Build the netbench ibverbs engine (InfiniBand/RoCE'
	@echo '                             via libibverbs, rdma-core 30 or newer). By default,'
	@echo '                             it gets enabled when libibverbs development files'
	@echo '                             are found.'
	@echo '   RDMACM_SUPPORT=0|1      - Build the netbench rdmacm engine (RDMA CM connections'
	@echo '                             via librdmacm, needs IBVERBS_SUPPORT). By default, it'
	@echo '                             gets enabled when librdmacm development files are'
	@echo '                             found.'
	@echo '   LIBFABRIC_SUPPORT=0|1   - Build the netbench libfabric engine (see'
	@echo '                             BUILD_LIBFABRIC). (Default: 0)'
	@echo '   S3_AWSCRT=0|1           - Build S3 support based on AWS Common Runtime (CRT)'
	@echo '                             instead of external libraries like libcurl. Only'
	@echo '                             effective together with S3_SUPPORT=1. Not'
	@echo '                             compatible with BUILD_STATIC=1. (Default: 0)'
	@echo '   S3_SUPPORT=0|1          - Build with S3 support. This will fetch a AWS SDK'
	@echo '                             git repo of over 1GB size. (Default: 0)'
	@echo '   SPDK_SUPPORT=0|1        - Build with SPDK NVMe-oF initiator support. This'
	@echo '                             will fetch a git repo of over 1GB size.'
	@echo '                             (Default: 0)'
	@echo '   USE_MIMALLOC=0|1        - Use Microsoft mimalloc library for memory'
	@echo '                             allocation management. Recommended when using'
	@echo '                             musl-libc. (Default: 0)'
	@echo
	@echo 'Contributed Plugins (see contrib/plugins/README.md):'
	$(foreach plugin,$(PLUGIN_NAMES),$(PLUGIN_HELP_$(plugin))$(newline))
	@echo
	@echo 'Optional Compile/Link Arguments:'
	@echo '   BUILD_VERBOSE=1         - Enable verbose build output.'
	@echo '   BUILD_STATIC=1          - Generate a static binary without dependencies.'
	@echo '                             (Tested only on Alpine Linux.)'
	@echo '   BUILD_DEBUG=1           - Include debug info in executable.'
	@echo '   CXX=<string>            - Path to alternative C++ compiler. (Default: g++)'
	@echo '   CXX_FLAVOR=<string>     - C++ standard compiler flag. (Default: c++17)'
	@echo '   CXXFLAGS_EXTRA=<string> - Additional C++ compiler flags.'
	@echo '   LDFLAGS_EXTRA=<string>  - Additional linker flags.'
	@echo
	@echo 'Include and library paths:'
	@echo '   AWS_LIB_DIR=<path>         - If this is set in combination with S3_SUPPORT=1'
	@echo '                                then link against pre-built libs in given dir'
	@echo '                                instead of building the AWS SDK CPP.'
	@echo '   AWS_INCLUDE_DIR=<path>     - Include files path for AWS_LIB_DIR.'
	@echo '                                (Default: /usr/include")'
	@echo '   CUDA_INCLUDE_PATH=<path>   - Path to directory containing cuda_runtime.h.'
	@echo '                                (Default: search under /usr/local/cuda*")'
	@echo '   CUDA_LIB_PATH=<path>       - Path to directory containing libcudart.so.'
	@echo '                                (Default: search under /usr/local/cuda*")'
	@echo '   CUFILE_INCLUDE_PATH=<path> - Path to directory containing cufile.h.'
	@echo '                                (Default: search under /usr/local/cuda*")'
	@echo '   CUFILE_LIB_PATH=<path>     - Path to directory containing libcufile.so.'
	@echo '                                (Default: search under /usr/local/cuda*")'
	@echo
	@echo 'Targets:'
	@echo '   all (default)     - Build executable'
	@echo '   clean             - Remove build artifacts'
	@echo '   clean-all         - Remove build artifacts and external sources'
	@echo '   install           - Install executable to /usr/local/bin'
	@echo '   uninstall         - Uninstall executable from /usr/local/bin'
	@echo '   rpm               - Create RPM package file'
	@echo '   deb               - Create Debian package file'
	@echo '   help              - Print this help message'
	@echo
	@echo 'Note: Changing an optional build feature rebuilds what depends on it. "make clean-all"'
	@echo '  is only needed to build the downloaded externals differently (e.g. BUILD_STATIC).'
	@echo '  "make install", "make deb" and "make rpm" take an existing executable as it is.'


# for multi-line variable expansion in recipes (e.g. plugin help)
define newline


endef

.PHONY: clean clean-all clean-externals clean-packaging clean-buildhelpers clean-s3rdma-tools deb \
externals features-info FORCE help prepare-buildroot rpm version


.DEFAULT_GOAL := all


# Include dependency files
ifneq ($(DEPENDENCY_FILES),)
include $(DEPENDENCY_FILES)
endif
