// SPDX-FileCopyrightText: 2020-2026 Sven Breuner and elbencho contributors
// SPDX-License-Identifier: GPL-3.0-only

#include <chrono>
#include <fcntl.h>
#include <iterator>
#include <string>
#include <sys/mman.h>
#include <sys/socket.h>

#include "Common.h"
#include "LocalWorker.h"
#include "Logger.h"
#include "PathStore.h"
#include "toolkits/FileTk.h"
#include "toolkits/random/RandAlgoSelectorTk.h"
#include "toolkits/S3Tk.h"
#include "toolkits/StringTk.h"
#include "toolkits/TranslatorTk.h"
#include "WorkerException.h"
#include "WorkersSharedData.h"
#include "workers/Worker.h"

#ifdef CUDA_SUPPORT
	#include <cuda_runtime.h>
#endif

#ifdef S3_SUPPORT
	#include <aws/core/auth/AWSCredentialsProvider.h>
	#include <aws/core/utils/HashingUtils.h>
	#include <aws/core/utils/memory/stl/AWSString.h>
	#include <aws/core/utils/StringUtils.h>
	#include <aws/core/utils/threading/Executor.h>
	#include <aws/core/utils/UUID.h>
	#include INCLUDE_AWS_S3(model/AbortMultipartUploadRequest.h)
	#include INCLUDE_AWS_S3(model/BucketLocationConstraint.h)
	#include INCLUDE_AWS_S3(model/CompleteMultipartUploadRequest.h)
	#include INCLUDE_AWS_S3(model/CreateBucketRequest.h)
	#include INCLUDE_AWS_S3(model/CreateMultipartUploadRequest.h)
	#include INCLUDE_AWS_S3(model/DeleteBucketRequest.h)
	#include INCLUDE_AWS_S3(model/DeleteBucketTaggingRequest.h)
	#include INCLUDE_AWS_S3(model/DeleteObjectRequest.h)
    #include INCLUDE_AWS_S3(model/DeleteObjectTaggingRequest.h)
	#include INCLUDE_AWS_S3(model/DeleteObjectsRequest.h)
	#include INCLUDE_AWS_S3(model/GetBucketAclRequest.h)
	#include INCLUDE_AWS_S3(model/GetBucketTaggingRequest.h)
	#include INCLUDE_AWS_S3(model/GetObjectAclRequest.h)
	#include INCLUDE_AWS_S3(model/GetObjectRequest.h)
    #include INCLUDE_AWS_S3(model/GetObjectTaggingRequest.h)
    #include INCLUDE_AWS_S3(model/GetObjectLockConfigurationRequest.h)
    #include INCLUDE_AWS_S3(model/GetBucketVersioningRequest.h)
	#include INCLUDE_AWS_S3(model/HeadObjectRequest.h)
	#include INCLUDE_AWS_S3(model/HeadBucketRequest.h)
	#include INCLUDE_AWS_S3(model/ListObjectsV2Request.h)
	#include INCLUDE_AWS_S3(model/Object.h)
    #include INCLUDE_AWS_S3(model/ObjectLockRule.h)
	#include INCLUDE_AWS_S3(model/PutBucketAclRequest.h)
	#include INCLUDE_AWS_S3(model/PutBucketTaggingRequest.h)
	#include INCLUDE_AWS_S3(model/PutObjectAclRequest.h)
	#include INCLUDE_AWS_S3(model/PutObjectRequest.h)
    #include INCLUDE_AWS_S3(model/PutObjectTaggingRequest.h)
    #include INCLUDE_AWS_S3(model/PutObjectLockConfigurationRequest.h)
    #include INCLUDE_AWS_S3(model/PutBucketVersioningRequest.h)
	#include INCLUDE_AWS_S3(model/UploadPartRequest.h)
#endif

#define PATH_BUF_LEN					64
#define MKDIR_MODE						0777
#define INTERRUPTION_CHECK_INTERVAL		128
#define AIO_MAX_WAIT_SEC				5
#define AIO_MAX_EVENTS					4  // max number of events to retrieve in io_getevents()
#define NETBENCH_CONNECT_TIMEOUT_SEC	20 // max time for servers to wait and clients to retry
#define NETBENCH_RECEIVE_TIMEOUT_SEC	20 // max time to wait for incoming data on client & server
#define NETBENCH_SHORT_POLL_TIMEOUT_SEC	2  // time to check for interrupts in longer poll wait loops

#define TAG_CHECKSUM_LEN        2

#define TAG_KEY_MEDIUM_NAME     "medium"
#define TAG_KEY_LONG_NAME       "long"
#define TAG_VALUE_MEDIUM_LEN    100
#define TAG_VALUE_LONG_LEN      256

#define RETENTION_PERIOD_DAYS   1

#ifdef S3_SUPPORT
    #ifdef S3_AWSCRT
        namespace S3 = Aws::S3Crt::Model;
    #else
        namespace S3 = Aws::S3::Model;
        using S3Errors = Aws::S3::S3Errors;
    #endif // S3_AWSCRT

    S3UploadStore LocalWorker::s3SharedUploadStore; // singleton for shared uploads

    /**
     * Context variables for asynchronous GetObject requests.
     */
    class S3AsyncDownloadContext
    {
        public:

        /**
         * Note that this does not init all members - some need to be set later (because the
         * values are not ready yet at the time of object creation).
         */
        S3AsyncDownloadContext(uint64_t currentOffset, size_t blockSize) :
            currentOffset(currentOffset), blockSize(blockSize),
            partCompleteFuture(partCompletePromise.get_future() )
        {}

        uint64_t currentOffset;
        size_t blockSize;

        std::chrono::steady_clock::time_point ioStartT;

        S3::GetObjectRequest request;
        std::promise<S3::GetObjectOutcome> partCompletePromise;
        std::future<S3::GetObjectOutcome> partCompleteFuture;
    };

    /**
     * Context variables for parts of a asynchronous multipart upload.
     */
    class S3AsyncUploadPartContext
    {
        public:

        /**
         * Note that this does not init all members - some need to be set later (because the
         * values are not ready yet at the time of object creation).
         */
        S3AsyncUploadPartContext(uint64_t currentOffset, size_t blockSize, size_t partNum) :
            currentOffset(currentOffset), blockSize(blockSize), partNum(partNum),
            partCompleteFuture(partCompletePromise.get_future() )
        {
            completedPart.SetPartNumber(partNum);
        }

        uint64_t currentOffset;
        size_t blockSize;
        size_t partNum;

        std::chrono::steady_clock::time_point ioStartT;

        S3::UploadPartRequest uploadPartRequest;
        std::promise<S3::UploadPartOutcome> partCompletePromise;
        std::future<S3::UploadPartOutcome> partCompleteFuture;
        S3::CompletedPart completedPart;
    };

#endif // S3_SUPPORT


SocketVec LocalWorker::serverSocketVec; // singleton netbench sockets vec for all local threads


LocalWorker::LocalWorker(WorkersSharedData* workersSharedData, size_t workerRank) :
	Worker(workersSharedData, workerRank), opsLog(workersSharedData->progArgs, workerRank)
{
	nullifyPhaseFunctionPointers();

	fileHandles.fdVec.resize(1);
	fileHandles.cuFileHandleDataVec.resize(1);
}

LocalWorker::~LocalWorker()
{
	/* note: Most of the cleanup is done in cleanup() instead of here, because we need to release
	 	handles etc. before the destructor is called. This is because in service mode, this object
		needs to survive to provide statistics and might only get deleted when the next benchmark
		run starts, but we can't keep the handles until then. */
}


/**
 * Entry point for the thread.
 * Kick off the work that this worker has to do. Each phase is sychronized to wait for notification
 * by coordinator.
 *
 * Ensure that cleanup() is called when this method finishes.
 */
void LocalWorker::run()
{
	try
	{
		buuids::uuid currentBenchID = buuids::nil_uuid();

		// preparation phase
		preparePhase();

		// signal coordinator that our preparations phase is done
		phaseFinished = true; // before incNumWorkersDone(), as Coordinator can reset after done inc
		incNumWorkersDone();

		for( ; ; )
		{
			// wait for coordinator to set new bench ID to signal us that we are good to start
			waitForNextPhase(currentBenchID);

			currentBenchID = workersSharedData->currentBenchID;
			bool doInfiniteIOLoop = progArgs->getDoInfiniteIOLoop();

			do // for infinite I/O loop
			{
				initThreadPhaseVars();
				initPhaseFileHandleVecs();
				initPhaseRWOffsetGen();
				initPhaseFunctionPointers();

                // (note: order of bench phases is defined in Coordinator::runBenchmarks() )
				switch(workersSharedData->currentBenchPhase)
				{
					case BenchPhase_TERMINATE:
					{
						LOGGER(Log_DEBUG, "Terminating as requested. Rank: " << workerRank << "; "
							"(Offset: " << progArgs->getRankOffset() << ")" << std::endl);
						incNumWorkersDone();
						return;
					} break;

					case BenchPhase_CREATEDIRS:
					case BenchPhase_DELETEDIRS:
					case BenchPhase_STATDIRS:
					{
                        if(progArgs->getBenchPathType() != BenchPathType_DIR)
                            throw WorkerException("Directory creation and deletion are not "
                                "available in file and block device mode.");

                        if(progArgs->getBenchMode() == BenchMode_HDFS)
                            hdfsDirModeIterateDirs();
                        else
                        if(progArgs->getBenchMode() == BenchMode_S3)
                            s3ModeIterateBuckets();
                        else
                            progArgs->getTreeFilePath().empty() ?
                                dirModeIterateDirs() : dirModeIterateCustomDirs();
					} break;

                    case BenchPhase_GET_S3_BUCKET_MD:
                    case BenchPhase_PUT_S3_BUCKET_MD:
                    case BenchPhase_DEL_S3_BUCKET_MD:
                    {
                        s3ModeIterateBuckets();
                    } break;

					case BenchPhase_CREATEFILES:
					case BenchPhase_READFILES:
					{
                        if(progArgs->getBenchPathType() == BenchPathType_DIR)
                        {
                            if(progArgs->getBenchMode() == BenchMode_NETBENCH)
                                netbenchDoTransfer();
                            else
                            if(progArgs->getBenchMode() == BenchMode_HDFS)
                                hdfsDirModeIterateFiles();
                            else
                            if(progArgs->getBenchMode() == BenchMode_S3)
                                progArgs->getTreeFilePath().empty() ?
                                    s3ModeIterateObjects() : s3ModeIterateCustomObjects();
                            else
                                progArgs->getTreeFilePath().empty() ?
                                    dirModeIterateFiles() : dirModeIterateCustomFiles();
                        }
                        else
                        {
                            if(progArgs->getUseRandomOffsets() || progArgs->getUseStridedAccess() )
                                fileModeIterateFilesRand();
                            else
                                fileModeIterateFilesSeq();
                        }
					} break;

                    case BenchPhase_S3MPUCOMPLETE:
                    {
                        s3ModeIterateAndCompleteMpuIDs();
                    } break;

                    case BenchPhase_GET_S3_OBJECT_MD:
                    case BenchPhase_PUT_S3_OBJECT_MD:
                    case BenchPhase_DEL_S3_OBJECT_MD:
                    {
                        progArgs->getTreeFilePath().empty() ?
                            s3ModeIterateObjects() : s3ModeIterateCustomObjects();
                    } break;

					case BenchPhase_STATFILES:
					{
                        if(progArgs->getBenchPathType() != BenchPathType_DIR)
                            throw WorkerException("File stat operation not available in file and "
                                "block device mode.");

                        if(progArgs->getBenchMode() == BenchMode_HDFS)
                            hdfsDirModeIterateFiles();
                        else
                        if(progArgs->getBenchMode() == BenchMode_S3)
                            progArgs->getTreeFilePath().empty() ?
                                s3ModeIterateObjects() : s3ModeIterateCustomObjects();
                        else
                            progArgs->getTreeFilePath().empty() ?
                                dirModeIterateFiles() : dirModeIterateCustomFiles();
					} break;

					case BenchPhase_PUTBUCKETACL:
					case BenchPhase_GETBUCKETACL:
					{
						s3ModeIterateBuckets();
					} break;

					case BenchPhase_PUTOBJACL:
					case BenchPhase_GETOBJACL:
					{
						progArgs->getTreeFilePath().empty() ?
							s3ModeIterateObjects() : s3ModeIterateCustomObjects();
					} break;

					case BenchPhase_LISTOBJECTS:
					{
						s3ModeListObjects();
					} break;

					case BenchPhase_LISTOBJPARALLEL:
					{
						if(!progArgs->getTreeFilePath().empty() )
							throw WorkerException("Parallel object listing is not available in "
								"custom tree mode.");

						s3ModeListObjParallel();
					} break;

					case BenchPhase_MULTIDELOBJ:
					{
						s3ModeListAndMultiDeleteObjects();
					} break;

					case BenchPhase_DELETEFILES:
					{
                        if(progArgs->getBenchPathType() == BenchPathType_DIR)
                        {
                            if(progArgs->getBenchMode() == BenchMode_HDFS)
                                hdfsDirModeIterateFiles();
                            else
                            if(progArgs->getBenchMode() == BenchMode_S3)
                                progArgs->getTreeFilePath().empty() ?
                                    s3ModeIterateObjects() : s3ModeIterateCustomObjects();
                            else
                                progArgs->getTreeFilePath().empty() ?
                                    dirModeIterateFiles() : dirModeIterateCustomFiles();
                        }
                        else
                            fileModeDeleteFiles();
					} break;

					case BenchPhase_SYNC:
					{
						anyModeSync();
						doInfiniteIOLoop = false;
					} break;

					case BenchPhase_DROPCACHES:
					{
						anyModeDropCaches();
						doInfiniteIOLoop = false;
					} break;

					default:
					{ // should never happen
						throw WorkerException("Unknown/invalid next phase type: " +
							std::to_string(workersSharedData->currentBenchPhase) );
					} break;

				} // end of switch

				checkInterruptionRequest(); // for infinite loop workers with no work

			} while(doInfiniteIOLoop && workerGotPhaseWork); // end of infinite loop

			// let coordinator know that we are done
			finishPhase();

		} // end of for loop

	}
	catch(WorkerInterruptedException& e)
	{
		// whoever interrupted us will have a reason for it, so we don't print at normal level here
		ErrLogger(Log_DEBUG, progArgs->getRunAsService() ) << "Interrupted exception. " <<
			"WorkerRank: " << workerRank << std::endl;

		/* check if called twice. (happens on interrupted waitForNextPhase() while other workers
			haven't finished the previous phase, i.e. during the end game of a phase.) */
		if(!phaseFinished)
		{
			s3ModeAbortUnfinishedSharedUploads(); // abort unfinished S3 uploads
			finishPhase(); // let coordinator know that we are done
		}

		return;
	}
	catch(std::exception& e)
	{
		ErrLogger(Log_NORMAL, progArgs->getRunAsService() ) << e.what() << std::endl;

		s3ModeAbortUnfinishedSharedUploads(); // abort unfinished S3 uploads
	}

	incNumWorkersDoneWithError();
}

/**
 * Run all the preparations in the run() method that are needed before we can announce readiness
 * for an actual benchmark run.
 */
void LocalWorker::preparePhase()
{
    applyNumaAndCoreBinding();

    opsLog.openLogFile();

    initThreadFDVec();
    initThreadCuFileHandleDataVec();
    initThreadMmapVec();

    allocIOBuffer();
    allocGPUIOBuffer();

    prepareCustomTreePathStores();

    initLibAio();
    initS3Client();
    initHDFS();
    initNetBench();

#ifdef S3_SUPPORT
    s3SharedUploadStore.setProgArgs(progArgs, workerRank);
#endif // S3_SUPPORT
}

/**
 * Update finish time values, then signal coordinator that we're done.
 *
 * Nothing should run after this, because coordinator will assume that it can reset things after
 * the done counter has been increased inside this function.
 */
void LocalWorker::finishPhase()
{
	if(!workerGotPhaseWork)
		elapsedUSecVec.resize(0);
	else
	{
		std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();

		std::chrono::microseconds elapsedDurationUSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(now - workersSharedData->phaseStartT);
		uint64_t finishElapsedUSec = elapsedDurationUSec.count();

		elapsedUSecVec.resize(1);
		elapsedUSecVec[0] = finishElapsedUSec;
	}

	phaseFinished = true; // before incNumWorkersDone() because Coordinator can reset after inc

	incNumWorkersDone();
}

void LocalWorker::initLibAio()
{
#ifdef LIBAIO_SUPPORT

    const size_t maxIODepth = progArgs->getIODepth();

    if(maxIODepth < 2)
        return; // no libaio needed

    libaioContext.ioContext = (io_context_t)0; // zeroing required by io_queue_init
    libaioContext.iocbVec.resize(maxIODepth);
    libaioContext.iocbPointerVec.resize(maxIODepth);
    libaioContext.ioStartTimeVec.resize(maxIODepth);

    int initRes = io_queue_init(maxIODepth, &libaioContext.ioContext);
    IF_UNLIKELY(initRes)
        throw WorkerException(std::string("Initializing async IO (io_queue_init) failed. ") +
            "SysErr: " + strerror(-initRes) ); // (io_queue_init returns negative errno)

#endif // LIBAIO_SUPPORT
}

void LocalWorker::uninitLibAio()
{
#ifdef LIBAIO_SUPPORT

    const size_t maxIODepth = progArgs->getIODepth();

    if(maxIODepth < 2)
        return; // no libaio needed

    if(libaioContext.ioContext != NULL)
        io_queue_release(libaioContext.ioContext);

#endif // LIBAIO_SUPPORT
}

/**
 * Initialize AWS S3 SDK & S3 client object. Intended to be called at the start of each benchmark
 * phase. Will do nothing if not built with S3 support or no S3 endpoints defined.
 *
 * S3 endpoints get assigned round-robin to workers based on workerRank.
 */
void LocalWorker::initS3Client()
{
#ifdef S3_SUPPORT

	if(progArgs->getBenchMode() != BenchMode_S3)
		return; // nothing to do

    if(progArgs->getUseS3ClientSingleton() )
    { // using shared singleton s3 client instead of per-worker s3 client instances
        s3Client = progArgs->getS3ClientSingleton();
        s3EndpointStr = progArgs->getS3SingletonEndpointStr();
    }
    else
    { // using per-worker s3 client instances
        s3Client = S3Tk::initS3Client(progArgs, workerRank, &isInterruptionRequested,
            &s3EndpointStr);
    }

    useS3SSE = progArgs->getUseS3SSE();

    s3SSECKey = progArgs->getS3SSECKey();
    if(!s3SSECKey.empty() )
        s3SSECKeyMD5 = S3Tk::computeKeyMD5(s3SSECKey);

    s3SSEKMSKey = progArgs->getS3SSEKMSKey();

    if(progArgs->getS3ChecksumAlgo().empty() )
        s3ChecksumAlgorithm = S3ChecksumAlgorithm::NOT_SET;
    else
    {
        /* GetChecksumAlgorithmForName returns hash value for invalid algo names, so we need to
            confirm the algo by checking the valid algo enum values. */

        s3ChecksumAlgorithm = S3ChecksumAlgorithmMapper::GetChecksumAlgorithmForName(
            progArgs->getS3ChecksumAlgo() );

        switch(s3ChecksumAlgorithm)
        {
            case S3ChecksumAlgorithm::CRC32:
            case S3ChecksumAlgorithm::CRC32C:
            case S3ChecksumAlgorithm::SHA1:
            case S3ChecksumAlgorithm::SHA256:
            //case S3ChecksumAlgorithm::CRC64NVME: // crc64nvme not supported by aws sdk currently
                // because of missing uploadPartRequest.SetChecksumCRC64NVME for
                // S3Tk::addUploadPartRequestChecksum()
                break;
            default:
                throw WorkerException(std::string("Invalid S3 checksum algorithm: ") +
                    progArgs->getS3ChecksumAlgo() );
        }
    }

#endif // S3_SUPPORT
}

/**
 * Free S3 client object. Intended to be called at the end of each benchmark phase.
 */
void LocalWorker::uninitS3Client()
{
#ifdef S3_SUPPORT

    if(progArgs->getBenchMode() != BenchMode_S3)
		return; // nothing to do

	// s3Client is a std::shared_ptr, so reset() will cleanup the client object
	// (note: this could also be the shared singleton s3 client from ProgArgs)
	s3Client.reset();

#endif // S3_SUPPORT
}

void LocalWorker::initHDFS()
{
#ifdef HDFS_SUPPORT

	if(!progArgs->getUseHDFS() )
		return; // nothing to do

	hdfsFSHandle = hdfsConnect("default", 0);
	if(!hdfsFSHandle)
		throw WorkerException("Unable to connect to HDFS using \"default\" config.");

#endif // HDFS_SUPPORT
}

void LocalWorker::uninitHDFS()
{
#ifdef HDFS_SUPPORT

	if(!progArgs->getUseHDFS() )
		return; // nothing to do

	if(!hdfsFSHandle)
		return; // nothing to do

	hdfsDisconnect(hdfsFSHandle);

	hdfsFSHandle = NULL;

#endif // HDFS_SUPPORT
}

/**
 * Wrapper to initialize network benchmark mode servers and clients.
 */
void LocalWorker::initNetBench()
{
	if(!progArgs->getRunAsService() || (progArgs->getBenchMode() != BenchMode_NETBENCH) )
		return; // nothing to do

	const size_t hostIndex =
		progArgs->getRankOffset() / progArgs->getNumThreads(); // zero-based
	const bool hostIsServer = (hostIndex < progArgs->getNumNetBenchServers() );

	if(hostIsServer)
		initNetBenchServer();
	else
		initNetBenchClient();
}

/**
 * Initialize network benchmark mode server. First thread of a server will open service port +1000
 * to accept conns from clients. First worker of a server accepts connections for all worker
 * threads.
 */
void LocalWorker::initNetBenchServer()
{
	if(!progArgs->getRunAsService() || (progArgs->getBenchMode() != BenchMode_NETBENCH) )
		return; // nothing to do

	const unsigned listenTimeoutMS = NETBENCH_CONNECT_TIMEOUT_SEC * 1000;
	const unsigned short listenPort = progArgs->getServicePort() + 1000;

	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();

	const size_t hostIndex =
		progArgs->getRankOffset() / progArgs->getNumThreads(); // zero-based
	const bool hostIsServer = (hostIndex < progArgs->getNumNetBenchServers() );
	const size_t numClients = (progArgs->getNumDataSetThreads() / progArgs->getNumThreads() ) -
		progArgs->getNumNetBenchServers();

	if(hostIsServer && (localWorkerRank != 0) )
		return; // nothing to do. (only first worker of server accepts all client conns)

	const unsigned numConnsTotal = (numClients * progArgs->getNumThreads() );

	unsigned numConnsToWaitFor = (numConnsTotal / progArgs->getNumNetBenchServers() );

	if( (numConnsTotal % progArgs->getNumNetBenchServers() ) &&
		(hostIndex < (numConnsTotal % progArgs->getNumNetBenchServers() ) ) )
		numConnsToWaitFor += 1;

	// prepare listen socket

	BasicSocket listenSock(AF_INET, SOCK_STREAM);

	// (note: buf size has to be set before listen() to be applied to new accepted sockets)
	if(progArgs->getSockRecvBufSize() )
	{
		LOGGER(Log_VERBOSE, "Changing sock recv buf size. Old value: " <<
			listenSock.getSoRcvBuf() << std::endl);

		listenSock.setSoRcvBuf(progArgs->getSockRecvBufSize() );
	}

	if(progArgs->getSockSendBufSize() )
	{
		LOGGER(Log_VERBOSE, "Changing sock send buf size. Old value: " <<
			listenSock.getSoSndBuf() << std::endl);

		listenSock.setSoSndBuf(progArgs->getSockSendBufSize() );
	}

	listenSock.setSoReuseAddr(true);
	listenSock.bind(listenPort);

	listenSock.listen();

	// wait for incoming connections

	LOGGER(Log_VERBOSE, "netbench init: "
		"Listening for client connections at: " << listenPort << std::endl);

	while(serverSocketVec.size() < numConnsToWaitFor)
	{
		bool haveIncomingConn = listenSock.waitForIncomingData(listenTimeoutMS);
		if(!haveIncomingConn) // timeout
			throw WorkerException("Timed out waiting for client connections. "
				"Received connections: " + std::to_string(serverSocketVec.size() ) + "; "
				"Expected connections: " + std::to_string(numConnsToWaitFor) + "; "
				"Timeout in ms: " + std::to_string(listenTimeoutMS) );

		struct sockaddr_in peer;
		socklen_t peerStructSize = sizeof(peer);
		BasicSocket* newSocket = (BasicSocket*)listenSock.accept(
			(struct sockaddr*)&peer, &peerStructSize);

		LOGGER(Log_VERBOSE, "netbench init: "
			"Accepted new connection from: " << newSocket->getPeername() << "; "
			"Connected: " << (serverSocketVec.size() + 1) << " / " << numConnsToWaitFor <<
			std::endl);

		newSocket->setSoKeepAlive(true);
		newSocket->setTcpNoDelay(true);

		serverSocketVec.push_back(newSocket);
	}
}

/**
 * Initialize network benchmark client mode. Each client worker opens one connection. Client threads
 * connect round-robin to the different servers, so that a single client with multiple threads can
 * talk to multiple servers.
 */
void LocalWorker::initNetBenchClient()
{
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();

	// prepare socket

	clientSocket = new BasicSocket(AF_INET, SOCK_STREAM);

	clientSocket->setSoKeepAlive(true);
	clientSocket->setTcpNoDelay(true);

	if(progArgs->getSockRecvBufSize() )
	{
		LOGGER(Log_VERBOSE, "Changing sock recv buf size. Old value: " <<
			clientSocket->getSoRcvBuf() << std::endl);

		clientSocket->setSoRcvBuf(progArgs->getSockRecvBufSize() );
	}

	if(progArgs->getSockSendBufSize() )
	{
		LOGGER(Log_VERBOSE, "Changing sock send buf size. Old value: " <<
			clientSocket->getSoSndBuf() << std::endl);

		clientSocket->setSoSndBuf(progArgs->getSockSendBufSize() );
	}

	if(!progArgs->getNetDevsVec().empty() )
	{ // round-robin binding of sockets to user-given network devices
		unsigned netDevIdx = (localWorkerRank % progArgs->getNetDevsVec().size() );
		clientSocket->setSoBindToDevice(progArgs->getNetDevsVec()[netDevIdx].c_str() );
	}

	/* note: clientWorkerRank and serverOffset have to remain in sync with the number of
		 connections that each server expects. */

	/* clients connect round-robin to different servers
		(next client continues where the previous client stopped) */

	const NetBenchServerAddrVec& serversVec = progArgs->getNetBenchServers();
	const size_t clientWorkerRank = workerRank -
		(serversVec.size() * progArgs->getNumThreads() );
	const size_t serversOffset = clientWorkerRank % serversVec.size();
	const NetBenchServerAddr& serverAddr = serversVec[serversOffset];

	// start time for connection retry timeout
	std::chrono::steady_clock::time_point connectStartT = std::chrono::steady_clock::now();

	// connection attempt(s)
	for( ; ; )
	{
		try
		{
			LOGGER(Log_DEBUG, "netbench init: "
				"Connecting to: " << serverAddr.host.c_str() << ":" <<
				serverAddr.port << "; " <<
				"WorkerRank: " << workerRank << std::endl);

			clientSocket->connect(serverAddr.host.c_str(), serverAddr.port);

			LOGGER(Log_VERBOSE, "netbench init: "
				"Established connection to: " << clientSocket->getPeername() << "; " <<
				"WorkerRank: " << workerRank << std::endl);

			break; // connection successful if no exception thrown
		}
		catch(SocketConnectException& e)
		{ // server might just not be ready yet, thus retry if not timed out yet

			// calculate elapsed time to check retry timeout
			std::chrono::steady_clock::time_point connectEndT =
				std::chrono::steady_clock::now();
			std::chrono::seconds connectElapsedSecs =
				std::chrono::duration_cast<std::chrono::seconds>
				(connectEndT - connectStartT);

			if(connectElapsedSecs.count() > NETBENCH_CONNECT_TIMEOUT_SEC)
				throw;

			// wait 500ms before the next retry to avoid flooding
			std::this_thread::sleep_for(std::chrono::milliseconds(500));
		}

	} // end of connection retry loop
}

/**
 * Cleanup client sockets from netbench mode. Server sockets are shared across all threads and
 * thus get cleaned up in uninitNetBenchAfterPhaseDone().
 */
void LocalWorker::uninitNetBench()
{
	if(!progArgs->getRunAsService() || (progArgs->getBenchMode() != BenchMode_NETBENCH) )
		return; // nothing to do

	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const size_t hostIndex =
		progArgs->getRankOffset() / progArgs->getNumThreads(); // zero-based
	const bool hostIsServer = (hostIndex < progArgs->getNumNetBenchServers() );

	if(hostIsServer && (localWorkerRank != 0) )
		return; // nothing to do. (only first worker of server closes all client conns)

	if(hostIsServer)
	{
		/* this cleanup happens in uninitNetBenchAfterPhaseDone() because some workers might still
		 	 be running at this point. */
	}

	if(!hostIsServer && (clientSocket != NULL) )
	{ // this is a client => just one socket to disconnect
		try { clientSocket->shutdown(); } catch(...) {}
		delete(clientSocket);
		clientSocket = NULL;
	}
}

/**
 * Late cleanup for shared server sockets. Client sockets get cleaned up in uninitNetBench() because
 * they are not shared across all workers.
 */
void LocalWorker::uninitNetBenchAfterPhaseDone()
{
	if(!progArgs->getRunAsService() || (progArgs->getBenchMode() != BenchMode_NETBENCH) )
		return; // nothing to do

	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const size_t hostIndex =
		progArgs->getRankOffset() / progArgs->getNumThreads(); // zero-based
	const bool hostIsServer = (hostIndex < progArgs->getNumNetBenchServers() );

	if(hostIsServer && (localWorkerRank != 0) )
		return; // nothing to do. (only first worker of server closes all client conns)

	if(hostIsServer)
	{
		for(Socket* sock : serverSocketVec)
		{
			try { sock->shutdown(); } catch(...) {}
			delete(sock); // destructor contains close()
		}

		serverSocketVec.clear();
		serverSocketVec.shrink_to_fit();
	}

}

/**
 * If progArgs::useNoFDSharing is set, initialize threadFDVec with separate open files in file/bdev
 * mode. Otherwise do nothing.
 *
 * Note: It's assumed that progArgs did all the basic checks (e.g. to find out if all paths refer
 * to the same type) and that progArgs also takes care of truncate options.
 *
 * @throw WorkerException on error, e.g. if open failed.
 */
void LocalWorker::initThreadFDVec()
{
	if(!progArgs->getUseNoFDSharing() )
		return; // shared FDs, so nothing to do

	if(progArgs->getBenchPathType() == BenchPathType_DIR)
		return; // nothing to do in dir mode

	const StringVec& benchPathsVec = progArgs->getBenchPaths();

	fileHandles.threadFDVec.reserve(benchPathsVec.size() );

	// check if each given path exists as dir and add it to pathFDsVec
	// note: keep flags in sync with ProgArgs::prepareBenchPathFDsVec
	for(std::string path : benchPathsVec)
	{
		int fd;
		int openFlags = 0;

		if(progArgs->getRunCreateFilesPhase() || progArgs->getRunDeleteFilesPhase() )
			openFlags |= O_RDWR;
		else
			openFlags |= O_RDONLY;

#if !defined(__APPLE__)
        if(progArgs->getUseDirectIO() )
            openFlags |= O_DIRECT;
#endif // !apple

		if(progArgs->getRunCreateFilesPhase() )
			openFlags |= O_CREAT;

	    OPLOG_PRE_OP("open", path.c_str(), 0, 0);

        fd = open(path.c_str(), openFlags, MKFILE_MODE);

	    OPLOG_POST_OP("open", path.c_str(), 0, 0, fd == -1);

		if(fd == -1)
			throw WorkerException("Unable to open benchmark path: " + path + "; "
				"SysErr: " + strerror(errno) );

		fileHandles.threadFDVec.push_back(fd);
	}

}

/**
 * If progArgs::useNoFDSharing is set, close FDs of threadFDVec. Otherwise do nothing.
 */
void LocalWorker::uninitThreadFDVec()
{
	for(int fd : fileHandles.threadFDVec)
	{
        OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

        int closeRes = close(fd);

        OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

		if(closeRes == -1)
			ERRLOGGER(Log_NORMAL, "Error on file close. "
				"FD: " << fd << "; "
				"SysErr: " << strerror(errno) << std::endl);
	}

	fileHandles.threadFDVec.resize(0);
}

/**
 * Similar to threadFDVec, here we init thread-local cuFile handles for progArgs::useNoFDSharing.
 *
 * @throw WorkerException if cuFile handle registration fails.
 */
void LocalWorker::initThreadCuFileHandleDataVec()
{
	for(int fd : fileHandles.threadFDVec)
	{
		// add new element to vec and reference it
		fileHandles.threadCuFileHandleDataVec.resize(
			fileHandles.threadCuFileHandleDataVec.size() + 1);
		CuFileHandleData& cuFileHandleData =
			fileHandles.threadCuFileHandleDataVec[fileHandles.threadCuFileHandleDataVec.size() - 1];

		if(!progArgs->getUseCuFile() )
			continue; // no registration to be done if cuFile API is not used

		// note: cleanup won't be a prob if reg not done, as CuFileHandleData can handle that case

		cuFileHandleData.registerHandle<WorkerException>(fd);
	}
}

/**
 * Deregsiter threadCuFileHandleVec entries.
 */
void LocalWorker::uninitThreadCuFileHandleDataVec()
{
	for(CuFileHandleData& cuFileHandleData : fileHandles.threadCuFileHandleDataVec)
		cuFileHandleData.deregisterHandle();

	fileHandles.threadCuFileHandleDataVec.resize(0); // reset vec before reuse in service mode
}

/**
 * Init fileHandles.mmapVec from progArgs->getBenchPathFDs() if this is a random phase with files
 * as bench paths. Otherwise the init will happen later in initPhaseFileHandleVecs().
 */
void LocalWorker::initThreadMmapVec()
{
	if(!progArgs->getUseMmap() )
		return; // nothing to do

	if(progArgs->getBenchPathType() == BenchPathType_DIR)
		return; // init will happen in initPhaseFileHandleVecs()

	if(!progArgs->getUseRandomOffsets() && !progArgs->getUseStridedAccess() )
		return; // init will happen in initPhaseFileHandleVecs()

	fileHandles.mmapVec = progArgs->getMmapVec();
}

/**
 * Unmap fileHandles.mmapVec entries after a run of possibly multiple phases.
 */
void LocalWorker::uninitThreadMmapVec()
{
	if( (progArgs->getBenchPathType() != BenchPathType_DIR) &&
		(progArgs->getUseRandomOffsets() || progArgs->getUseStridedAccess() ) )
	{ // random file/bdev mode: we copied mappings from progArgs, so don't unmap here
		fileHandles.mmapVec.resize(0);

		return;
	}

	// if we got here then we're not in random file/bdev mode, so we did our own mapping in worker

	for(char*& mmapPtr : fileHandles.mmapVec)
	{
		if(mmapPtr == MAP_FAILED)
			continue;

		int unmapRes = munmap(mmapPtr, progArgs->getFileSize() );

		if(unmapRes == -1)
			ERRLOGGER(Log_NORMAL, "File memory unmap failed. "
				"SysErr: " << strerror(errno) << std::endl);

		mmapPtr = (char*)MAP_FAILED;
	}

	fileHandles.mmapVec.resize(0);
}


/**
 * Init thread-local phase values.
 */
void LocalWorker::initThreadPhaseVars()
{
    const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
    const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
    const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
        (localWorkerRank < progArgs->getNumRWMixReadThreads() ) );

    /* in rwmix threads mode, we set benchPhase to BenchPhase_READFILES, because the corresponding
       workers will only be reading */
    if(isRWMixedReader)
        benchPhase = BenchPhase_READFILES;
    else
        benchPhase = globalBenchPhase;
}

/**
 * Init fileHandle vectors for current phase.
 */
void LocalWorker::initPhaseFileHandleVecs()
{
	const BenchPathType benchPathType = progArgs->getBenchPathType();

	fileHandles.errorFDVecIdx = -1; // clear ("-1" means "not set")

	// reset/clear all vecs
	fileHandles.fdVec.resize(0);
	fileHandles.fdVecPtr = NULL;
	fileHandles.cuFileHandleDataVec.resize(0);
	fileHandles.cuFileHandleDataPtrVec.resize(0);


	if(benchPathType == BenchPathType_DIR)
	{
		/* in dir mode, there is only one currently active file per worker.
			files will be dynamically opened in the dir mode file iter method and their fd will be
			stored in fileHandles.fdVec[0] */

		fileHandles.fdVec.resize(1);
		fileHandles.fdVec[0] = -1; // clear/reset

		fileHandles.fdVecPtr = &fileHandles.fdVec;

		// fileHandles.cuFileHandleDataVec[0] will be used for current file

		fileHandles.cuFileHandleDataVec.resize(1);

		fileHandles.cuFileHandleDataPtrVec.push_back(&fileHandles.cuFileHandleDataVec[0] );

		fileHandles.mmapVec.resize(1, (char*)MAP_FAILED);
	}
	else
	if(!progArgs->getUseRandomOffsets() && !progArgs->getUseStridedAccess() )
	{
		/* in sequential file/bdev mode, there is only one currently active file per worker.
			original file FDs will be taken from progArgs or fileHandles.threadFDVec, but FD will be
			copied to fileHandles.fdVec[0] for the single current file. */

		fileHandles.fdVec.resize(1);
		fileHandles.fdVec[0] = -1; // clear/reset, will be set dynamically to current file

		fileHandles.fdVecPtr = &fileHandles.fdVec;

		fileHandles.cuFileHandleDataPtrVec.resize(1); // set dynamically to current file

		fileHandles.mmapVec.resize(1, (char*)MAP_FAILED); /* fileModeIterateFilesSeq() will do the
			mmap() dynamically when this thread switches to a new file */
	}
	else
	{
		// in random file/bdev mode, rwBlockSized/aioBlockSized randomly select FDs from given set

		fileHandles.fdVecPtr = fileHandles.threadFDVec.empty() ?
			&progArgs->getBenchPathFDs() : &fileHandles.threadFDVec;

		CuFileHandleDataVec& cuFileHandleDataVec = fileHandles.threadCuFileHandleDataVec.empty() ?
			progArgs->getCuFileHandleDataVec() : fileHandles.threadCuFileHandleDataVec;

		for(size_t i=0; i < cuFileHandleDataVec.size(); i++)
			fileHandles.cuFileHandleDataPtrVec.push_back(&(cuFileHandleDataVec[i]) );

		// note: nothing for fileHandles.mmapVec here; init was done in initThreadMmapVec
	}
}

/**
 * Prepare read/write file contents offset generator for dirModeIterateFiles and
 * fileModeIterateFiles.
 *
 * Note: rwOffsetGen will always be set to an object here (even if phase is not read or write) to
 * prevent extra NULL pointer checks in file/dir loops.
 */
void LocalWorker::initPhaseRWOffsetGen()
{
	const size_t blockSize = progArgs->getBlockSize();
	const uint64_t fileSize = progArgs->getFileSize();
	const bool isWritePhase = (workersSharedData->currentBenchPhase == BenchPhase_CREATEFILES);

	/* in dir mode randAmount is file size, for file/bdev it's the total amount for this thread
		across all given files/bdevs */
	const uint64_t randomAmount = (progArgs->getBenchPathType() == BenchPathType_DIR) ?
		fileSize : progArgs->getRandomAmount() / progArgs->getNumDataSetThreads();

	// init random algos
	randBlockVarAlgo = RandAlgoSelectorTk::stringToAlgo(progArgs->getBlockVarianceAlgo() );
	randBlockVarReseed = std::make_unique<RandAlgoXoshiro256ss>();

	// init algo for random offsets within files
    randOffsetAlgo = RandAlgoSelectorTk::stringToAlgo(progArgs->getRandOffsetAlgo().empty() ?
        RANDALGO_BALANCED_SEQUENTIAL_STR : progArgs->getRandOffsetAlgo() );

	// note: in some cases these defs get overridden per-file later (e.g. for custom tree)

	if(progArgs->getDoReverseSeqOffsets() || getS3ModeDoReverseSeqFallback() ) // seq backward
		rwOffsetGen = std::make_unique<OffsetGenReverseSeq>(
			fileSize, 0, blockSize);
	else
	if(!progArgs->getUseRandomOffsets() && !progArgs->getUseStridedAccess() ) // sequential forward
		rwOffsetGen = std::make_unique<OffsetGenSequential>(
			fileSize, 0, blockSize);
	else
	if(progArgs->getUseRandomUnaligned() ) // random unaligned
	{
        rwOffsetGen = std::make_unique<OffsetGenRandom>(randomAmount, *randOffsetAlgo,
            fileSize, 0, blockSize);
	}
	else // random aligned
	{
	    if(!progArgs->getRandOffsetAlgo().empty() || !isWritePhase )
	        rwOffsetGen = std::make_unique<OffsetGenRandomAligned>(randomAmount, *randOffsetAlgo,
	            fileSize, 0, blockSize);
	    else
	    { // random aligned writes without explicit algo selection => use full coverage algo
	        rwOffsetGen = std::make_unique<OffsetGenRandomAlignedFullCoverageV2>(
	            randomAmount, fileSize, 0, blockSize);
	    }
	} // end of random aligned
}

/**
 * Just set all phase-dependent function pointers to NULL.
 */
void LocalWorker::nullifyPhaseFunctionPointers()
{
	funcRWBlockSized = NULL;
	funcPositionalWrite = NULL;
	funcPositionalRead = NULL;
	funcAioRwPrepper = NULL;
	funcPreWriteCudaMemcpy = NULL;
	funcPostReadCudaMemcpy = NULL;
	funcPreWriteCudaMemcpy = NULL;
	funcPreWriteBlockModifier = NULL;
	funcPostReadBlockChecker = NULL;
	funcCuFileHandleReg = NULL;
	funcCuFileHandleDereg = NULL;
	funcRWRateLimiter = NULL;
}

/**
 * Prepare read/write function pointers for dirModeIterateFiles and fileModeIterateFiles.
 */
void LocalWorker::initPhaseFunctionPointers()
{
    const size_t ioDepth = progArgs->getIODepth();
    const bool useHDFS = (progArgs->getBenchMode() == BenchMode_HDFS);
    const bool useMmap = progArgs->getUseMmap();
    const bool useCuFileAPI = progArgs->getUseCuFile();
    const BenchPathType benchPathType = progArgs->getBenchPathType();
    const bool integrityCheckEnabled = (progArgs->getIntegrityCheckSalt() != 0);
    const bool areGPUsGiven = !progArgs->getGPUIDsVec().empty();
    const bool doDirectVerify = progArgs->getDoDirectVerify();
    const bool doReadInline = progArgs->getDoReadInline();
    const unsigned blockVariancePercent = progArgs->getBlockVariancePercent();
    const unsigned rwMixReadPercent = progArgs->getRWMixReadPercent();
    const uint64_t perThreadWriteRateLimitBps = progArgs->getLimitWriteBps();
    const uint64_t perThreadReadRateLimitBps = progArgs->getLimitReadBps();
    const size_t numRWMixReadThreads = progArgs->getNumRWMixReadThreads();
    const size_t numRWMixWriteThreads = progArgs->getNumThreads() - numRWMixReadThreads;
    const unsigned rwMixThreadsReadPercent = progArgs->getRWMixThreadsReadPercent();
    const size_t blockSize = progArgs->getBlockSize();

	nullifyPhaseFunctionPointers(); // set all function pointers to NULL


	// independent of whether current phase is read or write...
	// (these need to be set above the phase-dependent settings because those can override

	if(useHDFS)
		funcPositionalWrite = &LocalWorker::hdfsWriteWrapper;
	else
	if(useMmap)
		funcPositionalWrite = &LocalWorker::mmapWriteWrapper;
	else
	if(useCuFileAPI)
		funcPositionalWrite = &LocalWorker::cuFileWriteWrapper;
	else
		funcPositionalWrite = &LocalWorker::pwriteWrapper;

	if(useHDFS)
		funcPositionalRead = &LocalWorker::hdfsReadWrapper;
	else
	if(useMmap)
		funcPositionalRead = &LocalWorker::mmapReadWrapper;
	else
	if(useCuFileAPI)
		funcPositionalRead = &LocalWorker::cuFileReadWrapper;
	else
		funcPositionalRead = &LocalWorker::preadWrapper;


	// phase-dependent settings...

	if(benchPhase == BenchPhase_CREATEFILES)
	{
		funcRWBlockSized = (ioDepth == 1) ?
			&LocalWorker::rwBlockSized : &LocalWorker::aioBlockSized;

		funcAioRwPrepper = (ioDepth == 1) ? NULL : &LocalWorker::aioWritePrepper;

		if(rwMixReadPercent && funcAioRwPrepper)
			funcAioRwPrepper = &LocalWorker::aioRWMixPrepper;

		funcPreWriteCudaMemcpy = (areGPUsGiven && !useCuFileAPI) ?
			&LocalWorker::cudaMemcpyGPUToHost : &LocalWorker::noOpCudaMemcpy;
		funcPostReadCudaMemcpy = &LocalWorker::noOpCudaMemcpy;

		if(areGPUsGiven && integrityCheckEnabled)
			funcPreWriteCudaMemcpy = &LocalWorker::cudaMemcpyHostToGPU;

		if(integrityCheckEnabled)
			funcPreWriteBlockModifier = &LocalWorker::preWriteIntegrityCheckFillBuf;
		else
		if(blockVariancePercent && areGPUsGiven)
			funcPreWriteBlockModifier = &LocalWorker::preWriteBufRandRefillCuda;
		else
		if(blockVariancePercent)
			funcPreWriteBlockModifier = &LocalWorker::preWriteBufRandRefill;
		else
			funcPreWriteBlockModifier = &LocalWorker::noOpIntegrityCheck;

		funcPostReadBlockChecker = &LocalWorker::noOpIntegrityCheck;

		if(doDirectVerify || doReadInline)
		{
			if(!useCuFileAPI)
				funcPositionalWrite = &LocalWorker::pwriteAndReadWrapper;
			else
			{
				funcPositionalWrite = &LocalWorker::cuFileWriteAndReadWrapper;
				funcPostReadCudaMemcpy = &LocalWorker::cudaMemcpyGPUToHost;
			}

			if(doDirectVerify)
				funcPostReadBlockChecker = &LocalWorker::postReadIntegrityCheckVerifyBuf;
		}

	    // rate limiter / balancer

        if(numRWMixReadThreads && rwMixThreadsReadPercent)
        { // rate balancer between reader and writer threads
            rateLimiterRWMixThreads.initStart(
                rwMixThreadsReadPercent, numRWMixReadThreads, numRWMixWriteThreads, blockSize);

            funcRWRateLimiter = &LocalWorker::preRWRateBalanceLimiterForWriters;
        }
        else
        if(perThreadWriteRateLimitBps)
        { // plain per-thread rate limiter
            funcRWRateLimiter = &LocalWorker::preRWRateLimiter;
            rateLimiter.initStart(perThreadWriteRateLimitBps);
        }
        else // no rate limit
            funcRWRateLimiter = &LocalWorker::noOpRateLimiter;
	}
	else // BenchPhase_READFILES (and others which don't use these function pointers)
	{
        // (note: this also applies to rwmixthreads readers in a write phase)

		funcRWBlockSized = (ioDepth == 1) ?
			&LocalWorker::rwBlockSized : &LocalWorker::aioBlockSized;

		funcAioRwPrepper = (ioDepth == 1) ? NULL : &LocalWorker::aioReadPrepper;

		funcPreWriteCudaMemcpy = &LocalWorker::noOpCudaMemcpy;
		funcPostReadCudaMemcpy = (areGPUsGiven && !useCuFileAPI) ?
			&LocalWorker::cudaMemcpyHostToGPU : &LocalWorker::noOpCudaMemcpy;

		if(useCuFileAPI && integrityCheckEnabled)
			funcPostReadCudaMemcpy = &LocalWorker::cudaMemcpyGPUToHost;

		funcPreWriteBlockModifier = &LocalWorker::noOpIntegrityCheck;
		funcPostReadBlockChecker = integrityCheckEnabled ?
			&LocalWorker::postReadIntegrityCheckVerifyBuf : &LocalWorker::noOpIntegrityCheck;

        // rate limiter / balancer

        if(numRWMixReadThreads && rwMixThreadsReadPercent)
        { // rate balancer between reader and writer threads
            rateLimiterRWMixThreads.initStart(
                rwMixThreadsReadPercent, numRWMixReadThreads, numRWMixWriteThreads, blockSize);

            funcRWRateLimiter = &LocalWorker::preRWRateBalanceLimiterForReaders;
        }
        else
        if(perThreadReadRateLimitBps)
        { // plain per-thread rate limiter
            funcRWRateLimiter = &LocalWorker::preRWRateLimiter;
            rateLimiter.initStart(perThreadReadRateLimitBps);
        }
        else // no rate limit
            funcRWRateLimiter = &LocalWorker::noOpRateLimiter;
	}

	// independent of whether current phase is read or write...

	if(useCuFileAPI)
	{
		funcCuFileHandleReg = (benchPathType == BenchPathType_DIR) ?
			&LocalWorker::dirModeCuFileHandleReg : &LocalWorker::noOpCuFileHandleReg;
		funcCuFileHandleDereg = (benchPathType == BenchPathType_DIR) ?
			&LocalWorker::dirModeCuFileHandleDereg : &LocalWorker::noOpCuFileHandleDereg;
	}
	else
	{
		funcCuFileHandleReg = &LocalWorker::noOpCuFileHandleReg;
		funcCuFileHandleDereg = &LocalWorker::noOpCuFileHandleDereg;
	}

}

/**
 * Allocate aligned I/O buffer and fill with random data.
 *
 * @throw WorkerException if allocation fails.
 */
void LocalWorker::allocIOBuffer()
{
	if(!progArgs->getBlockSize() )
		return; // nothing to do here

	if( (progArgs->getBenchMode() == BenchMode_S3) && !progArgs->getRunCreateFilesPhase() &&
		progArgs->getUseS3FastRead() )
		return; // nothing to do if read to /dev/null is set and no writes to be done

	// alloc number of IO buffers matching iodepth
	for(size_t i=0; i < progArgs->getIODepth(); i++)
	{
		char* ioBuf;

		// alloc I/O buffer appropriately aligned for O_DIRECT
		int allocAlignedRes = posix_memalign( (void**)&ioBuf, sysconf(_SC_PAGESIZE),
			progArgs->getBlockSize() );

		if(allocAlignedRes)
			throw WorkerException("Aligned memory allocation failed. "
				"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
				"Page size: " + std::to_string(sysconf(_SC_PAGESIZE) ) + "; "
				"SysErr: " + strerror(allocAlignedRes) ); // yes, not errno here

		ioBufVec.push_back(ioBuf);

		// fill buffer with random data to ensure it's really alloc'ed (and not "sparse")
		RandAlgoXoshiro256ss randGen;
		randGen.fillBuf(ioBuf, progArgs->getBlockSize() );
	}

    LOGGER(Log_DEBUG, "Allocated IO buffers for ioBufVec. "
        "Rank: " << workerRank << "; "
        "Number of buffers: " << ioBufVec.size() << std::endl);
}

/**
 * Allocate GPU I/O buffer and fill with random data.
 *
 * @throw WorkerException if allocation fails.
 */
void LocalWorker::allocGPUIOBuffer()
{
	if(!progArgs->getBlockSize() )
		return; // nothing to do here

	if(progArgs->getGPUIDsVec().empty() )
	{ // gpu bufs won't be accessed, but gpuIOBufVec elems might be passed e.g. to noOpCudaMemcpy
		gpuIOBufVec.resize(progArgs->getIODepth(), NULL);
		return;
	}

#ifndef CUDA_SUPPORT

	throw WorkerException("GPU given, but this executable was built without CUDA support.");

#else // CUDA_SUPPORT

	size_t gpuIndex = workerRank % progArgs->getGPUIDsVec().size();

	gpuID = progArgs->getGPUIDsVec()[gpuIndex];

	LOGGER(Log_DEBUG, "Initializing GPU buffer. "
		"Rank: " << workerRank << "; "
		"GPU ID: " << gpuID << std::endl);

	// set the GPU that this worker thread will use
	cudaError_t setDevRes = cudaSetDevice(gpuID);

	if(setDevRes != cudaSuccess)
		throw WorkerException("Setting CUDA device failed. "
			"GPU ID: " + std::to_string(gpuID) + "; "
			"CUDA Error: " + cudaGetErrorString(setDevRes) );

	curandStatus_t createGenRes = curandCreateGenerator(&gpuRandGen, CURAND_RNG_PSEUDO_DEFAULT);
	if(createGenRes != CURAND_STATUS_SUCCESS)
		throw WorkerException("Initialization of GPU/CUDA random number generator failed. "
			"curand error code: " + std::to_string(createGenRes) );

	curandStatus_t seedRes = curandSetPseudoRandomGeneratorSeed(gpuRandGen,
		( (uint64_t)std::random_device()() << 32) | (uint32_t)std::random_device()() );
	if(seedRes != CURAND_STATUS_SUCCESS)
		throw WorkerException("Seeding GPU/CUDA random number generator failed. "
			"curand error code: " + std::to_string(seedRes) );

	// alloc number of GPU IO buffers matching iodepth
	for(size_t i=0; i < progArgs->getIODepth(); i++)
	{
		void* gpuIOBuf;

		cudaError_t allocRes = cudaMalloc(&gpuIOBuf, progArgs->getBlockSize() );

		if(allocRes != cudaSuccess)
			throw WorkerException("GPU memory allocation failed. "
				"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
				"GPU ID: " + std::to_string(gpuID) + "; "
				"CUDA Error: " + cudaGetErrorString(allocRes) );

		gpuIOBufVec.push_back( (char*)gpuIOBuf);

		if(progArgs->getUseCuHostBufReg() )
		{
			cudaError_t regRes = cudaHostRegister(ioBufVec[i], progArgs->getBlockSize(),
				cudaHostRegisterDefault);

			if(regRes != cudaSuccess)
				throw WorkerException("Registration of host buffer via cudaHostRegister failed. "
					"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
					"CUDA Error: " + cudaGetErrorString(regRes) );
		}

		cudaError_t copyRes = cudaMemcpy(gpuIOBuf, ioBufVec[i], progArgs->getBlockSize(),
			cudaMemcpyHostToDevice);

		if(copyRes != cudaSuccess)
			throw WorkerException("Initialization of GPU buffer via cudaMemcpy failed. "
				"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
				"GPU ID: " + std::to_string(gpuID) + "; "
				"CUDA Error: " + cudaGetErrorString(copyRes) );
	}

    LOGGER(Log_DEBUG, "Allocated GPU buffers for gpuIOBufVec. "
        "Rank: " << workerRank << "; "
        "Number of buffers: " << gpuIOBufVec.size() << std::endl);

#endif // CUDA_SUPPORT

#ifndef CUFILE_SUPPORT

	if(progArgs->getUseCuFile() )
		throw WorkerException("cuFile API requested, but this executable was built without cuFile "
			"support.");

#else // CUFILE_SUPPORT

	// register GPU buffer for DMA
	for(char* gpuIOBuf : gpuIOBufVec)
	{
		if(!progArgs->getUseCuFile() || !gpuIOBuf || !progArgs->getUseGPUBufReg() )
			continue;

		CUfileError_t registerRes = cuFileBufRegister(gpuIOBuf, progArgs->getBlockSize(), 0);

		if(registerRes.err != CU_FILE_SUCCESS)
			throw WorkerException(std::string(
				"GPU DMA buffer registration via cuFileBufRegister failed. ") +
				"GPU ID: " + std::to_string(gpuID) + "; "
				"cuFile Error: " + CUFILE_ERRSTR(registerRes.err) );
	}

#endif // CUFILE_SUPPORT
}

/**
 * Prepare paths for custom tree mode for this worker.
 *
 * This also loads the s3 mpu sharing file for services, because this requires the custom tree but
 * has to be done before randomization of the objects.
 *
 * @throw WorkerException on error
 */
void LocalWorker::prepareCustomTreePathStores()
{
	if(progArgs->getTreeFilePath().empty() )
		return; // nothing to do here

	const bool throwOnSmallerThanBlockSize = !progArgs->getNoDirectIOCheck() &&
		progArgs->getUseDirectIO() && progArgs->getUseRandomOffsets();

	const size_t numThreads = progArgs->getNumThreads();
	const size_t numDataSetThreads = progArgs->getNumDataSetThreads();

	progArgs->getCustomTreeDirs().getWorkerSublistNonShared(workerRank,
		numDataSetThreads, false, customTreeDirs);

	progArgs->getCustomTreeFilesNonShared().getWorkerSublistNonShared(workerRank,
		numDataSetThreads, throwOnSmallerThanBlockSize, customTreeFiles);

    // (note: ProgAgs uses the same condition in prepareCustomTreePathStores() )
	if(progArgs->getRunAsService() && (progArgs->getBenchMode() == BenchMode_S3) &&
		progArgs->getRunCreateFilesPhase() && (numDataSetThreads != numThreads) &&
        !progArgs->getUseS3MPUSharing() )
	{
		/* shared s3 uploads of an object possible, but only among threads within the same
			service instance (due to the uploadID not beeing shared among service instances).
			in this case, ProgArgs prepared a special shared file tree that only contains files for
			this service instance. thus we use only the local ranks, not the global ranks. */

		size_t workerRankLocal = workerRank - progArgs->getRankOffset();

		progArgs->getCustomTreeFilesShared().getWorkerSublistShared(workerRankLocal,
			numThreads, throwOnSmallerThanBlockSize, customTreeFiles);
	}
	else
	{
		// this is the normal case: fair sharing across all services/workers based on blocksize

        if(progArgs->getUseCustomTreeRoundRobin() )
            progArgs->getCustomTreeFilesShared().getWorkerSublistSharedRoundRobin(workerRank,
                progArgs->getNumDataSetThreads(), throwOnSmallerThanBlockSize, customTreeFiles);
        else
            progArgs->getCustomTreeFilesShared().getWorkerSublistShared(workerRank,
                progArgs->getNumDataSetThreads(), throwOnSmallerThanBlockSize, customTreeFiles);
	}

	if(progArgs->getUseCustomTreeRandomize() )
		customTreeFiles.randomShuffle();
}

/**
 * Release all allocated objects, handles etc.
 *
 * This needs to be called when run() ends. The things in here would usually be done in the
 * LocalWorker destructor, but especially in service mode we need the LocalWorker object to still
 * exist (to query phase results) while all ressources need to be released, because the LocalWorker
 * object will only be deleted when and if the next benchmark starts.
 */
void LocalWorker::cleanup()
{
	// delete rwOffsetGen (unique ptr) to eliminate any references to progArgs data etc.
	rwOffsetGen.reset();

	uninitNetBench();
	uninitHDFS();
	uninitS3Client();
    uninitLibAio();

	// reset custom tree mode path store
	customTreeFiles.clear();

#ifdef CUFILE_SUPPORT
	// deregister GPU buffers for DMA
	for(char* gpuIOBuf : gpuIOBufVec)
	{
		if(!progArgs->getUseCuFile() || !gpuIOBuf || !progArgs->getUseGPUBufReg() )
			continue;

		CUfileError_t deregRes = cuFileBufDeregister(gpuIOBuf);

		if(deregRes.err != CU_FILE_SUCCESS)
			ERRLOGGER(Log_VERBOSE, "GPU DMA buffer deregistration via cuFileBufDeregister failed. "
				"GPU ID: " << gpuID << "; "
				"cuFile Error: " << CUFILE_ERRSTR(deregRes.err) << std::endl);
	}
#endif

#ifdef CUDA_SUPPORT
	// cuda-free gpu memory buffers
	for(char* gpuIOBuf : gpuIOBufVec)
	{
		if(gpuIOBuf)
			cudaFree(gpuIOBuf);
	}

	// cuda-unregister host buffers
	if(progArgs->getUseCuHostBufReg() && !progArgs->getGPUIDsVec().empty() )
	{
		for(char* ioBuf : ioBufVec)
		{
			if(!ioBuf)
				continue;

			cudaError_t unregRes = cudaHostUnregister(ioBuf);

			if(unregRes != cudaSuccess)
				ERRLOGGER(Log_VERBOSE,
					"CPU DMA buffer deregistration via cudaHostUnregister failed. "
					"GPU ID: " << gpuID << "; "
					"CUDA Error: " << cudaGetErrorString(unregRes) << std::endl);
		}
	}

	if(gpuRandGen)
	{
		curandDestroyGenerator(gpuRandGen);
		gpuRandGen = NULL;
	}
#endif

	// free host memory buffers
	for(char* ioBuf : ioBufVec)
		SAFE_FREE(ioBuf);

	uninitThreadMmapVec();
	uninitThreadCuFileHandleDataVec();
	uninitThreadFDVec();

	opsLog.closeLogFile();
}

/**
 * Late cleanup after all workers are done with the current phase. This gets called by the
 * WorkerManager for all threads, so it's non-parallel (in contrast to LocalWorker::cleanup() ) and
 * thus should only be used for cleanup that can't be done while some workers are still running,
 * e.g. cleanup of shared data structures for all workers.
 *
 * Note: This can be called more than once after the same phase, e.g. in service mode if user
 * sends phase interrupt request more than once.
 *
 * Note: This is called after each phase, so don't free anything here that might be used for
 * multiple phases, e.g. in standalone mode.
 */
void LocalWorker::cleanupAfterPhaseDone()
{
	uninitNetBenchAfterPhaseDone();
}

/**
 * Loop around pread/pwrite to use user-defined block size instead of full given count in one call.
 * Reads/writes the pre-allocated ioBuf. Uses rwOffsetGen for next offset and block size.
 *
 * If this->fileHandles contains multiple FDs then they will be treated as described in
 * calcFileIdxAndOffsetStriped().
 *
 * @return similar to pread/pwrite.
 */
int64_t LocalWorker::rwBlockSized()
{
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const unsigned rwMixReadPercent = progArgs->getRWMixReadPercent();
    const uint64_t fileSize = progArgs->getFileSize();
    const bool isSingleFile = (fileHandles.fdVecPtr->size() == 1);
    const unsigned short fileLockType = progArgs->getFLockType();

	while(rwOffsetGen->getNumBytesLeftToSubmit() )
	{
        const uint64_t rwOffsetGenNext = rwOffsetGen->getNextOffset();
        const size_t currentBlockSize = rwOffsetGen->getNextBlockSizeToSubmit();
        uint64_t currentOffset;
        size_t fileHandleIdx;
        bool isRWMixRead = false;
        ssize_t rwRes;

        calcFileIdxAndOffsetStriped(rwOffsetGenNext, fileSize, isSingleFile,
            fileHandleIdx, currentOffset);

		((*this).*funcRWRateLimiter)(currentBlockSize, isInterruptionRequested);

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		((*this).*funcPreWriteBlockModifier)(ioBufVec[0], gpuIOBufVec[0], currentBlockSize,
		    currentOffset);
		((*this).*funcPreWriteCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], currentBlockSize);

		if(benchPhase == BenchPhase_READFILES)
		{ // this is a read, but could be a rwmix read thread
			isRWMixRead = (benchPhase != globalBenchPhase);

            FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandleIdx], fileLockType,
                currentOffset, currentBlockSize, false /*isWrite*/, false /*isUnlock*/, NULL);

			rwRes = ((*this).*funcPositionalRead)(
				fileHandleIdx, ioBufVec[0], currentBlockSize, currentOffset);
		}
		else // this is a write or rwmixpct read
		if(rwMixReadPercent &&
			( ( (workerRank + numIOPSSubmitted) % 100) < rwMixReadPercent) )
		{ // this is a rwmix read
			isRWMixRead = true;

            FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandleIdx], fileLockType,
                currentOffset, currentBlockSize, false /*isWrite*/, false /*isUnlock*/, NULL);

            rwRes = ((*this).*funcPositionalRead)(
				fileHandleIdx, ioBufVec[0], currentBlockSize, currentOffset);
		}
		else
		{ // this is a plain write
            FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandleIdx], fileLockType,
                currentOffset, currentBlockSize, true /*isWrite*/, false /*isUnlock*/, NULL);

			rwRes = ((*this).*funcPositionalWrite)(
				fileHandleIdx, ioBufVec[0], currentBlockSize, currentOffset);
		}

		IF_UNLIKELY(rwRes <= 0)
		{ // unexpected result
			ERRLOGGER(Log_NORMAL, "IO failed: " << "blockSize: " << currentBlockSize << "; " <<
				"currentOffset:" << currentOffset << "; " <<
				"leftToSubmit:" << rwOffsetGen->getNumBytesLeftToSubmit() << "; " <<
				"rank:" << workerRank << "; " <<
				"return code: " << rwRes << "; " <<
				"errno: " << errno << std::endl);

			fileHandles.errorFDVecIdx = fileHandleIdx;

	        FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandleIdx], fileLockType,
	            currentOffset, currentBlockSize, true /*ignored*/, true /*isUnlock*/, NULL);

	        return (rwRes < 0) ?
				rwRes :
				(rwOffsetGen->getNumBytesTotal() - rwOffsetGen->getNumBytesLeftToSubmit() );
		}

        FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandleIdx], fileLockType,
            currentOffset, currentBlockSize, true /*ignored*/, true /*isUnlock*/, NULL);

		((*this).*funcPostReadCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], currentBlockSize);
		((*this).*funcPostReadBlockChecker)(ioBufVec[0], gpuIOBufVec[0], currentBlockSize,
		    currentOffset);

		// calc io operation latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		// iops lat & num done
		if(isRWMixRead)
		{ // inc special rwmix read stats
			iopsLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
			atomicLiveOpsReadMix.numBytesDone += rwRes;
			atomicLiveOpsReadMix.numIOPSDone++;
		}
		else
		{
			iopsLatHisto.addLatency(ioElapsedMicroSec.count() );
			atomicLiveOps.numBytesDone += rwRes;
			atomicLiveOps.numIOPSDone++;
		}

		numIOPSSubmitted++;
		rwOffsetGen->addBytesSubmitted(rwRes);

		checkInterruptionRequest();
	}

	return rwOffsetGen->getNumBytesTotal();
}

/**
 * Loop around libaio read/write to use user-defined block size instead of full file size in one
 * call.
 * Reads/writes the pre-allocated ioBuf. Uses iodepth from progArgs.
 *
 * If this->fileHandles contains multiple FDs then they will be treated as a striped single range,
 * so sequential IOs would be done round-robin. Thus, this is not suitable if serial processing
 * of files is needed.
 *
 * @return similar to pread/pwrite.
 * @throw WorkerException on async IO framework errors.
 */
int64_t LocalWorker::aioBlockSized()
{
#ifndef LIBAIO_SUPPORT

	throw WorkerException("Async IO via libaio requested, but this executable was built without "
		"libaio support.");

#else // LIBAIO_SUPPORT

	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t maxIODepth = progArgs->getIODepth();
	const size_t fileHandlesVecSize = fileHandles.fdVecPtr->size();
    const uint64_t fileSize = progArgs->getFileSize();
    const bool isSingleFile = (fileHandlesVecSize == 1);
    const unsigned short fileLockType = progArgs->getFLockType();

	size_t numPending = 0; // num requests submitted and pending for completion
	size_t numBytesDone = 0; // after successfully completed requests

	struct io_event ioEvents[AIO_MAX_EVENTS];
	struct timespec ioTimeout;

	// P H A S E 1: initial seed of io submissions up to full ioDepth

	while(rwOffsetGen->getNumBytesLeftToSubmit() && (numPending < maxIODepth) )
	{
		const uint64_t rwOffsetGenNext = rwOffsetGen->getNextOffset();
        const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
		const size_t ioVecIdx = numPending; // iocbVec index

		uint64_t currentOffset;
        size_t fileHandlesIdx;

        calcFileIdxAndOffsetStriped(rwOffsetGenNext, fileSize, isSingleFile,
            fileHandlesIdx, currentOffset);

        const int fd = (*fileHandles.fdVecPtr)[fileHandlesIdx];

		libaioContext.iocbPointerVec[ioVecIdx] = &libaioContext.iocbVec[ioVecIdx];

        ((*this).*funcAioRwPrepper)(&libaioContext.iocbVec[ioVecIdx], fd, ioBufVec[ioVecIdx],
            blockSize, currentOffset);
        libaioContext.iocbVec[ioVecIdx].data = (void*)ioVecIdx; /* the vec index of this request;
            ioctl.data is caller's private data returned after io_getevents as ioEvents[].data */

        libaioContext.ioStartTimeVec[ioVecIdx] = std::chrono::steady_clock::now();

        bool hadToWait = ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);
        IF_UNLIKELY(hadToWait) // invalidate start time of all pending due to rate limiter wait
            for(std::chrono::steady_clock::time_point& startT : libaioContext.ioStartTimeVec)
                startT = std::chrono::steady_clock::time_point::min();

        ((*this).*funcPreWriteBlockModifier)(ioBufVec[ioVecIdx], gpuIOBufVec[ioVecIdx], blockSize,
            currentOffset);
        ((*this).*funcPreWriteCudaMemcpy)(ioBufVec[ioVecIdx], gpuIOBufVec[ioVecIdx], blockSize);

        FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandlesIdx], fileLockType,
            currentOffset, blockSize, libaioContext.iocbVec[ioVecIdx].aio_lio_opcode==IO_CMD_PWRITE,
            false /*isUnlock*/, NULL);

        int submitRes = io_submit(libaioContext.ioContext, 1,
            &libaioContext.iocbPointerVec[ioVecIdx] );
        IF_UNLIKELY(submitRes != 1)
        {
            FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandlesIdx], fileLockType,
                currentOffset, blockSize, true /*ignored*/, true /*isUnlock*/, NULL);

            throw WorkerException(std::string("Async IO submission (io_submit) failed. ") +
                "NumRequests: " + std::to_string(numPending) + "; "
                "ReturnCode: " + std::to_string(submitRes) + "; "
                "SysErr: " + strerror(-submitRes) ); // (io_submit returns negative errno)
        }

		numPending++;
		numIOPSSubmitted++;
		rwOffsetGen->addBytesSubmitted(blockSize);
	}


	// P H A S E 2: wait for submissions to complete and submit new requests if bytes left

	while(numPending)
	{
		ioTimeout.tv_sec = AIO_MAX_WAIT_SEC;
		ioTimeout.tv_nsec = 0;

        int eventsRes = io_getevents(libaioContext.ioContext, 1, AIO_MAX_EVENTS, ioEvents,
            &ioTimeout);
		IF_UNLIKELY(!eventsRes)
		{ // timeout expired; that's ok, as we set a short timeout to check interruptions
			checkInterruptionRequest();
			continue;
		}
		else
		IF_UNLIKELY(eventsRes < 0)
		{
			throw WorkerException(std::string("Getting async IO events (io_getevents) failed. ") +
				"NumPending: " + std::to_string(numPending) + "; "
				"ReturnCode: " + std::to_string(eventsRes) + "; "
				"Wait time: " + std::to_string(AIO_MAX_WAIT_SEC) + "; "
				"Wait time left: " + std::to_string(ioTimeout.tv_sec) + "; "
				"SysErr: " + strerror(-eventsRes) ); // (io_getevents returns negative errno)
		}

		// check result of completed iocbs and reuse them if any blocks left to submit

		for(int eventIdx = 0; eventIdx < eventsRes; eventIdx++)
		{
			// ioEvents[].res2 is negative errno for aio framework errors, 0 means no error
			// ioEvents[].res is actually read/written bytes when res2==0 or negative errno
			/* note: all messy with res/res2, because defined as ulong, but examples need them
				interpreted as int for errors, which can overlap valid partial writes on 64bit */

			IF_UNLIKELY(ioEvents[eventIdx].res2 ||
				(ioEvents[eventIdx].res != ioEvents[eventIdx].obj->u.c.nbytes) )
			{ // unexpected result
				if(ioEvents[eventIdx].res2)
					throw WorkerException(std::string("Async IO framework error. ") +
						"NumPending: " + std::to_string(numPending) + "; "
						"res: " + std::to_string(ioEvents[eventIdx].res) + "; "
						"res2: " + std::to_string(ioEvents[eventIdx].res2) + "; "
						"IO size: " + std::to_string(ioEvents[eventIdx].obj->u.c.nbytes) + "; "
						"SysErr: " + strerror(-(int)ioEvents[eventIdx].res2) );

	            FileTk::flock<WorkerException>(ioEvents[eventIdx].obj->aio_fildes, fileLockType,
	                ioEvents[eventIdx].obj->u.c.offset, ioEvents[eventIdx].obj->u.c.nbytes,
	                true /*ignored*/, true /*isUnlock*/, NULL);

				if( (int)ioEvents[eventIdx].res < 0)
				{
					errno = -(int)ioEvents[eventIdx].res; // res is negative errno
					return -1;
				}

				// partial read/write, so return what we got so far
				return (numBytesDone + ioEvents[eventIdx].res);
			}

            FileTk::flock<WorkerException>(ioEvents[eventIdx].obj->aio_fildes, fileLockType,
                ioEvents[eventIdx].obj->u.c.offset, ioEvents[eventIdx].obj->u.c.nbytes,
                true /*ignored*/, true /*isUnlock*/, NULL);

			const size_t ioVecIdx = (size_t)ioEvents[eventIdx].data; // caller priv data is vec idx

			((*this).*funcPostReadCudaMemcpy)(ioBufVec[ioVecIdx], gpuIOBufVec[ioVecIdx],
				ioEvents[eventIdx].obj->u.c.nbytes);
			((*this).*funcPostReadBlockChecker)( (char*)ioEvents[eventIdx].obj->u.c.buf,
				gpuIOBufVec[ioVecIdx], ioEvents[eventIdx].obj->u.c.nbytes,
				ioEvents[eventIdx].obj->u.c.offset);

			// calc io operation latency
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - libaioContext.ioStartTimeVec[ioVecIdx] );

			numBytesDone += ioEvents[eventIdx].res;

			// inc special rwmix read stats
			if(	(ioEvents[eventIdx].obj->aio_lio_opcode == IO_CMD_PREAD) &&
				(globalBenchPhase == BenchPhase_CREATEFILES) )
			{ // this is a read in a write phase => inc rwmix read stats
                // don't count latency if this I/O had to wait for rate limiter
                IF_LIKELY(libaioContext.ioStartTimeVec[ioVecIdx] !=
                    std::chrono::steady_clock::time_point::min() )
                    iopsLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );

				atomicLiveOpsReadMix.numBytesDone += ioEvents[eventIdx].res;
				atomicLiveOpsReadMix.numIOPSDone++;
			}
			else
			{
                // don't count latency if this I/O had to wait for rate limiter
                IF_LIKELY(libaioContext.ioStartTimeVec[ioVecIdx] !=
                    std::chrono::steady_clock::time_point::min() )
                    iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

				atomicLiveOps.numBytesDone += ioEvents[eventIdx].res;
				atomicLiveOps.numIOPSDone++;
			}

			checkInterruptionRequest();

			if(!rwOffsetGen->getNumBytesLeftToSubmit() )
			{
				numPending--;
				continue;
			}

			// request complete, so reuse iocb for the next request...

			const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
			const uint64_t rwOffsetGenNext = rwOffsetGen->getNextOffset();

			uint64_t currentOffset;
            size_t fileHandlesIdx;

            calcFileIdxAndOffsetStriped(rwOffsetGenNext, fileSize, isSingleFile,
                fileHandlesIdx, currentOffset);

			const int fd = (*fileHandles.fdVecPtr)[fileHandlesIdx];

			((*this).*funcAioRwPrepper)(ioEvents[eventIdx].obj, fd, ioBufVec[ioVecIdx], blockSize,
				currentOffset);
			ioEvents[eventIdx].obj->data = (void*)ioVecIdx; // caller's private data

            libaioContext.ioStartTimeVec[ioVecIdx] = std::chrono::steady_clock::now();

            bool hadToWait = ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);
            IF_UNLIKELY(hadToWait) // invalidate start time of all pending due to rate limiter wait
                for(std::chrono::steady_clock::time_point& startT : libaioContext.ioStartTimeVec)
                    startT = std::chrono::steady_clock::time_point::min();

            ((*this).*funcPreWriteBlockModifier)(ioBufVec[ioVecIdx], gpuIOBufVec[ioVecIdx],
                blockSize, currentOffset);
            ((*this).*funcPreWriteCudaMemcpy)(ioBufVec[ioVecIdx], gpuIOBufVec[ioVecIdx], blockSize);

            FileTk::flock<WorkerException>( (*fileHandles.fdVecPtr)[fileHandlesIdx], fileLockType,
                currentOffset, blockSize,
                libaioContext.iocbVec[ioVecIdx].aio_lio_opcode==IO_CMD_PWRITE,
                false /*isUnlock*/, NULL);

			int submitRes = io_submit(
				libaioContext.ioContext, 1, &libaioContext.iocbPointerVec[ioVecIdx] );
			IF_UNLIKELY(submitRes != 1)
			{
				throw WorkerException(std::string("Async IO resubmission (io_submit) failed. ") +
					"NumRequests: " + std::to_string(numPending) + "; "
					"ReturnCode: " + std::to_string(submitRes) + "; "
					"SysErr: " + strerror(-submitRes) ); // (io_submit returns negative errno)
			}

			numIOPSSubmitted++;
			rwOffsetGen->addBytesSubmitted(blockSize);

		} // end of for loop to resubmit completed iocbs

	} // end of while loop until all blocks completed

	return rwOffsetGen->getNumBytesTotal();

#endif // LIBAIO_SUPPORT
}

/**
 * Calculate the next file and offset within file for rwBlockSized and aioBlockSized.
 *
 * If only one file in fileHandlesVec then rwOffsetGenNext is the next offset. Otherwise the set
 * of files is treated like a single virtual large file with one file after the other and all
 * files are expected to have progArgs->getFileSize() length.
 *
 * @fileSize progArgs->getFileSize().
 * @isSingleFile true if there is only one file in fileHandlesVec.
 * @outFileOffset offset within outNextFileIdx.
 * @outFileIdx index within fileHandlesVec.
 */
void LocalWorker::calcFileIdxAndOffsetStriped(const uint64_t rwOffsetGenNext,
    const uint64_t fileSize, const bool isSingleFile,
    size_t& outFileIdx, uint64_t& outFileOffset)
{
    if(isSingleFile)
    { // single file
        outFileIdx = 0;
        outFileOffset = rwOffsetGenNext;
    }
    else
    { // multiple files => treat them as single serial block range, one file after the other
        outFileIdx = rwOffsetGenNext / fileSize;
        outFileOffset = rwOffsetGenNext % fileSize;
    }

    LOGGER_DEBUG_BUILD(__func__ << ": " <<
        "workerRank: " << workerRank << "; " <<
        "rwOffsetGenNext: " << rwOffsetGenNext << "; " <<
        "fileSize: " << fileSize << "; " <<
        "isSingleFile: " << isSingleFile << "; " <<
        "outFileIdx: " << outFileIdx << "; " <<
        "outFileOffset: " << outFileOffset <<
        std::endl);
}

/**
 * Noop for the case when no integrity check selected by user.
 */
void LocalWorker::noOpIntegrityCheck(char* hostIOBuf, char* gpuIOBuf, size_t bufLen,
	off_t fileOffset)
{
	return; // noop
}

/**
 * Fill buf with unsigned 64bit values made of offset plus integrity check salt.
 *
 * @bufLen buf len to fill with checksums
 * @fileOffset file offset for buf
 */
void LocalWorker::preWriteIntegrityCheckFillBuf(char* hostIOBuf, char* gpuIOBuf, size_t bufLen,
	off_t fileOffset)
{
	const size_t checkSumLen = sizeof(uint64_t);
	const uint64_t checkSumSalt = progArgs->getIntegrityCheckSalt();

	size_t numBytesDone = 0;
	size_t numBytesLeft = bufLen;
	off_t currentOffset = fileOffset;

	/* note: fileOffset and bufLen are not guaranteed to be a multiple of uint64_t (e.g. if
	   blocksize is 1 byte). We also want to support writes and verficiation reads to use different
	   block size (bufLen). So we only copy the relevant part of the uint64_t to buf. */

	while(numBytesLeft)
	{
		/* checksum value is always calculated aligned to 8 byte block size, even if we only copy a
		   partial block. */

		// (note: after the 1st loop pass, the remaining offsets will be 8 byte aligned.)

		// 8 byte aligned offset as basis for checksum value calculation
		off_t checkSumStartOffset = currentOffset - (currentOffset % checkSumLen);

		uint64_t checkSum = checkSumStartOffset + checkSumSalt;

		char* checkSumArray = (char*)&checkSum; // byte-addressable array for checksum value
		off_t checkSumArrayStartIdx = currentOffset - checkSumStartOffset;
		size_t checkSumCopyLen = std::min(
			(uint64_t)numBytesLeft, (uint64_t)checkSumLen - checkSumArrayStartIdx);

		memcpy(&hostIOBuf[numBytesDone], &checkSumArray[checkSumArrayStartIdx], checkSumCopyLen);

		numBytesDone += checkSumCopyLen;
		numBytesLeft -= checkSumCopyLen;
		currentOffset += checkSumCopyLen;
	}
}

/**
 * Verify buffer contents as counterpart to preWriteIntegrityCheckFillBuf.
 *
 * @bufLen buf len to fill with checksums
 * @fileOffset file offset for buf
 * @throw WorkerException if verification fails.
 */
void LocalWorker::postReadIntegrityCheckVerifyBuf(char* hostIOBuf, char* gpuIOBuf, size_t bufLen,
	off_t fileOffset)
{
	IF_UNLIKELY(!bufLen)
		return;

	char* verifyBuf = (char*)malloc(bufLen);

	IF_UNLIKELY(!verifyBuf)
		throw WorkerException("Buffer alloc for verification buffer failed. "
			"Size: " + std::to_string(bufLen) );

	// fill verifyBuf with the correct data
	preWriteIntegrityCheckFillBuf(verifyBuf, gpuIOBuf, bufLen, fileOffset);

	// compare correct data to actual data
	int compareRes = memcmp(hostIOBuf, verifyBuf, bufLen);

	if(!compareRes)
	{ // buffers are equal, so all good
		free(verifyBuf);
		return;
	}

	// verification failed, find exact mismatch offset
	for(size_t i=0; i < bufLen; i++)
	{
		if(verifyBuf[i] == hostIOBuf[i])
			continue;

		// we found the exact offset for mismatch

		unsigned expectedVal = (unsigned char)verifyBuf[i];
		unsigned actualVal = (unsigned char)hostIOBuf[i];

		free(verifyBuf);

		throw WorkerException("Data verification failed. "
			"Offset: " + std::to_string(fileOffset + i) + "; "
			"Expected value: " + std::to_string(expectedVal) + "; "
			"Actual value: " + std::to_string(actualVal) );
	}
}

/**
 * Fill buffer with given value. In contrast to memset() this can fill 64bit values to at least
 * make simple dedupe less likely among all the different non-variable block remainders.
 */
void LocalWorker::bufFill(char* buf, uint64_t fillValue, size_t bufLen)
{
	size_t numBytesDone = 0;

	for(uint64_t i=0; i < (bufLen / sizeof(uint64_t) ); i++)
	{
		uint64_t* uint64Buf = (uint64_t*)buf;
		*uint64Buf = fillValue;

		buf += sizeof(uint64_t);
		numBytesDone += sizeof(uint64_t);
	}

	if(numBytesDone == bufLen)
		return; // all done, complete buffer filled

	// we have a remainder to fill, which can only be smaller than sizeof(uint64_t)
	memcpy(buf, &fillValue, bufLen - numBytesDone);
}

/**
 * Refill some percentage of the buffer with random data. The percentage to refill is defined via
 * progArgs::blockVariancePercent.
 */
void LocalWorker::preWriteBufRandRefill(char* hostIOBuf, char* gpuIOBuf, size_t bufLen,
	off_t fileOffset)
{
	// note: this same logic is used in aioRWMixPrepper/pwriteRWMixWrapper
	if( ( (workerRank + numIOPSSubmitted) % 100) < progArgs->getRWMixReadPercent() )
		return; // this is a read in rwmix mode, so no need for refill in this round

	// refill buffer with random data

	const unsigned blockVariancePercent = progArgs->getBlockVariancePercent();
	const uint64_t varFillLen = (bufLen * blockVariancePercent) / 100;
	const size_t constFillRemainderLen = bufLen - varFillLen;

	randBlockVarAlgo->fillBuf(hostIOBuf, varFillLen);

	if(!constFillRemainderLen)
		return;

	// fill remainder of buffer with same 64bit value
	// note: rand algo is used to defeat simple dedupe across remainders of different blocks
	bufFill(&hostIOBuf[varFillLen], randBlockVarAlgo->next(), constFillRemainderLen);
}

/**
 * Refill some percentage of the GPU buffer with random data. The percentage to refill is defined
 * via progArgs::blockVariancePercent.
 */
void LocalWorker::preWriteBufRandRefillCuda(char* hostIOBuf, char* gpuIOBuf, size_t bufLen,
	off_t fileOffset)
{
#ifndef CUDA_SUPPORT

	throw WorkerException("preWriteBufRandRefillCuda called, but this executable was built without "
		"CUDA support.");

#else // CUDA_SUPPORT

	// note: this same logic is used in aioRWMixPrepper/pwriteRWMixWrapper
	if( ( (workerRank + numIOPSSubmitted) % 100) < progArgs->getRWMixReadPercent() )
		return; // this is a read in rwmix mode, so no need for refill in this round

	// refill buffer with random data

	const unsigned blockVariancePercent = progArgs->getBlockVariancePercent();
	uint64_t varFillLen = (bufLen * blockVariancePercent) / 100;

	if(varFillLen % sizeof(int) ) // curandGenerate can only fill full 32bit values
		varFillLen -= (varFillLen % sizeof(int) );

	const size_t constFillRemainderLen = bufLen - varFillLen;

	curandStatus_t randGenRes = curandGenerate(gpuRandGen, (unsigned int*)gpuIOBuf,
		varFillLen / sizeof(int) );

	IF_UNLIKELY(randGenRes != CURAND_STATUS_SUCCESS)
		throw WorkerException("Random number generation via GPU/CUDA failed. "
			"curand error code: " + std::to_string(randGenRes) );

	if(!constFillRemainderLen)
		return;

	/* fill remainder of host buffer with same 64bit value and copy over to gpu (in lack of a good
		way of filling a buffer with a 64bit value directly on the gpu) */
	// note: rand algo is used to defeat simple dedupe across remainders of different blocks
	bufFill(&hostIOBuf[varFillLen], randBlockVarAlgo->next(), constFillRemainderLen);
	cudaMemcpyHostToGPU(&hostIOBuf[varFillLen], &gpuIOBuf[varFillLen], constFillRemainderLen);

#endif // CUDA_SUPPORT
}


/**
 * Simple wrapper for io_prep_pwrite().
 */
void LocalWorker::aioWritePrepper(struct iocb* iocb, int fd, void* buf, size_t count,
	long long offset)
{
#ifndef LIBAIO_SUPPORT

	throw WorkerException("Async IO via libaio requested, but this executable was built without "
		"libaio support.");

#else // LIBAIO_SUPPORT

    OPLOG_PRE_OP("aiowrite", std::to_string(fd), offset, count);

	io_prep_pwrite(iocb, fd, buf, count, offset);

#endif // LIBAIO_SUPPORT
}

/**
 * Simple wrapper for io_prep_pread().
 */
void LocalWorker::aioReadPrepper(struct iocb* iocb, int fd, void* buf, size_t count,
	long long offset)
{
#ifndef LIBAIO_SUPPORT

	throw WorkerException("Async IO via libaio requested, but this executable was built without "
		"libaio support.");

#else // LIBAIO_SUPPORT

    OPLOG_PRE_OP("aioread", std::to_string(fd), offset, count);

	io_prep_pread(iocb, fd, buf, count, offset);

#endif // LIBAIO_SUPPORT
}

/**
 * Within a write phase, send user-defined pecentage of block reads for mixed r/w.
 *
 * Parameters are similar to io_prep_p{write,read}.
 */
void LocalWorker::aioRWMixPrepper(struct iocb* iocb, int fd, void* buf, size_t count,
	long long offset)
{
#ifndef LIBAIO_SUPPORT

	throw WorkerException("Async IO via libaio requested, but this executable was built without "
		"libaio support.");

#else // LIBAIO_SUPPORT

	// example: 40% means 40 out of 100 submitted blocks will be reads, the remaining 60 are writes

	/* note: keep in mind that this also needs to work with lots of small files, so percentage needs
		to work between different files. (numIOPSSubmitted ensures that below; numIOPSDone would not
		work for this because aio would not inc counter directly on submission.) */

	// note: workerRank is used to have skew between different worker threads
	// note: this same logic is used in preWriteBufRandRefill/preWriteBufRandRefillFast
	if( ( (workerRank + numIOPSSubmitted) % 100) >= progArgs->getRWMixReadPercent() )
    {
        OPLOG_PRE_OP("aiowrite", std::to_string(fd), offset, count);

        io_prep_pwrite(iocb, fd, buf, count, offset);
    }
	else
    {
        OPLOG_PRE_OP("aioread", std::to_string(fd), offset, count);

        io_prep_pread(iocb, fd, buf, count, offset);
    }

#endif // LIBAIO_SUPPORT
}

/**
 * Noop for cases where no rate limit selected by user.
 *
 * @return true if we had to wait, false if we are good to go immediately.
 */
bool LocalWorker::noOpRateLimiter(size_t rwSize, std::atomic_bool& isInterruptionRequested)
{
    return false; // noop
}

/**
 * Rate limiter before writes/reads in case rate limit was selected by user.
 *
 * @return true if we had to wait, false if we are good to go immediately.
 */
bool LocalWorker::preRWRateLimiter(size_t rwSize, std::atomic_bool& isInterruptionRequested)
{
    return rateLimiter.wait(rwSize);
}

/**
 * Rate limiter before reads in case rwmix threads rate balance was selected by user.
 *
 * @return true if we had to wait, false if we are good to go immediately.
 */
bool LocalWorker::preRWRateBalanceLimiterForReaders(size_t rwSize,
    std::atomic_bool& isInterruptionRequested)
{
    return rateLimiterRWMixThreads.waitRead(rwSize, isInterruptionRequested);
}

/**
 * Rate limiter before writes in case rwmix threads rate balance was selected by user.
 *
 * @return true if we had to wait, false if we are good to go immediately.
 */
bool LocalWorker::preRWRateBalanceLimiterForWriters(size_t rwSize,
    std::atomic_bool& isInterruptionRequested)
{
    return rateLimiterRWMixThreads.waitWrite(rwSize, isInterruptionRequested);
}

/**
 * Noop for cases where preWriteCudaMemcpy & postReadCudaMemcpy are not appropriate.
 */
void LocalWorker::noOpCudaMemcpy(void* hostIOBuf, void* gpuIOBuf, size_t count)
{
	return; // noop
}

/**
 * Copy hostIOBuf to gpuIOBuf. This is e.g. to simulate transfer of file contents to GPU buffer
 * after file read.
 *
 * @count number of bytes to copy.
 * @throw WorkerException if buffer copy fails.
 */
void LocalWorker::cudaMemcpyGPUToHost(void* hostIOBuf, void* gpuIOBuf, size_t count)
{
#ifdef CUDA_SUPPORT

	cudaError_t copyRes = cudaMemcpy(hostIOBuf, gpuIOBuf, count, cudaMemcpyDeviceToHost);

	IF_UNLIKELY(copyRes != cudaSuccess)
		throw WorkerException("Initialization of GPU buffer via memcpy failed. "
			"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
			"Byte count: " + std::to_string(count) + "; "
			"GPU ID: " + std::to_string(gpuID) + "; "
			"CUDA Error: " + cudaGetErrorString(copyRes) );

#endif // CUDA_SUPPORT
}

/**
 * Copy hostIOBuf to gpuIOBuf. This is e.g. to simulate transfer of file contents to GPU buffer
 * after a file read.
 *
 * @count number of bytes to copy.
 * @throw WorkerException if buffer copy fails.
 */
void LocalWorker::cudaMemcpyHostToGPU(void* hostIOBuf, void* gpuIOBuf, size_t count)
{
#ifdef CUDA_SUPPORT

	cudaError_t copyRes = cudaMemcpy(gpuIOBuf, hostIOBuf, count, cudaMemcpyHostToDevice);

	IF_UNLIKELY(copyRes != cudaSuccess)
		throw WorkerException("Initialization of GPU buffer via memcpy failed. "
			"Buffer size: " + std::to_string(progArgs->getBlockSize() ) + "; "
			"Byte count: " + std::to_string(count) + "; "
			"GPU ID: " + std::to_string(gpuID) + "; "
			"CUDA Error: " + cudaGetErrorString(copyRes) );

#endif // CUDA_SUPPORT
}

/**
 * Noop cuFile handle register for cases where executable is built without CUFILE_SUPPORT or where
 * cuFile API was not selected by user.
 */
void LocalWorker::noOpCuFileHandleReg(int fd, CuFileHandleData& handleData)
{
	return; // noop
}

/**
 * Noop cuFile handle /deregister for cases where executable is built without CUFILE_SUPPORT or
 * where cuFile API was not selected by user.
 */
void LocalWorker::noOpCuFileHandleDereg(CuFileHandleData& handleData)
{
	return; // noop
}

/**
 * cuFile handle register as preparation for cuFileRead/Write in dir mode. Call cuFileHandleDereg
 * when done with file access.
 *
 * @fd posix file handle to register for cuFile access
 * @outHandleData the registered handle on success, in which case outHandleData.isCuFileRegistered
 * 		will be set to true.
 * @throw WorkerException if registration fails.
 */
void LocalWorker::dirModeCuFileHandleReg(int fd, CuFileHandleData& outHandleData)
{
	outHandleData.registerHandle<WorkerException>(fd);
}

/**
 * Counterpart to cuFileHandleReg to deregister a file handle in dir mode. This is safe to call even
 * if registration was not called or if it failed, based on handleData.isCuFileRegistered.
 *
 * @handleData the same that was passed to cuFileHandleReg before.
 */
void LocalWorker::dirModeCuFileHandleDereg(CuFileHandleData& handleData)
{
	handleData.deregisterHandle();
}

/**
 * Wrapper for positional sync read.
 */
ssize_t LocalWorker::preadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
	const int fd = (*fileHandles.fdVecPtr)[fileHandleIdx];

	OPLOG_PRE_OP("pread", std::to_string(fd), offset, nbytes);

	ssize_t preadRes = pread(fd, buf, nbytes, offset);

	OPLOG_POST_OP("pread", std::to_string(fd), offset, nbytes, preadRes == -1);

	return preadRes;
}

/**
 * Wrapper for positional sync write.
 */
ssize_t LocalWorker::pwriteWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
	const int fd = (*fileHandles.fdVecPtr)[fileHandleIdx];

	OPLOG_PRE_OP("pwrite", std::to_string(fd), offset, nbytes);

	ssize_t pwriteRes = pwrite(fd, buf, nbytes, offset);

	OPLOG_POST_OP("pwrite", std::to_string(fd), offset, nbytes, pwriteRes <= 0);

	return pwriteRes;
}

/**
 * Wrapper for positional sync write followed by an immediate read of the same block.
 */
ssize_t LocalWorker::pwriteAndReadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes,
	off_t offset)
{
	const int fd = (*fileHandles.fdVecPtr)[fileHandleIdx];

	OPLOG_PRE_OP("pwrite", std::to_string(fd), offset, nbytes);

	ssize_t pwriteRes = pwrite(fd, buf, nbytes, offset);

	OPLOG_POST_OP("pwrite", std::to_string(fd), offset, nbytes, pwriteRes <= 0);

	IF_UNLIKELY(pwriteRes <= 0)
		return pwriteRes;

	OPLOG_PRE_OP("pread", std::to_string(fd), offset, nbytes);

	ssize_t preadRes = pread(fd, buf, pwriteRes, offset);

	OPLOG_POST_OP("pread", std::to_string(fd), offset, nbytes, preadRes == -1);

	return preadRes;
}

/**
 * Within a write phase, send user-defined pecentage of block reads for mixed r/w.
 *
 * Parameters and return value are similar to p{write,read}.
 */
ssize_t LocalWorker::pwriteRWMixWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
	// example: 40% means 40 out of 100 submitted blocks will be reads, the remaining 60 are writes

	/* note: keep in mind that this also needs to work with lots of small files, so percentage needs
		to work between different files. (numIOPSSubmitted ensures that below; numIOPSDone would not
		work for this because aio would not inc counter directly on submission.) */

	const int fd = (*fileHandles.fdVecPtr)[fileHandleIdx];

	ssize_t ioRes;

	// note: workerRank is used to have skew between different worker threads
	if( ( (workerRank + numIOPSSubmitted) % 100) >= progArgs->getRWMixReadPercent() )
	{
		OPLOG_PRE_OP("pwrite", std::to_string(fd), offset, nbytes);

		ioRes = pwrite(fd, buf, nbytes, offset);

		OPLOG_POST_OP("pwrite", std::to_string(fd), offset, nbytes, ioRes <= 0);
	}
	else
	{
		OPLOG_PRE_OP("pread", std::to_string(fd), offset, nbytes);

		ioRes = pread(fd, buf, nbytes, offset);

		OPLOG_POST_OP("pread", std::to_string(fd), offset, nbytes, ioRes == -1);
	}

	return ioRes;
}

/**
 * Wrapper for positional sync cuFile read.
 *
 * @fd ignored, using cuFileHandleData.cfr_handle instead.
 * @buf ignored, using gpuIOBufVec[0] instead.
 */
ssize_t LocalWorker::cuFileReadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
#ifndef CUFILE_SUPPORT
	throw WorkerException("cuFileReadWrapper called, but this executable was built without cuFile "
		"API support");
#else
	OPLOG_PRE_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t ioRes = cuFileRead(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
		gpuIOBufVec[0], nbytes, offset, 0);

	OPLOG_POST_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes, ioRes == -1);

	return ioRes;
#endif
}

/**
 * Wrapper for positional sync cuFile write.
 *
 * @fd ignored, using cuFileHandleData.cfr_handle instead.
 * @buf ignored, using gpuIOBufVec[0] instead.
 */
ssize_t LocalWorker::cuFileWriteWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
#ifndef CUFILE_SUPPORT
	throw WorkerException("cuFileWriteWrapper called, but this executable was built without cuFile "
		"API support");
#else
	OPLOG_PRE_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t ioRes = cuFileWrite(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
		gpuIOBufVec[0], nbytes, offset, 0);

	OPLOG_POST_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes, ioRes <= 0);

	return ioRes;
#endif
}

/**
 * Wrapper for positional sync cuFile write followed by an immediate cuFile read of the same block.
 */
ssize_t LocalWorker::cuFileWriteAndReadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes,
	off_t offset)
{
#ifndef CUFILE_SUPPORT
	throw WorkerException("cuFileWriteAndReadWrapper called, but this executable was built without "
		"cuFile API support");
#else
	OPLOG_PRE_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t writeRes =
		cuFileWrite(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
			gpuIOBufVec[0], nbytes, offset, 0);

	OPLOG_POST_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes, writeRes <= 0);

	IF_UNLIKELY(writeRes <= 0)
		return writeRes;

	OPLOG_PRE_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t readRes = cuFileRead(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
		gpuIOBufVec[0], writeRes, offset, 0);

	OPLOG_POST_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes, readRes == -1);

	return readRes;
#endif
}

/**
 * Within a write phase, send user-defined pecentage of block reads for mixed r/w.
 *
 * Parameters and return value are similar to p{write,read}.
 */
ssize_t LocalWorker::cuFileRWMixWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
#ifndef CUFILE_SUPPORT
	throw WorkerException("cuFileRWMixWrapper called, but this executable was built without cuFile "
		"API support");
#else
	// example: 40% means 40 out of 100 submitted blocks will be reads, the remaining 60 are writes

	/* note: keep in mind that this also needs to work with lots of small files, so percentage needs
		to work between different files. (numIOPSSubmitted ensures that below; numIOPSDone would not
		work for this because aio would not inc counter directly on submission.) */

	ssize_t ioRes;

	// note: workerRank is used to have skew between different worker threads
	if( ( (workerRank + numIOPSSubmitted) % 100) >= progArgs->getRWMixReadPercent() )
	{
		OPLOG_PRE_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes);

		ioRes = cuFileWrite(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
			gpuIOBufVec[0], nbytes, offset, 0);

		OPLOG_POST_OP("cuFileWrite", std::to_string(fileHandleIdx), offset, nbytes, ioRes <= 0);
	}
	else
	{
		OPLOG_PRE_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes);

		ioRes = cuFileRead(fileHandles.cuFileHandleDataPtrVec[fileHandleIdx]->cfr_handle,
			gpuIOBufVec[0], nbytes, offset, 0);

		OPLOG_POST_OP("cuFileRead", std::to_string(fileHandleIdx), offset, nbytes, ioRes == -1);
}

	return ioRes;
#endif
}

/**
 * Wrapper for positional sync read for HDFS.
 */
ssize_t LocalWorker::hdfsReadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
#ifndef HDFS_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but built without hdfs support");
#else
	OPLOG_PRE_OP("hdfsPread", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t ioRes = hdfsPread(hdfsFSHandle, hdfsFileHandle, offset, buf, nbytes);

	OPLOG_POST_OP("hdfsPread", std::to_string(fileHandleIdx), offset, nbytes, ioRes != -1);

	return ioRes;
#endif // HDFS_SUPPORT
}

/**
 * Wrapper for positional sync write for HDFS.
 *
 * HDFS does not support seeking for writes, so offset is ignored.
 */
ssize_t LocalWorker::hdfsWriteWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
#ifndef HDFS_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but built without hdfs support");
#else
	OPLOG_PRE_OP("hdfsWrite", std::to_string(fileHandleIdx), offset, nbytes);

	ssize_t ioRes = hdfsWrite(hdfsFSHandle, hdfsFileHandle, buf, nbytes);

	OPLOG_POST_OP("hdfsWrite", std::to_string(fileHandleIdx), offset, nbytes, ioRes <= 0);

	return ioRes;
#endif // HDFS_SUPPORT
}

/**
 * Wrapper for positional sync read via mmap.
 */
ssize_t LocalWorker::mmapReadWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
	memcpy(buf, &(fileHandles.mmapVec[fileHandleIdx][offset]), nbytes);

	return nbytes;
}

/**
 * Wrapper for positional sync write via mmap.
 */
ssize_t LocalWorker::mmapWriteWrapper(size_t fileHandleIdx, void* buf, size_t nbytes, off_t offset)
{
	memcpy(&(fileHandles.mmapVec[fileHandleIdx][offset]), buf, nbytes);

	return nbytes;
}

/**
 * Iterate over all directories to create or remove them.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::dirModeIterateDirs()
{
	if(progArgs->getNumDirs() == 0)
		return; // nothing to do

	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t numDirs = progArgs->getNumDirs();
	const IntVec& pathFDs = progArgs->getBenchPathFDs();
	const StringVec& pathVec = progArgs->getBenchPaths();
	const bool ignoreDelErrors = progArgs->getDoDirSharing() ?
		true : progArgs->getIgnoreDelErrors(); // in dir share mode, all workers mk/del all dirs
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */

	// create rank dir inside each pathFD
	if(benchPhase == BenchPhase_CREATEDIRS)
	{
		for(unsigned pathFDsIndex = 0; pathFDsIndex < pathFDs.size(); pathFDsIndex++)
		{
			// create rank dir for current pathFD...

			checkInterruptionRequest();

			// generate path
			int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu", workerDirRank);
			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("mkdir path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) );

		    OPLOG_PRE_OP("mkdirat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0);

            int mkdirRes = mkdirat(pathFDs[pathFDsIndex], currentPath.data(), MKDIR_MODE);

		    OPLOG_POST_OP("mkdirat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0,
		        mkdirRes == -1);

			if( (mkdirRes == -1) && (errno != EEXIST) )
				throw WorkerException(std::string("Rank directory creation failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
					"SysErr: " + strerror(errno) );
		}
	}

	// create user-specified number of directories round-robin across all given bench paths
	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		checkInterruptionRequest();

		// generate current dir path
		int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu",
			workerDirRank, dirIndex);
		IF_UNLIKELY(printRes >= PATH_BUF_LEN)
			throw WorkerException("mkdir path too long for static buffer. "
				"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
				"dirIndex: " + std::to_string(dirIndex) + "; "
				"workerRank: " + std::to_string(workerRank) );

		unsigned pathFDsIndex = (workerRank + dirIndex) % pathFDs.size();

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if(benchPhase == BenchPhase_CREATEDIRS)
		{ // create dir
            OPLOG_PRE_OP("mkdirat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0);

            int mkdirRes = mkdirat(pathFDs[pathFDsIndex], currentPath.data(), MKDIR_MODE);

            OPLOG_POST_OP("mkdirat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0,
                mkdirRes == -1);

			if( (mkdirRes == -1) && (errno != EEXIST) )
				throw WorkerException(std::string("Directory creation failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
					"SysErr: " + strerror(errno) );
		}

		if(benchPhase == BenchPhase_DELETEDIRS)
		{ // remove dir
		    OPLOG_PRE_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0);

		    int rmdirRes = unlinkat(pathFDs[pathFDsIndex], currentPath.data(), AT_REMOVEDIR);

            OPLOG_POST_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0,
                rmdirRes == -1);

			if( (rmdirRes == -1) && ( (errno != ENOENT) || !ignoreDelErrors) )
				throw WorkerException(std::string("Directory deletion failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
					"SysErr: " + strerror(errno) );
		}

		// calc entry operations latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

		atomicLiveOps.numEntriesDone++;
	} // end of for loop


	// delete rank dir inside each pathFD
	if(benchPhase == BenchPhase_DELETEDIRS)
	{
		for(unsigned pathFDsIndex = 0; pathFDsIndex < pathFDs.size(); pathFDsIndex++)
		{
			// delete rank dir for current pathFD...

			checkInterruptionRequest();

			// generate path
			int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu", workerDirRank);
			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("mkdir path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) );

            OPLOG_PRE_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0);

			int rmdirRes = unlinkat(pathFDs[pathFDsIndex], currentPath.data(), AT_REMOVEDIR);

            OPLOG_POST_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0,
                rmdirRes == -1);

			if( (rmdirRes == -1) && ( (errno != ENOENT) || !ignoreDelErrors) )
				throw WorkerException(std::string("Directory deletion failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
					"SysErr: " + strerror(errno) );
		}
	}

}

/**
 * In directory mode with custom tree, for creation we iterate over a fair share per worker and
 * and create parents as needed (because there is no communication between workers that
 * guarantees that all parents have been created). For deletion, first worker of each instance
 * iterates over all dirs remove them (because we have no way to guarantee otherwise that all
 * subdirs under a certain dir have been deleted).
 *
 * Note: With a custom tree, multiple benchmark paths are not supported (because otherwise we
 * 	can't ensure in file creation phase that the matching parent dir has been created for the
 * 	current bench path).
 *
 * @throw WorkerException on error.
 */
void LocalWorker::dirModeIterateCustomDirs()
{
	const int benchPathFD = progArgs->getBenchPathFDs()[0];
	const std::string benchPathStr = progArgs->getBenchPaths()[0];
	const bool ignoreDelErrors = true; // in custom tree mode, all workers mk/del all dirs
	const PathList& customTreePaths = (benchPhase == BenchPhase_DELETEDIRS) ?
		progArgs->getCustomTreeDirs().getPaths() : customTreeDirs.getPaths();
	const bool reverseOrder = (benchPhase == BenchPhase_DELETEDIRS);
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool thisWorkerDoesDelDirs = progArgs->getIsServicePathShared() ?
		(workerRank == 0) : (localWorkerRank == 0);

	IF_UNLIKELY(customTreePaths.empty() )
		return; // nothing to do here

	if( (benchPhase == BenchPhase_DELETEDIRS) && !thisWorkerDoesDelDirs)
	{ // only first worker iterates over all dirs for delete, others do nothing
		workerGotPhaseWork = false;
		return;
	}

	/* note on reverse: dirs are ordered by path length, so that parent dirs come before their
		subdirs. for tree removal, we need to remove subdirs first, hence the reverse order */

	PathList::const_iterator forwardIter = customTreePaths.cbegin();
	PathList::const_reverse_iterator reverseIter = customTreePaths.crbegin();

	// create user-specified directories round-robin across all given bench paths
	for( ; ; )
	{
		checkInterruptionRequest();

		const PathStoreElem& currentPathElem = reverseOrder ? *reverseIter : *forwardIter;

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if(benchPhase == BenchPhase_CREATEDIRS)
		{ // create dir
			int mkdirRes = FileTk::mkdiratBottomUp(
				benchPathFD, currentPathElem.path.c_str(), MKDIR_MODE);

			if( (mkdirRes == -1) && (errno != EEXIST) )
				throw WorkerException(std::string("Directory creation failed. ") +
					"Path: " + benchPathStr + "/" + currentPathElem.path + "; "
					"SysErr: " + strerror(errno) );
		}

		if(benchPhase == BenchPhase_DELETEDIRS)
		{ // remove dir
            OPLOG_PRE_OP("unlinkat", benchPathStr + "/" + currentPathElem.path, 0, 0);

		    int rmdirRes = unlinkat(benchPathFD, currentPathElem.path.c_str(), AT_REMOVEDIR);

            OPLOG_POST_OP("unlinkat", benchPathStr + "/" + currentPathElem.path, 0, 0,
                rmdirRes == -1);

			if( (rmdirRes == -1) && ( (errno != ENOENT) || !ignoreDelErrors) )
				throw WorkerException(std::string("Directory deletion failed. ") +
					"Path: " + benchPathStr + "/" + currentPathElem.path + "; "
					"SysErr: " + strerror(errno) );
		}

		// calc entry operations latency. (for create, this includes open/rw/close.)
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

		atomicLiveOps.numEntriesDone++;

		// advance iterator and check for end of list
		if(reverseOrder)
		{
			reverseIter++;
			if(reverseIter == customTreePaths.crend() )
				break;
		}
		else
		{
			forwardIter++;
			if(forwardIter == customTreePaths.cend() )
				break;
		}
	} // end of for loop
}

/**
 * This is for directory mode. Iterate over all files to create/read/remove them.
 * By default, this uses a unique dir per worker and fills up each dir before moving on to the next.
 * If dir sharing is enabled, all workers will use dirs of rank 0.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::dirModeIterateFiles()
{
	const bool haveSubdirs = (progArgs->getNumDirs() > 0);
	const size_t numDirs = haveSubdirs ? progArgs->getNumDirs() : 1; // set 1 to run dir loop once
	const size_t numFiles = progArgs->getNumFiles();
	const uint64_t fileSize = progArgs->getFileSize();
	const IntVec& pathFDs = progArgs->getBenchPathFDs();
	const StringVec& pathVec = progArgs->getBenchPaths();
	const int openFlags = getDirModeOpenFlags(benchPhase);
	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
		(localWorkerRank < progArgs->getNumRWMixReadThreads() ) );
	const bool useMmap = progArgs->getUseMmap();
	const bool doStatInline = progArgs->getDoStatInline();

	int& fd = fileHandles.fdVec[0];
	CuFileHandleData& cuFileHandleData = fileHandles.cuFileHandleDataVec[0];

	// walk over each unique dir per worker

	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		// occasional interruption check
		IF_UNLIKELY( (dirIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		// fill up this dir with all files before moving on to the next dir

		for(size_t fileIndex = 0; fileIndex < numFiles; fileIndex++)
		{
			// occasional interruption check
			IF_UNLIKELY( (fileIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
				checkInterruptionRequest();

			// generate current dir path
			int printRes;

			if(haveSubdirs)
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-f%zu",
					workerDirRank, dirIndex, workerRank, fileIndex);
			else
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-f%zu",
					workerRank, fileIndex);

			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("file path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) + "; "
					"dirIndex: " + std::to_string(dirIndex) + "; "
					"fileIndex: " + std::to_string(fileIndex) );

			unsigned pathFDsIndex = (workerRank + dirIndex) % pathFDs.size();

			rwOffsetGen->reset(); // reset for next file

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			if( (benchPhase == BenchPhase_CREATEFILES) || (benchPhase == BenchPhase_READFILES) )
			{
				fd = dirModeOpenAndPrepFile(benchPhase, pathFDs, pathFDsIndex,
					currentPath.data(), openFlags, fileSize);

				// try-block to ensure that fd is closed in case of exception
				try
				{
					((*this).*funcCuFileHandleReg)(fd, cuFileHandleData); // reg cuFile handle

					if(doStatInline)
					{ // inline stat (i.e. stat immediately after file open)
						struct stat statBuf;

					    OPLOG_PRE_OP("fstat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0,
					        0);

                        int statRes = fstat(fd, &statBuf);

					    OPLOG_POST_OP("fstat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0,
					        0, statRes == -1);

						IF_UNLIKELY(statRes == -1)
							throw WorkerException(std::string("Inline file stat failed. ") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"SysErr: " + strerror(errno) );
					}

					if(benchPhase == BenchPhase_CREATEFILES)
					{

						int64_t writeRes = ((*this).*funcRWBlockSized)();

						IF_UNLIKELY(writeRes == -1)
							throw WorkerException(std::string("File write failed. ") +
								( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
									"Can be caused by directIO misalignment. " : "") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"SysErr: " + strerror(errno) );

						IF_UNLIKELY( (size_t)writeRes != fileSize)
							throw WorkerException(std::string("Unexpected short file write. ") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"Bytes written: " + std::to_string(writeRes) + "; "
								"Expected written: " + std::to_string(fileSize) + "; "
                                "Hint: Consider initial sequential write or adding "
                                    "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
					}

					if(benchPhase == BenchPhase_READFILES)
					{
						ssize_t readRes = ((*this).*funcRWBlockSized)();

						IF_UNLIKELY(readRes == -1)
							throw WorkerException(std::string("File read failed. ") +
								( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
									"Can be caused by directIO misalignment. " : "") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"SysErr: " + strerror(errno) );

						IF_UNLIKELY( (size_t)readRes != fileSize)
							throw WorkerException(std::string("Unexpected short file read. ") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"Bytes read: " + std::to_string(readRes) + "; "
								"Expected read: " + std::to_string(fileSize) + "; "
                                "Hint: Consider initial sequential write or adding "
                                    "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
					}
				}
				catch(...)
				{
					// release memory mapping
					if(useMmap && (fileHandles.mmapVec[0] != MAP_FAILED) )
					{
						munmap(fileHandles.mmapVec[0], fileSize);
						fileHandles.mmapVec[0] = (char*)MAP_FAILED;
					}

					((*this).*funcCuFileHandleDereg)(cuFileHandleData); // dereg cuFile handle

					OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

					int closeRes = close(fd);

					OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

					throw;
				}

				// release memory mapping
				if(useMmap)
				{
					int unmapRes = munmap(fileHandles.mmapVec[0], fileSize);

					IF_UNLIKELY(unmapRes == -1)
						ERRLOGGER(Log_NORMAL, "File memory unmap failed. " <<
							"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
							"SysErr: " << strerror(errno) << std::endl);

					fileHandles.mmapVec[0] = (char*)MAP_FAILED;
				}

				((*this).*funcCuFileHandleDereg)(cuFileHandleData); // deReg cuFile handle

                OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

				int closeRes = close(fd);

                OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

				IF_UNLIKELY(closeRes == -1)
					throw WorkerException(std::string("File close failed. ") +
						"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
						"FD: " + std::to_string(fd) + "; "
						"SysErr: " + strerror(errno) );
			}

			if(benchPhase == BenchPhase_STATFILES)
			{
				struct stat statBuf;

				int statRes = fstatat(pathFDs[pathFDsIndex], currentPath.data(), &statBuf, 0);

				IF_UNLIKELY(statRes == -1)
					throw WorkerException(std::string("File stat failed. ") +
						"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
						"SysErr: " + strerror(errno) );
			}

			if(benchPhase == BenchPhase_DELETEFILES)
			{
                OPLOG_PRE_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0);

                int unlinkRes = unlinkat(pathFDs[pathFDsIndex], currentPath.data(), 0);

	            OPLOG_POST_OP("unlinkat", pathVec[pathFDsIndex] + "/" + currentPath.data(), 0, 0,
	                unlinkRes == -1);

				if( (unlinkRes == -1) && (!progArgs->getIgnoreDelErrors() || (errno != ENOENT) ) )
					throw WorkerException(std::string("File delete failed. ") +
						"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
						"SysErr: " + strerror(errno) );
			}

			// calc entry operations latency. (for create, this includes open/rw/close.)
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			// inc special rwmix thread stats
			if(isRWMixedReader)
			{
				entriesLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOpsReadMix.numEntriesDone++;
			}
			else
			{
				entriesLatHisto.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOps.numEntriesDone++;
			}

		} // end of files for loop
	} // end of dirs for loop

}

/**
 * This is for directory mode with custom files. Iterate over all files to create/read/remove them.
 * Each worker uses a subset of the files from the non-shared tree and parts of files from the
 * shared tree.
 *
 * Note: With a custom tree, multiple benchmark paths are not supported (because otherwise we
 * 	can't ensure in file creation phase that the matching parent dir has been created for the
 * 	current bench path).
 *
 * @throw WorkerException on error.
 */
void LocalWorker::dirModeIterateCustomFiles()
{
	const IntVec& benchPathFDs = progArgs->getBenchPathFDs();
	const unsigned benchPathFDIdx = 0; // multiple bench paths not supported with custom tree
	const int benchPathFD = progArgs->getBenchPathFDs()[0];
	const std::string benchPathStr = progArgs->getBenchPaths()[0];
	const int openFlags = getDirModeOpenFlags(benchPhase);
	const bool ignoreDelErrors = true; // shared files are unliked by all workers, so no errs
	const PathList& customTreePaths = customTreeFiles.getPaths();
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
		(localWorkerRank < progArgs->getNumRWMixReadThreads() ) );
	const bool useMmap = progArgs->getUseMmap();
	const bool doStatInline = progArgs->getDoStatInline();

	int& fd = fileHandles.fdVec[0];
	CuFileHandleData& cuFileHandleData = fileHandles.cuFileHandleDataVec[0];


	// check if this worker has anything to do in this round
    IF_UNLIKELY(customTreePaths.empty() )
    {
        LOGGER(Log_DEBUG, "got no work in this round. workerRank: " << workerRank << std::endl);
        workerGotPhaseWork = false;
        return;
    }


	unsigned short numFilesDone = 0; // just for occasional interruption check (so short is ok)

	// walk over custom tree part of this worker

	for(const PathStoreElem& currentPathElem : customTreePaths)
	{
		// occasional interruption check
		if( (numFilesDone % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		const char* currentPath = currentPathElem.path.c_str();

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if( (benchPhase == BenchPhase_CREATEFILES) || (benchPhase == BenchPhase_READFILES) )
		{
			const uint64_t rangeLen = currentPathElem.rangeLen;
			const uint64_t fileOffset = currentPathElem.rangeStart;

			rwOffsetGen->reset(rangeLen, fileOffset);

			fd = dirModeOpenAndPrepFile(benchPhase, benchPathFDs, benchPathFDIdx,
				currentPathElem.path.c_str(), openFlags, currentPathElem.totalLen);

			// try-block to ensure that fd is closed in case of exception
			try
			{
				((*this).*funcCuFileHandleReg)(fd, cuFileHandleData); // reg cuFile handle

				if(doStatInline)
				{ // inline stat (i.e. stat immediately after file open)
					struct stat statBuf;

					int statRes = fstat(fd, &statBuf);

					IF_UNLIKELY(statRes == -1)
						throw WorkerException(std::string("File stat failed. ") +
							"Path: " + benchPathStr + "/" + currentPath + "; "
							"SysErr: " + strerror(errno) );
				}

				if(benchPhase == BenchPhase_CREATEFILES)
				{
					int64_t writeRes = ((*this).*funcRWBlockSized)();

					IF_UNLIKELY(writeRes == -1)
						throw WorkerException(std::string("File write failed. ") +
							( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
								"Can be caused by directIO misalignment. " : "") +
							"Path: " + benchPathStr + "/" + currentPath + "; "
							"SysErr: " + strerror(errno) );

					IF_UNLIKELY( (size_t)writeRes != currentPathElem.rangeLen)
						throw WorkerException(std::string("Unexpected short file write. ") +
							"Path: " + benchPathStr + "/" + currentPath + "; "
							"Bytes written: " + std::to_string(writeRes) + "; "
							"Expected written: " + std::to_string(rangeLen) + "; "
                            "Hint: Consider initial sequential write or adding "
                                "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
				}

				if(benchPhase == BenchPhase_READFILES)
				{
					ssize_t readRes = ((*this).*funcRWBlockSized)();

					IF_UNLIKELY(readRes == -1)
						throw WorkerException(std::string("File read failed. ") +
							( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
								"Can be caused by directIO misalignment. " : "") +
							"Path: " + benchPathStr + "/" + currentPath + "; "
							"SysErr: " + strerror(errno) );

					IF_UNLIKELY( (size_t)readRes != rangeLen)
						throw WorkerException(std::string("Unexpected short file read. ") +
							"Path: " + benchPathStr + "/" + currentPath + "; "
							"Bytes read: " + std::to_string(readRes) + "; "
							"Expected read: " + std::to_string(rangeLen) + "; "
                            "Hint: Consider initial sequential write or adding "
                                "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
				}
			}
			catch(...)
			{
				// release memory mapping
				if(useMmap && (fileHandles.mmapVec[0] != MAP_FAILED) )
				{
					munmap(fileHandles.mmapVec[0], currentPathElem.totalLen);
					fileHandles.mmapVec[0] = (char*)MAP_FAILED;
				}

				((*this).*funcCuFileHandleDereg)(cuFileHandleData); // dereg cuFile handle

                OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

                int closeRes = close(fd);

                OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

                throw;
			}

			// release memory mapping
			if(useMmap)
			{
				int unmapRes = munmap(fileHandles.mmapVec[0], currentPathElem.totalLen);

				IF_UNLIKELY(unmapRes == -1)
					ERRLOGGER(Log_NORMAL, "File memory unmap failed. " <<
							"Path: " + benchPathStr + "/" + currentPath + "; "
						"SysErr: " << strerror(errno) << std::endl);

				fileHandles.mmapVec[0] = (char*)MAP_FAILED;
			}

			((*this).*funcCuFileHandleDereg)(cuFileHandleData); // deReg cuFile handle

            OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

            int closeRes = close(fd);

            OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

			IF_UNLIKELY(closeRes == -1)
				throw WorkerException(std::string("File close failed. ") +
					"Path: " + benchPathStr + "/" + currentPath + "; "
					"FD: " + std::to_string(fd) + "; "
					"SysErr: " + strerror(errno) );
		}

		if(benchPhase == BenchPhase_STATFILES)
		{
			struct stat statBuf;

			int statRes = fstatat(benchPathFD, currentPath, &statBuf, 0);

			IF_UNLIKELY(statRes == -1)
				throw WorkerException(std::string("File stat failed. ") +
					"Path: " + benchPathStr + "/" + currentPath + "; "
					"SysErr: " + strerror(errno) );
		}

		if(benchPhase == BenchPhase_DELETEFILES)
		{
            OPLOG_PRE_OP("unlinkat", benchPathStr + "/" + currentPath, 0, 0);

            int unlinkRes = unlinkat(benchPathFD, currentPath, 0);

            OPLOG_POST_OP("unlinkat", benchPathStr + "/" + currentPath, 0, 0, unlinkRes == -1);

			if( (unlinkRes == -1) && (!ignoreDelErrors || (errno != ENOENT) ) )
				throw WorkerException(std::string("File delete failed. ") +
					"Path: " + benchPathStr + "/" + currentPath + "; "
					"SysErr: " + strerror(errno) );
		}

		// calc entry operations latency. (for create, this includes open/rw/close.)
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		// inc entry lat & num done count
		if(currentPathElem.totalLen == currentPathElem.rangeLen)
		{ // entry lat & done is only meaningful for fully processed entries
			if(isRWMixedReader)
			{
				entriesLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOpsReadMix.numEntriesDone++;
			}
			else
			{
				entriesLatHisto.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOps.numEntriesDone++;
			}
		}

		numFilesDone++;

	} // end of tree elements for-loop

}

/**
 * This is for file/bdev mode. Send random I/Os round-robin to all given files across full file
 * range.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::fileModeIterateFilesRand()
{
	// the total file range used for all workers

    const BenchPhase benchPhase = workersSharedData->currentBenchPhase;
    const bool isWritePhase = (benchPhase == BenchPhase_CREATEFILES);
    const size_t fileHandleVecSize = fileHandles.fdVecPtr->size();
    const size_t blockSize = progArgs->getBlockSize();
    const uint64_t fileSize = progArgs->getFileSize();
    const uint64_t numBlocksPerFile = fileSize / blockSize;
    const uint64_t numBlocksTotal = numBlocksPerFile * fileHandleVecSize;
    const size_t numDataSetThreads = progArgs->getNumDataSetThreads();
    const uint64_t randomAmount = progArgs->getRandomAmount() / numDataSetThreads;

    // treat all files as virtual file-serial range of blocks and instantiate selected offset gen...

    const uint64_t rangeLen = blockSize * (numBlocksTotal / numDataSetThreads);
    const uint64_t rangeOffset = workerRank * blockSize * (numBlocksTotal / numDataSetThreads);

    if(progArgs->getUseStridedAccess() )
        rwOffsetGen = std::make_unique<OffsetGenStrided>(rangeLen, blockSize * workerRank,
            blockSize, numDataSetThreads);
    else
    if(progArgs->getUseRandomUnaligned() )
        rwOffsetGen = std::make_unique<OffsetGenRandom>(randomAmount, *randOffsetAlgo,
            rangeLen, rangeOffset, blockSize);
    else
    if(!progArgs->getRandOffsetAlgo().empty() || !isWritePhase )
        rwOffsetGen = std::make_unique<OffsetGenRandomAligned>(randomAmount, *randOffsetAlgo,
            rangeLen, rangeOffset, blockSize);
    else
    { // random aligned writes without explicit algo selection => use full coverage algo
        rwOffsetGen = std::make_unique<OffsetGenRandomAlignedFullCoverageV2>(
            randomAmount, rangeLen, rangeOffset, blockSize);
    }

    // let funcRWBlockSized do the actual work of reading/writing all blocks...

	if(benchPhase == BenchPhase_CREATEFILES)
	{
		ssize_t writeRes = ((*this).*funcRWBlockSized)();

		IF_UNLIKELY(writeRes == -1)
			throw WorkerException(std::string("File write failed. ") +
				( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
					"Can be caused by directIO misalignment. " : "") +
				fileModeLogPathFromFileHandlesErr() +
				"SysErr: " + strerror(errno) );

		IF_UNLIKELY( (size_t)writeRes != rwOffsetGen->getNumBytesTotal() )
			throw WorkerException(std::string("Unexpected short file write. ") +
				fileModeLogPathFromFileHandlesErr() +
				"Bytes written: " + std::to_string(writeRes) + "; "
				"Expected written: " + std::to_string(rwOffsetGen->getNumBytesTotal() ) + "; "
                "Hint: Consider initial sequential write or adding "
                    "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
	}

	if(benchPhase == BenchPhase_READFILES)
	{
		ssize_t readRes = ((*this).*funcRWBlockSized)();

		IF_UNLIKELY(readRes == -1)
			throw WorkerException(std::string("File read failed. ") +
				( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
					"Can be caused by directIO misalignment. " : "") +
				fileModeLogPathFromFileHandlesErr() +
				"SysErr: " + strerror(errno) );

		IF_UNLIKELY( (size_t)readRes != rwOffsetGen->getNumBytesTotal() )
			throw WorkerException(std::string("Unexpected short file read. ") +
				fileModeLogPathFromFileHandlesErr() +
				"Bytes read: " + std::to_string(readRes) + "; "
				"Expected read: " + std::to_string(rwOffsetGen->getNumBytesTotal() ) + "; "
                "Hint: Consider initial sequential write or adding "
                    "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
	}

}

/**
 * This is for file/bdev mode. Iterate over all files to create/write or read them with sequential
 * I/O.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::fileModeIterateFilesSeq()
{
	const IntVec& pathFDs = fileHandles.threadFDVec.empty() ?
		progArgs->getBenchPathFDs() : fileHandles.threadFDVec;
	CuFileHandleDataVec& cuFileHandleDataVec = fileHandles.threadCuFileHandleDataVec.empty() ?
		progArgs->getCuFileHandleDataVec() : fileHandles.threadCuFileHandleDataVec;
	const size_t numFiles = pathFDs.size();
	const uint64_t fileSize = progArgs->getFileSize();
	const size_t blockSize = progArgs->getBlockSize();
	const size_t numThreads = progArgs->getNumDataSetThreads();
	const bool useMmap = progArgs->getUseMmap();

	const uint64_t numBlocksPerFile = (fileSize / blockSize) +
		( (fileSize % blockSize) ? 1 : 0);

	const uint64_t numBlocksTotal = numBlocksPerFile * numFiles; // total for all files
	const uint64_t standardWorkerNumBlocks = numBlocksTotal / numThreads;

	// note: last worker might need to write up to "numThreads-1" more blocks than the others
	uint64_t thisWorkerNumBlocks = standardWorkerNumBlocks;
	if( (workerRank == (numThreads-1) ) && (numBlocksTotal % numThreads) )
		thisWorkerNumBlocks = numBlocksTotal - (standardWorkerNumBlocks * (numThreads-1) );

	// indices of start and end block. (end block is not inclusive.)
	uint64_t startBlock = workerRank * standardWorkerNumBlocks;
	uint64_t endBlock = startBlock + thisWorkerNumBlocks;

	LOGGER(Log_DEBUG, "workerRank: " << workerRank << "; "
		"numFiles: " << numFiles << "; "
		"dataSetThreads: " << numThreads << "; "
		"blocksTotal: " << numBlocksTotal << "; "
		"blocksPerFile: " << numBlocksPerFile << "; "
		"standardWorkerNumBlocks: " << standardWorkerNumBlocks << "; "
		"thisWorkerNumBlocks: " << thisWorkerNumBlocks << "; "
		"startBlock: " << startBlock << "; "
		"endBlock: " << endBlock << "; " << std::endl);


	// check if worker has anything to do in this round
	IF_UNLIKELY(startBlock >= endBlock)
	{
        LOGGER(Log_DEBUG, "got no work in this round. workerRank: " << workerRank << std::endl);

	    workerGotPhaseWork = false;
	    return;
	}


	uint64_t currentBlockIdx = startBlock;

	// iterate over global block range for this worker thread
	// (note: "global block range" means that different blocks can refer to different files)
	while(currentBlockIdx < endBlock)
	{
		// find the file index and inner file block index for current global block index
		const uint64_t currentFileIndex = currentBlockIdx / numBlocksPerFile;
		fileHandles.fdVec[0] = pathFDs[currentFileIndex];
		fileHandles.cuFileHandleDataPtrVec[0] = &(cuFileHandleDataVec[currentFileIndex]);

		const uint64_t currentBlockInFile = currentBlockIdx % numBlocksPerFile;
		const uint64_t currentIOStart = currentBlockInFile * blockSize;

		// calc byte offset in file and range length
		const uint64_t remainingWorkerLen = (endBlock - currentBlockIdx) * blockSize;
		const uint64_t remainingFileLen = fileSize - (currentBlockInFile * blockSize);
		const uint64_t currentIOLen = std::min(remainingWorkerLen, remainingFileLen);

		// prep offset generator for current file range
		rwOffsetGen->reset(currentIOLen, currentIOStart);

		FileTk::fadvise<WorkerException>(fileHandles.fdVec[0], progArgs->getFadviseFlags(),
			progArgs->getBenchPaths()[currentFileIndex].c_str() );

		// prep memory mapping
		if(useMmap)
		{
			int protectionMode = (benchPhase == BenchPhase_READFILES) ?
				PROT_READ : (PROT_WRITE | PROT_READ);

			fileHandles.mmapVec[0] = (char*)FileTk::mmapAndMadvise<WorkerException>(
				fileSize, protectionMode, MAP_SHARED, fileHandles.fdVec[0],
				progArgs->getMadviseFlags(), progArgs->getBenchPaths()[currentFileIndex].c_str() );
		}

		// (try-block for munmap on error)
		try
		{
			// write/read our range of this file

			if(benchPhase == BenchPhase_CREATEFILES)
			{
				ssize_t writeRes = ((*this).*funcRWBlockSized)();

				IF_UNLIKELY(writeRes == -1)
					throw WorkerException(std::string("File write failed. ") +
						( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
							"Can be caused by directIO misalignment. " : "") +
						"Path: " + progArgs->getBenchPaths()[currentFileIndex] + "; "
						"SysErr: " + strerror(errno) );

				IF_UNLIKELY( (size_t)writeRes != currentIOLen)
					throw WorkerException(std::string("Unexpected short file write. ") +
						"Path: " + progArgs->getBenchPaths()[currentFileIndex] + "; "
						"Bytes written: " + std::to_string(writeRes) + "; "
						"Expected written: " + std::to_string(currentIOLen) + "; "
                        "Hint: Consider initial sequential write or adding "
                            "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
			}

			if(benchPhase == BenchPhase_READFILES)
			{
				ssize_t readRes = ((*this).*funcRWBlockSized)();

				IF_UNLIKELY(readRes == -1)
					throw WorkerException(std::string("File read failed. ") +
						( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
							"Can be caused by directIO misalignment. " : "") +
						"Path: " + progArgs->getBenchPaths()[currentFileIndex] + "; "
						"SysErr: " + strerror(errno) );

				IF_UNLIKELY( (size_t)readRes != currentIOLen)
					throw WorkerException(std::string("Unexpected short file read. ") +
						"Path: " + progArgs->getBenchPaths()[currentFileIndex] + "; "
						"Bytes read: " + std::to_string(readRes) + "; "
						"Expected read: " + std::to_string(currentIOLen) + "; "
                        "Hint: Consider initial sequential write or adding "
                            "\"--" ARG_TRUNCTOSIZE_LONG "\" to ensure full file size.");
			}

			// calc completed number of blocks to inc for next loop pass
			const uint64_t numBlocksDone = (currentIOLen / blockSize) +
				( (currentIOLen % blockSize) ? 1 : 0);

			LOGGER_DEBUG_BUILD("  w" << workerRank << " f" << currentFileIndex <<
				" b" << currentBlockInFile << " " <<
				currentIOStart << " - " << (currentIOStart+currentIOLen) << std::endl);

			currentBlockIdx += numBlocksDone;
		}
		catch(...)
		{
			// release memory mapping
			if(useMmap)
			{
				munmap(fileHandles.mmapVec[0], fileSize);
				fileHandles.mmapVec[0] = (char*)MAP_FAILED;
			}

			throw;
		}

		// release memory mapping
		if(useMmap)
		{
			int unmapRes = munmap(fileHandles.mmapVec[0], fileSize);

			IF_UNLIKELY(unmapRes == -1)
				ERRLOGGER(Log_NORMAL, "File memory unmap failed. " <<
					"Path: " + progArgs->getBenchPaths()[currentFileIndex] + "; "
					"SysErr: " << strerror(errno) << std::endl);

			fileHandles.mmapVec[0] = (char*)MAP_FAILED;
		}

	} // end of global blocks while-loop
}

/**
 * This is for file mode. Each thread tries to delete all given files.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::fileModeDeleteFiles()
{
	const StringVec& benchPaths = progArgs->getBenchPaths();
	const size_t numFiles = benchPaths.size();

	// walk over all files and delete each of them
	// (note: each worker starts with a different file (based on workerRank) to spread the load)

	for(size_t fileIndex = 0; fileIndex < numFiles; fileIndex++)
	{
		// occasional interruption check
		if( (fileIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		// delete current file

		const std::string& path =
			benchPaths[ (workerRank + fileIndex) % numFiles];

		int unlinkRes = unlink(path.c_str() );

		// (note: all threads try to delete all files, so ignore ENOENT)
		if( (unlinkRes == -1) && (errno != ENOENT) )
			throw WorkerException(std::string("File delete failed. ") +
				"Path: " + path + "; "
				"SysErr: " + strerror(errno) );

		atomicLiveOps.numEntriesDone++;

	} // end of files for loop

}

/**
 * This is for file/bdev mode. Retrieve path for log message based on fileHandles.errorFDVecIdx.
 *
 * The result will be "Path: /some/path; " based on progArgs benchPathsVec or "Path: unavailable; "
 * if errorFDVecIdx is not set ("==-1").
 */
std::string LocalWorker::fileModeLogPathFromFileHandlesErr()
{
	if(fileHandles.errorFDVecIdx == -1)
		return std::string("Path: unavailable; ");

	return "Path: " + progArgs->getBenchPaths()[fileHandles.errorFDVecIdx] + "; ";
}

/**
 * Iterate over all buckets to create or remove them. Each worker processes its own subset of
 * buckets.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeIterateBuckets()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const StringVec& bucketVec = progArgs->getBenchPaths();
	const size_t numBuckets = bucketVec.size();
	const size_t numDataSetThreads = progArgs->getNumDataSetThreads();

    workerGotPhaseWork = false; // not all workers might get work

	for(unsigned bucketIndex = workerRank;
		bucketIndex < numBuckets;
		bucketIndex += numDataSetThreads)
	{
		checkInterruptionRequest();

		workerGotPhaseWork = true;
        const auto& bucketName = bucketVec[bucketIndex];

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if(benchPhase == BenchPhase_CREATEDIRS)
			s3ModeCreateBucket(bucketName);

        if(benchPhase == BenchPhase_PUT_S3_BUCKET_MD)
        {
            if (progArgs->getDoS3BucketTagging())
                s3ModeCreateBucketTagging(bucketName);

            if (progArgs->getDoS3ObjectLockConfiguration())
                s3ModePutObjectLockConfiguration(bucketName);

            if (progArgs->getDoS3BucketVersioning())
                s3ModePutBucketVersioning(bucketName, false);
        }

		if(benchPhase == BenchPhase_PUTBUCKETACL)
			s3ModePutBucketAcl(bucketName);

		if(benchPhase == BenchPhase_GETBUCKETACL)
			s3ModeGetBucketAcl(bucketName);

		if(benchPhase == BenchPhase_STATDIRS)
            s3ModeHeadBucket(bucketName);

        if(benchPhase == BenchPhase_GET_S3_BUCKET_MD)
        {
            if (progArgs->getDoS3BucketTagging())
                s3ModeGetBucketTagging(bucketName);

            if (progArgs->getDoS3ObjectLockConfiguration())
                s3ModeGetObjectLockConfiguration(bucketName);

            if (progArgs->getDoS3BucketVersioning())
                s3ModeGetBucketVersioning(bucketName);
        }

        if(benchPhase == BenchPhase_DEL_S3_BUCKET_MD)
        {
            if (progArgs->getDoS3BucketVersioning())
                s3ModePutBucketVersioning(bucketName, true);

            // Disable lock configuration
            if (progArgs->getDoS3ObjectLockConfiguration())
                s3ModePutObjectLockConfiguration(bucketName, true);

            if (progArgs->getDoS3BucketTagging())
                s3ModeDeleteBucketTagging(bucketName);
        }

        // delete buckets
		if(benchPhase == BenchPhase_DELETEDIRS)
            s3ModeDeleteBucket(bucketName);

		// calc entry operations latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

		atomicLiveOps.numEntriesDone++;
	}

#endif // S3_SUPPORT
}

/**
 * This is for s3 mode. Iterate over all objects to create/read/remove them.
 * By default, this uses a unique "dir" (i.e. prefix with slashes inside a bucket) per worker and
 * fills up each dir before moving on to the next. If dir sharing is enabled, all workers will use
 * dirs of rank 0.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeIterateObjects()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	if( (benchPhase == BenchPhase_READFILES) && progArgs->getUseS3RandObjSelect() )
	{
		s3ModeIterateObjectsRand();
		return;
	}

	const bool haveSubdirs = (progArgs->getNumDirs() > 0);
	const size_t numDirs = haveSubdirs ? progArgs->getNumDirs() : 1; // set 1 to run dir loop once
	const size_t numFiles = progArgs->getNumFiles();
	const uint64_t fileSize = progArgs->getFileSize();
	const size_t blockSize = progArgs->getBlockSize();
	const StringVec& bucketVec = progArgs->getBenchPaths();
	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */
	std::string objectPrefix = progArgs->getS3ObjectPrefix();
	const bool objectPrefixRand = progArgs->getUseS3ObjectPrefixRand();
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
		(localWorkerRank < progArgs->getNumRWMixReadThreads() ) );

	// walk over each unique dir per worker

	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		// occasional interruption check
		IF_UNLIKELY( (dirIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		// fill up this dir with all files before moving on to the next dir

		for(size_t fileIndex = 0; fileIndex < numFiles; fileIndex++)
		{
			// occasional interruption check
			IF_UNLIKELY( (fileIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
				checkInterruptionRequest();

			// generate current dir path
			int printRes;

			if(haveSubdirs)
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-f%zu",
					workerDirRank, dirIndex, workerRank, fileIndex);
			else
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-f%zu",
					workerRank, fileIndex);

			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("object path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) + "; "
					"dirIndex: " + std::to_string(dirIndex) + "; "
					"fileIndex: " + std::to_string(fileIndex) );

			if(objectPrefixRand)
				objectPrefix = getS3RandObjectPrefix(
					workerRank, dirIndex, fileIndex, progArgs->getS3ObjectPrefix() );

			unsigned bucketIndex = (workerRank + dirIndex) % bucketVec.size();
			std::string currentObjectPath = objectPrefix + currentPath.data();


			rwOffsetGen->reset(); // reset for next file

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			if( (benchPhase == BenchPhase_CREATEFILES) && !isRWMixedReader)
			{
				if(blockSize < fileSize)
					s3ModeUploadObjectMultiPart(bucketVec[bucketIndex], currentObjectPath);
				else
					s3ModeUploadObjectSinglePart(bucketVec[bucketIndex], currentObjectPath);
			}

            if (benchPhase == BenchPhase_PUT_S3_OBJECT_MD)
            {
                if (progArgs->getDoS3ObjectTagging())
                    s3ModePutObjectTags(bucketVec[bucketIndex], currentObjectPath);
            }

            if( (benchPhase == BenchPhase_READFILES) || isRWMixedReader)
            {
                s3ModeDownloadObject(bucketVec[bucketIndex], currentObjectPath,
                    isRWMixedReader);
            }

			if(benchPhase == BenchPhase_STATFILES)
                s3ModeStatObject(bucketVec[bucketIndex], currentObjectPath);

            if(benchPhase == BenchPhase_GET_S3_OBJECT_MD)
            {
                if (progArgs->getDoS3ObjectTagging())
                    s3ModeGetObjectTags(bucketVec[bucketIndex], currentObjectPath);
            }

			if(benchPhase == BenchPhase_PUTOBJACL)
				s3ModePutObjectAcl(bucketVec[bucketIndex], currentObjectPath);

			if(benchPhase == BenchPhase_GETOBJACL)
				s3ModeGetObjectAcl(bucketVec[bucketIndex], currentObjectPath);

            if(benchPhase == BenchPhase_DEL_S3_OBJECT_MD)
            {
                if (progArgs->getDoS3ObjectTagging())
                    s3ModeDeleteObjectTags(bucketVec[bucketIndex], currentObjectPath);
            }

			if(benchPhase == BenchPhase_DELETEFILES)
				s3ModeDeleteObject(bucketVec[bucketIndex], currentObjectPath);

			// calc entry operations latency. (for create, this includes open/rw/close.)
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			// entry lat & num done count
			if(isRWMixedReader)
			{
				entriesLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOpsReadMix.numEntriesDone++;
			}
			else
			{
				entriesLatHisto.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOps.numEntriesDone++;
			}

		} // end of files for loop
	} // end of dirs for loop

#endif // S3_SUPPORT
}

/**
 * This is for s3 mode and only valid for reads. Randomly selects the next object and does one
 * random offset read within each object. Number of ops is defined by ProgArgs::randomAmount.
 *
 * This inits all of the used random generators (for offset, dir index, file index) internally.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeIterateObjectsRand()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const bool haveSubdirs = (progArgs->getNumDirs() > 0);
	const size_t numDirs = haveSubdirs ? progArgs->getNumDirs() : 1; // set 1 to run dir loop once
	const size_t numFiles = progArgs->getNumFiles();
	const uint64_t fileSize = progArgs->getFileSize();
	const size_t blockSize = progArgs->getBlockSize();
	const StringVec& bucketVec = progArgs->getBenchPaths();
	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */
	std::string objectPrefix = progArgs->getS3ObjectPrefix();
	const bool objectPrefixRand = progArgs->getUseS3ObjectPrefixRand();

    // init random generators for dir & file index selection

    std::unique_ptr<RandAlgoInterface> randIndexAlgo =
        RandAlgoSelectorTk::stringToAlgo(progArgs->getRandOffsetAlgo().empty() ?
            RANDALGO_BALANCED_SEQUENTIAL_STR : progArgs->getRandOffsetAlgo() );

	OffsetGenRandom randDirIndexGen(~(uint64_t)0, *randIndexAlgo, numDirs, 0, 1);
	OffsetGenRandom randFileIndexGen(~(uint64_t)0, *randIndexAlgo, numFiles, 0, 1);

	// init random offset gen for one read per file

	const uint64_t randomAmount = progArgs->getRandomAmount() / progArgs->getNumDataSetThreads();
	if(!progArgs->getUseRandomUnaligned() ) // random aligned
	{
		rwOffsetGen = std::make_unique<OffsetGenRandomAligned>(blockSize, *randOffsetAlgo,
			fileSize, 0, blockSize);
	}
	else // random unaligned
		rwOffsetGen = std::make_unique<OffsetGenRandom>(blockSize, *randOffsetAlgo,
			fileSize, 0, blockSize);

	// randomly select objects and do one random offset read from each

	uint64_t numBytesDone = 0;
	uint64_t interruptCheckBytes = blockSize * INTERRUPTION_CHECK_INTERVAL;

	while(numBytesDone < randomAmount)
	{
		// occasional interruption check
		IF_UNLIKELY( (numBytesDone % interruptCheckBytes) == 0)
			checkInterruptionRequest();

		const size_t dirIndex = randDirIndexGen.getNextOffset();
		const size_t fileIndex = randFileIndexGen.getNextOffset();
		const uint64_t currentBlockSize = rwOffsetGen->getNextBlockSizeToSubmit();

		// generate current dir path
		int printRes;

		if(haveSubdirs)
			printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-f%zu",
				workerDirRank, dirIndex, workerRank, fileIndex);
		else
			printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-f%zu",
				workerRank, fileIndex);

		IF_UNLIKELY(printRes >= PATH_BUF_LEN)
			throw WorkerException("object path too long for static buffer. "
				"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
				"workerRank: " + std::to_string(workerRank) + "; "
				"dirIndex: " + std::to_string(dirIndex) + "; "
				"fileIndex: " + std::to_string(fileIndex) );

		if(objectPrefixRand)
			objectPrefix = getS3RandObjectPrefix(
				workerRank, dirIndex, fileIndex, progArgs->getS3ObjectPrefix() );

		const unsigned bucketIndex = (workerRank + dirIndex) % bucketVec.size();
		std::string currentObjectPath = objectPrefix + currentPath.data();

		s3ModeDownloadObject(bucketVec[bucketIndex], currentObjectPath, false);

		rwOffsetGen->reset(); // reset for next rand read op
		numBytesDone += currentBlockSize;
	}


#endif // S3_SUPPORT
}

/**
 * This is for s3 mode with custom tree. Iterate over all objects to create/read/remove them. Each
 * worker uses a subset of the files from the non-shared tree and parts of files from the shared
 * tree.
 *
 * Note: With a custom tree, multiple benchmark paths are not supported. This is a limitation of
 * dirModeIterateCustomFiles() and we keep the same limitation here for compatibility.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeIterateCustomObjects()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const std::string bucketName = progArgs->getBenchPaths()[0];
	const size_t blockSize = progArgs->getBlockSize();
    const bool useS3MpuSharing = progArgs->getUseS3MPUSharing();
	const PathList& customTreePaths = customTreeFiles.getPaths();
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
		(localWorkerRank < progArgs->getNumRWMixReadThreads() ) );
    std::string objectPrefix = progArgs->getS3ObjectPrefix();


    // check if this worker has anything to do in this round
    IF_UNLIKELY(customTreePaths.empty() )
    {
        LOGGER(Log_DEBUG, "got no work in this round. workerRank: " << workerRank << std::endl);

        workerGotPhaseWork = false;
        return;
    }


	unsigned short numFilesDone = 0; // just for occasional interruption check (so "short" is ok)

	// walk over custom tree part of this worker

	for(const PathStoreElem& currentPathElem : customTreePaths)
	{
		// occasional interruption check
		if( (numFilesDone % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if( (benchPhase == BenchPhase_CREATEFILES) || (benchPhase == BenchPhase_READFILES) )
		{
			const uint64_t fileSize = currentPathElem.totalLen;
			const uint64_t fileOffset = currentPathElem.rangeStart;
			const uint64_t rangeLen = currentPathElem.rangeLen;

			rwOffsetGen->reset(rangeLen, fileOffset);

            LOGGER_DEBUG_BUILD(__func__ << ":" << __LINE__ << ": " <<
                "obj: " << objectPrefix + currentPathElem.path << "; "
                "rangeStart: " << fileOffset << "; "
                "rangeLen: " << rangeLen << "; " <<
                "fSize: " << fileSize << "; " << std::endl);

			if(benchPhase == BenchPhase_CREATEFILES)
			{
                // (useS3MpuSharing is to prevent creation of new MPU IDs outside of S3UploadStore)
				if( (rangeLen < fileSize) || useS3MpuSharing)
					s3ModeUploadObjectMultiPartShared(bucketName,
					    objectPrefix + currentPathElem.path, fileSize);
				else
				{ // this worker uploads the whole object
					if(blockSize < fileSize)
						s3ModeUploadObjectMultiPart(bucketName,
						    objectPrefix + currentPathElem.path);
					else
						s3ModeUploadObjectSinglePart(bucketName,
						    objectPrefix + currentPathElem.path);
				}
			}

            if(benchPhase == BenchPhase_READFILES)
                s3ModeDownloadObject(bucketName, objectPrefix + currentPathElem.path, false);
		}

		if(benchPhase == BenchPhase_STATFILES)
			s3ModeStatObject(bucketName, objectPrefix + currentPathElem.path);

		if(benchPhase == BenchPhase_PUTOBJACL)
			s3ModePutObjectAcl(bucketName, objectPrefix + currentPathElem.path);

		if(benchPhase == BenchPhase_GETOBJACL)
			s3ModeGetObjectAcl(bucketName, objectPrefix + currentPathElem.path);

		if(benchPhase == BenchPhase_DELETEFILES)
			s3ModeDeleteObject(bucketName, objectPrefix + currentPathElem.path);

		// calc entry operations latency. (for create, this includes open/rw/close.)
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		// entry lat & num done count
		if(currentPathElem.totalLen == currentPathElem.rangeLen)
		{ // entry lat & done is only meaningful for fully processed entries
			if(isRWMixedReader)
			{
				entriesLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOpsReadMix.numEntriesDone++;
			}
			else
			{
				entriesLatHisto.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOps.numEntriesDone++;
			}
		}

		numFilesDone++;

	} // end of tree elements for-loop

#endif // S3_SUPPORT
}

/**
 * Iterate over all MPU IDs from service MPU sharing mode and mark them as completed. Each worker
 * processes its own subset of MPU IDs.
 *
 * @throw WorkerException on error.
 */
 void LocalWorker::s3ModeIterateAndCompleteMpuIDs()
 {
 #ifndef S3_SUPPORT
     throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
 #else

     const StringVec& bucketVec = progArgs->getBenchPaths();
     const size_t numBuckets = bucketVec.size();
     const std::string bucketName = bucketVec.empty() ? "" : bucketVec[0];
     const size_t numDataSetThreads = progArgs->getNumDataSetThreads();
     const PathList& pathList = progArgs->getCustomTreeFilesShared().getPaths();
     const size_t pathListLen = pathList.size();
     const std::string objectPrefix = progArgs->getS3ObjectPrefix();
     const uint64_t objectSize = progArgs->getFileSize();

     workerGotPhaseWork = false; // not all workers might get work

     // sanity check: should never happen
     IF_UNLIKELY(!progArgs->getUseS3MPUSharing() )
        throw WorkerException("MPU sharing completion phase started without MPU sharing being "
            "selected.");

     // sanity check: should never happen
     IF_UNLIKELY(numBuckets != 1)
        throw WorkerException("MPU sharing completion phase detected invalid number of buckets. "
            "NumBuckets: " + std::to_string(numBuckets) );

    // each worker rank completes its own subset of MPU IDs
    unsigned pathIndex = workerRank;
    for(PathListCIter pathListIter = std::next(pathList.begin(), pathIndex);
         pathIndex < pathListLen;
         pathIndex += numDataSetThreads, std::advance(pathListIter, numDataSetThreads) )
    {
        checkInterruptionRequest();

        workerGotPhaseWork = true;

        std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

        const std::string objectName = objectPrefix + pathListIter->path;

        std::string uploadID = s3SharedUploadStore.getMultipartUploadID(bucketName, objectName,
            s3Client, opsLog);

        s3ModeQueryAndFinishMultipartUpload(bucketName, objectName, uploadID, objectSize);

        // calc entry operations latency
        std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
        std::chrono::microseconds ioElapsedMicroSec =
            std::chrono::duration_cast<std::chrono::microseconds>
            (ioEndT - ioStartT);

        entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

        atomicLiveOps.numEntriesDone++;
    }

 #endif // S3_SUPPORT
 }

#ifdef S3_SUPPORT
/**
 * Throw an informative WorkerException if the s3 request outcome has the error flag set.
 *
 * @outcome s3 request outcome.
 * @failMessage human-friendly error message, e.g. "Object upload failed."
 * @objectName name of object to which this error applies, can be empty.
 * @throw WorkerException on error.
 */
template <typename R>
void LocalWorker::s3ModeThrowOnError(
        const Aws::Utils::Outcome<R, S3ErrorType>& outcome,
        const std::string& failMessage,
        const std::string& bucketName,
        const std::string& objectName)
{
    IF_LIKELY(outcome.IsSuccess() )
        return;

    const auto s3Error = outcome.GetError();

    std::stringstream errStr;
        errStr << failMessage << std::endl <<
        "Endpoint: " << s3EndpointStr << std::endl <<
        "Bucket: " << bucketName << std::endl <<
        (objectName.empty() ? std::string("") : ("Object: " + objectName + "\n" )) <<
        "Exception: " << s3Error.GetExceptionName() << std::endl <<
        "Message: " << s3Error.GetMessage() << std::endl <<
        "HTTP Error Code: " << (int)s3Error.GetResponseCode() << " (" <<
            TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) << ")" <<
            std::endl <<
        "Request ID: " << s3Error.GetRequestId() << std::endl;

    throw WorkerException(errStr.str() );
}
#endif // S3_SUPPORT

/**
 * Add server-side encryption to s3 request.
 */
template <typename REQUESTTYPE>
void LocalWorker::s3ModeAddServerSideEncryption(REQUESTTYPE& request)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

    if(this->useS3SSE)
        request.WithServerSideEncryption(S3::ServerSideEncryption::AES256);
    else
    if (!s3SSECKey.empty())
    {
        request.WithSSECustomerAlgorithm("AES256")
            .WithSSECustomerKey(s3SSECKey)
            .WithSSECustomerKeyMD5(s3SSECKeyMD5);
    }
    else
    if(!s3SSEKMSKey.empty())
    {
        request.WithServerSideEncryption(S3::ServerSideEncryption::aws_kms)
            .WithSSEKMSKeyId(s3SSEKMSKey);
    }

#endif // S3_SUPPORT
}

/**
 * Add checksum algorithm to given request if an algorithm has been explicitly selected.
 */
template <typename REQUESTTYPE>
void LocalWorker::s3ModeAddChecksumAlgorithm(REQUESTTYPE& request)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else
    IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
        request.SetChecksumAlgorithm(s3ChecksumAlgorithm);
#endif // S3_SUPPORT
}

/**
 * Create given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeCreateBucket(std::string bucketName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

    OPLOG_PRE_OP("S3CreateBucket", bucketName, 0, 0);

    S3::CreateBucketRequest createRequest;
    createRequest.SetBucket(bucketName);

    // Check if multi-credentials are being used and set ACL to public-read-write
    if(!progArgs->getS3CredentialsFile().empty() || !progArgs->getS3CredentialsList().empty())
    {
        createRequest.SetACL(S3::BucketCannedACL::public_read_write);
        LOGGER(Log_DEBUG, "Setting bucket ACL to public-read-write for multi-credentials job" << std::endl);
    }

    const auto createOutcome = s3Client->CreateBucket(createRequest);

    OPLOG_POST_OP("S3CreateBucket", bucketName, 0, 0, !createOutcome.IsSuccess());

    if (!createOutcome.IsSuccess())
	{
		auto s3Error = createOutcome.GetError();

        // bucket already existing is not an error
        if (s3Error.GetErrorType() != S3Errors::BUCKET_ALREADY_OWNED_BY_YOU &&
            s3Error.GetErrorType() != S3Errors::BUCKET_ALREADY_EXISTS)
        {
            s3ModeThrowOnError(createOutcome, "Bucket creation failed.", bucketName);
        }
    }

#endif // S3_SUPPORT
}

/**
 * Request attributes of given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeHeadBucket(std::string bucketName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else
    OPLOG_PRE_OP("HeadBucket", bucketName, 0, 0);

	const auto outcome = s3Client->HeadBucket(S3::HeadBucketRequest().WithBucket(bucketName));

    OPLOG_POST_OP("HeadBucket", bucketName, 0, 0, !outcome.IsSuccess());

    s3ModeThrowOnError(outcome, "Head bucket request failed.", bucketName);

#endif // S3_SUPPORT
}

/**
 * Create tags for given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeCreateBucketTagging(const std::string& bucketName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else
    const auto tagging = S3::Tagging().WithTagSet(
        {
            S3::Tag().WithKey(TAG_KEY_MEDIUM_NAME)
                     .WithValue(StringTk::generateRandomS3TagValue(bucketName, TAG_VALUE_MEDIUM_LEN)),
            S3::Tag().WithKey(TAG_KEY_LONG_NAME)
                     .WithValue(StringTk::generateRandomS3TagValue(bucketName, TAG_VALUE_LONG_LEN)),
        }
    );

    S3::PutBucketTaggingRequest taggingRequest;
    taggingRequest.WithBucket(bucketName).WithTagging(tagging);

    s3ModeAddChecksumAlgorithm(taggingRequest);

    OPLOG_PRE_OP("PutBucketTagging", bucketName, 0, 0);

    const auto taggingOutcome = s3Client->PutBucketTagging(taggingRequest);

    OPLOG_POST_OP("PutBucketTagging", bucketName, 0, 0, !taggingOutcome.IsSuccess());

    s3ModeThrowOnError(taggingOutcome, "Put bucket tagging failed.", bucketName);

#endif // S3_SUPPORT
}

/**
 * Get tags for given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeGetBucketTagging(const std::string& bucketName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else
    OPLOG_PRE_OP("GetBucketTagging", bucketName, 0, 0);

    const auto outcome = s3Client->GetBucketTagging(S3::GetBucketTaggingRequest().WithBucket(bucketName));

    OPLOG_POST_OP("GetBucketTagging", bucketName, 0, 0, !outcome.IsSuccess());

    s3ModeThrowOnError(outcome, "Get bucket tagging failed.", bucketName);

    if (!progArgs->getDoS3BucketTaggingVerify())
        return;

    for (const auto& tag : outcome.GetResult().GetTagSet())
    {
        if (!StringTk::verifyRandomS3TagValue(tag.GetValue(), bucketName))
        {
            std::stringstream errStr;
            errStr << "Bucket tag value is corrupted (invalid checksum). "
                   << "Bucket: " << bucketName << "; "
                   << "Tag: " << tag.GetKey() << "=" << tag.GetValue() << std::endl;
            throw WorkerException(errStr.str());
        }
    }

#endif // S3_SUPPORT
}

/**
 * Delete bucket tags for given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeDeleteBucketTagging(const std::string& bucketName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

    OPLOG_PRE_OP("DeleteBucketTagging", bucketName, 0, 0);

    const auto outcome = s3Client->DeleteBucketTagging(
        S3::DeleteBucketTaggingRequest().WithBucket(bucketName)
    );

    OPLOG_POST_OP("DeleteBucketTagging", bucketName, 0, 0, !outcome.IsSuccess());

    s3ModeThrowOnError(outcome, "Delete bucket tagging failed.", bucketName);

#endif // S3_SUPPORT
}

/**
 * Delete given S3 bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeDeleteBucket(const std::string& bucketName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

	const bool ignoreDelErrors = progArgs->getIgnoreDelErrors();

	S3::DeleteBucketRequest request;
	request.SetBucket(bucketName);

	OPLOG_PRE_OP("S3DeleteBucket", bucketName, 0, 0);

	S3::DeleteBucketOutcome deleteOutcome = s3Client->DeleteBucket(request);

	OPLOG_POST_OP("S3DeleteBucket", bucketName, 0, 0, !deleteOutcome.IsSuccess() );

	if(!deleteOutcome.IsSuccess() )
	{
		auto s3Error = deleteOutcome.GetError();

		if( (s3Error.GetErrorType() != S3Errors::NO_SUCH_BUCKET) || !ignoreDelErrors)
            s3ModeThrowOnError(deleteOutcome, "Bucket deletion failed.", bucketName);
	}

#endif // S3_SUPPORT
}

/**
 * Put ACL of given S3 object.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModePutBucketAcl(std::string bucketName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this build is without S3 support");
#else

	S3::PutBucketAclRequest request;
	request.WithBucket(bucketName);

    TranslatorTk::applyS3PutAclRequestGrants<S3::BucketCannedACL>(progArgs, request);

	OPLOG_PRE_OP("S3PutBucketAcl", bucketName, 0, 0);

	S3::PutBucketAclOutcome outcome = s3Client->PutBucketAcl(request);

	OPLOG_POST_OP("S3PutBucketAcl", bucketName, 0, 0, !outcome.IsSuccess() );

    s3ModeThrowOnError(outcome, "Putting bucket ACL failed.", bucketName);

#endif // S3_SUPPORT
}

/**
 * Get ACL of given S3 object.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeGetBucketAcl(std::string bucketName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this build is without S3 support");
#else

	bool doS3AclVerify = progArgs->getDoS3AclVerify();

	S3::GetBucketAclRequest request;
	request.WithBucket(bucketName);

	OPLOG_PRE_OP("S3GetBucketAcl", bucketName, 0, 0);

	S3::GetBucketAclOutcome outcome = s3Client->GetBucketAcl(request);

	OPLOG_POST_OP("S3GetBucketAcl", bucketName, 0, 0, !outcome.IsSuccess() );

    s3ModeThrowOnError(outcome, "Getting bucket ACL failed.", bucketName);

	IF_UNLIKELY(doS3AclVerify)
	{
        // check canned ACL as special grantee...

        S3::BucketCannedACL cannedAcl = S3::BucketCannedACLMapper::GetBucketCannedACLForName(
            progArgs->getS3AclGrantee() );

        /* note: checking for ::NOT_SET alone here is not enough, because GetObjectCannedACLForName()
            can return other values if granteeStr doesn't match another enum value. */
        switch( (S3::ObjectCannedACL)cannedAcl)
        {
            case S3::ObjectCannedACL::private_:
            case S3::ObjectCannedACL::public_read:
            case S3::ObjectCannedACL::public_read_write:
            case S3::ObjectCannedACL::authenticated_read:
            case S3::ObjectCannedACL::aws_exec_read:
            case S3::ObjectCannedACL::bucket_owner_read:
            case S3::ObjectCannedACL::bucket_owner_full_control:
            { // found canned ACL as special grantee
                throw WorkerException("Verification of canned ACLs is not supported.");
            }

            case S3::ObjectCannedACL::NOT_SET:
            default:
            { // normal grantee, not a canned ACL
                break;
            }
        }

        // check list of grants...

		std::vector<S3::Grant> verifyGrants;
		TranslatorTk::getS3ObjectAclGrants(progArgs, verifyGrants);

		const std::vector<S3::Grant>& outcomeGrants = outcome.GetResult().GetGrants();

		// iterate over all grants that need to be verified
		for(S3::Grant& verifyGrant : verifyGrants)
		{
			bool grantFound = false;

			// iterate over all outcome grants to see if any grant matches current verifyGrant
			for(const S3::Grant& outcomeGrant : outcomeGrants)
			{
				if( (outcomeGrant.GetGrantee().GetID() ==
						verifyGrant.GetGrantee().GetID() ) ||
					(outcomeGrant.GetGrantee().GetEmailAddress() ==
						verifyGrant.GetGrantee().GetEmailAddress() ) ||
					(outcomeGrant.GetGrantee().GetURI() ==
						verifyGrant.GetGrantee().GetURI() ) )
				{ // grantee matches => check if permission also matches
					if(outcomeGrant.GetPermission() == verifyGrant.GetPermission() )
					{ // permission matches
						grantFound = true;
						break;
					}
				}
			}

			if(!grantFound)
				throw WorkerException(std::string("S3 ACL verification failed. ") +
					"Endpoint: " + s3EndpointStr + "; "
					"Bucket: " + bucketName + "; "
					"Grantee ID: " + verifyGrant.GetGrantee().GetID() + "; "
					"Grantee Email: " + verifyGrant.GetGrantee().GetEmailAddress() + "; "
					"Grantee URI: " + verifyGrant.GetGrantee().GetURI() + "; "
					"Permission: " + TranslatorTk::s3AclPermissionToStr(
						verifyGrant.GetPermission() ) );
		}
	} // end of verifcation

#endif // S3_SUPPORT
}

void LocalWorker::s3ModeGetBucketVersioning(const std::string& bucketName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    OPLOG_PRE_OP("GetBucketVersioning", bucketName, 0, 0);

    const auto bucketVersioningOutcome = s3Client->GetBucketVersioning(
            S3::GetBucketVersioningRequest().WithBucket(bucketName)
    );

    OPLOG_POST_OP("GetBucketVersioning", bucketName, 0, 0, !bucketVersioningOutcome.IsSuccess());

    s3ModeThrowOnError(bucketVersioningOutcome, "Get bucket versioning failed.", bucketName);

    const auto bucketVersioningStatus = bucketVersioningOutcome.GetResult().GetStatus();

    if (!progArgs->getDoS3BucketVersioningVerify())
        return;

    const auto statusNameMapper =
        S3::BucketVersioningStatusMapper::GetNameForBucketVersioningStatus;
    const auto expectedStatus = S3::BucketVersioningStatus::Suspended;

    IF_UNLIKELY(bucketVersioningStatus != expectedStatus)
    {
        std::stringstream errStr;
        errStr << "Bucket versioning is set to '" << statusNameMapper(bucketVersioningStatus)
               << "' but the expected value was '" << statusNameMapper(expectedStatus) << "'." << std::endl
               << "Bucket: " << bucketName << ';';
        throw WorkerException(errStr.str());
    }

#endif // S3_SUPPORT
}

void LocalWorker::s3ModePutBucketVersioning(const std::string& bucketName, bool enable)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    S3::VersioningConfiguration versioningConfiguration;

    versioningConfiguration.SetStatus(enable ? S3::BucketVersioningStatus::Enabled
                                             : S3::BucketVersioningStatus::Suspended);

    OPLOG_PRE_OP("PutBucketVersioning", bucketName, 0, 0);

    const auto putBucketVersioningOutcome = s3Client->PutBucketVersioning(
            S3::PutBucketVersioningRequest()
                    .WithBucket(bucketName)
                    .WithVersioningConfiguration(versioningConfiguration)
    );

    OPLOG_POST_OP("PutBucketVersioning", bucketName, 0, 0, !putBucketVersioningOutcome.IsSuccess());

    s3ModeThrowOnError(putBucketVersioningOutcome, "Put bucket versioning failed.", bucketName);
#endif // S3_SUPPORT
}

/**
 * Singlepart upload of an S3 object to an existing bucket. This assumes that progArgs fileSize
 * is not larger than blockSize. Or in other words: This can only upload objects consisting of a
 * single block.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeUploadObjectSinglePart(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const uint64_t currentOffset = rwOffsetGen->getNextOffset();
	const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
	const bool doS3AclPutInline = progArgs->getDoS3AclPutInline();
	const bool ignoreS3Errors = progArgs->getIgnoreS3Errors();

	std::shared_ptr<Aws::IOStream> s3MemStream;

    if(blockSize)
    {
        s3MemStream = std::make_shared<S3MemoryStream>(
            (unsigned char*) (blockSize ? ioBufVec[0] : NULL), blockSize);

        ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);
    }

	std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

	if(blockSize)
	{
		((*this).*funcPreWriteBlockModifier)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);
		((*this).*funcPreWriteCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);
	}

	S3::PutObjectRequest request;
	request.WithBucket(bucketName)
		.WithKey(objectName)
		.WithContentLength(blockSize);

    if(blockSize)
        request.SetBody(s3MemStream);

    if(doS3AclPutInline)
        TranslatorTk::applyS3PutObjectAclGrants(progArgs, request);

    s3ModeAddServerSideEncryption(request);

    IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
        S3Tk::addUploadPartRequestChecksum(request, NULL,
            s3ChecksumAlgorithm, (unsigned char*) ioBufVec[0], blockSize);

    request.SetDataSentEventHandler(
        [&](const Aws::Http::HttpRequest* request, long long numBytes)
        { atomicLiveOps.numBytesDone += numBytes; } );

    #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
        request.SetContinueRequestHandler( [&](const Aws::Http::HttpRequest* request)
            { return !isInterruptionRequested.load(); } );
    #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

	OPLOG_PRE_OP("S3PutObject", bucketName + "/" + objectName, currentOffset, blockSize);

	S3::PutObjectOutcome outcome = s3Client->PutObject(request);

	OPLOG_POST_OP("S3PutObject", bucketName + "/" + objectName, currentOffset, blockSize,
		!outcome.IsSuccess() );

	checkInterruptionRequest(); // (placed here to avoid outcome check on interruption)

	IF_UNLIKELY(!outcome.IsSuccess() && !ignoreS3Errors)
        s3ModeThrowOnError(outcome, "Object upload failed.", bucketName, objectName);

	if(blockSize)
	{
		((*this).*funcPostReadCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);
		((*this).*funcPostReadBlockChecker)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);
	}

	// calc io operation latency
	std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
	std::chrono::microseconds ioElapsedMicroSec =
		std::chrono::duration_cast<std::chrono::microseconds>
		(ioEndT - ioStartT);

	iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

	numIOPSSubmitted++;
	rwOffsetGen->addBytesSubmitted(blockSize);
	atomicLiveOps.numIOPSDone++;

#endif // S3_SUPPORT
}

/**
 * Block-sized multipart upload of an S3 object to an existing bucket.
 *
 * This will delegate to s3ModeUploadObjectMultiPartAsync() if iodepth > 1.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeUploadObjectMultiPart(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const bool doS3AclPutInline = progArgs->getDoS3AclPutInline();
	const bool ignoreS3Errors = progArgs->getIgnoreS3Errors();
    const bool s3NoMpuCompletion = progArgs->getS3NoMpuCompletion();
    const size_t s3MpuSizeVariance = progArgs->getS3MpuSizeVariance();

    // S T E P 0: hand over to async function if iodepth is given

    if(progArgs->getIODepth() > 1)
    {
        s3ModeUploadObjectMultiPartAsync(bucketName, objectName);
        return;
    }

	// S T E P 1: retrieve multipart upload ID from server

	S3::CreateMultipartUploadRequest createMultipartUploadRequest;
	createMultipartUploadRequest.SetBucket(bucketName);
	createMultipartUploadRequest.SetKey(objectName);

    s3ModeAddServerSideEncryption(createMultipartUploadRequest);
    s3ModeAddChecksumAlgorithm(createMultipartUploadRequest);

    if(doS3AclPutInline)
        TranslatorTk::applyS3PutObjectAclGrants(progArgs, createMultipartUploadRequest);

    LOGGER_DEBUG_BUILD(__func__ << ":" << __LINE__ << ": "
        "Worker: " << workerRank << "; " <<
        "Obj: " << bucketName + "/" + objectName << std::endl);

    OPLOG_PRE_OP("S3CreateMultipartUpload", bucketName + "/" + objectName, 0, 0);

	auto createMultipartUploadOutcome = s3Client->CreateMultipartUpload(
		createMultipartUploadRequest);

	OPLOG_POST_OP("S3CreateMultipartUpload", bucketName + "/" + objectName, 0, 0,
		!createMultipartUploadOutcome.IsSuccess() );

	IF_UNLIKELY(!createMultipartUploadOutcome.IsSuccess() && !ignoreS3Errors)
		s3ModeThrowOnError(createMultipartUploadOutcome, "Multipart upload creation failed.",
			bucketName, objectName);

	Aws::String uploadID = createMultipartUploadOutcome.GetResult().GetUploadId();

	S3::CompletedMultipartUpload completedMultipartUpload;

	// S T E P 2: upload one block-sized part in each loop pass

    uint64_t currentPartNum = 0; // valid range is 1..10K

	while(rwOffsetGen->getNumBytesLeftToSubmit() )
	{
        const uint64_t currentOffset = rwOffsetGen->getNextOffset();
        size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();

        /* note: in normal forward mode, blockSize is variable because of s3MpuSizeVariance, so we
            can't use the currentPartNum formula with fixed blockSize from reverse mode. */
        currentPartNum = (progArgs->getDoReverseSeqOffsets() || getS3ModeDoReverseSeqFallback() ) ?
            1 + (currentOffset / rwOffsetGen->getBlockSize() ) : (currentPartNum+1);

        /* note: rwOffsetGen->getBlockSize() comparison is to prevent random subtract from last part
            (to avoid risk of two trailing parts of less than usual 5MiB min allowed part size). */
        if(s3MpuSizeVariance && (blockSize == rwOffsetGen->getBlockSize() ) )
        {
            const size_t randBlockSizeVar = randBlockVarReseed->next() % s3MpuSizeVariance;

            IF_LIKELY(randBlockSizeVar < blockSize)
                blockSize = blockSize - randBlockSizeVar;
        }

        /* note: streamBuf (member of S3MemoryStream) needs to be initialized in loop to
            have the exact remaining blockSize as len. otherwise the AWS SDK will send full
            streamBuf len despite smaller contentLength in UploadPartRequest. */
        std::shared_ptr<Aws::IOStream> s3MemStream = std::make_shared<S3MemoryStream>(
            (unsigned char*) ioBufVec[0], blockSize);

		((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		((*this).*funcPreWriteBlockModifier)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);
		((*this).*funcPreWriteCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);

		// prepare part upload

		S3::CompletedPart completedPart;
		S3::UploadPartRequest uploadPartRequest;
		uploadPartRequest.WithBucket(bucketName)
			.WithKey(objectName)
			.WithUploadId(uploadID)
			.WithPartNumber(currentPartNum)
			.WithContentLength(blockSize);

        // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
        if(!s3SSECKey.empty() )
            uploadPartRequest.WithSSECustomerAlgorithm("AES256")
                    .WithSSECustomerKey(s3SSECKey)
                    .WithSSECustomerKeyMD5(s3SSECKeyMD5);

        IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
            S3Tk::addUploadPartRequestChecksum(uploadPartRequest, &completedPart,
                s3ChecksumAlgorithm, (unsigned char*) ioBufVec[0], blockSize);

		uploadPartRequest.SetBody(s3MemStream);

		uploadPartRequest.SetDataSentEventHandler(
			[&](const Aws::Http::HttpRequest* request, long long numBytes)
			{ atomicLiveOps.numBytesDone += numBytes; } );

        #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
            uploadPartRequest.SetContinueRequestHandler( [&](const Aws::Http::HttpRequest* request)
                { return !isInterruptionRequested.load(); } );
        #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

		OPLOG_PRE_OP("S3UploadPart", bucketName + "/" + objectName, currentOffset, blockSize);

		auto uploadPartOutcome = s3Client->UploadPart(uploadPartRequest);

		/* note: there is no way to tell the server about the offset of a part within an object, so
			concurrent part uploads might add overhead on completion for the S3 server to assemble
			the full object in correct parts order. */

		OPLOG_POST_OP("S3UploadPart", bucketName + "/" + objectName, currentOffset, blockSize,
			!uploadPartOutcome.IsSuccess() );

		checkInterruptionRequest( // (placed here to avoid outcome check on interruption)
			[&] { s3ModeAbortMultipartUpload(bucketName, objectName, uploadID); } );

		IF_UNLIKELY(!uploadPartOutcome.IsSuccess() )
		{
			s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

			if (!ignoreS3Errors)
			{
                auto s3Error = uploadPartOutcome.GetError();

                throw WorkerException(std::string("Multipart part upload failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketName + "; "
                    "Object: " + objectName + "; "
                    "Part: " + std::to_string(currentPartNum) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); "
                    "Request ID: " + s3Error.GetRequestId() );
			}
		}

		// mark part as completed

		completedPart.SetPartNumber(currentPartNum);
		auto partETag = uploadPartOutcome.GetResult().GetETag();
		completedPart.SetETag(partETag);

		completedMultipartUpload.AddParts(completedPart);

		((*this).*funcPostReadCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);
		((*this).*funcPostReadBlockChecker)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);

		// calc io operation latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

		numIOPSSubmitted++;
		rwOffsetGen->addBytesSubmitted(blockSize);
		atomicLiveOps.numIOPSDone++;
	}

	// S T E P 3: submit upload completion

    IF_UNLIKELY(s3NoMpuCompletion)
        return; // user-defined skip of completion message

    if(progArgs->getDoReverseSeqOffsets() || getS3ModeDoReverseSeqFallback() )
    { // we need to reverse the parts vector for ascending order
        const Aws::Vector<S3::CompletedPart>& reversePartsVec = completedMultipartUpload.GetParts();
        Aws::Vector<S3::CompletedPart> forwardPartsVec(reversePartsVec.size() );

        std::reverse_copy(std::begin(reversePartsVec), std::end(reversePartsVec),
            std::begin(forwardPartsVec) );

        completedMultipartUpload.SetParts(forwardPartsVec);
    }

	S3::CompleteMultipartUploadRequest completionRequest;
	completionRequest.WithBucket(bucketName)
		.WithKey(objectName)
		.WithUploadId(uploadID)
		.WithMultipartUpload(completedMultipartUpload);

    // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
    if(!s3SSECKey.empty() )
        completionRequest.WithSSECustomerAlgorithm("AES256")
                .WithSSECustomerKey(s3SSECKey)
                .WithSSECustomerKeyMD5(s3SSECKeyMD5);

    OPLOG_PRE_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedMultipartUpload.GetParts().size() );

	auto completionOutcome = s3Client->CompleteMultipartUpload(completionRequest);

    OPLOG_POST_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedMultipartUpload.GetParts().size(), !completionOutcome.IsSuccess() );

	IF_UNLIKELY(!completionOutcome.IsSuccess())
	{
        auto s3Error = completionOutcome.GetError();

        if (!(progArgs->getS3IgnoreMultipartUpload404() &&
              s3Error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND))
        {
            s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

            if (!ignoreS3Errors)
            {
                throw WorkerException(std::string("Multipart upload completion failed. ") +
                     "Endpoint: " + s3EndpointStr + "; "
                     "Bucket: " + bucketName + "; "
                     "Object: " + objectName + "; "
                     "NumParts: " + std::to_string(completedMultipartUpload.GetParts().size() ) +
                        "; "
                     "Exception: " + s3Error.GetExceptionName() + "; " +
                     "Message: " + s3Error.GetMessage() + "; " +
                     "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                         TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                         "); "
                     "Request ID: " + s3Error.GetRequestId() );
            }
        }
	}

#endif // S3_SUPPORT
}

/**
 * Async block-sized multipart upload of an S3 object to an existing bucket.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeUploadObjectMultiPartAsync(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    const bool doS3AclPutInline = progArgs->getDoS3AclPutInline();
    const bool ignoreS3Errors = progArgs->getIgnoreS3Errors();
    const bool s3NoMpuCompletion = progArgs->getS3NoMpuCompletion();
    const unsigned ioDepth = progArgs->getIODepth();

    // S T E P 1: retrieve multipart upload ID from server

    S3::CreateMultipartUploadRequest createMultipartUploadRequest;
    createMultipartUploadRequest.SetBucket(bucketName);
    createMultipartUploadRequest.SetKey(objectName);

    s3ModeAddServerSideEncryption(createMultipartUploadRequest);
    s3ModeAddChecksumAlgorithm(createMultipartUploadRequest);

    if(doS3AclPutInline)
        TranslatorTk::applyS3PutObjectAclGrants(progArgs, createMultipartUploadRequest);

    LOGGER_DEBUG_BUILD(__func__ << ":" << __LINE__ << ": "
        "Worker: " << workerRank << "; " <<
        "Obj: " << bucketName + "/" + objectName << std::endl);

    OPLOG_PRE_OP("S3CreateMultipartUpload", bucketName + "/" + objectName, 0, 0);

    auto createMultipartUploadOutcome = s3Client->CreateMultipartUpload(
        createMultipartUploadRequest);

    OPLOG_POST_OP("S3CreateMultipartUpload", bucketName + "/" + objectName, 0, 0,
        !createMultipartUploadOutcome.IsSuccess() );

    IF_UNLIKELY(!createMultipartUploadOutcome.IsSuccess() && !ignoreS3Errors)
        s3ModeThrowOnError(createMultipartUploadOutcome, "Multipart upload creation failed.",
            bucketName, objectName);

    Aws::String uploadID = createMultipartUploadOutcome.GetResult().GetUploadId();


    // S T E P 2: upload parts

    S3::CompletedMultipartUpload completedMultipartUpload;
    std::vector<S3AsyncUploadPartContext> partCompletionsVec;

    partCompletionsVec.reserve(ioDepth); /* (reserve is important to not invalidate pointers for
        async callbacks on vector grow) */

    /* NOTE on try-catch: before we can exit this function we need to wait for all async tasks to
        complete because they might still be accessing members of partCompletionsVec. */
    try
    {
        while(rwOffsetGen->getNumBytesLeftToSubmit() )
        {
            // S T E P 2.1: submit parts asynchronously up to iodepth

            for(unsigned currentIODepth = 0;
                (currentIODepth < ioDepth) && rwOffsetGen->getNumBytesLeftToSubmit();
                currentIODepth++)
            {
                const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
                const uint64_t currentOffset = rwOffsetGen->getNextOffset();
                const uint64_t currentPartNum =
                    1 + (currentOffset / rwOffsetGen->getBlockSize() ); // +1 for 1-based part range

                S3AsyncUploadPartContext& asyncPartContext = partCompletionsVec.emplace_back(
                    currentOffset, blockSize, currentPartNum);

                /* note: streamBuf (member of S3MemoryStream) needs to be initialized in loop to
                    have the exact remaining blockSize as len. otherwise the AWS SDK will send full
                    streamBuf len despite smaller contentLength in UploadPartRequest. */
                std::shared_ptr<Aws::IOStream> s3MemStream = std::make_shared<S3MemoryStream>(
                    (unsigned char*) ioBufVec[currentIODepth], blockSize);

                ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

                asyncPartContext.ioStartT = std::chrono::steady_clock::now();

                ((*this).*funcPreWriteBlockModifier)(ioBufVec[currentIODepth],
                    gpuIOBufVec[currentIODepth], blockSize, currentOffset);
                ((*this).*funcPreWriteCudaMemcpy)(ioBufVec[currentIODepth],
                    gpuIOBufVec[currentIODepth], blockSize);

                // prepare part upload

                S3::UploadPartRequest& uploadPartRequest = asyncPartContext.uploadPartRequest;
                uploadPartRequest.WithBucket(bucketName)
                    .WithKey(objectName)
                    .WithUploadId(uploadID)
                    .WithPartNumber(currentPartNum)
                    .WithContentLength(blockSize);

                // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
                if(!s3SSECKey.empty() )
                    uploadPartRequest.WithSSECustomerAlgorithm("AES256")
                            .WithSSECustomerKey(s3SSECKey)
                            .WithSSECustomerKeyMD5(s3SSECKeyMD5);

                IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
                    S3Tk::addUploadPartRequestChecksum(uploadPartRequest,
                        &asyncPartContext.completedPart, s3ChecksumAlgorithm,
                        (unsigned char*) ioBufVec[currentIODepth], blockSize);

                uploadPartRequest.SetBody(s3MemStream);

                uploadPartRequest.SetDataSentEventHandler(
                    [&atomicLiveOps = atomicLiveOps]
                    (const Aws::Http::HttpRequest* request, long long numBytes)
                    { atomicLiveOps.numBytesDone += numBytes; } );

                #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
                    uploadPartRequest.SetContinueRequestHandler(
                        [&isInterruptionRequested = isInterruptionRequested]
                        (const Aws::Http::HttpRequest* request)
                        { return !isInterruptionRequested.load(); } );
                #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

                OPLOG_PRE_OP("S3UploadPartAsync", bucketName + "/" + objectName, currentOffset,
                    blockSize);

                s3Client->UploadPartAsync(uploadPartRequest,
                    [partCompletePromise = &asyncPartContext.partCompletePromise]
                    (const S3Client*, const S3::UploadPartRequest&,
                    S3::UploadPartOutcome outcome,
                    const std::shared_ptr<const Aws::Client::AsyncCallerContext>&)
                    { partCompletePromise->set_value(std::move(outcome) ); });

                numIOPSSubmitted++;
                rwOffsetGen->addBytesSubmitted(asyncPartContext.blockSize);

            } // enf of step 2.1: async submit for-loop

            // S T E P 2.2: handle completion of submitted parts

            for(unsigned currentIODepth = 0;
                currentIODepth < partCompletionsVec.size();
                currentIODepth++)
            {
                S3AsyncUploadPartContext& asyncPartContext = partCompletionsVec[currentIODepth];

                // wait for part upload to complete ("future.get()" blocks)
                S3::UploadPartOutcome uploadPartOutcome = asyncPartContext.partCompleteFuture.get();

                OPLOG_POST_OP("S3UploadPartAsync", bucketName + "/" + objectName,
                    asyncPartContext.currentOffset, asyncPartContext.blockSize,
                    !uploadPartOutcome.IsSuccess() );

                // (interrupt check placed here to avoid outcome check on interruption)
                checkInterruptionRequest([this, &bucketName, &objectName, &uploadID]
                    { s3ModeAbortMultipartUpload(bucketName, objectName, uploadID); } );

                IF_UNLIKELY(!uploadPartOutcome.IsSuccess() )
                {
                    s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

                    if (!ignoreS3Errors)
                    {
                        auto s3Error = uploadPartOutcome.GetError();

                        throw WorkerException(std::string("Multipart part upload failed. ") +
                            "Endpoint: " + s3EndpointStr + "; "
                            "Bucket: " + bucketName + "; "
                            "Object: " + objectName + "; "
                            "Part: " + std::to_string(asyncPartContext.partNum) + "; "
                            "Exception: " + s3Error.GetExceptionName() + "; " +
                            "Message: " + s3Error.GetMessage() + "; " +
                            "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                                TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                                "); "
                            "Request ID: " + s3Error.GetRequestId() );
                    }
                }

                // mark part as completed

                auto partETag = uploadPartOutcome.GetResult().GetETag();
                asyncPartContext.completedPart.SetETag(partETag);

                completedMultipartUpload.AddParts(asyncPartContext.completedPart);

                ((*this).*funcPostReadCudaMemcpy)(
                    ioBufVec[currentIODepth], gpuIOBufVec[currentIODepth],
                    asyncPartContext.blockSize);
                ((*this).*funcPostReadBlockChecker)(
                    ioBufVec[currentIODepth], gpuIOBufVec[currentIODepth],
                    asyncPartContext.blockSize, asyncPartContext.currentOffset);

                // calc io operation latency
                std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
                std::chrono::microseconds ioElapsedMicroSec =
                    std::chrono::duration_cast<std::chrono::microseconds>
                    (ioEndT - asyncPartContext.ioStartT);

                iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

                atomicLiveOps.numIOPSDone++;
            } // end of step 2.2: wait for part batch completion for-loop

            checkInterruptionRequest([this, &bucketName, &objectName, &uploadID]
                { s3ModeAbortMultipartUpload(bucketName, objectName, uploadID); } );

            // reset completions vec for next round
            partCompletionsVec.clear();

        } // end of step 2 loop: submit parts + wait for completions
    }
    catch(...)
    {
        interruptExecution(); // for SetContinueRequestHandler()

        // wait for all parts to complete ("future.get()" blocks)
        for(unsigned i = 0; i < partCompletionsVec.size(); i++)
        {
            /* ignore errors because we're already handling an error (and ".get()" will also throw
                an exception if value has already been retrieved) */
            try { partCompletionsVec[i].partCompleteFuture.get(); }
            catch(...) {}
        }

        /* ignore errors because we're already handling an error (and ".get()" will also throw
            an exception if value has already been retrieved) */
        try { s3ModeAbortMultipartUpload(bucketName, objectName, uploadID); }
        catch(...) {}

        throw;
    }

    // S T E P 3: submit upload completion

    IF_UNLIKELY(s3NoMpuCompletion)
        return; // user-defined skip of completion message

    if(progArgs->getDoReverseSeqOffsets() || getS3ModeDoReverseSeqFallback() )
    { // we need to reverse the parts vector for ascending order
        const Aws::Vector<S3::CompletedPart>& reversePartsVec = completedMultipartUpload.GetParts();
        Aws::Vector<S3::CompletedPart> forwardPartsVec(reversePartsVec.size() );

        std::reverse_copy(std::begin(reversePartsVec), std::end(reversePartsVec),
            std::begin(forwardPartsVec) );

        completedMultipartUpload.SetParts(forwardPartsVec);
    }

    S3::CompleteMultipartUploadRequest completionRequest;
    completionRequest.WithBucket(bucketName)
        .WithKey(objectName)
        .WithUploadId(uploadID)
        .WithMultipartUpload(completedMultipartUpload);

    // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
    if(!s3SSECKey.empty() )
        completionRequest.WithSSECustomerAlgorithm("AES256")
                .WithSSECustomerKey(s3SSECKey)
                .WithSSECustomerKeyMD5(s3SSECKeyMD5);

    OPLOG_PRE_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedMultipartUpload.GetParts().size() );

    auto completionOutcome = s3Client->CompleteMultipartUpload(completionRequest);

    OPLOG_POST_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedMultipartUpload.GetParts().size(), !completionOutcome.IsSuccess() );

    IF_UNLIKELY(!completionOutcome.IsSuccess())
    {
        auto s3Error = completionOutcome.GetError();

        if (!(progArgs->getS3IgnoreMultipartUpload404() &&
              s3Error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND))
        {
            s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

            if (!ignoreS3Errors)
            {
                throw WorkerException(std::string("Multipart upload completion failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketName + "; "
                    "Object: " + objectName + "; "
                    "NumParts: " + std::to_string(
                        completedMultipartUpload.GetParts().size() ) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); " +
                    "Request ID: " + s3Error.GetRequestId() );
            }
        }
	}

#endif // S3_SUPPORT
}

/**
 * Block-sized multipart upload of an S3 object to an existing bucket, shared by multiple workers.
 *
 * This will delegate to s3ModeUploadObjectMultiPartSharedAsync() if iodepth > 1.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeUploadObjectMultiPartShared(std::string bucketName, std::string objectName,
	uint64_t objectTotalSize)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    const bool s3NoMpuCompletion = progArgs->getS3NoMpuCompletion();

    // S T E P 0: hand over to async function if iodepth is given

    if(progArgs->getIODepth() > 1)
    {
        s3ModeUploadObjectMultiPartSharedAsync(bucketName, objectName, objectTotalSize);
        return;
    }

	// S T E P 1: retrieve multipart upload ID from server

	Aws::String uploadID = s3SharedUploadStore.getMultipartUploadID(
		bucketName, objectName, s3Client, opsLog);

	std::unique_ptr<Aws::Vector<S3::CompletedPart>> allCompletedParts; // need sort before send

	// S T E P 2: upload one block-sized part in each loop pass

	while(rwOffsetGen->getNumBytesLeftToSubmit() )
	{
		const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
		const uint64_t currentOffset = rwOffsetGen->getNextOffset();
		const uint64_t currentPartNum =
			1 + (currentOffset / rwOffsetGen->getBlockSize() ); // +1 because valid range is 1..10K

        /* note: streamBuf (member of S3MemoryStream) needs to be initialized in loop to
            have the exact remaining blockSize as len. otherwise the AWS SDK will send full
            streamBuf len despite smaller contentLength in UploadPartRequest. */
        std::shared_ptr<Aws::IOStream> s3MemStream = std::make_shared<S3MemoryStream>(
            (unsigned char*) ioBufVec[0], blockSize);

		((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		((*this).*funcPreWriteBlockModifier)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);
		((*this).*funcPreWriteCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);

		// prepare part upload

		S3::CompletedPart completedPart;
		S3::UploadPartRequest uploadPartRequest;
		uploadPartRequest.WithBucket(bucketName)
			.WithKey(objectName)
			.WithUploadId(uploadID)
			.WithPartNumber(currentPartNum)
			.WithContentLength(blockSize);

        IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
            S3Tk::addUploadPartRequestChecksum(uploadPartRequest, &completedPart,
                s3ChecksumAlgorithm, (unsigned char*) ioBufVec[0], blockSize);

		uploadPartRequest.SetBody(s3MemStream);

		uploadPartRequest.SetDataSentEventHandler(
			[&](const Aws::Http::HttpRequest* request, long long numBytes)
			{ atomicLiveOps.numBytesDone += numBytes; } );

        #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
            uploadPartRequest.SetContinueRequestHandler( [&](const Aws::Http::HttpRequest* request)
                { return !isInterruptionRequested.load(); } );
        #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

		OPLOG_PRE_OP("S3UploadPart", bucketName + "/" + objectName, currentOffset, blockSize);

		auto uploadPartOutcome = s3Client->UploadPart(uploadPartRequest);

		OPLOG_POST_OP("S3UploadPart", bucketName + "/" + objectName, currentOffset, blockSize,
			!uploadPartOutcome.IsSuccess() );

		checkInterruptionRequest(); /* (placed here to avoid outcome check on interruption; abort
			message will be sent during s3SharedUploadStore cleanup) */

		IF_UNLIKELY(!uploadPartOutcome.IsSuccess() )
		{
			// (note: abort message will be sent during s3SharedUploadStore cleanup)

			auto s3Error = uploadPartOutcome.GetError();

            throw WorkerException(std::string("Shared multipart part upload failed. ") +
                "Endpoint: " + s3EndpointStr + "; "
                "Bucket: " + bucketName + "; "
                "Object: " + objectName + "; "
                "Part: " + std::to_string(currentPartNum) + "; "
                "UploadID: " + uploadID + "; "
                "Rank: " + std::to_string(workerRank) + "; "
                "Exception: " + s3Error.GetExceptionName() + "; " +
                "Message: " + s3Error.GetMessage() + "; " +
                "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                    TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                    "); " +
                "Request ID: " + s3Error.GetRequestId() );
        }

		// mark part as completed

		completedPart.SetPartNumber(currentPartNum);
		auto partETag = uploadPartOutcome.GetResult().GetETag();
		completedPart.SetETag(partETag);

        LOGGER_DEBUG_BUILD(__func__ << __LINE__ << ": worker=" << workerRank << "; "
            "blocksize=" << blockSize << "; "
            "totalSize=" << objectTotalSize << "; " << "objectName=" << objectName << std::endl);

		allCompletedParts = s3SharedUploadStore.addCompletedPart(
			bucketName, objectName, blockSize, objectTotalSize, completedPart);

		((*this).*funcPostReadCudaMemcpy)(ioBufVec[0], gpuIOBufVec[0], blockSize);
		((*this).*funcPostReadBlockChecker)(ioBufVec[0], gpuIOBufVec[0], blockSize, currentOffset);

		// calc io operation latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

		numIOPSSubmitted++;
		rwOffsetGen->addBytesSubmitted(blockSize);
		atomicLiveOps.numIOPSDone++;

		// sanity check afer rwOffsetGen update
		IF_UNLIKELY(allCompletedParts && rwOffsetGen->getNumBytesLeftToSubmit() )
			throw WorkerException(std::string("Shared multipart upload logic error. ") +
				"Completion returned by upload store, but bytes left to submit. "
				"Endpoint: " + s3EndpointStr + "; "
				"Bucket: " + bucketName + "; "
				"Object: " + objectName + "; "
				"Part: " + std::to_string(currentPartNum) + "; "
				"BytesLeft: " + std::to_string(rwOffsetGen->getNumBytesLeftToSubmit() ) + "; " +
				"ObjectSize: " + std::to_string(objectTotalSize) );
	}

	// S T E P 3: submit upload completion

	IF_UNLIKELY(s3NoMpuCompletion)
	    return; // user-defined skip of completion message

	if(!allCompletedParts)
		return; // another worker still needs to upload parts, so no completion yet

	// (note: part vec must be in ascending order)
	std::sort(allCompletedParts->begin(), allCompletedParts->end(),
		[](const S3::CompletedPart& a, const S3::CompletedPart& b) -> bool
		{ return a.GetPartNumber() < b.GetPartNumber(); } );

	S3::CompletedMultipartUpload completedMultipartUpload;
	completedMultipartUpload.SetParts(*allCompletedParts);

	S3::CompleteMultipartUploadRequest completionRequest;
	completionRequest.WithBucket(bucketName)
		.WithKey(objectName)
		.WithUploadId(uploadID)
		.WithMultipartUpload(completedMultipartUpload);

	OPLOG_PRE_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0, objectTotalSize);

	auto completionOutcome = s3Client->CompleteMultipartUpload(completionRequest);

	OPLOG_POST_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0, objectTotalSize,
		!completionOutcome.IsSuccess() );


    IF_UNLIKELY(!completionOutcome.IsSuccess() )
    {
        auto s3Error = completionOutcome.GetError();

        if (!(progArgs->getS3IgnoreMultipartUpload404() &&
              s3Error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND))
        {
            s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

            throw WorkerException(std::string("Shared multipart upload completion failed. ") +
                "Endpoint: " + s3EndpointStr + "; "
                "Bucket: " + bucketName + "; "
                "Object: " + objectName + "; "
                "NumParts: " + std::to_string(completedMultipartUpload.GetParts().size() ) + "; "
                "Exception: " + s3Error.GetExceptionName() + "; " +
                "Message: " + s3Error.GetMessage() + "; " +
                "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                    TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                    "); " +
                "Request ID: " + s3Error.GetRequestId() );
        }
    }

#endif // S3_SUPPORT
}

/**
 * Async block-sized multipart upload of an S3 object to an existing bucket, shared by multiple
 * workers.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeUploadObjectMultiPartSharedAsync(std::string bucketName,
    std::string objectName, uint64_t objectTotalSize)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    const bool s3NoMpuCompletion = progArgs->getS3NoMpuCompletion();
    const unsigned ioDepth = progArgs->getIODepth();

	// S T E P 1: retrieve multipart upload ID from server

	Aws::String uploadID = s3SharedUploadStore.getMultipartUploadID(
		bucketName, objectName, s3Client, opsLog);

	std::unique_ptr<Aws::Vector<S3::CompletedPart>> allCompletedParts; // need sort before send

	// S T E P 2: upload parts

    std::vector<S3AsyncUploadPartContext> partCompletionsVec;

    partCompletionsVec.reserve(ioDepth); /* (reserve is important to not invalidate pointers for
        async callbacks on vector grow) */

    /* NOTE on try-catch: before we can exit this function we need to wait for all async tasks to
        complete because they might still be accessing members of partCompletionsVec. */
    try
    {
        while(rwOffsetGen->getNumBytesLeftToSubmit() )
        {
            // S T E P 2.1: submit parts asynchronously up to iodepth

            for(unsigned currentIODepth = 0;
                (currentIODepth < ioDepth) && rwOffsetGen->getNumBytesLeftToSubmit();
                currentIODepth++)
            {
                const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();
                const uint64_t currentOffset = rwOffsetGen->getNextOffset();
                const uint64_t currentPartNum =
                    1 + (currentOffset / rwOffsetGen->getBlockSize() ); // +1 for 1-based part range

                S3AsyncUploadPartContext& asyncPartContext = partCompletionsVec.emplace_back(
                    currentOffset, blockSize, currentPartNum);

                /* note: streamBuf (member of S3MemoryStream) needs to be initialized in loop to
                    have the exact remaining blockSize as len. otherwise the AWS SDK will send full
                    streamBuf len despite smaller contentLength in UploadPartRequest. */
                std::shared_ptr<Aws::IOStream> s3MemStream = std::make_shared<S3MemoryStream>(
                    (unsigned char*) ioBufVec[currentIODepth], blockSize);

                ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

                asyncPartContext.ioStartT = std::chrono::steady_clock::now();

                ((*this).*funcPreWriteBlockModifier)(ioBufVec[currentIODepth],
                    gpuIOBufVec[currentIODepth], blockSize, currentOffset);
                ((*this).*funcPreWriteCudaMemcpy)(ioBufVec[currentIODepth],
                    gpuIOBufVec[currentIODepth], blockSize);

                // prepare part upload

                S3::UploadPartRequest& uploadPartRequest = asyncPartContext.uploadPartRequest;
                uploadPartRequest.WithBucket(bucketName)
                    .WithKey(objectName)
                    .WithUploadId(uploadID)
                    .WithPartNumber(currentPartNum)
                    .WithContentLength(blockSize);

                // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
                if(!s3SSECKey.empty() )
                    uploadPartRequest.WithSSECustomerAlgorithm("AES256")
                            .WithSSECustomerKey(s3SSECKey)
                            .WithSSECustomerKeyMD5(s3SSECKeyMD5);

                IF_UNLIKELY(s3ChecksumAlgorithm != S3ChecksumAlgorithm::NOT_SET)
                    S3Tk::addUploadPartRequestChecksum(uploadPartRequest,
                        &asyncPartContext.completedPart, s3ChecksumAlgorithm,
                        (unsigned char*) ioBufVec[currentIODepth], blockSize);

                uploadPartRequest.SetBody(s3MemStream);

                uploadPartRequest.SetDataSentEventHandler(
                    [&atomicLiveOps = atomicLiveOps]
                    (const Aws::Http::HttpRequest* request, long long numBytes)
                    { atomicLiveOps.numBytesDone += numBytes; } );

                #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
                    uploadPartRequest.SetContinueRequestHandler(
                        [&isInterruptionRequested = isInterruptionRequested]
                        (const Aws::Http::HttpRequest* request)
                        { return !isInterruptionRequested.load(); } );
                #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

                OPLOG_PRE_OP("S3UploadPartAsync", bucketName + "/" + objectName, currentOffset,
                    blockSize);

                s3Client->UploadPartAsync(uploadPartRequest,
                    [partCompletePromise = &asyncPartContext.partCompletePromise]
                    (const S3Client*, const S3::UploadPartRequest&,
                    S3::UploadPartOutcome outcome,
                    const std::shared_ptr<const Aws::Client::AsyncCallerContext>&)
                    { partCompletePromise->set_value(std::move(outcome) ); });

                numIOPSSubmitted++;
                rwOffsetGen->addBytesSubmitted(asyncPartContext.blockSize);

            } // enf of step 2.1: async submit for-loop

            // S T E P 2.2: handle completion of submitted parts

            for(unsigned currentIODepth = 0;
                currentIODepth < partCompletionsVec.size();
                currentIODepth++)
            {
                S3AsyncUploadPartContext& asyncPartContext = partCompletionsVec[currentIODepth];

                // wait for part upload to complete ("future.get()" blocks)
                S3::UploadPartOutcome uploadPartOutcome = asyncPartContext.partCompleteFuture.get();

                OPLOG_POST_OP("S3UploadPartAsync", bucketName + "/" + objectName,
                    asyncPartContext.currentOffset, asyncPartContext.blockSize,
                    !uploadPartOutcome.IsSuccess() );

                checkInterruptionRequest(); /* (placed here to avoid outcome check on interruption;
                    abort message will be sent during s3SharedUploadStore cleanup) */

                IF_UNLIKELY(!uploadPartOutcome.IsSuccess() )
                {
                    // (note: abort message will be sent during s3SharedUploadStore cleanup)

                    auto s3Error = uploadPartOutcome.GetError();

                    throw WorkerException(std::string("Shared multipart part upload failed. ") +
                        "Endpoint: " + s3EndpointStr + "; "
                        "Bucket: " + bucketName + "; "
                        "Object: " + objectName + "; "
                        "Part: " + std::to_string(asyncPartContext.partNum) + "; "
                        "UploadID: " + uploadID + "; "
                        "Rank: " + std::to_string(workerRank) + "; "
                        "Exception: " + s3Error.GetExceptionName() + "; " +
                        "Message: " + s3Error.GetMessage() + "; " +
                        "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) +
                            " (" +
                            TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                            "); " +
                        "Request ID: " + s3Error.GetRequestId() );
                }

                // mark part as completed

                auto partETag = uploadPartOutcome.GetResult().GetETag();
                asyncPartContext.completedPart.SetETag(partETag);

                allCompletedParts = s3SharedUploadStore.addCompletedPart(bucketName, objectName,
                    asyncPartContext.blockSize, objectTotalSize, asyncPartContext.completedPart);

                ((*this).*funcPostReadCudaMemcpy)(
                    ioBufVec[currentIODepth], gpuIOBufVec[currentIODepth],
                    asyncPartContext.blockSize);
                ((*this).*funcPostReadBlockChecker)(
                    ioBufVec[currentIODepth], gpuIOBufVec[currentIODepth],
                    asyncPartContext.blockSize, asyncPartContext.currentOffset);

                // calc io operation latency
                std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
                std::chrono::microseconds ioElapsedMicroSec =
                    std::chrono::duration_cast<std::chrono::microseconds>
                    (ioEndT - asyncPartContext.ioStartT);

                iopsLatHisto.addLatency(ioElapsedMicroSec.count() );

                atomicLiveOps.numIOPSDone++;

                // sanity check afer rwOffsetGen update
                IF_UNLIKELY(allCompletedParts && rwOffsetGen->getNumBytesLeftToSubmit() )
                    throw WorkerException(std::string("Shared multipart upload logic error. ") +
                        "Completion returned by upload store, but bytes left to submit. "
                        "Endpoint: " + s3EndpointStr + "; "
                        "Bucket: " + bucketName + "; "
                        "Object: " + objectName + "; "
                        "Part: " + std::to_string(asyncPartContext.partNum) + "; "
                        "BytesLeft: " + std::to_string(rwOffsetGen->getNumBytesLeftToSubmit() ) +
                            "; " +
                        "ObjectSize: " + std::to_string(objectTotalSize) );
            } // end of step 2.2: wait for part batch completion for-loop

            checkInterruptionRequest(); /* (placed here to avoid outcome check on interruption;
                abort message will be sent during s3SharedUploadStore cleanup) */

            // reset completions vec for next round
            partCompletionsVec.clear();

        } // end of step 2 loop: submit parts + wait for completions
    }
    catch(...)
    {
        interruptExecution(); // for SetContinueRequestHandler()

        // wait for all parts to complete ("future.get()" blocks)
        for(unsigned i = 0; i < partCompletionsVec.size(); i++)
        {
            /* ignore errors because we're already handling an error (and ".get()" will also throw
                an exception if value has already been retrieved) */
            try { partCompletionsVec[i].partCompleteFuture.get(); }
            catch(...) {}
        }

        /* note: no s3AbortMultipartUpload() here, abort message will be sent during
            s3SharedUploadStore cleanup */

        throw;
    }

	// S T E P 3: submit upload completion

	IF_UNLIKELY(s3NoMpuCompletion)
	    return; // user-defined skip of completion message

	if(!allCompletedParts)
		return; // another worker still needs to upload parts, so no completion yet

	// (note: part vec must be in ascending order)
	std::sort(allCompletedParts->begin(), allCompletedParts->end(),
		[](const S3::CompletedPart& a, const S3::CompletedPart& b) -> bool
		{ return a.GetPartNumber() < b.GetPartNumber(); } );

	S3::CompletedMultipartUpload completedMultipartUpload;
	completedMultipartUpload.SetParts(*allCompletedParts);

	S3::CompleteMultipartUploadRequest completionRequest;
	completionRequest.WithBucket(bucketName)
		.WithKey(objectName)
		.WithUploadId(uploadID)
		.WithMultipartUpload(completedMultipartUpload);

	OPLOG_PRE_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0, objectTotalSize);

	auto completionOutcome = s3Client->CompleteMultipartUpload(completionRequest);

	OPLOG_POST_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0, objectTotalSize,
		!completionOutcome.IsSuccess() );


    IF_UNLIKELY(!completionOutcome.IsSuccess() )
    {
        auto s3Error = completionOutcome.GetError();

        if (!(progArgs->getS3IgnoreMultipartUpload404() &&
              s3Error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND))
        {
            s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

            throw WorkerException(std::string("Shared multipart upload completion failed. ") +
                "Endpoint: " + s3EndpointStr + "; "
                "Bucket: " + bucketName + "; "
                "Object: " + objectName + "; "
                "NumParts: " + std::to_string(completedMultipartUpload.GetParts().size() ) + "; "
                "Exception: " + s3Error.GetExceptionName() + "; " +
                "Message: " + s3Error.GetMessage() + "; " +
                "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                    TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                    "); " +
                "Request ID: " + s3Error.GetRequestId() );
        }
    }

#endif // S3_SUPPORT
}

/**
 * Retrieves all parts of a multipart upload based on given uploadID, verifies the total size,
 * and completes the upload.
 *
 * @param bucketName The name of the S3 bucket.
 * @param objectKey The key (path/filename) of the S3 object.
 * @param uploadId The Multipart Upload ID string.
 * @param expectedTotalSize The combined size of all uploaded parts in bytes.
 * @return true if the upload was successfully completed, false otherwise.
 */
void LocalWorker::s3ModeQueryAndFinishMultipartUpload(std::string bucketName,
    std::string objectName, std::string uploadID, uint64_t expectedTotalSize)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

    S3::ListPartsRequest listPartsRequest;
    listPartsRequest.SetBucket(bucketName);
    listPartsRequest.SetKey(objectName);
    listPartsRequest.SetUploadId(uploadID);

    std::vector<S3::CompletedPart> completedParts;
    uint64_t actualTotalSize = 0; // sum of parts sizes from listing
    bool isTruncated = true; // for pagination >1000 parts
    int nextPartNumberMarker = 0; // for pagination >1000 parts

    LOGGER_DEBUG_BUILD("Retrieving parts for Upload ID: '" << uploadID << "'" <<
        "Worker: " << workerRank << "Obj: " << bucketName + "/" + objectName << std::endl);

    // STEP 1: Retrieve all uploaded parts (loop on pagination)

    while(isTruncated)
    {
        OPLOG_PRE_OP("S3ListParts", bucketName + "/" + objectName, nextPartNumberMarker, 0);

        auto outcome = s3Client->ListParts(listPartsRequest);

        OPLOG_POST_OP("S3ListParts", bucketName + "/" + objectName, nextPartNumberMarker,
            outcome.IsSuccess() ? outcome.GetResult().GetParts().size() : 0, !outcome.IsSuccess() );

        s3ModeThrowOnError(outcome, "Listing parts of unfinished MPU failed.", bucketName,
            objectName);

        const auto& result = outcome.GetResult();

        for(const auto& part : result.GetParts() )
        {
            S3::CompletedPart completedPart;
            completedPart.SetPartNumber(part.GetPartNumber() );
            completedPart.SetETag(part.GetETag() );

            completedParts.push_back(completedPart);
            actualTotalSize += part.GetSize();
        }

        isTruncated = result.GetIsTruncated();
        if(isTruncated)
        { // pagination
            nextPartNumberMarker = result.GetNextPartNumberMarker();
            listPartsRequest.SetPartNumberMarker(nextPartNumberMarker);
        }
    }

    LOGGER_DEBUG_BUILD("Retrieved parts: " << completedParts.size() << "; "
        "Worker: " << workerRank << std::endl);

    // STEP 2: Verify the total size against the expected size

    IF_UNLIKELY(actualTotalSize != expectedTotalSize)
    {
        throw WorkerException("MPU parts listing result does not match expected size. "
            "Bucket: " + bucketName + "; "
            "Object: " + objectName + "; "
            "ExpectedSize: " + std::to_string(expectedTotalSize) + "; "
            "ActualSize: " + std::to_string(actualTotalSize) + "; "
            "ListedNumParts: " + std::to_string(completedParts.size() ) + "; "
            "MpuID: " + uploadID);
    }

    // STEP 3: Assemble the completion structure

    S3::CompletedMultipartUpload completedUpload;
    completedUpload.SetParts(completedParts);

    S3::CompleteMultipartUploadRequest completeRequest;
    completeRequest.SetBucket(bucketName);
    completeRequest.SetKey(objectName);
    completeRequest.SetUploadId(uploadID);
    completeRequest.SetMultipartUpload(completedUpload);

    LOGGER_DEBUG_BUILD("Sending CompleteMultipartUpload request... "
        "Worker: " << workerRank << std::endl);

    // STEP 4: Send the completion request

    OPLOG_PRE_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedUpload.GetParts().size() );

    auto completeOutcome = s3Client->CompleteMultipartUpload(completeRequest);

    OPLOG_POST_OP("S3CompleteMultipartUpload", bucketName + "/" + objectName, 0,
        completedUpload.GetParts().size(), !completeOutcome.IsSuccess() );

    s3ModeThrowOnError(completeOutcome, "Error completing multipart upload.", bucketName,
        objectName);

    LOGGER_DEBUG_BUILD("Successfully completed multipart upload for object: " << objectName << " "
        "Worker: " << workerRank << std::endl);

#endif // S3_SUPPORT
}

/**
 * Abort an incomplete S3 multipart upload.
 *
 * @return true if abort request succeeded, false otherwise.
 */
bool LocalWorker::s3ModeAbortMultipartUpload(std::string bucketName, std::string objectName,
	std::string uploadID)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    if(!s3Client)
        return false;

    S3::AbortMultipartUploadRequest abortMultipartUploadRequest;

	abortMultipartUploadRequest.SetBucket(bucketName);
	abortMultipartUploadRequest.SetKey(objectName);
	abortMultipartUploadRequest.SetUploadId(uploadID);

	OPLOG_PRE_OP("S3AbortMultipartUpload", bucketName + "/" + objectName, 0, 0);

	auto abortOutcome = s3Client->AbortMultipartUpload(abortMultipartUploadRequest);

	OPLOG_POST_OP("S3AbortMultipartUpload", bucketName + "/" + objectName, 0, 0,
		!abortOutcome.IsSuccess() );

	return abortOutcome.IsSuccess();

#endif // S3_SUPPORT
}

/**
 * Abort unfinished shared multipart uploads e.g. after interruption or error. This does not throw
 * an exception on error, because it's intended for the cleanup after a previous error or
 * interruption.
 */
void LocalWorker::s3ModeAbortUnfinishedSharedUploads()
{
#ifdef S3_SUPPORT

    bool useS3MpuSharing = progArgs->getUseS3MPUSharing();

	// loop until no more unfinished objects are returned
	for( ; ; )
	{
		std::string bucketName;
		std::string objectName;
		std::string uploadID;

		s3SharedUploadStore.getNextUnfinishedUpload(bucketName, objectName, uploadID);

		if(uploadID.empty() )
			return; // no unfinished uploads left to abort

        if(!s3Client)
        { // this can happen if an exception gets thrown in preparePhase() before initS3Client()
            LOGGER(Log_DEBUG, "Skipping abort attempt of unfinished S3 MPUs because S3 client not "
                "initialized");
            return;
        }

		LOGGER(Log_DEBUG, "Aborting unfinished shared multipart upload. "
			"Rank: " << workerRank << "; "
			"Endpoint: " << s3EndpointStr << "; "
			"Bucket: " << bucketName << "; "
			"Object: " << objectName << "; " << std::endl);

		bool abortSuccess = s3ModeAbortMultipartUpload(bucketName, objectName, uploadID);

		if(abortSuccess)
			continue;

		// aborting unfinished upload failed
        // (this is normal in svc mpu sharing mode, where all services cancel all shared mpu ids)
		ERRLOGGER(useS3MpuSharing ? Log_DEBUG : Log_NORMAL,
            "Aborting unfinished shared multipart upload failed. "
			"Rank: " << workerRank << "; "
			"Endpoint: " << s3EndpointStr << "; "
			"Bucket: " << bucketName << "; "
			"Object: " << objectName << "; "
			"UploadID: " << uploadID << "; " << std::endl);
	}

#endif // S3_SUPPORT
}

/**
 * Block-sized download of an S3 object.
 *
 * This will delegate to s3ModeDownloadObjectAsync() if iodepth > 1.
 *
 * @isRWMixedReader true if this is a reader of a mixed read/write phase, so that the corresponding
 * 		statistics get increased.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeDownloadObject(std::string bucketName, std::string objectName,
	const bool isRWMixedReader)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const bool useS3FastRead = progArgs->getUseS3FastRead();
	const bool ignoreS3Errors = progArgs->getIgnoreS3Errors();

    // async delegation: hand over to async function if iodepth is given
    if(progArgs->getIODepth() > 1)
    {
        s3ModeDownloadObjectAsync(bucketName, objectName, isRWMixedReader);
        return;
    }

	// download one block-sized chunk in each loop pass
	while(rwOffsetGen->getNumBytesLeftToSubmit() )
	{
		const uint64_t currentOffset = rwOffsetGen->getNextOffset();
		const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();

		std::string objectRange = "bytes=" + std::to_string(currentOffset) + "-" +
			std::to_string(currentOffset+blockSize-1);

		char* ioBuf = useS3FastRead ? NULL : ioBufVec[0];
		char* gpuIOBuf = useS3FastRead ? NULL : gpuIOBufVec[0];

		((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		((*this).*funcPreWriteBlockModifier)(ioBuf, gpuIOBuf, blockSize, currentOffset);
		((*this).*funcPreWriteCudaMemcpy)(ioBuf, gpuIOBuf, blockSize);

		S3::GetObjectRequest request;
		request.WithBucket(bucketName)
			.WithKey(objectName)
			.WithRange(objectRange);

        // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
        if(!s3SSECKey.empty())
            request.WithSSECustomerAlgorithm("AES256")
                    .WithSSECustomerKey(s3SSECKey)
                    .WithSSECustomerKeyMD5(s3SSECKeyMD5);

        if(!useS3FastRead)
            request.SetResponseStreamFactory([&]()
            { /* note: this lambda will be called async after additional for-loop passes, so
                we can't rely on stack values from current loop pass to exist and thus need
                to pass vars by value instead of by reference. */

                S3MemoryStream* s3MemStream = new S3MemoryStream(
                    (unsigned char*)ioBuf, blockSize);

                /* note on seek: this factory will also get called in case of retries,
                    so we need to reset streamBuf position to beginning of buffer. */
                s3MemStream->seekp(0);
                s3MemStream->seekg(0);

                return s3MemStream;
            } );
        else
            request.SetResponseStreamFactory([&]()
            {
                return new Aws::FStream("/dev/null", std::ios_base::out | std::ios_base::binary);
            });

		request.SetDataReceivedEventHandler(
			[&](const Aws::Http::HttpRequest* request, Aws::Http::HttpResponse* response,
			long long numBytes)
			{
				if(isRWMixedReader)
					atomicLiveOpsReadMix.numBytesDone += numBytes;
				else
					atomicLiveOps.numBytesDone += numBytes;
			} );

        #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
            request.SetContinueRequestHandler( [&](const Aws::Http::HttpRequest* request)
                { return !isInterruptionRequested.load(); } );
        #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

		OPLOG_PRE_OP("S3GetObject", bucketName + "/" + objectName, currentOffset, blockSize);

		S3::GetObjectOutcome outcome = s3Client->GetObject(request);

		OPLOG_POST_OP("S3GetObject", bucketName + "/" + objectName, currentOffset, blockSize,
		    !outcome.IsSuccess() );

		checkInterruptionRequest(); // (placed here to avoid outcome check on interruption)

		IF_UNLIKELY(!outcome.IsSuccess() && !ignoreS3Errors)
            s3ModeThrowOnError(outcome, "Object download failed.", bucketName, objectName);

		IF_UNLIKELY( ( (size_t)outcome.GetResult().GetContentLength() < blockSize) &&
            !ignoreS3Errors)
		{
            throw WorkerException(std::string("Object too small. ") +
                "Endpoint: " + s3EndpointStr + "; "
                "Bucket: " + bucketName + "; "
                "Object: " + objectName + "; "
                "Offset: " + std::to_string(currentOffset) + "; "
                "Requested blocksize: " + std::to_string(blockSize) + "; "
                "Received length: " + std::to_string(outcome.GetResult().GetContentLength() ) );
		}

		((*this).*funcPostReadCudaMemcpy)(ioBuf, gpuIOBuf, blockSize);
		((*this).*funcPostReadBlockChecker)(ioBuf, gpuIOBuf, blockSize, currentOffset);

		// calc io operation latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		if(isRWMixedReader)
		{
			iopsLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
			atomicLiveOpsReadMix.numIOPSDone++;
		}
		else
		{
			iopsLatHisto.addLatency(ioElapsedMicroSec.count() );
			atomicLiveOps.numIOPSDone++;
		}

		numIOPSSubmitted++;
		rwOffsetGen->addBytesSubmitted(blockSize);
	}

#endif // S3_SUPPORT
}

/**
 * Async block-sized download of an S3 object.
 *
 * @isRWMixedReader true if this is a reader of a mixed read/write phase, so that the corresponding
 * 		statistics get increased.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeDownloadObjectAsync(std::string bucketName, std::string objectName,
    const bool isRWMixedReader)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    const bool useS3FastRead = progArgs->getUseS3FastRead();
    const bool ignoreS3Errors = progArgs->getIgnoreS3Errors();
    const unsigned ioDepth = progArgs->getIODepth();

    std::vector<S3AsyncDownloadContext> partCompletionsVec;

    partCompletionsVec.reserve(ioDepth); /* (reserve is important to not invalidate pointers for
        async callbacks on vector grow) */


    /* NOTE on try-catch: before we can exit this function we need to wait for all async tasks to
        complete because they might still be accessing members of partCompletionsVec. */
    try
    {
        // download block-sized chunks async up to iodepth in each loop pass
        while(rwOffsetGen->getNumBytesLeftToSubmit() )
        {
            // S T E P 1: submit parts asynchronously up to iodepth

            for(unsigned currentIODepth = 0;
                (currentIODepth < ioDepth) && rwOffsetGen->getNumBytesLeftToSubmit();
                currentIODepth++)
            {
                const uint64_t currentOffset = rwOffsetGen->getNextOffset();
                const size_t blockSize = rwOffsetGen->getNextBlockSizeToSubmit();

                S3AsyncDownloadContext& asyncPartContext = partCompletionsVec.emplace_back(
                    currentOffset, blockSize);

                std::string objectRange = "bytes=" + std::to_string(currentOffset) + "-" +
                    std::to_string(currentOffset+blockSize-1);

                char* ioBuf = useS3FastRead ? NULL : ioBufVec[currentIODepth];
                char* gpuIOBuf = useS3FastRead ? NULL : gpuIOBufVec[currentIODepth];

                ((*this).*funcRWRateLimiter)(blockSize, isInterruptionRequested);

                asyncPartContext.ioStartT = std::chrono::steady_clock::now();

                ((*this).*funcPreWriteBlockModifier)(ioBuf, gpuIOBuf, blockSize, currentOffset);
                ((*this).*funcPreWriteCudaMemcpy)(ioBuf, gpuIOBuf, blockSize);

                S3::GetObjectRequest& request = asyncPartContext.request;
                request.WithBucket(bucketName)
                    .WithKey(objectName)
                    .WithRange(objectRange);

                // (no s3ModeAddServerSideEncryptionHeaders() because this one is only for SSE-C)
                if(!s3SSECKey.empty())
                    request.WithSSECustomerAlgorithm("AES256")
                            .WithSSECustomerKey(s3SSECKey)
                            .WithSSECustomerKeyMD5(s3SSECKeyMD5);

                if(!useS3FastRead)
                    request.SetResponseStreamFactory([ioBuf, blockSize]()
                    { /* note: this lambda will be called async after additional for-loop passes, so
                        we can't rely on stack values from current loop pass to exist and thus need
                        to pass vars by value instead of by reference. */

                        S3MemoryStream* s3MemStream = new S3MemoryStream(
                            (unsigned char*)ioBuf, blockSize);

                        /* note on seek: this factory will also get called in case of retries,
                            so we need to reset streamBuf position to beginning of buffer. */
                        s3MemStream->seekp(0);
                        s3MemStream->seekg(0);

                        return s3MemStream;
                    } );
                else
                    request.SetResponseStreamFactory([]()
                    {
                        return new Aws::FStream("/dev/null",
                            std::ios_base::out | std::ios_base::binary);
                    });

                request.SetDataReceivedEventHandler(
                    [&isRWMixedReader, &atomicLiveOpsReadMix = atomicLiveOpsReadMix,
                        &atomicLiveOps = atomicLiveOps]
                    (const Aws::Http::HttpRequest* request, Aws::Http::HttpResponse* response,
                        long long numBytes)
                    {
                        if(isRWMixedReader)
                            atomicLiveOpsReadMix.numBytesDone += numBytes;
                        else
                            atomicLiveOps.numBytesDone += numBytes;
                    } );

                #if !defined(S3_AWSCRT) || AWS_SDK_AT_LEAST(1, 11, 708)
                    request.SetContinueRequestHandler(
                        [&isInterruptionRequested = isInterruptionRequested]
                        (const Aws::Http::HttpRequest* request)
                        { return !isInterruptionRequested.load(); } );
                #endif // !S3_AWSCRT or AWS SDK >= 1.11.708

                OPLOG_PRE_OP("S3GetObjectAsync", bucketName + "/" + objectName, currentOffset,
                    blockSize);

                s3Client->GetObjectAsync(request,
                    [partCompletePromise = &asyncPartContext.partCompletePromise]
                    (const S3Client*, const S3::GetObjectRequest&, S3::GetObjectOutcome outcome,
                        const std::shared_ptr<const Aws::Client::AsyncCallerContext>&)
                    { partCompletePromise->set_value(std::move(outcome)); });

                numIOPSSubmitted++;
                rwOffsetGen->addBytesSubmitted(blockSize);

            } // end of step 1: async submission loop


            // S T E P 2: handle completion of submitted part downloads

            for(unsigned currentIODepth = 0;
                currentIODepth < partCompletionsVec.size();
                currentIODepth++)
            {
                S3AsyncDownloadContext& asyncPartContext = partCompletionsVec[currentIODepth];

                // wait for this part to complete ("future.get()" blocks)
                S3::GetObjectOutcome outcome = asyncPartContext.partCompleteFuture.get();

                OPLOG_POST_OP("S3GetObjectAsync", bucketName + "/" + objectName,
                    asyncPartContext.currentOffset, asyncPartContext.blockSize,
                    !outcome.IsSuccess() );

                checkInterruptionRequest(); // (placed here to avoid outcome check on interruption)

                IF_UNLIKELY(!outcome.IsSuccess() && !ignoreS3Errors)
                    s3ModeThrowOnError(outcome, "Object download failed.", bucketName, objectName);

                IF_UNLIKELY(
                    ( (size_t)outcome.GetResult().GetContentLength() <
                        asyncPartContext.blockSize) &&
                    !ignoreS3Errors)
                {
                    throw WorkerException(std::string("Object too small. ") +
                        "Endpoint: " + s3EndpointStr + "; "
                        "Bucket: " + bucketName + "; "
                        "Object: " + objectName + "; "
                        "Offset: " + std::to_string(asyncPartContext.currentOffset) + "; "
                        "Requested blocksize: " + std::to_string(asyncPartContext.blockSize) + "; "
                        "Received length: " + std::to_string(
                            outcome.GetResult().GetContentLength() ) );
                }

                char* ioBuf = useS3FastRead ? NULL : ioBufVec[currentIODepth];
                char* gpuIOBuf = useS3FastRead ? NULL : gpuIOBufVec[currentIODepth];

                ((*this).*funcPostReadCudaMemcpy)(ioBuf, gpuIOBuf, asyncPartContext.blockSize);
                ((*this).*funcPostReadBlockChecker)(ioBuf, gpuIOBuf, asyncPartContext.blockSize,
                    asyncPartContext.currentOffset);

                // calc io operation latency
                std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
                std::chrono::microseconds ioElapsedMicroSec =
                    std::chrono::duration_cast<std::chrono::microseconds>
                    (ioEndT - asyncPartContext.ioStartT);

                if(isRWMixedReader)
                {
                    iopsLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
                    atomicLiveOpsReadMix.numIOPSDone++;
                }
                else
                {
                    iopsLatHisto.addLatency(ioElapsedMicroSec.count() );
                    atomicLiveOps.numIOPSDone++;
                }

            } // end of step 2: for-loop: async completion

            checkInterruptionRequest();

            // reset completions vec for next round
            partCompletionsVec.clear();

        } // end of while-loop

    }
    catch(...)
    {
        interruptExecution(); // for SetContinueRequestHandler()

        // wait for all parts to complete ("future.get()" blocks)
        for(unsigned i = 0; i < partCompletionsVec.size(); i++)
        {
            /* ignore errors because we're already handling an error (and ".get()" will also throw
                an exception if value has already been retrieved) */
            try { partCompletionsVec[i].partCompleteFuture.get(); }
            catch(...) {}
        }

        throw;
    }

#endif // S3_SUPPORT
}

/**
 * Retrieve object metadata by sending a HeadObject request (the equivalent of a stat() call in the
 * file world).
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeStatObject(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	S3::HeadObjectRequest request;
	request.WithBucket(bucketName)
		.WithKey(objectName);

	OPLOG_PRE_OP("S3HeadObject", bucketName + "/" + objectName, 0, 0);

	S3::HeadObjectOutcome outcome = s3Client->HeadObject(request);

    OPLOG_POST_OP("S3HeadObject", bucketName + "/" + objectName, 0, 0, !outcome.IsSuccess() );

    s3ModeThrowOnError(outcome, "Object metadata retrieval via HeadObject failed.", bucketName,
        objectName);

#endif // S3_SUPPORT
}

/**
 * Delete given S3 object.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeDeleteObject(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const bool ignoreDelErrors = progArgs->getIgnoreDelErrors();

	S3::DeleteObjectRequest request;
	request.WithBucket(bucketName)
		.WithKey(objectName);

	OPLOG_PRE_OP("S3DeleteObject", bucketName + "/" + objectName, 0, 0);

	S3::DeleteObjectOutcome outcome = s3Client->DeleteObject(request);

    OPLOG_POST_OP("S3DeleteObject", bucketName + "/" + objectName, 0, 0, !outcome.IsSuccess() );

	IF_UNLIKELY(!outcome.IsSuccess() &&
		(!ignoreDelErrors ||
			(outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) ) )
	{
        s3ModeThrowOnError(outcome, "Object deletion failed.", bucketName, objectName);
	}

#endif // S3_SUPPORT
}

/**
 * List objects in given buckets with user-defined limit for number of entries.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeListObjects()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const StringVec& bucketVec = progArgs->getBenchPaths();
	const size_t numBuckets = bucketVec.size();
	const size_t numDataSetThreads = progArgs->getNumDataSetThreads();
	std::string objectPrefix = progArgs->getS3ObjectPrefix();

	workerGotPhaseWork = false; // not all workers might get work

	for(unsigned bucketIndex = workerRank;
		bucketIndex < numBuckets;
		bucketIndex += numDataSetThreads)
	{
		uint64_t numObjectsLeft = progArgs->getS3ListObjNum();
		std::string nextContinuationToken;
		bool isTruncated; // true if S3 server reports more objects left to retrieve

		workerGotPhaseWork = true;

		do
		{
			checkInterruptionRequest();

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			S3::ListObjectsV2Request request;
			request.SetBucket(bucketVec[bucketIndex] );
			request.SetPrefix(objectPrefix);
			request.SetMaxKeys( (numObjectsLeft > 1000) ? 1000 : numObjectsLeft); // can't be >1000

			if(!nextContinuationToken.empty() )
				request.SetContinuationToken(nextContinuationToken);

			OPLOG_PRE_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
				request.GetMaxKeys() );

			S3::ListObjectsV2Outcome outcome = s3Client->ListObjectsV2(request);

            OPLOG_POST_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
                outcome.GetResult().GetKeyCount(), !outcome.IsSuccess() );

			IF_UNLIKELY(!outcome.IsSuccess() )
			{
				auto s3Error = outcome.GetError();

                throw WorkerException(std::string("Object listing v2 failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketVec[bucketIndex] + "; "
                    "Prefix: " + objectPrefix + "; "
                    "ContinuationToken: " + nextContinuationToken + "; "
                    "NumObjectsLeft: " + std::to_string(numObjectsLeft) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); " +
                    "Request ID: " + s3Error.GetRequestId() );
			}

			// calc entry operations latency
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

			unsigned keyCount = outcome.GetResult().GetKeyCount();

			atomicLiveOps.numEntriesDone += keyCount;
			numObjectsLeft -= keyCount;

			nextContinuationToken = outcome.GetResult().GetNextContinuationToken();
			isTruncated = outcome.GetResult().GetIsTruncated();

		} while(isTruncated && numObjectsLeft); // end of while numObjectsLeft loop
	}

#endif // S3_SUPPORT
}

/**
 * This is for s3 mode parallel listing of objects. Expects a dataset created via
 * s3ModeIterateObjects() and uses different prefixes per worker thread to parallelize, so that each
 * worker requests the listing of the dirs/objs that it created in s3ModeIterateObjects().
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeListObjParallel()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	const bool haveSubdirs = (progArgs->getNumDirs() > 0);
	const size_t numDirs = haveSubdirs ? progArgs->getNumDirs() : 1; // set 1 to run dir loop once
	const size_t numFiles = progArgs->getNumFiles();
	const StringVec& bucketVec = progArgs->getBenchPaths();
	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */
	std::string objectPrefix = progArgs->getS3ObjectPrefix();
	const bool objectPrefixRand = progArgs->getUseS3ObjectPrefixRand();
	const bool doListObjVerify = progArgs->getDoListObjVerify();


	// walk over each unique dir per worker

	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		uint64_t numObjectsLeft = numFiles;
		std::string nextContinuationToken;
		bool isTruncated; // true if S3 server reports more objects left to retrieve
		StringList receivedObjs; // for verification (if requested by user)
		StringSet expectedObjs; // for verification (if requested by user)

		// generate list prefix for current dir
		int printRes;

		if(haveSubdirs)
			printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-",
						workerDirRank, dirIndex, workerRank);
		else
			printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-",
						workerRank);

		IF_UNLIKELY(printRes >= PATH_BUF_LEN)
			throw WorkerException("object path too long for static buffer. "
				"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
				"workerRank: " + std::to_string(workerRank) + "; "
				"dirIndex: " + std::to_string(dirIndex) );

		unsigned bucketIndex = (workerRank + dirIndex) % bucketVec.size();
		std::string currentListPrefix = objectPrefix + currentPath.data();

		// build list of expected objs in dir for verification. (std::set for alphabetic order)
		for(size_t fileIndex = 0; doListObjVerify && (fileIndex < numFiles); fileIndex++)
		{
			int printRes;

			if(haveSubdirs)
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-f%zu",
					workerDirRank, dirIndex, workerRank, fileIndex);
			else
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-f%zu",
					workerRank, fileIndex);

			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("Verification object path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) + "; "
					"dirIndex: " + std::to_string(dirIndex) + "; "
					"fileIndex: " + std::to_string(fileIndex) );

			if(objectPrefixRand)
				objectPrefix = getS3RandObjectPrefix(
					workerRank, dirIndex, fileIndex, progArgs->getS3ObjectPrefix() );

			std::string currentObjectPath = objectPrefix + currentPath.data();

			expectedObjs.insert(currentObjectPath);
		}

		// receive listing of current dir
		do
		{
			checkInterruptionRequest();

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			S3::ListObjectsV2Request request;
			request.SetBucket(bucketVec[bucketIndex] );
			request.SetPrefix(currentListPrefix);
			request.SetMaxKeys( (numObjectsLeft > 1000) ? 1000 : numObjectsLeft); // can't be >1000

			if(!nextContinuationToken.empty() )
				request.SetContinuationToken(nextContinuationToken);

			OPLOG_PRE_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + currentListPrefix, 0,
				request.GetMaxKeys() );

			S3::ListObjectsV2Outcome outcome = s3Client->ListObjectsV2(request);

			OPLOG_POST_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + currentListPrefix, 0,
			    outcome.GetResult().GetKeyCount(), !outcome.IsSuccess() );

			IF_UNLIKELY(!outcome.IsSuccess() )
			{
				auto s3Error = outcome.GetError();

                throw WorkerException(std::string("Object listing v2 failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketVec[bucketIndex] + "; "
                    "Prefix: " + objectPrefix + "; "
                    "ContinuationToken: " + nextContinuationToken + "; "
                    "NumObjectsLeft: " + std::to_string(numObjectsLeft) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); " +
                    "Request ID: " + s3Error.GetRequestId() );
			}

			// calc entry operations latency
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

			unsigned keyCount = outcome.GetResult().GetKeyCount();

			atomicLiveOps.numEntriesDone += keyCount;
			numObjectsLeft -= keyCount;

			nextContinuationToken = outcome.GetResult().GetNextContinuationToken();
			isTruncated = outcome.GetResult().GetIsTruncated();

			// build list of received keys. (std::list to preserve order; must be alphabetic)
			IF_UNLIKELY(doListObjVerify)
				for(const S3::Object& obj : outcome.GetResult().GetContents() )
					receivedObjs.push_back(obj.GetKey() );

		} while(isTruncated && numObjectsLeft); // end of while numObjectsLeft in dir loop

		IF_UNLIKELY(doListObjVerify)
			s3ModeVerifyListing(expectedObjs, receivedObjs, bucketVec[bucketIndex],
				currentListPrefix);

	} // end of dirs for-loop

#endif // S3_SUPPORT
}

/**
 * Verify expected and received dir listing. This includes a verification of the entry order
 * inside the listing.
 *
 * @listPrefix the prefix that was used in the object listing request.
 *
 * @throw WorkerException on error (e.g. mismatch between expected and received).
 */
void LocalWorker::s3ModeVerifyListing(StringSet& expectedSet, StringList& receivedList,
	std::string bucketName, std::string listPrefix)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

	if(expectedSet.size() != receivedList.size() )
		throw WorkerException(std::string("Object listing v2 verification failed. ") +
			"Number of expected and number of received entries differ. "
			"Endpoint: " + s3EndpointStr + "; "
			"Bucket: " + bucketName + "; "
			"Prefix: " + listPrefix + "; "
			"NumObjectsExpected: " + std::to_string(expectedSet.size() ) + "; " +
			"NumObjectsReceived: " + std::to_string(receivedList.size() ) );

	uint64_t currentOffset = 0; // offset inside listing

	while(!expectedSet.empty() && !receivedList.empty() )
	{
		if(*expectedSet.begin() == *receivedList.begin() )
		{ // all good with this entry, so delete and move on to the next one
			expectedSet.erase(expectedSet.begin() );
			receivedList.erase(receivedList.begin() );

			currentOffset++;

			continue;
		}

		// entries differ, so verification failed

		throw WorkerException(std::string("Object listing v2 verification failed. ") +
			"Found object differs from expected object at offset. "
			"FoundObject: " + *receivedList.begin() + "; "
			"ExpectedObject: " + *expectedSet.begin() + "; "
			"ListingOffset: " + std::to_string(currentOffset) + "; "
			"Endpoint: " + s3EndpointStr + "; "
			"Bucket: " + bucketName + "; "
			"Prefix: " + listPrefix + "; "
			"NumObjectsExpected: " + std::to_string(expectedSet.size() ) + "; " +
			"NumObjectsReceived: " + std::to_string(receivedList.size() ) );
	}

#endif // S3_SUPPORT
}

/**
 * List objects and multi-delete them in given buckets with user-defined limit for number of
 * entries.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeListAndMultiDeleteObjects()
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this was built without S3 support");
#else

	const StringVec& bucketVec = progArgs->getBenchPaths();
	const size_t numBuckets = bucketVec.size();
	const size_t numDataSetThreads = progArgs->getNumDataSetThreads();
	const uint64_t numObjectsPerRequest = progArgs->getS3MultiDelObjNum();
	std::string objectPrefix = progArgs->getS3ObjectPrefix();
	const bool ignoreDelErrors = progArgs->getIgnoreDelErrors();

	workerGotPhaseWork = false; // not all workers might get work

	for(unsigned bucketIndex = workerRank;
		bucketIndex < numBuckets;
		bucketIndex += numDataSetThreads)
	{
		std::string nextContinuationToken;
		bool isTruncated; // true if S3 server reports more objects left to retrieve

		workerGotPhaseWork = true;

		do
		{
			checkInterruptionRequest();

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			// receive a batch of object names through listing...

			S3::ListObjectsV2Request listRequest;
			listRequest.SetBucket(bucketVec[bucketIndex] );
			listRequest.SetPrefix(objectPrefix);
			listRequest.SetMaxKeys(numObjectsPerRequest);

			if(!nextContinuationToken.empty() )
				listRequest.SetContinuationToken(nextContinuationToken);

			OPLOG_PRE_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
				numObjectsPerRequest);

			S3::ListObjectsV2Outcome listOutcome = s3Client->ListObjectsV2(listRequest);

            OPLOG_POST_OP("S3ListObjectsV2", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
                listOutcome.GetResult().GetKeyCount(), !listOutcome.IsSuccess() );

			IF_UNLIKELY(!listOutcome.IsSuccess() )
			{
				auto s3Error = listOutcome.GetError();

                throw WorkerException(std::string("Object listing v2 failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketVec[bucketIndex] + "; "
                    "Prefix: " + objectPrefix + "; "
                    "ContinuationToken: " + nextContinuationToken + "; "
                    "NumObjectsPerRequest: " + std::to_string(numObjectsPerRequest) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); " +
                    "Request ID: " + s3Error.GetRequestId() );
			}

			// check if we have anything to delete in this round
			if(!listOutcome.GetResult().GetKeyCount() )
			    break;

			// send multi-delete request for received batch of objects...

			S3::Delete deleteObjectList;

			for(const S3::Object& obj : listOutcome.GetResult().GetContents() )
				deleteObjectList.AddObjects(
					S3::ObjectIdentifier().WithKey(obj.GetKey() ) );

			S3::DeleteObjectsRequest delRequest;
			delRequest.SetBucket(bucketVec[bucketIndex] );
			delRequest.SetDelete(deleteObjectList);

            OPLOG_PRE_OP("S3DeleteObjects", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
                numObjectsPerRequest);

			S3::DeleteObjectsOutcome delOutcome = s3Client->DeleteObjects(delRequest);

            OPLOG_POST_OP("S3DeleteObjects", bucketVec[bucketIndex] + "/" + objectPrefix, 0,
                delOutcome.GetResult().GetDeleted().size(), !delOutcome.IsSuccess() );

			IF_UNLIKELY(!delOutcome.IsSuccess() &&
				(!ignoreDelErrors ||
					(delOutcome.GetError().GetResponseCode() ==
						Aws::Http::HttpResponseCode::NOT_FOUND) ) )
			{
				auto s3Error = delOutcome.GetError();

                throw WorkerException(std::string("DeleteObjects failed. ") +
                    "Endpoint: " + s3EndpointStr + "; "
                    "Bucket: " + bucketVec[bucketIndex] + "; "
                    "NumObjectsPerRequest: " + std::to_string(numObjectsPerRequest) + "; "
                    "Exception: " + s3Error.GetExceptionName() + "; " +
                    "Message: " + s3Error.GetMessage() + "; " +
                    "HTTP Error Code: " + std::to_string( (int)s3Error.GetResponseCode() ) + " (" +
                        TranslatorTk::httpErrorCodeToHumanStr( (int)s3Error.GetResponseCode() ) +
                        "); " +
                    "Request ID: " + s3Error.GetRequestId() );
			}

			// calc entry operations latency
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

			unsigned keyCount = delOutcome.GetResult().GetDeleted().size();

			atomicLiveOps.numEntriesDone += keyCount;

			nextContinuationToken = listOutcome.GetResult().GetNextContinuationToken();
			isTruncated = listOutcome.GetResult().GetIsTruncated();

		} while(isTruncated); // end of while numObjectsLeft loop
	}

#endif // S3_SUPPORT
}

/**
 * Put ACL of given S3 object.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModePutObjectAcl(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this build is without S3 support");
#else

	S3::PutObjectAclRequest request;
	request.WithBucket(bucketName)
		.WithKey(objectName);

    TranslatorTk::applyS3PutAclRequestGrants<S3::ObjectCannedACL>(progArgs, request);

	OPLOG_PRE_OP("S3PutObjectAcl", bucketName + "/" + objectName, 0, 0);

	S3::PutObjectAclOutcome outcome = s3Client->PutObjectAcl(request);

    OPLOG_POST_OP("S3PutObjectAcl", bucketName + "/" + objectName, 0, 0, !outcome.IsSuccess() );

    s3ModeThrowOnError(outcome, "Putting object ACL failed.", bucketName, objectName);

#endif // S3_SUPPORT
}

/**
 * Get ACL of given S3 object.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::s3ModeGetObjectAcl(std::string bucketName, std::string objectName)
{
#ifndef S3_SUPPORT
	throw WorkerException(std::string(__func__) + " called, but this build is without S3 support");
#else

    bool doS3AclVerify = progArgs->getDoS3AclVerify();

	S3::GetObjectAclRequest request;
	request.WithBucket(bucketName)
		.WithKey(objectName);

	OPLOG_PRE_OP("S3GetObjectAcl", bucketName + "/" + objectName, 0, 0);

	S3::GetObjectAclOutcome outcome = s3Client->GetObjectAcl(request);

    OPLOG_POST_OP("S3GetObjectAcl", bucketName + "/" + objectName, 0, 0, !outcome.IsSuccess() );

    s3ModeThrowOnError(outcome, "Getting object ACL failed.", bucketName, objectName);

	IF_UNLIKELY(doS3AclVerify)
	{
        // check canned ACL as special grantee...

        S3::BucketCannedACL cannedAcl = S3::BucketCannedACLMapper::GetBucketCannedACLForName(
            progArgs->getS3AclGrantee() );

        /* note: checking for ::NOT_SET alone here is not enough, because GetObjectCannedACLForName()
            can return other values if granteeStr doesn't match another enum value. */
        switch( (S3::ObjectCannedACL)cannedAcl)
        {
            case S3::ObjectCannedACL::private_:
            case S3::ObjectCannedACL::public_read:
            case S3::ObjectCannedACL::public_read_write:
            case S3::ObjectCannedACL::authenticated_read:
            case S3::ObjectCannedACL::aws_exec_read:
            case S3::ObjectCannedACL::bucket_owner_read:
            case S3::ObjectCannedACL::bucket_owner_full_control:
            { // found canned ACL as special grantee
                throw WorkerException("Verification of canned ACLs is not supported.");
            }

            case S3::ObjectCannedACL::NOT_SET:
            default:
            { // normal grantee, not a canned ACL
                break;
            }
        }

	    // check list of grants...

		std::vector<S3::Grant> verifyGrants;
		TranslatorTk::getS3ObjectAclGrants(progArgs, verifyGrants);

		const std::vector<S3::Grant>& outcomeGrants = outcome.GetResult().GetGrants();

		// iterate over all grants that need to be verified
		for(S3::Grant& verifyGrant : verifyGrants)
		{
			bool grantFound = false;

			// iterate over all outcome grants to see if any grant matches current verifyGrant
			for(const S3::Grant& outcomeGrant : outcomeGrants)
			{
				if( (outcomeGrant.GetGrantee().GetID() ==
						verifyGrant.GetGrantee().GetID() ) ||
					(outcomeGrant.GetGrantee().GetEmailAddress() ==
						verifyGrant.GetGrantee().GetEmailAddress() ) ||
					(outcomeGrant.GetGrantee().GetURI() ==
						verifyGrant.GetGrantee().GetURI() ) )
				{ // grantee matches => check if permission also matches
					if(outcomeGrant.GetPermission() == verifyGrant.GetPermission() )
					{ // permission matches
						grantFound = true;
						break;
					}
				}
			}

			if(!grantFound)
				throw WorkerException(std::string("S3 ACL verification failed. ") +
					"Endpoint: " + s3EndpointStr + "; "
					"Bucket: " + bucketName + "; "
					"Object: " + objectName + "; "
					"Grantee ID: " + verifyGrant.GetGrantee().GetID() + "; "
					"Grantee Email: " + verifyGrant.GetGrantee().GetEmailAddress() + "; "
					"Grantee URI: " + verifyGrant.GetGrantee().GetURI() + "; "
					"Permission: " + TranslatorTk::s3AclPermissionToStr(
						verifyGrant.GetPermission() ) );
		}
	} // end of verifcation

#endif // S3_SUPPORT
}


void LocalWorker::s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    OPLOG_PRE_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0);

    const auto getTagOutcome = s3Client->GetObjectTagging(
        S3::GetObjectTaggingRequest()
            .WithBucket(bucketName)
            .WithKey(objectName)
    );

    OPLOG_POST_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0, !getTagOutcome.IsSuccess());

    s3ModeThrowOnError(getTagOutcome, "Get object tagging failed.", bucketName, objectName);

    // Continue only if we need to verify
    if (!progArgs->getDoS3ObjectTaggingVerify())
        return;

    const auto& tagSet = getTagOutcome.GetResult().GetTagSet();

    IF_UNLIKELY(tagSet.empty())
    {
        std::stringstream errStr;
        errStr << "Object has no tags, but 1 tag was expected" << std::endl
               << "Bucket: " << bucketName << "; "
               << "Key: " << objectName << std::endl
               << "Tag: " << TAG_KEY_MEDIUM_NAME << std::endl;
        throw WorkerException(errStr.str());
    }

    const auto& firstTag = tagSet.front();

    IF_UNLIKELY(!StringTk::verifyRandomS3TagValue(firstTag.GetValue(), objectName))
    {
        std::stringstream errStr;
        errStr << "Random tag value is corrupted (invalid checksum). "
               << "Bucket: " << bucketName << "; "
               << "Key: " << objectName << std::endl
               << "Tag: " << firstTag.GetKey() << "=" << firstTag.GetValue() << std::endl;
        throw WorkerException(errStr.str());
    }
#endif // S3_SUPPORT
}

void LocalWorker::s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    const auto tag = S3::Tag()
        .WithKey(TAG_KEY_MEDIUM_NAME)
        .WithValue(StringTk::generateRandomS3TagValue(objectName, TAG_VALUE_MEDIUM_LEN));

    S3::PutObjectTaggingRequest request;
    request.WithBucket(bucketName)
          .WithKey(objectName)
          .WithTagging(S3::Tagging().AddTagSet(tag));

    s3ModeAddChecksumAlgorithm(request);

    OPLOG_PRE_OP("PutObjectTagging", bucketName + "/" + objectName, 0, TAG_VALUE_MEDIUM_LEN);

    const auto putTagOutcome = s3Client->PutObjectTagging(request);

    OPLOG_POST_OP("PutObjectTagging", bucketName + "/" + objectName,
                  0, TAG_VALUE_MEDIUM_LEN, !putTagOutcome.IsSuccess());

    s3ModeThrowOnError(putTagOutcome, "Put object tagging failed.", bucketName, objectName);

#endif // S3_SUPPORT
}

void LocalWorker::s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else

    OPLOG_PRE_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0);

    const auto delTagOutcome = s3Client->DeleteObjectTagging(
        S3::DeleteObjectTaggingRequest()
            .WithBucket(bucketName)
            .WithKey(objectName)
    );

    OPLOG_POST_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0, !delTagOutcome.IsSuccess());

    s3ModeThrowOnError(delTagOutcome, "Delete object tagging failed.", bucketName, objectName);

#endif // S3_SUPPORT
}

void LocalWorker::s3ModeGetObjectLockConfiguration(const std::string &bucketName)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    OPLOG_PRE_OP("GetObjectLockConfiguration", bucketName, 0, 0);

    const auto getObjLockOutcome = s3Client->GetObjectLockConfiguration(
        S3::GetObjectLockConfigurationRequest().WithBucket(bucketName)
    );

    OPLOG_POST_OP("GetObjectLockConfiguration", bucketName, 0, 0, !getObjLockOutcome.IsSuccess());

    s3ModeThrowOnError(getObjLockOutcome, "Get object lock configuration failed.", bucketName);

    // Continue only if we need to verify
    if (!progArgs->getDoS3ObjectLockConfigurationVerify())
        return;

    const auto objLockCfg = getObjLockOutcome.GetResult().GetObjectLockConfiguration();

    IF_UNLIKELY(!objLockCfg.ObjectLockEnabledHasBeenSet())
        throw WorkerException("Object lock has not been enabled for bucket: " + bucketName);

    IF_UNLIKELY(!objLockCfg.RuleHasBeenSet())
        throw WorkerException("Object lock rule has not been set for bucket: " + bucketName);

    const auto objRetentionRule = objLockCfg.GetRule();

    IF_UNLIKELY(!objRetentionRule.DefaultRetentionHasBeenSet())
        throw WorkerException("Object lock default retention has not been set for bucket: " + bucketName);

    const auto objDefaultRetention = objRetentionRule.GetDefaultRetention();

    IF_UNLIKELY(!objDefaultRetention.DaysHasBeenSet())
        throw WorkerException("Object lock retention days have not been set for bucket: " + bucketName);

    const auto retentionDays = objDefaultRetention.GetDays();

    IF_UNLIKELY(retentionDays != RETENTION_PERIOD_DAYS)
    {
        std::stringstream errStr;
        errStr << "Object lock default retention was set to '" << retentionDays << " days' in bucket '"
               << bucketName << "', but the expected value was '" << RETENTION_PERIOD_DAYS << " days'";
        throw WorkerException(errStr.str());
    }
#endif // S3_SUPPORT
}

void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName, bool unset)
{
#ifndef S3_SUPPORT
    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
#else
    S3::ObjectLockConfiguration objectLockCfg;

    if (unset)
        objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::NOT_SET);
    else
    {
        objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::Enabled);
        objectLockCfg.SetRule(
            S3::ObjectLockRule().WithDefaultRetention(
                S3::DefaultRetention().WithMode(S3::ObjectLockRetentionMode::COMPLIANCE).WithDays(1)
            )
        );
    }

    OPLOG_PRE_OP("PutObjectLockConfiguration", bucketName, 0, 0);

    const auto putObjLockOutcome = s3Client->PutObjectLockConfiguration(
        S3::PutObjectLockConfigurationRequest()
            .WithBucket(bucketName)
            .WithObjectLockConfiguration(objectLockCfg));

    OPLOG_POST_OP("PutObjectLockConfiguration", bucketName, 0, 0, !putObjLockOutcome.IsSuccess());

    s3ModeThrowOnError(putObjLockOutcome, "Put object lock configuration failed.", bucketName);
#endif // S3_SUPPORT
}

/**
 * In S3 mode, decide if we do fallback to reverse upload. This would be the case if this is a write
 * phase and user selected random offsets.
 *
 * @return true if we do fallback to reverse sequential, false otherwise.
 */
bool LocalWorker::getS3ModeDoReverseSeqFallback()
{
	if(progArgs->getUseRandomOffsets() &&
		(progArgs->getBenchMode() == BenchMode_S3) &&
		(benchPhase == BenchPhase_CREATEFILES) )
		return true;

	return false;
}

/**
 * In S3 mode, replace any sequence of at least 3 consecutive RAND_PREFIX_MARK_CHAR chars with a
 * random uppercase hex string based on worker rank, dir index and file index. It's based on these
 * so that we can calculate the same random values again later to find the files.
 *
 * Note: It's a good idea to check progArgs->getUseS3ObjectPrefixRand() to avoid calling this
 * 		unnecessairly.
 *
 * @objectPrefix string in which to repace the consecutive occurences of RAND_PREFIX_MARK_CHAR.
 * @return objectPrefix with replaced RAND_PREFIX_MARK_CHAR chars.
 */
std::string LocalWorker::getS3RandObjectPrefix(size_t workerRank, size_t dirIdx,
	size_t fileIdx, const std::string& objectPrefix)
{
	size_t threeMarksPos = objectPrefix.find(RAND_PREFIX_MARKS_SUBSTR);

	if(threeMarksPos == std::string::npos)
		return objectPrefix; // not found, so nothing to replace here

	std::string randObjectPrefix(objectPrefix); // the copy to replace chars

	// we don't want any zero-based to turn result to all-zero (e.g. "-n 0" would always be 0)
	workerRank++;
	dirIdx++;
	fileIdx++;

	uint64_t randomNum = RandAlgoGoldenPrime(workerRank * dirIdx * fileIdx).next();

	for(size_t i = threeMarksPos;
		(i < objectPrefix.size() ) && (objectPrefix[i] == RAND_PREFIX_MARK_CHAR);
		i++)
	{
		randObjectPrefix[i] = ( (char*)HEX_ALPHABET)[randomNum % HEX_ALPHABET_LEN];

		randomNum /= HEX_ALPHABET_LEN;
	}

	return randObjectPrefix;
}

/**
 * Return appropriate file open flags for the current benchmark phase in dir mode.
 *
 * @return flags for file open() in dir mode.
 */
int LocalWorker::getDirModeOpenFlags(BenchPhase benchPhase)
{
	int openFlags = 0;

	if(benchPhase == BenchPhase_CREATEFILES)
	{
		openFlags = O_CREAT | O_RDWR;

		if(progArgs->getDoTruncate() )
			openFlags |= O_TRUNC;
	}
	else
		openFlags = O_RDONLY;

#if !defined(__APPLE__)
    if(progArgs->getUseDirectIO() )
        openFlags |= O_DIRECT;
#endif // !apple

	return openFlags;
}

/**
 * Open file and prepare it for the actual IO, e.g. by truncating or preallocating it to
 * user-requested size.
 *
 * @benchPhase current benchmark phase.
 * @pathFDs progArgs->getBenchPathFDs.
 * @pathFDsIndex current index in pathFDs.
 * @relativePath path to open, relative to pathFD.
 * @openFlags as returned by getDirModeOpenFlags().
 * @fileSize progArgs->getFileSize().
 * @return filedescriptor of open file.
 * @throw WorkerException on error, in which case file is guaranteed to be closed.
 */
int LocalWorker::dirModeOpenAndPrepFile(BenchPhase benchPhase, const IntVec& pathFDs,
		unsigned pathFDsIndex, const char* relativePath, int openFlags, uint64_t fileSize)
{
    const bool useMmap = progArgs->getUseMmap();
    const std::string currentPath = progArgs->getBenchPaths()[pathFDsIndex] + "/" + relativePath;

    OPLOG_PRE_OP("openat", currentPath, 0, 0);

    int fd = openat(pathFDs[pathFDsIndex], relativePath, openFlags, MKFILE_MODE);

    OPLOG_POST_OP("openat", currentPath, 0, 0, fd == -1);

    IF_UNLIKELY(fd == -1)
    {
        if( (errno == ENOENT) && (openFlags & O_CREAT) && !progArgs->getRunCreateDirsPhase() &&
            (progArgs->getNumDirs() || !progArgs->getCustomTreeDirs().getNumPaths() ) &&
            (progArgs->getBenchPathType() == BenchPathType_DIR) )
            throw WorkerException(std::string("File create/open failed. ") +
                "Did you forget to enable directory creation ('--" ARG_CREATEDIRS_LONG "')? "  +
                "Path: " + currentPath + "; "
                "SysErr: " + strerror(errno) );
        else
            throw WorkerException(std::string("File open failed. ") +
                "Path: " + currentPath + "; "
                "SysErr: " + strerror(errno) );
    }

    // try block to ensure file close on error
    try
    {
        if(benchPhase == BenchPhase_CREATEFILES)
        {
            if(progArgs->getDoTruncToSize() )
            {
                int truncRes = ftruncate(fd, fileSize);
                if(truncRes == -1)
                    throw WorkerException("Unable to set file size through ftruncate. "
                        "Path: " + currentPath + "; "
                        "Size: " + std::to_string(fileSize) + "; "
                        "SysErr: " + strerror(errno) );
            }

            if(progArgs->getDoPreallocFile() )
            {
                #if defined(__APPLE__)
                    throw WorkerException("posix_fallocate is not supported on macOS. "
                        "Path: " + currentPath + "; "
                        "Size: " + std::to_string(fileSize) );
                #else // linux
                    // (note: posix_fallocate does not set errno.)
                    int preallocRes = posix_fallocate(fd, 0, fileSize);
                    if(preallocRes != 0)
                        throw WorkerException(
                            "Unable to preallocate file size through posix_fallocate. "
                            "File: " + currentPath + "; "
                            "Size: " + std::to_string(fileSize) + "; "
                            "SysErr: " + strerror(preallocRes) );
                #endif // linux
            }
        }

        FileTk::fadvise<WorkerException>(fd, progArgs->getFadviseFlags(), currentPath.c_str() );

        // create memory mapping
        if(useMmap)
        {
            int protectionMode = (benchPhase == BenchPhase_READFILES) ?
                PROT_READ : (PROT_WRITE | PROT_READ);

            fileHandles.mmapVec[0] =  (char*)FileTk::mmapAndMadvise<WorkerException>(
                fileSize, protectionMode, MAP_SHARED, fd, progArgs->getMadviseFlags(),
                currentPath.c_str() );
        }

        return fd;
    }
    catch(WorkerException& e)
    {
        // release memory mapping
        if(useMmap)
        {
            munmap(fileHandles.mmapVec[0], fileSize);
            fileHandles.mmapVec[0] = (char*)MAP_FAILED;
        }

        OPLOG_PRE_OP("close", std::to_string(fd), 0, 0);

        int closeRes = close(fd);

        OPLOG_POST_OP("close", std::to_string(fd), 0, 0, closeRes == -1);

        if(closeRes == -1)
            ERRLOGGER(Log_NORMAL, "File close failed. " <<
                "Path: " << currentPath << "; " <<
                "FD: " << std::to_string(fd) << "; " <<
                "SysErr: " << strerror(errno) << std::endl);

        throw;
    }
}

/**
 * Iterate over all directories in HDFS dir mode to create or remove them.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::hdfsDirModeIterateDirs()
{
#ifndef HDFS_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but built without hdfs support");
#else

	if(progArgs->getNumDirs() == 0)
		return; // nothing to do

	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t numDirs = progArgs->getNumDirs();
	const StringVec& pathVec = progArgs->getBenchPaths();
	const bool ignoreDelErrors = progArgs->getDoDirSharing() ?
		true : progArgs->getIgnoreDelErrors(); // in dir share mode, all workers mk/del all dirs
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */

	// create rank dir inside each pathFD
	if(benchPhase == BenchPhase_CREATEDIRS)
	{
		for(unsigned pathFDsIndex = 0; pathFDsIndex < pathVec.size(); pathFDsIndex++)
		{
			// create rank dir for current pathFD...

			checkInterruptionRequest();

			// generate path
			int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu", workerDirRank);
			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("mkdir path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) );

			std::string fullPath = pathVec[pathFDsIndex] + "/" + currentPath.data();

			int mkdirRes = hdfsCreateDirectory(hdfsFSHandle, fullPath.c_str() );

			if( (mkdirRes == -1) && (errno != EEXIST) )
				throw WorkerException(std::string("Rank directory creation failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
		}
	}

	// create user-specified number of directories round-robin across all given bench paths
	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		checkInterruptionRequest();

		// generate current dir path
		int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu",
			workerDirRank, dirIndex);
		IF_UNLIKELY(printRes >= PATH_BUF_LEN)
			throw WorkerException("mkdir path too long for static buffer. "
				"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
				"dirIndex: " + std::to_string(dirIndex) + "; "
				"workerRank: " + std::to_string(workerRank) );

		unsigned pathFDsIndex = (workerRank + dirIndex) % pathVec.size();

		std::string fullPath = pathVec[pathFDsIndex] + "/" + currentPath.data();

		std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

		if(benchPhase == BenchPhase_CREATEDIRS)
		{ // create dir
			int mkdirRes = hdfsCreateDirectory(hdfsFSHandle, fullPath.c_str() );

			if( (mkdirRes == -1) && (errno != EEXIST) )
				throw WorkerException(std::string("Directory creation failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
		}

		if(benchPhase == BenchPhase_DELETEDIRS)
		{ // remove dir
			int rmdirRes = hdfsDelete(hdfsFSHandle, fullPath.c_str(), 0 /* recursive */ );

			if( (rmdirRes == -1) && !ignoreDelErrors) // hdfs doesn't have a meaningful error code
				throw WorkerException(std::string("Directory deletion failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
		}

		// calc entry operations latency
		std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
		std::chrono::microseconds ioElapsedMicroSec =
			std::chrono::duration_cast<std::chrono::microseconds>
			(ioEndT - ioStartT);

		entriesLatHisto.addLatency(ioElapsedMicroSec.count() );

		atomicLiveOps.numEntriesDone++;
	} // end of for loop


	// delete rank dir inside each pathFD
	if(benchPhase == BenchPhase_DELETEDIRS)
	{
		for(unsigned pathFDsIndex = 0; pathFDsIndex < pathVec.size(); pathFDsIndex++)
		{
			// delete rank dir for current pathFD...

			checkInterruptionRequest();

			// generate path
			int printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu", workerDirRank);
			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("mkdir path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) );

			std::string fullPath = pathVec[pathFDsIndex] + "/" + currentPath.data();

			int rmdirRes = hdfsDelete(hdfsFSHandle, fullPath.c_str(), 0 /* recursive */ );

			if( (rmdirRes == -1) && !ignoreDelErrors) // hdfs doesn't have a meaningful error code
				throw WorkerException(std::string("Directory deletion failed. ") +
					"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
		}
	}

#endif // HDFS_SUPPORT
}

/**
 * This is for HDFS directory mode. Iterate over all files to create/read/remove them.
 * By default, this uses a unique dir per worker and fills up each dir before moving on to the next.
 * If dir sharing is enabled, all workers will use dirs of rank 0.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::hdfsDirModeIterateFiles()
{
#ifndef HDFS_SUPPORT
	throw WorkerException(std::string(__func__) + "called, but built without hdfs support");
#else

	const bool haveSubdirs = (progArgs->getNumDirs() > 0);
	const size_t numDirs = haveSubdirs ? progArgs->getNumDirs() : 1; // set 1 to run dir loop once
	const size_t numFiles = progArgs->getNumFiles();
	const uint64_t fileSize = progArgs->getFileSize();
	const StringVec& pathVec = progArgs->getBenchPaths();
	const int openFlags = (benchPhase == BenchPhase_CREATEFILES) ? O_WRONLY : O_RDONLY;
	std::array<char, PATH_BUF_LEN> currentPath;
	const size_t workerDirRank = progArgs->getDoDirSharing() ? 0 : workerRank; /* for dir sharing,
		all workers use the dirs of worker rank 0 */
	const BenchPhase globalBenchPhase = workersSharedData->currentBenchPhase;
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();
	const bool isRWMixedReader = ( (globalBenchPhase == BenchPhase_CREATEFILES) &&
		(localWorkerRank < progArgs->getNumRWMixReadThreads() ) );

	// walk over each unique dir per worker

	for(size_t dirIndex = 0; dirIndex < numDirs; dirIndex++)
	{
		// occasional interruption check
		IF_UNLIKELY( (dirIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
			checkInterruptionRequest();

		// fill up this dir with all files before moving on to the next dir

		for(size_t fileIndex = 0; fileIndex < numFiles; fileIndex++)
		{
			// occasional interruption check
			IF_UNLIKELY( (fileIndex % INTERRUPTION_CHECK_INTERVAL) == 0)
				checkInterruptionRequest();

			// generate current dir path
			int printRes;

			if(haveSubdirs)
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu/d%zu/r%zu-f%zu",
					workerDirRank, dirIndex, workerRank, fileIndex);
			else
				printRes = snprintf(currentPath.data(), PATH_BUF_LEN, "r%zu-f%zu",
					workerRank, fileIndex);

			IF_UNLIKELY(printRes >= PATH_BUF_LEN)
				throw WorkerException("file path too long for static buffer. "
					"Buffer size: " + std::to_string(PATH_BUF_LEN) + "; "
					"workerRank: " + std::to_string(workerRank) + "; "
					"dirIndex: " + std::to_string(dirIndex) + "; "
					"fileIndex: " + std::to_string(fileIndex) );

			unsigned pathFDsIndex = (workerRank + dirIndex) % pathVec.size();

			std::string fullPath = pathVec[pathFDsIndex] + "/" + currentPath.data();

			rwOffsetGen->reset(); // reset for next file

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			if( (benchPhase == BenchPhase_CREATEFILES) || (benchPhase == BenchPhase_READFILES) )
			{
				hdfsFileHandle = hdfsOpenFile(hdfsFSHandle, fullPath.c_str(), openFlags, 0, 0, 0);

				IF_UNLIKELY(hdfsFileHandle == NULL) // hdfs doesn't provide a meaningful error code
					throw WorkerException(std::string("File open failed. ") +
						"Path: " + fullPath);

				if(progArgs->getUseDirectIO() )
					hdfsUnbufferFile(hdfsFileHandle);

				// try-block to ensure that fd is closed in case of exception
				try
				{
					if(benchPhase == BenchPhase_CREATEFILES)
					{
						int64_t writeRes = ((*this).*funcRWBlockSized)();

						IF_UNLIKELY(writeRes == -1)
							throw WorkerException(std::string("File write failed. ") +
								( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
									"Can be caused by directIO misalignment. " : "") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"SysErr: " + strerror(errno) );

						IF_UNLIKELY( (size_t)writeRes != fileSize)
							throw WorkerException(std::string("Unexpected short file write. ") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"Bytes written: " + std::to_string(writeRes) + "; "
								"Expected written: " + std::to_string(fileSize) );
					}

					if(benchPhase == BenchPhase_READFILES)
					{
						ssize_t readRes = ((*this).*funcRWBlockSized)();

						IF_UNLIKELY(readRes == -1)
							throw WorkerException(std::string("File read failed. ") +
								( (progArgs->getUseDirectIO() && (errno == EINVAL) ) ?
									"Can be caused by directIO misalignment. " : "") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"SysErr: " + strerror(errno) );

						IF_UNLIKELY( (size_t)readRes != fileSize)
							throw WorkerException(std::string("Unexpected short file read. ") +
								"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() + "; "
								"Bytes read: " + std::to_string(readRes) + "; "
								"Expected read: " + std::to_string(fileSize) );
					}
				}
				catch(...)
				{ // ensure that we don't leak an open file fd
					hdfsCloseFile(hdfsFSHandle, hdfsFileHandle);
					throw;
				}

				int closeRes = hdfsCloseFile(hdfsFSHandle, hdfsFileHandle);

				IF_UNLIKELY(closeRes == -1) // hdfs doesn't provide a meaningful error code
					throw WorkerException(std::string("File close failed. ") +
						"Path: " + fullPath);
			}

			if(benchPhase == BenchPhase_STATFILES)
			{
				hdfsFileInfo* fileInfo = hdfsGetPathInfo(hdfsFSHandle, fullPath.c_str() );

				if(fileInfo == NULL) // hdfs doesn't provide a meaningful error code
					throw WorkerException(std::string("File stat failed. ") +
						"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
			}

			if(benchPhase == BenchPhase_DELETEFILES)
			{
				int unlinkRes =  hdfsDelete(hdfsFSHandle, fullPath.c_str(), 0 /* recursive */ );

				if( (unlinkRes == -1) && !progArgs->getIgnoreDelErrors() )
					throw WorkerException(std::string("File delete failed. ") +
						"Path: " + pathVec[pathFDsIndex] + "/" + currentPath.data() );
			}

			// calc entry operations latency. (for create, this includes open/rw/close.)
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			// inc special rwmix thread stats
			if(isRWMixedReader)
			{
				entriesLatHistoReadMix.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOpsReadMix.numEntriesDone++;
			}
			else
			{
				entriesLatHisto.addLatency(ioElapsedMicroSec.count() );
				atomicLiveOps.numEntriesDone++;
			}

		} // end of files for loop
	} // end of dirs for loop


#endif // HDFS_SUPPORT
}

/**
 * In netbench mode, this is the wrapper to start either server or client mode (or none) for this
 * worker thread. "None" would be the case if this was a server, but we don't have enough client
 * connections to feed all of the server threads (e.g. 1 client with single thread and 2 servers).
 */
void LocalWorker::netbenchDoTransfer()
{
	if(serverSocketVec.size() )
		netbenchDoTransferServer();
	else
	if(clientSocket)
		netbenchDoTransferClient();
	else // neither server nor client: clients don't have enough threads for all servers
	{
		LOGGER(Log_DEBUG, "This worker is neither initialized as server nor as client. "
			"Rank: " + std::to_string(workerRank) );

		workerGotPhaseWork = false;
	}
}

/**
 * In netbench mode, this worker owns its fair share of incoming client connections and polls all
 * of them until we have received a complete blocksized package from one, in which case a
 * respsized reply is sent back.
 */
void LocalWorker::netbenchDoTransferServer()
{
	const int pollShortTimeoutSecs = NETBENCH_SHORT_POLL_TIMEOUT_SEC; // to re-check for interrupt
	const uint64_t transferBytesPerConn = progArgs->getFileSize();
	const size_t blockSize = progArgs->getBlockSize();
	const size_t respSize = progArgs->getNetBenchRespSize();
	size_t transferBufSize = std::max(blockSize, respSize);
	std::unique_ptr<char[]> transferBuf(new char [transferBufSize] );
	const size_t localWorkerRank = workerRank - progArgs->getRankOffset();

	// get our own subset of sockets (each n-th socket, where n is number of local threads)

	SocketVec workerSocketVec;

	for(size_t i = localWorkerRank; i < serverSocketVec.size(); i += progArgs->getNumThreads() )
		workerSocketVec.push_back(serverSocketVec[i] );

	if(workerSocketVec.empty() )
	{ // this worker didn't get any work

		LOGGER(Log_DEBUG, "This server worker didn't get any sockets. "
			"Rank: " + std::to_string(workerRank) );

		workerGotPhaseWork = false;
		return;
	}

	UInt64Vec transferredBytesVec(workerSocketVec.size(), 0); // to check when we're done

	// build pollFDVec

	std::vector<struct pollfd> pollFDVec; // for poll()

	pollFDVec.reserve(workerSocketVec.size() );

	for(BasicSocket* sock : workerSocketVec)
		pollFDVec.push_back( { sock->getFD(), POLLIN, 0 } );


	// at the end of a tranfer, all serverSocketVec sockets will have been erased
	while(workerSocketVec.size() )
	{
		// wait for incoming data on any of the client connections

		int pollRes = 0;

		for(int elapsedSecs=0;
			!pollRes && (elapsedSecs < NETBENCH_RECEIVE_TIMEOUT_SEC);
			elapsedSecs += pollShortTimeoutSecs)
		{
			// (this short loop exists to more quickly detect user interrupt requests)
			checkInterruptionRequest();

			pollRes = poll(pollFDVec.data(), pollFDVec.size(), pollShortTimeoutSecs * 1000);
		}

		if(!pollRes)
			throw WorkerException("Server: Waiting for incoming data timed out. "
				"Rank: " + std::to_string(workerRank) + "; "
				"1st peer: " + workerSocketVec.at(0)->getPeername() + "; "
				"Num peers: " + std::to_string(workerSocketVec.size() ) );
		else
		if(pollRes == -1)
			throw WorkerException(std::string("Server: poll() failed. ") +
				"SysErr: " + std::strerror(errno) );

		// process the events that poll() returned

		int numEventsProcessed=0;

		for(unsigned i=0; (numEventsProcessed < pollRes) && (i < pollFDVec.size() ); i++)
		{
			if(!pollFDVec[i].revents)
				continue;

			// we have an event for this socket

			numEventsProcessed++;

			try
			{
				const uint64_t bytesLeft = transferBytesPerConn - transferredBytesVec[i];
				const size_t currentBlockSize = (bytesLeft < blockSize) ?
					bytesLeft : blockSize;

				// allow partial block read to not stall other sockets with available data

				ssize_t recvRes = workerSocketVec[i]->recvT(
					transferBuf.get(), currentBlockSize, 0, NETBENCH_RECEIVE_TIMEOUT_SEC);

				transferredBytesVec[i] += recvRes;

				atomicLiveOpsReadMix.numBytesDone += recvRes;

				// send single response byte after a complete block transfer
				if( ( (transferredBytesVec[i] % blockSize) == 0) ||
					(transferredBytesVec[i] == transferBytesPerConn) )
				{
					((*this).*funcPreWriteBlockModifier)(
						transferBuf.get(), gpuIOBufVec[0], respSize, transferredBytesVec[i] );

					workerSocketVec[i]->send(transferBuf.get(), respSize, 0);

					atomicLiveOps.numBytesDone += respSize;
					atomicLiveOpsReadMix.numIOPSDone++;
				}
			}
			catch(SocketDisconnectException& e)
			{
				if(transferredBytesVec[i] != transferBytesPerConn)
				{ // unexpected premature disconnect => probably ctrl+c
					LOGGER(Log_VERBOSE,"Server: Unexpected disconnect: " <<
							workerSocketVec[i]->getPeername() << "; " <<
						"Transferred bytes: " << transferredBytesVec[i] << "; " <<
						"Expected bytes: " << transferBytesPerConn << "; "
						"Message: " << e.what() << std::endl);

					transferredBytesVec.erase(transferredBytesVec.begin() + i);
					pollFDVec.erase(pollFDVec.begin() + i);
					workerSocketVec.erase(
						workerSocketVec.begin() + i); // destructor contains close()

					continue; // skip check below because elem [i] has been erased
				}
			}
			catch(SocketException& e)
			{ // avoid making noise if we have e.g. incomplete send() because of ctrl+c
				checkInterruptionRequest();
				throw;
			}

			if(transferredBytesVec[i] == transferBytesPerConn)
			{ // everything done with this connection
				LOGGER(Log_DEBUG,"Server: Transfer finished: " <<
					workerSocketVec[i]->getPeername() << std::endl);

				// (note: no sock shutdown() here because of possible infloop)

				transferredBytesVec.erase(transferredBytesVec.begin() + i);
				pollFDVec.erase(pollFDVec.begin() + i);
				workerSocketVec.erase(
					workerSocketVec.begin() + i); // destructor contains close()
			}

		} // end of poll() returned events loop
	} // end of while(serverSocketVec.size() )
}

/**
 * In netbench mode, this worker owns a single connection exclusively and keeps sending
 * "blocksize" to the server and waits "respsize" response after each block.
 */
void LocalWorker::netbenchDoTransferClient()
{
	const int pollShortTimeoutSecs = NETBENCH_SHORT_POLL_TIMEOUT_SEC; // to re-check for interrupt
	const uint64_t transferBytesPerConn = progArgs->getFileSize();
	const size_t blockSize = progArgs->getBlockSize();
	const size_t respSize = progArgs->getNetBenchRespSize();
	size_t transferBufSize = std::max(blockSize, respSize);
	std::unique_ptr<char[]> transferBuf(new char [transferBufSize] );

	// each loop is one blocksize transfer
	for(uint64_t transferredBytes = 0; transferredBytes != transferBytesPerConn; )
	{
		checkInterruptionRequest();

		try
		{
			const uint64_t bytesLeft = transferBytesPerConn - transferredBytes;
			const size_t currentBlockSize = (bytesLeft < blockSize) ?
				bytesLeft : blockSize;

			((*this).*funcRWRateLimiter)(currentBlockSize, isInterruptionRequested);

			std::chrono::steady_clock::time_point ioStartT = std::chrono::steady_clock::now();

			((*this).*funcPreWriteBlockModifier)(
				transferBuf.get(), gpuIOBufVec[0], currentBlockSize, transferredBytes);

			clientSocket->send(transferBuf.get(), currentBlockSize, 0);

			transferredBytes += currentBlockSize;

			// receive single response byte after a complete block transfer

			int recvRes = 0;

			for(int elapsedSecs=0;
				!recvRes && (elapsedSecs < NETBENCH_RECEIVE_TIMEOUT_SEC);
				elapsedSecs += pollShortTimeoutSecs)
			{
				// (this short loop exists to more quickly detect user interrupt requests)
				checkInterruptionRequest();

				try
				{
					recvRes = clientSocket->recvExactT(
						transferBuf.get(), respSize, 0, pollShortTimeoutSecs * 1000);

					atomicLiveOpsReadMix.numBytesDone += recvRes;
				}
				catch(SocketTimeoutException& e)
				{ /* ignore for pollShortTimeoutSecs. (NETBENCH_RECEIVE_TIMEOUT_SEC handled
						below.) */
				}
			}

			if(!recvRes)
				throw WorkerException("Client: Waiting for incoming data timed out. "
					"Peer: " + clientSocket->getPeername() + "; "
					"Transferred: " + std::to_string(transferredBytes) + " / " +
						std::to_string(transferBytesPerConn) + "; "
					"Timeout: " + std::to_string(NETBENCH_RECEIVE_TIMEOUT_SEC) + "s");

			// calc io operation latency
			std::chrono::steady_clock::time_point ioEndT = std::chrono::steady_clock::now();
			std::chrono::microseconds ioElapsedMicroSec =
				std::chrono::duration_cast<std::chrono::microseconds>
				(ioEndT - ioStartT);

			// iops lat & num done
			iopsLatHisto.addLatency(ioElapsedMicroSec.count() );
			atomicLiveOps.numBytesDone += currentBlockSize;
			atomicLiveOps.numIOPSDone++;
		}
		catch(SocketDisconnectException& e)
		{
			if(transferredBytes != transferBytesPerConn)
			{ // unexpected premature disconnect => probably ctrl+c
				LOGGER(Log_VERBOSE,"Client: Unexpected disconnect: " <<
					clientSocket->getPeername() << "; " <<
					"Transferred bytes: " << transferredBytes << "; " <<
					"Expected bytes: " << transferBytesPerConn << "; "
					"Message: " << e.what() << std::endl);

				break;
			}
		}
		catch(SocketException& e)
		{ // avoid making noise if we have e.g. incomplete send() because of ctrl+c
			checkInterruptionRequest();
			throw;
		}

		if(transferredBytes == transferBytesPerConn)
		{ // everything done with this connection
			LOGGER(Log_DEBUG,"Client: Transfer finished: " <<
				clientSocket->getPeername() << std::endl);

			// (note: no sock shutdown() here because of possible infloop)
		}

	} // end of for-loop for each block

}

/**
 * Calls the general sync() command to commit dirty pages from the linux page cache to stable
 * storage.
 *
 * Only the first worker of this instance does this, otherwise the kernel-level spinlocks of the
 * page cache make the sync extremely slow.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::anyModeSync()
{
	// don't do anything if this is not the first worker thread of this instance
	if(workerRank != progArgs->getRankOffset() )
	{
		workerGotPhaseWork = false;
		return;
	}

#ifndef SYNCFS_SUPPORT

		sync();

#else // SYNCFS_SUPPORT

	const IntVec& pathFDs = progArgs->getBenchPathFDs();
	const StringVec& pathVec = progArgs->getBenchPaths();

	for(size_t i=0; i < pathFDs.size(); i++)
	{
		// (workerRank offset is to let different workers sync different file systems in parallel)
		size_t currentIdx = (i + workerRank) % pathFDs.size();
		int currentFD = pathFDs[currentIdx];

		int syncRes = syncfs(currentFD);

		if(syncRes == -1)
			throw WorkerException(std::string("Cache sync failed. ") +
				"Path: " + pathVec[currentIdx] + "; "
				"SysErr: " + strerror(errno) );
	}

#endif // SYNCFS_SUPPORT
}

/**
 * Prints 3 to /proc/sys/vm/drop_caches to drop cached data from the Linux page cache.
 *
 * Only the first worker of this instance does this, otherwise the kernel-level spinlocks of the
 * page cache make the flush extremely slow.
 *
 * @throw WorkerException on error.
 */
void LocalWorker::anyModeDropCaches()
{
	// don't do anything if this is not the first worker thread of this instance
	if(workerRank != progArgs->getRankOffset() )
	{
		workerGotPhaseWork = false;
		return;
	}

	std::string dropCachesPath = "/proc/sys/vm/drop_caches";
	std::string dropCachesValStr = "3"; // "3" to drop page cache, dentries and inodes

	int fd = open(dropCachesPath.c_str(), O_WRONLY);

	if(fd == -1)
		throw WorkerException(std::string("Opening virtual drop_caches file failed. ") +
			"Path: " + dropCachesPath + "; "
			"SysErr: " + strerror(errno) );

	ssize_t writeRes = write(fd, dropCachesValStr.c_str(), dropCachesValStr.size() );

	if(writeRes == -1)
	{
		close(fd);

		throw WorkerException(std::string("Writing to cache drop command file failed. ") +
			"Path: " + dropCachesPath + "; "
			"SysErr: " + strerror(errno) );
	}

	close(fd);
}
