// SPDX-FileCopyrightText: 2020-2025 Sven Breuner and elbencho contributors
// SPDX-License-Identifier: GPL-3.0-only

#include <chrono>
#include <client_http.hpp>
#include <csignal>
#include <iostream>
#include <thread>

#include "Coordinator.h"
#include "Common.h"
#include "HTTPServiceSWS.h"
#include "HTTPServiceUWS.h"
#include "Logger.h"
#include "ProgArgs.h"
#include "ProgException.h"
#include "toolkits/S3Tk.h"
#include "toolkits/SignalTk.h"
#include "workers/WorkerException.h"
#include "workers/WorkersSharedData.h"


#define SIGINT_GRACEFUL_SHUTDOWN_TIME_SECS  5 // secs to wait in case of repeated interrupt sigs


/**
 * The entry point to start coordinating benchmarks.
 *
 * @return value is suitable to use as application main() return value, so 0 on success and non-zero
 * 		otherwise.
 */
int Coordinator::main()
{
	int retVal = EXIT_SUCCESS;

	try
	{
		SignalTk::registerFaultSignalHandlers(progArgs);

		// HTTP service mode
		// note: no other threads may be running when HTTPService.startServer() daemonizes.
		if(progArgs.getRunAsService() )
		{
			LoggerBase::enableErrHistory();

			if(progArgs.getUseAlternativeHTTPService() )
			{
				#ifndef ALTHTTPSVC_SUPPORT
					throw ProgException("This executable was built without support for alternative "
						"HTTP service.");
				#else
					HTTPServiceUWS(progArgs, workerManager, statistics).startServer();
				#endif // ALTHTTPSVC_SUPPORT
			}
			else
				HTTPServiceSWS(progArgs, workerManager, statistics).startServer();

			LOGGER(Log_VERBOSE, "HTTP service stopped." << std::endl);

			workerManager.interruptAndNotifyWorkers();
			goto joinall_and_exit;
		}

		S3Tk::initS3Global(&progArgs); // inits threads and thus after potential service daemonize

        waitForServicesReady(); // wait for service instances to become reachable

        workerManager.prepareThreads();

		/* register signal handlers for clean worker stop and stats print after ctrl+c. This is not
			done in service mode, because a service shall just quit on an interrupt signal. */
		registerInterruptSignalHandlers();

		// special preparations if we're running in master mode
		if(!progArgs.getHostsVec().empty() )
		{
			// check if we were just called to tell service hosts to interrupt
			if(progArgs.getInterruptServices() || progArgs.getQuitServices() )
			{
				// tell RemoteWorkers to terminate
				workerManager.startNextPhase(BenchPhase_TERMINATE);
				goto joinall_and_exit;
			}

			// check that services accepted given settings and things are consistent across services
			workerManager.checkServiceBenchPathInfos();
		}

		if(progArgs.hasUserRequestedDryRun() )
			statistics.printDryRunInfo();
		else
		{
			waitForUserDefinedStartTime();

			runBenchmarks();
		}

		// signal workers to self-terminate
		workerManager.startNextPhase(BenchPhase_TERMINATE);
	}
	catch(WorkerException& e)
	{
		/* WorkerExceptions in main thread are only for errors which have already been noticed and
		   logged by workers themselves, so we don't print anything here. */
	}
	catch(ProgInterruptedException& e)
	{ // interrupted by signal, e.g. user pressed ctrl+c
		Logger() << e.what() << std::endl;
		workerManager.interruptAndNotifyWorkers();
		retVal = EXIT_FAILURE;
	}
	catch(ProgTimeLimitException& e)
	{ // user-defined phase time limit expired
		Logger() << e.what() << std::endl;
		workerManager.interruptAndNotifyWorkers();
		// this is a user-defined time limit, not an error, so no retVal=EXIT_FAILURE here.
	}
	catch(ProgException& e)
	{
		ErrLogger(Log_NORMAL, false) << e.what() << std::endl;
		workerManager.interruptAndNotifyWorkers();
		retVal = EXIT_FAILURE;
	}

joinall_and_exit:

	workerManager.joinAllThreads();

	if(!LoggerBase::getErrHistory().empty() )
	{
		std::cerr << LoggerBase::getErrHistory();
		LoggerBase::clearErrHistory();
	}

	progArgs.resetBenchPath(); // important here to release shared s3 client before s3 sdk uninit

	S3Tk::uninitS3Global(&progArgs);

	if( (retVal != EXIT_SUCCESS) || workerManager.getNumWorkersDoneWithError() )
		return EXIT_FAILURE;

	return EXIT_SUCCESS;
}

/**
 * Wait for user-defined start time (if any).
 *
 * @throw ProgException if user-defined start time has already passed.
 */
void Coordinator::waitForUserDefinedStartTime()
{
	if(progArgs.getStartTime() && (time(NULL) > progArgs.getStartTime() ) )
		throw ProgException("Defined start time has already passed. Aborting.");

	statistics.printLiveCountdown();

	while(time(NULL) < progArgs.getStartTime() )
		usleep(1000);
}

/**
 * Wait for all given services to be reachable.
 * This is a no-op if service interrupt or service quit commands are given.
 */
void Coordinator::waitForServicesReady()
{
    const size_t maxWaitSecs = progArgs.getSvcReadyWaitSec();
    const size_t retryWaitMS = 1000; // 1 sec
    const std::chrono::steady_clock::time_point endWaitT =
        std::chrono::steady_clock::now() + std::chrono::seconds(maxWaitSecs);

    bool waitMsgLogged = false;
    std::string notReadyServiceHost; // hostname of a service that is not ready yet

    // check if we have to wait
    if(progArgs.getHostsVec().empty() || progArgs.getInterruptServices() ||
        progArgs.getQuitServices() || !progArgs.getSvcReadyWaitSec() )
        return;

    LOGGER(Log_DEBUG, "Starting services ready wait... " << std::endl);

    // loop until either all services are reachable or until max wait time exceeded
    for( ; ; )
    {
        notReadyServiceHost = ""; // will be set for logging when we encounter an unreachable svc

        for(const std::string& host : progArgs.getHostsVec() )
        {
            try
            {
                SimpleWeb::Client<SimpleWeb::HTTP> httpClient(host);
                httpClient.config.timeout = maxWaitSecs;
                httpClient.config.timeout_connect = maxWaitSecs;

                auto response = httpClient.request("GET", HTTPCLIENTPATH_STATUS);

                if(response->status_code == SimpleWeb::status_code(
                    SimpleWeb::StatusCode::success_ok) )
                    continue;
            }
            catch(std::exception& e) { }

            // this service is not yet ready
            notReadyServiceHost = host;
            break;
        }

        // check if all services are ready
        if(notReadyServiceHost.empty() )
            return;

        // check if max wait time exceeded
        if(std::chrono::steady_clock::now() >= endWaitT)
            throw ProgException("Timed out waiting for services to become ready. "
                "Unreachable service: " + notReadyServiceHost);

        if(!waitMsgLogged)
        {
            waitMsgLogged = true;
            LOGGER(Log_NORMAL, "NOTE: Waiting for services to become reachable... "
                "(Max wait time: " << maxWaitSecs << "s)" << std::endl);
        }

        // sleep before retry
        std::this_thread::sleep_for(std::chrono::milliseconds(retryWaitMS) );
    }
}

/**
 * Check if WorkersSharedData flags for interrupt signal or phase time limit are set.
 * This is intended to be called immediately before and after a benchmark phase.
 *
 * @throw ProgInterruptedException if interrupt flag set or phase timeout flag set.
 */
void Coordinator::checkInterruptionBetweenPhases()
{
	if(WorkersSharedData::gotUserInterruptSignalT)
		throw ProgInterruptedException("Terminating due to interrupt signal.");

	if(WorkersSharedData::isPhaseTimeExpired)
		throw ProgTimeLimitException("Terminating due to phase time limit.");
}

/**
 * Run coordinated benchmark through workers, print live stats and wait for workers to finish.
 *
 * @throw WorkerException on worker error; ProgException on main thread error.
 */
void Coordinator::runBenchmarkPhase(BenchPhase newBenchPhase)
{
	// don't start next phase if interrupted (e.g. by ctrl+c) or timeout for previous phase expired
	checkInterruptionBetweenPhases();

	workerManager.startNextPhase(newBenchPhase);

	statistics.printLiveStats();

	workerManager.waitForWorkersDone();

	if(!LoggerBase::getErrHistory().empty() )
	{
		std::cerr << LoggerBase::getErrHistory();
		LoggerBase::clearErrHistory();
	}

	// print stats
	statistics.printPhaseResults();

	workerManager.cleanupWorkersAfterPhaseDone();

	// check again for interrupted or timeout (might be last phase and we want to return an error)
	checkInterruptionBetweenPhases();
}

/**
 * Run sync and drop caches phases if selected by user.
 */
void Coordinator::runSyncAndDropCaches()
{
	// temporarily disable phase time limit, as sync and cache drop cannot be interrupted this way
	size_t oldTimeLimitTecs = progArgs.getTimeLimitSecs();
	progArgs.setTimeLimitSecs(0);

	if(progArgs.getRunSyncPhase() )
		runBenchmarkPhase(BenchPhase_SYNC);

	if(progArgs.getRunDropCachesPhase() )
		runBenchmarkPhase(BenchPhase_DROPCACHES);

	// restore user-defined phase time limit
	progArgs.setTimeLimitSecs(oldTimeLimitTecs);
}

/**
 * Run coordinated benchmarks through workers according to user selection.
 *
 * @throw WorkerException on worker error; ProgException on main thread error.
 */
void Coordinator::runBenchmarks()
{
	struct BenchPhaseConfig
	{
	    BenchPhase benchPhase;
	    bool runPhase;
	};

	// array of all existing bench phases
	/* note: multiple phases can be selected for a single run and this array defines the order in
		which they run, so make sure to have reasonable ordering here (e.g. creates before
		deletes). */
	std::array allBenchPhasesArray
	{
		BenchPhaseConfig { BenchPhase_CREATEDIRS, progArgs.getRunCreateDirsPhase() },
        BenchPhaseConfig { BenchPhase_PUTBUCKETACL, progArgs.getRunS3BucketAclPut() },
        BenchPhaseConfig { BenchPhase_PUT_S3_BUCKET_MD, progArgs.getRunS3PutBucketMetadata() },
		BenchPhaseConfig { BenchPhase_STATDIRS, progArgs.getRunS3StatDirs() },
        BenchPhaseConfig { BenchPhase_GET_S3_BUCKET_MD, progArgs.getRunS3GetBucketMetadata() },
		BenchPhaseConfig { BenchPhase_CREATEFILES, progArgs.getRunCreateFilesPhase() },
        BenchPhaseConfig { BenchPhase_S3MPUCOMPLETE, progArgs.getRunS3MPUSharingCompletionPhase() },
        BenchPhaseConfig { BenchPhase_PUTOBJACL, progArgs.getRunS3AclPut() },
        BenchPhaseConfig { BenchPhase_PUT_S3_OBJECT_MD, progArgs.getRunS3PutObjectMetadata() },
		BenchPhaseConfig { BenchPhase_STATFILES, progArgs.getRunStatFilesPhase() },
        BenchPhaseConfig { BenchPhase_GET_S3_OBJECT_MD, progArgs.getRunS3GetObjectMetadata() },
		BenchPhaseConfig { BenchPhase_GETOBJACL, progArgs.getRunS3AclGet() },
		BenchPhaseConfig { BenchPhase_LISTOBJECTS, progArgs.getRunListObjPhase() },
		BenchPhaseConfig { BenchPhase_LISTOBJPARALLEL, progArgs.getRunListObjParallelPhase() },
		BenchPhaseConfig { BenchPhase_READFILES, progArgs.getRunReadPhase() },
        BenchPhaseConfig { BenchPhase_DEL_S3_OBJECT_MD, progArgs.getRunS3DelObjectMetadata() },
        BenchPhaseConfig { BenchPhase_MULTIDELOBJ, progArgs.getRunMultiDelObjPhase() },
		BenchPhaseConfig { BenchPhase_DELETEFILES, progArgs.getRunDeleteFilesPhase() },
        BenchPhaseConfig { BenchPhase_DEL_S3_BUCKET_MD, progArgs.getRunS3DelBucketMetadata() },
        BenchPhaseConfig { BenchPhase_GETBUCKETACL, progArgs.getRunS3BucketAclGet() },
		BenchPhaseConfig { BenchPhase_DELETEDIRS, progArgs.getRunDeleteDirsPhase() },
	};

	// vector of enabled bench phases

	std::vector<BenchPhase> enabledBenchPhasesVec;

	for(BenchPhaseConfig benchPhaseConfig : allBenchPhasesArray)
	{
		if(benchPhaseConfig.runPhase)
			enabledBenchPhasesVec.push_back(benchPhaseConfig.benchPhase);
	}


	for(size_t iterationIndex = 0; iterationIndex < progArgs.getIterations(); iterationIndex++)
	{
        if(progArgs.getIterations() > 1)
            LOGGER(Log_NORMAL, "[Starting iteration " <<
                (iterationIndex+1) << " of " << progArgs.getIterations() << "...]" << std::endl);

		statistics.printPhaseResultsTableHeader();

		runSyncAndDropCaches();

		for(unsigned benchPhaseIdx=0; benchPhaseIdx < enabledBenchPhasesVec.size(); benchPhaseIdx++)
		{
			// run actual test phase
			runBenchmarkPhase(enabledBenchPhasesVec[benchPhaseIdx] );

			// run special "sync" and "drop caches" phases
			runSyncAndDropCaches();

			if(benchPhaseIdx < (enabledBenchPhasesVec.size() - 1) )
			{
				// delay between phases
				if(progArgs.getNextPhaseDelaySecs() )
					sleep(progArgs.getNextPhaseDelaySecs() );

				// rotate hosts (which requires new prep phase for ranks)
				rotateHosts();
			}
		}
	}
}

/**
 * Stop and restart all workers after inter-phase hosts rotation. This is necessary because we need
 * to run the prep phase again to update the worker ranks.
 *
 * This is a no-op if the user didn't request hosts rotation.
 */
void Coordinator::rotateHosts()
{
	if(progArgs.getHostsVec().empty() ||
		!progArgs.getRotateHostsNum() ||
		(progArgs.getBenchMode() == BenchMode_NETBENCH) )
		return;

	workerManager.interruptAndNotifyWorkers();
	workerManager.joinAllThreads();
	workerManager.cleanupWorkersAfterPhaseDone();
	workerManager.deleteThreads();

	if(!LoggerBase::getErrHistory().empty() )
	{
		std::cerr << LoggerBase::getErrHistory();
		LoggerBase::clearErrHistory();
	}

	progArgs.rotateHosts();

	/* note: workerManager.prepareThreads() blocks signal interrupt signals for spawned threads
		so we don't need to reset signal handlers here. */

	workerManager.prepareThreads();
}

/**
 * Set interrupted flag as a friendly ask for coordinator/workers to self-terminate. On repeated
 * interrupt signals, we wait for a grace period before actually interrupting before resetting to
 * default signal handler, so that next attempt to interrupt will be successful if friendly way
 * did not work out.
 *
 * The wait time is because we want to give the components a chance to self-terminate gracefully
 * by sending stop requests to remove service instances and by switching out of live stats
 * full-screen mode.
 */
void Coordinator::handleInterruptSignal(int signal)
{
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);

    if(!WorkersSharedData::gotUserInterruptSignalT)
    { // first interrupt signal => set timestamp to engage graceful shutdown

        WorkersSharedData::gotUserInterruptSignalT = ts.tv_sec;

        return;
    }

    // repeated interrupt signal => check if enough time has passed to engage graceful shutdown

    uint32_t timeSinceFirstInterrupt = ts.tv_sec - WorkersSharedData::gotUserInterruptSignalT;

    if(timeSinceFirstInterrupt > SIGINT_GRACEFUL_SHUTDOWN_TIME_SECS)
    {
        // reset signal handler to default, so that next interrupt request will definitely interrupt
        std::signal(signal, SIG_DFL);
    }
}

/**
 * Register handler of SIGINT/SIGTERM.
 *
 * Note: This method is "public" because Statistics.cpp needs to call it after ftxui has been
 * 	initialized.
 */
void Coordinator::registerInterruptSignalHandlers()
{
	std::signal(SIGINT, handleInterruptSignal);
	std::signal(SIGTERM, handleInterruptSignal);
}



