haiku.git

/*
 * Copyright 2017-2025, Andrew Lindesay <apl@lindesay.co.nz>.
 * All rights reserved. Distributed under the terms of the MIT License.
 */


#include "AbstractServerProcess.h"

#include <errno.h>
#include <math.h>
#include <string.h>
#include <unistd.h>

#include <AutoDeleter.h>
#include <BufferedDataIO.h>
#include <File.h>
#include <FileIO.h>
#include <HttpTime.h>
#include <UrlProtocolRoster.h>

#include <support/ZlibCompressionAlgorithm.h>

#include "DataIOUtils.h"
#include "HaikuDepotConstants.h"
#include "Logger.h"
#include "ServerHelper.h"
#include "ServerSettings.h"
#include "StandardMetaDataJsonEventListener.h"
#include "StorageUtils.h"


using namespace BPrivate::Network;


#define MAX_REDIRECTS 3
#define MAX_FAILURES 2


// 30 seconds
#define TIMEOUT_MICROSECONDS 3e+7


static const size_t kFileBufferSize = 10 * 1024;

static const bigtime_t kServerUnavailableRetryDelayMinimum = 1000 * 1000 * 5;
	// 5 seconds
static const bigtime_t kServerUnavailableRetryDelayMaximum = 1000 * 1000 * 120;
	// 120 seconds
static const bigtime_t kServerUnavailableRetryDelayDefault = 1000 * 1000 * 10;
	// 10 seconds

// #pragma mark - Interface ReadProgressDataIO


class ReadProgressDataIO : public BDataIO
{
public:
								ReadProgressDataIO(AbstractServerProcess* serverProcess,
									BDataIO* delegate, size_t bytesTotal);
	virtual						~ReadProgressDataIO();

	virtual	ssize_t				Read(void* buffer, size_t size);
	virtual	ssize_t				Write(const void* buffer, size_t size);

	virtual	status_t			Flush();

private:
			AbstractServerProcess*
								fServerProcess;
			BDataIO*			fDelegate;
			size_t				fBytesRead;
			size_t				fBytesTotal;
};


// #pragma mark - Interface LoggingUrlProtocolListener


class LoggingUrlProtocolListener : public BUrlProtocolListener
{
public:
								LoggingUrlProtocolListener(AbstractServerProcess* serverProcess,
									BString traceLoggingIdentifier, bool traceLogging);
	virtual						~LoggingUrlProtocolListener();

			size_t				ContentLength();

	virtual	void				DownloadProgress(BUrlRequest* caller,
									off_t bytesReceived, off_t bytesTotal);
	virtual	void				BytesWritten(BUrlRequest* caller,
									size_t bytesWritten);
	virtual	void				DebugMessage(BUrlRequest* caller,
									BUrlProtocolDebugMessage type,
									const char* text);

private:
			AbstractServerProcess*
								fServerProcess;
			bool				fTraceLogging;
			BString				fTraceLoggingIdentifier;
			size_t				fContentLength;
};


// #pragma mark - Implementation for ReadProgressDataIO


ReadProgressDataIO::ReadProgressDataIO(AbstractServerProcess* serverProcess, BDataIO* delegate,
	size_t bytesTotal)
	:
	fServerProcess(serverProcess),
	fDelegate(delegate),
	fBytesRead(0),
	fBytesTotal(bytesTotal)
{
}


ReadProgressDataIO::~ReadProgressDataIO()
{
}


ssize_t
ReadProgressDataIO::Read(void* buffer, size_t size)
{
	ssize_t actualRead = fDelegate->Read(buffer, size);

	if (actualRead > 0) {
		fBytesRead += actualRead;

		if (fBytesTotal > 0.0f) {
			float progress = static_cast<float>(fBytesRead) / static_cast<float>(fBytesTotal);
			fServerProcess->SetDataProcessingProgress(progress);
		}
	}

	return actualRead;
}


ssize_t
ReadProgressDataIO::Write(const void* buffer, size_t size)
{
	return B_NOT_SUPPORTED;
}


status_t
ReadProgressDataIO::Flush()
{
	return B_OK;
}


// #pragma mark - Implementation for LoggingUrlProtocolListener


LoggingUrlProtocolListener::LoggingUrlProtocolListener(AbstractServerProcess* serverProcess,
	BString traceLoggingIdentifier, bool traceLogging)
	:
	fServerProcess(serverProcess),
	fTraceLogging(traceLogging),
	fTraceLoggingIdentifier(traceLoggingIdentifier),
	fContentLength(0)
{
}


LoggingUrlProtocolListener::~LoggingUrlProtocolListener()
{
}


void
LoggingUrlProtocolListener::BytesWritten(BUrlRequest* caller, size_t bytesWritten)
{
	fContentLength += bytesWritten;
}


void
LoggingUrlProtocolListener::DownloadProgress(BUrlRequest* caller, off_t bytesReceived,
	off_t bytesTotal)
{
	float progress = static_cast<float>(bytesReceived) / static_cast<float>(bytesTotal);
	fServerProcess->SetDownloadProgress(progress);
}


void
LoggingUrlProtocolListener::DebugMessage(BUrlRequest* caller, BUrlProtocolDebugMessage type,
	const char* text)
{
	HDTRACE("url->file <%s>; %s", fTraceLoggingIdentifier.String(), text);
}


size_t
LoggingUrlProtocolListener::ContentLength()
{
	return fContentLength;
}


// #pragma mark - AbstractServerProcess


AbstractServerProcess::AbstractServerProcess(uint32 options)
	:
	AbstractProcess(),
	fOptions(options),
	fRequest(NULL),
	fDownloadProgress(kProgressIndeterminate),
	fDataProcessingProgress(kProgressIndeterminate)
{
}


AbstractServerProcess::~AbstractServerProcess()
{
}


float
AbstractServerProcess::Progress()
{
	if (fDataProcessingProgress >= 0.0f)
		return 0.5f + (0.5f * fminf(fDataProcessingProgress, 1.0));
	if (fDownloadProgress >= 0.0f)
		return 0.5f * fminf(fDownloadProgress, 1.0);
	return kProgressIndeterminate;
}


void
AbstractServerProcess::SetDownloadProgress(float value)
{
	if (fDownloadProgress == 1.0 || (!_ShouldProcessProgress() && value != 1.0))
		return;

	if (fDataProcessingProgress < 0.0f)
		fDownloadProgress = value;
	else
		fDownloadProgress = 1.0;

	_NotifyChanged();
}


void
AbstractServerProcess::SetDataProcessingProgress(float value)
{
	if (fDataProcessingProgress == 1.0 || (!_ShouldProcessProgress() && value != 1.0))
		return;

	fDownloadProgress = 1.0;
	fDataProcessingProgress = value;
	_NotifyChanged();
}


bool
AbstractServerProcess::HasOption(uint32 flag)
{
	return (fOptions & flag) == flag;
}


bool
AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready)
{
	return !HasOption(SERVER_PROCESS_NO_NETWORKING)
		&& !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready);
}


status_t
AbstractServerProcess::StopInternal()
{
	if (fRequest != NULL)
		return fRequest->Stop();

	return AbstractProcess::StopInternal();
}


status_t
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue)
{
	BPath metaDataPath;
	BString jsonPath;

	status_t result = GetStandardMetaDataPath(metaDataPath);

	if (result != B_OK)
		return result;

	GetStandardMetaDataJsonPath(jsonPath);

	return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath);
}


status_t
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue, const BPath& metaDataPath,
	const BString& jsonPath)
{
	headerValue.SetTo("");
	struct stat s;

	if (-1 == stat(metaDataPath.Path(), &s)) {
		if (ENOENT != errno)
			return B_ERROR;

		return B_ENTRY_NOT_FOUND;
	}

	if (s.st_size == 0)
		return B_BAD_VALUE;

	StandardMetaData metaData;
	status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath);

	if (result == B_OK) {
		SetIfModifiedSinceHeaderValueFromMetaData(headerValue, metaData);
	} else {
		HDERROR("unable to parse the meta-data date and time from [%s] - cannot set the "
				"'If-Modified-Since' header",
			metaDataPath.Path());
	}

	return result;
}


/*static*/ void
AbstractServerProcess::SetIfModifiedSinceHeaderValueFromMetaData(BString& headerValue,
	const StandardMetaData& metaData)
{
	// An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
	BDateTime modifiedDateTime = metaData.GetDataModifiedTimestampAsDateTime();
	BHttpTime modifiedHttpTime(modifiedDateTime);
	headerValue.SetTo(modifiedHttpTime.ToString(B_HTTP_TIME_FORMAT_COOKIE));
}


status_t
AbstractServerProcess::PopulateMetaData(StandardMetaData& metaData, const BPath& path,
	const BString& jsonPath)
{
	StandardMetaDataJsonEventListener listener(jsonPath, metaData);
	status_t result = ParseJsonFromFileWithListener(&listener, path);

	if (result != B_OK)
		return result;

	result = listener.ErrorStatus();

	if (result != B_OK)
		return result;

	if (!metaData.IsPopulated()) {
		HDERROR("the meta data was read from [%s], but no values were extracted", path.Path());
		return B_BAD_DATA;
	}

	return B_OK;
}


/* static */ bool
AbstractServerProcess::_LooksLikeGzip(const char* pathStr)
{
	int l = strlen(pathStr);
	return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3);
}


/*!	Note that a B_OK return code from this method may not indicate that the
	listening process went well.  One has to see if there was an error in
	the listener.
*/

status_t
AbstractServerProcess::ParseJsonFromFileWithListener(BJsonEventListener* listener,
	const BPath& path)
{
	const char* pathStr = path.Path();

	bool fileExists = false;
	off_t fileSize = 0;
	status_t existsResult = StorageUtils::ExistsObject(path, &fileExists, NULL, &fileSize);

	if (existsResult != B_OK) {
		HDERROR("[%s] unable to get the length of the data file [%s]", Name(), pathStr);
		return existsResult;
	}

	if (!fileExists) {
		HDERROR("[%s] missing data file [%s]", Name(), pathStr);
		return B_ENTRY_NOT_FOUND;
	}

	if (fileSize == 0) {
		HDERROR("[%s] empty data file [%s]", Name(), pathStr);
		return B_BAD_DATA;
	}

	FILE* file = fopen(pathStr, "rb");

	if (file == NULL) {
		HDERROR("[%s] unable to find the data file at [%s]", Name(), pathStr);
		return B_ENTRY_NOT_FOUND;
	}

	BFileIO rawInput(file, true); // takes ownership
	ReadProgressDataIO readProgress(this, &rawInput, fileSize);
	BDataIO* bufferedRawInput = new BBufferedDataIO(readProgress, kFileBufferSize, false, true);
	ObjectDeleter<BDataIO> bufferedRawInputDeleter(bufferedRawInput);

	// if the file extension ends with '.gz' then the data will be
	// compressed and the algorithm needs to decompress the data as
	// it is parsed.

	if (_LooksLikeGzip(pathStr)) {
		BDataIO* gzDecompressedInput = NULL;
		BZlibDecompressionParameters* zlibDecompressionParameters
			= new BZlibDecompressionParameters();

		status_t result = BZlibCompressionAlgorithm().CreateDecompressingInputStream(
			bufferedRawInput, zlibDecompressionParameters, gzDecompressedInput);

		if (B_OK != result)
			return result;

		ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput);
		BPrivate::BJson::Parse(gzDecompressedInput, listener);
	} else {
		BPrivate::BJson::Parse(bufferedRawInput, listener);
	}

	return B_OK;
}


/*!	In order to reduce the chance of failure half way through downloading a
	large file, this method will download the file to a temporary file and
	then it can rename the file to the final target file.
*/

status_t
AbstractServerProcess::DownloadToLocalFileAtomically(const BPath& targetFilePath, const BUrl& url)
{
	BPath temporaryFilePath(tmpnam(NULL), NULL, true);
	status_t result = DownloadToLocalFile(temporaryFilePath, url, 0, 0);

	// if the data is coming in as .gz, but is not stored as .gz then
	// the data should be decompressed in the temporary file location
	// before being shifted into place.

	if (result == B_OK && _LooksLikeGzip(url.Path()) && !_LooksLikeGzip(targetFilePath.Path()))
		result = _DeGzipInSitu(temporaryFilePath);

	// not copying if the data has not changed because the data will be
	// zero length.  This is if the result is APP_ERR_NOT_MODIFIED.

	if (result == B_OK) {
		// if the file is zero length then assume that something has
		// gone wrong.
		off_t size;
		bool hasFile;

		result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL, &size);

		if (result == B_OK && hasFile && size > 0) {
			if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) {
				HDINFO("[%s] did rename [%s] --> [%s]", Name(), temporaryFilePath.Path(),
					targetFilePath.Path());
				result = B_IO_ERROR;
			}
		}
	}

	if (IsSuccess(result))
		SetDownloadProgress(1.0);

	return result;
}


/*static*/ status_t
AbstractServerProcess::_DeGzipInSitu(const BPath& path)
{
	const char* tmpPath = tmpnam(NULL);
	status_t result = B_OK;

	{
		BFile file(path.Path(), O_RDONLY);
		BFile tmpFile(tmpPath, O_WRONLY | O_CREAT);

		BDataIO* gzDecompressedInput = NULL;
		BZlibDecompressionParameters* zlibDecompressionParameters
			= new BZlibDecompressionParameters();

		result = BZlibCompressionAlgorithm().CreateDecompressingInputStream(&file,
			zlibDecompressionParameters, gzDecompressedInput);

		if (result == B_OK) {
			ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput);
			result = DataIOUtils::CopyAll(&tmpFile, gzDecompressedInput);
		}
	}

	if (result == B_OK) {
		if (rename(tmpPath, path.Path()) != 0) {
			HDERROR("unable to move the uncompressed data into place");
			result = B_ERROR;
		}
	} else {
		HDERROR("it was not possible to decompress the data");
	}

	return result;
}


/*!	This method will return the time that the client should wait before trying to call the server
	again. If the HTTP response has a `Retry-After` header then this is observed (within some
	bound) and otherwise sensible defaults are employed.
*/
/*static*/ bigtime_t
AbstractServerProcess::_ServerUnavailableRetryDelay(const BHttpHeaders& responseHeaders)
{
	const char* retryAfterC = responseHeaders["Retry-After"];
	bigtime_t result = kServerUnavailableRetryDelayDefault;

	if (retryAfterC != NULL) {
		// The HDS server will only ever use the numerical delay in seconds.

		bool allDigits = true;

		for (int i = strlen(retryAfterC) - 1; i >= 0 && allDigits; i--) {
			if (!isdigit(retryAfterC[i]))
				allDigits = false;
		}

		if (allDigits)
			result = static_cast<bigtime_t>(atoi(retryAfterC)) * 1000 * 1000;
		else
			HDERROR("`Retry-After` header [%s] unable to be handled", retryAfterC);
	}

	if (result < kServerUnavailableRetryDelayMinimum)
		result = kServerUnavailableRetryDelayMinimum;

	if (result > kServerUnavailableRetryDelayMaximum)
		result = kServerUnavailableRetryDelayMaximum;

	return result;
}


status_t
AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, const BUrl& url,
	uint32 redirects, uint32 failures)
{
	if (WasStopped())
		return B_CANCELED;

	if (redirects > MAX_REDIRECTS) {
		HDINFO("[%s] exceeded %d redirects --> failure", Name(), MAX_REDIRECTS);
		return B_IO_ERROR;
	}

	if (failures > MAX_FAILURES) {
		HDINFO("[%s] exceeded %d failures", Name(), MAX_FAILURES);
		return B_IO_ERROR;
	}

	HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(),
		targetFilePath.Path());

	LoggingUrlProtocolListener listener(this, Name(), Logger::IsTraceEnabled());
	BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT);
	status_t err = targetFile.InitCheck();
	if (err != B_OK)
		return err;

	BHttpHeaders headers;
	ServerSettings::AugmentHeaders(headers);

	BString ifModifiedSinceHeader;
	status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue(ifModifiedSinceHeader);

	if (ifModifiedSinceHeaderStatus == B_OK && ifModifiedSinceHeader.Length() > 0)
		headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);

	thread_id thread;

	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &targetFile, &listener);
	if (request == NULL)
		return B_NO_MEMORY;

	fRequest = dynamic_cast<BHttpRequest*>(request);
	if (fRequest == NULL) {
		delete request;
		return B_ERROR;
	}

	fRequest->SetHeaders(headers);
	fRequest->SetMaxRedirections(0);
	fRequest->SetTimeout(TIMEOUT_MICROSECONDS);
	fRequest->SetStopOnError(false);
		// without this we don't get headers on 5xx HTTP responses.
	thread = fRequest->Run();

	wait_for_thread(thread, NULL);

	const BHttpResult& result = dynamic_cast<const BHttpResult&>(fRequest->Result());
	int32 statusCode = result.StatusCode();
	const BHttpHeaders responseHeaders = result.Headers();
	const char* locationC = responseHeaders["Location"];
	BString location;

	if (locationC != NULL)
		location.SetTo(locationC);

	delete fRequest;
	fRequest = NULL;

	if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
		HDINFO("[%s] did complete streaming data [%" B_PRIdSSIZE " bytes]", Name(),
			listener.ContentLength());
		return B_OK;
	}

	if (responseHeaders.CountHeaders() == 0)
		HDTRACE("no headers returned in response");

	if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
		if (location.Length() != 0) {
			BUrl redirectUrl(result.Url(), location);
			HDINFO("[%s] will redirect to; %s", Name(), redirectUrl.UrlString().String());
			return DownloadToLocalFile(targetFilePath, redirectUrl, redirects + 1, 0);
		}

		HDERROR("[%s] unable to find 'Location' header for redirect", Name());
		return B_IO_ERROR;
	}

	switch (statusCode) {
		case B_HTTP_STATUS_NOT_MODIFIED:
			HDINFO("[%s] remote data has not changed since [%s] so was not downloaded", Name(),
				ifModifiedSinceHeader.String());
			return HD_ERR_NOT_MODIFIED;
		case B_HTTP_STATUS_PRECONDITION_FAILED:
			ServerHelper::NotifyClientTooOld(responseHeaders);
			return HD_CLIENT_TOO_OLD;
		case B_HTTP_STATUS_SERVICE_UNAVAILABLE:
		{
			// 503 is special; it means that the server is temporarily unable to respond and
			// maybe able to in the near future. The client should observe the `Retry-After`
			// header and try again soon.
			bigtime_t delay = _ServerUnavailableRetryDelay(responseHeaders);
			HDINFO("[%" B_PRId32 "] response from server; will delay for %" B_PRId32 " seconds",
				statusCode, static_cast<int32>(delay / (1000 * 1000)));
			snooze(delay);
			return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1);
		}
		default:
			if (statusCode == 0 || (statusCode / 100) == 5) {
				HDERROR("error response from server [%" B_PRId32 "] --> retry...", statusCode);
				return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1);
			}

			HDERROR("[%s] unexpected response from server [%" B_PRId32 "]", Name(), statusCode);
			return B_IO_ERROR;
	}
}


status_t
AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath)
{
	if (0 == remove(currentFilePath.Path()))
		return B_OK;

	return B_IO_ERROR;
}


/*!	When a file is damaged or corrupted in some way, the file should be 'moved
	aside' so that it is not involved in the next update.  This method will
	create such an alternative 'damaged' file location and move this file to
	that location.
*/

status_t
AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath)
{
	BPath damagedFilePath;
	BString damagedLeaf;

	damagedLeaf.SetToFormat("%s__damaged", currentFilePath.Leaf());
	currentFilePath.GetParent(&damagedFilePath);
	damagedFilePath.Append(damagedLeaf.String());

	if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) {
		HDERROR("[%s] unable to move damaged file [%s] aside to [%s]", Name(),
			currentFilePath.Path(), damagedFilePath.Path());
		return B_IO_ERROR;
	}

	HDINFO("[%s] did move damaged file [%s] aside to [%s]", Name(), currentFilePath.Path(),
		damagedFilePath.Path());

	return B_OK;
}


bool
AbstractServerProcess::IsSuccess(status_t e)
{
	return e == B_OK || e == HD_ERR_NOT_MODIFIED;
}