From 6cf03ad3dc98fe96893aec41b91b8f92c0a82e93 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 Aug 2021 10:10:58 -0700 Subject: Initial draft of FileStream --- core/CMakeLists.txt | 3 +- core/src/Utils/IO/FileStream.cpp | 248 +++++++++++++++++++++++++++++++++++++++ core/src/Utils/IO/FileStream.hpp | 86 ++++++++++++++ core/src/Utils/IO/fwd.hpp | 4 + 4 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 core/src/Utils/IO/FileStream.cpp create mode 100644 core/src/Utils/IO/FileStream.hpp (limited to 'core') diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 535c05b..346713d 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -61,7 +61,7 @@ add_source_group(Model.Workflow add_source_group(Model.Workflow.Nodes src/Model/Workflow/Nodes/DocumentNodes.cpp src/Model/Workflow/Nodes/UserInputNodes.cpp - src/Model/Workflow/Nodes/NumericNodes.cpp#include "DataStream.hpp" + src/Model/Workflow/Nodes/NumericNodes.cpp src/Model/Workflow/Nodes/TextNodes.cpp ) @@ -85,6 +85,7 @@ list(APPEND CPLT_CORE_SOURCES # UI add_source_group(Utils src/Utils/IO/Archive.cpp src/Utils/IO/DataStream.cpp + src/Utils/IO/FileStream.cpp src/Utils/Sigslot.cpp src/Utils/StandardDirectories.cpp src/Utils/Time.cpp diff --git a/core/src/Utils/IO/FileStream.cpp b/core/src/Utils/IO/FileStream.cpp new file mode 100644 index 0000000..bc95b7e --- /dev/null +++ b/core/src/Utils/IO/FileStream.cpp @@ -0,0 +1,248 @@ +#include "FileStream.hpp" + +#include +#include + +#if PLATFORM_WIN32 + +// TODO + +#elif PLATFORM_MACOS || PLATFORM_LINUX +# include +# include +# include +# include + +InputFileStream::InputFileStream(const std::filesystem::path& path) + : mOsFileHandle{ 0 } +{ + auto fd = reinterpret_cast(mOsFileHandle); + *fd = open(path.c_str(), O_RDONLY); +} + +InputFileStream::~InputFileStream() +{ + auto fd = reinterpret_cast(mOsFileHandle); + close(*fd); +} + +OutputFileStream::OutputFileStream(const std::filesystem::path& path, WriteMode mode) + : mOsFileHandle{ 0 } +{ + auto fd = reinterpret_cast(mOsFileHandle); + + int flags = O_WRONLY | O_CREAT; + switch (mode) { + case AppendFile: flags |= O_APPEND; break; + case TruncateFile: flags |= O_TRUNC; break; + } + + *fd = open(path.c_str(), flags, 0644); +} + +OutputFileStream::~OutputFileStream() +{ + auto fd = reinterpret_cast(mOsFileHandle); + close(*fd); +} + +static IoResult::ErrorKind MapErrnoToIoResult(int err) +{ + switch (err) { + // TODO + case EFAULT: return IoResult::ERR_UnexpectedEof; + case EPERM: return IoResult::ERR_PermissionDenied; + case ENOSPC: return IoResult::ERR_OutOfSpace; + case EIO: return IoResult::ERR_Other; + + default: + std::cerr << "Unimplemented POSIX errno " << err << ", report bug immediately.\n"; + std::abort(); + } +} + +static IoResult ReadBytesDirect(const char* osFileHandle, size_t byteCount, std::byte* bytes) +{ + int fd = *reinterpret_cast(osFileHandle); + int status = read(fd, bytes, byteCount); + + if (status == -1) { + int err = errno; + return IoResult{ + .Error = ::MapErrnoToIoResult(err), + .SystemError = err, + .BytesMoved = 0, + }; + } else { + return IoResult{ + .Error = IoResult::ERR_None, + .SystemError = 0, + .BytesMoved = (size_t)status, // Equal to number of bytes read + }; + } +} + +static IoResult WriteBytesDirect(const char* osFileHandle, size_t byteCount, const std::byte* bytes) +{ + int fd = *reinterpret_cast(osFileHandle); + int status = write(fd, bytes, byteCount); + + if (status == -1) { + int err = errno; + return IoResult{ + .Error = ::MapErrnoToIoResult(err), + .SystemError = err, + .BytesMoved = 0, + }; + } else { + return IoResult{ + .Error = IoResult::ERR_None, + .SystemError = 0, + .BytesMoved = (size_t)status, // Equal to number of bytes read + }; + } +} + +#else +# error "Unsupported target platform." +#endif + +int InputFileStream::GetReadInSize() const +{ + return mReadInSize; +} + +void InputFileStream::SetReadInSize(int size) +{ + if (size > mReadInSize) { + mReadInSize = size; + mBuffer = std::make_unique(size); + } +} + +bool InputFileStream::IsEof() const +{ + return mEof; +} + +IoResult InputFileStream::ReadBytes(size_t bufferLength, std::byte* buffer) +{ + // TODO reduce duplicated code + + auto bytesMoved = std::min(mAvailableBytes, bufferLength); + + // On first call after construction, mFirstByteIdx will equal to mReadInSize, i.e. bytesAvailable == 0 + // and this call to std::memcpy will be no-op + std::memcpy(buffer, &mBuffer[mFirstByteIdx], bytesMoved); + mFirstByteIdx += (int)bytesMoved; + mAvailableBytes -= (int)bytesMoved; + buffer += bytesMoved; + + size_t bytesLeft = bufferLength - bytesMoved; + if (bytesLeft > mReadInSize) { + // Our buffer can't handle rest of the request, just skip the buffering step + + // Read rest of the data into buffer + { + auto result = ::ReadBytesDirect(mOsFileHandle, bytesLeft, buffer); + bytesMoved += result.BytesMoved; + + if (result.Error == IoResult::ERR_None) { + if (result.BytesMoved < mReadInSize) { + mEof = true; + } + } else { + goto end; + } + } + + // Refill our buffer + { + auto result = ::ReadBytesDirect(mOsFileHandle, mReadInSize, mBuffer.get()); + mFirstByteIdx = 0; + mAvailableBytes = (int)result.BytesMoved; + + if (result.Error == IoResult::ERR_None) { + if (result.BytesMoved < mReadInSize) { + mEof = true; + } + } else { + goto end; + } + } + } else if (bytesLeft > 0) { + // Our buffer can handle rest of the request, first buffer than supply the requested data + + // Refill our buffer + { + auto result = ::ReadBytesDirect(mOsFileHandle, mReadInSize, mBuffer.get()); + mFirstByteIdx = 0; + mAvailableBytes = (int)result.BytesMoved; + + if (result.Error == IoResult::ERR_None) { + if (result.BytesMoved < mReadInSize) { + mEof = true; + } + } else { + goto end; + } + } + + // Copy data into buffer + { + std::memcpy(buffer, &mBuffer[mFirstByteIdx], bytesLeft); + mFirstByteIdx += (int)bytesLeft; + bytesMoved += bytesLeft; + buffer += bytesLeft; + } + } else { + // Request completed already + } + +end: + return IoResult{ + .Error = IoResult::ERR_None, + .SystemError = 0, + .BytesMoved = bytesMoved, + }; +} + +int OutputFileStream::GetMaxBufferSize() const +{ + return mMaxBufferSize; +} + +void OutputFileStream::SetMaxBufferSize(int maxSize) +{ + FlushBuffer(); + if (maxSize > mMaxBufferSize) { + mMaxBufferSize = maxSize; + mBuffer = std::make_unique(maxSize); + } +} + +IoResult OutputFileStream::WriteBytes(size_t bufferLength, const std::byte* buffer) +{ + if (bufferLength + mCurrentBufferSize > mMaxBufferSize) { + FlushBuffer(); + + if (bufferLength > mMaxBufferSize) { + return ::WriteBytesDirect(mOsFileHandle, bufferLength, buffer); + } + } + + std::memcpy(mBuffer.get() + mCurrentBufferSize, buffer, bufferLength); + mCurrentBufferSize += (int)bufferLength; + + return IoResult{ + .Error = IoResult::ERR_None, + .SystemError = 0, + .BytesMoved = bufferLength, + }; +} + +void OutputFileStream::FlushBuffer() +{ + ::WriteBytesDirect(mOsFileHandle, mCurrentBufferSize, mBuffer.get()); + mCurrentBufferSize = 0; +} diff --git a/core/src/Utils/IO/FileStream.hpp b/core/src/Utils/IO/FileStream.hpp new file mode 100644 index 0000000..224f3b6 --- /dev/null +++ b/core/src/Utils/IO/FileStream.hpp @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include + +struct IoResult +{ + enum ErrorKind + { + ERR_None, + ERR_PermissionDenied, + ERR_UnexpectedEof, + ERR_Unsupported, + ERR_OutOfSpace, + ERR_Other, + }; + + ErrorKind Error; + int32_t SystemError; + size_t BytesMoved; +}; + +class InputFileStream +{ +private: + alignas(void*) char mOsFileHandle[sizeof(void*)]; + + // mBuffer is always mReadInSize size + std::unique_ptr mBuffer; + int mReadInSize = 1024; + + int mFirstByteIdx = 0; + int mAvailableBytes = 0; + + bool mEof = false; + +public: + InputFileStream(const std::filesystem::path& path); + ~InputFileStream(); + + InputFileStream(const InputFileStream&) = delete; + InputFileStream& operator=(const InputFileStream&) = delete; + InputFileStream(InputFileStream&&) = default; + InputFileStream& operator=(InputFileStream&&) = default; + + int GetReadInSize() const; + void SetReadInSize(int size); + + bool IsEof() const; + + IoResult ReadBytes(size_t bufferLength, std::byte* buffer); +}; + +class OutputFileStream +{ +public: + enum WriteMode + { + AppendFile, + TruncateFile, + }; + +private: + alignas(void*) char mOsFileHandle[sizeof(void*)]; + std::unique_ptr mBuffer; + int mMaxBufferSize = 1024; + int mCurrentBufferSize = 0; + +public: + OutputFileStream(const std::filesystem::path& path, WriteMode mode); + ~OutputFileStream(); + + OutputFileStream(const OutputFileStream&) = delete; + OutputFileStream& operator=(const OutputFileStream&) = delete; + OutputFileStream(OutputFileStream&&) = default; + OutputFileStream& operator=(OutputFileStream&&) = default; + + int GetMaxBufferSize() const; + void SetMaxBufferSize(int maxSize); + + IoResult WriteBytes(size_t bufferLength, const std::byte* buffer); + + void FlushBuffer(); +}; diff --git a/core/src/Utils/IO/fwd.hpp b/core/src/Utils/IO/fwd.hpp index 5bfdb20..9f1492b 100644 --- a/core/src/Utils/IO/fwd.hpp +++ b/core/src/Utils/IO/fwd.hpp @@ -7,3 +7,7 @@ class DataArchive; class BaseDataStream; class InputDataStream; class OutputDataStream; + +// FileStream.hpp +class InputFileStream; +class OutputFileStream; -- cgit v1.2.3-70-g09d2