#include "info.h"
#include "madvise.h"
#include "defaults.h"
#include "hi_lo.h"
#include <util/generic/yexception.h>
#include <util/generic/singleton.h>
#if defined(_win_)
#include "winint.h"
#elif defined(_unix_)
#include <sys/types.h>
#include <sys/mman.h>
#if !defined(_linux_)
#ifdef MAP_POPULATE
#error unlisted platform supporting MAP_POPULATE
#endif
#define MAP_POPULATE 0
#endif
#if !defined(_freebsd_)
#ifdef MAP_NOCORE
#error unlisted platform supporting MAP_NOCORE
#endif
#define MAP_NOCORE 0
#endif
#else
#error todo
#endif
#include <util/generic/utility.h>
#include <util/system/sanitizers.h>
#include "filemap.h"
#undef PAGE_SIZE
#undef GRANULARITY
#ifdef _win_
#define MAP_FAILED ((void*)(LONG_PTR)-1)
#endif
namespace {
struct TSysInfo {
inline TSysInfo()
: GRANULARITY_(CalcGranularity())
, PAGE_SIZE_(NSystemInfo::GetPageSize())
{
}
static inline const TSysInfo& Instance() {
return *Singleton<TSysInfo>();
}
static inline size_t CalcGranularity() noexcept {
#if defined(_win_)
SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo);
return sysInfo.dwAllocationGranularity;
#else
return NSystemInfo::GetPageSize();
#endif
}
const size_t GRANULARITY_;
const size_t PAGE_SIZE_;
};
}
#define GRANULARITY (TSysInfo::Instance().GRANULARITY_)
#define PAGE_SIZE (TSysInfo::Instance().PAGE_SIZE_)
const TString& TMemoryMapCommon::UnknownFileName() {
static const TString unknownFileName = "Unknown_file_name";
return unknownFileName;
}
static inline i64 DownToGranularity(i64 offset) noexcept {
return offset & ~((i64)(GRANULARITY - 1));
}
#if defined(_unix_)
static int ModeToMmapFlags(TMemoryMapCommon::EOpenMode mode) {
int flags = MAP_NOCORE;
if ((mode & TMemoryMapCommon::oAccessMask) == TMemoryMapCommon::oCopyOnWr) {
flags |= MAP_PRIVATE;
} else {
flags |= MAP_SHARED;
}
if (mode & TMemoryMapCommon::oPopulate) {
flags |= MAP_POPULATE;
}
return flags;
}
static int ModeToMmapProt(TMemoryMapCommon::EOpenMode mode) {
int prot = PROT_READ;
if ((mode & TMemoryMapCommon::oAccessMask) != TMemoryMapCommon::oRdOnly) {
prot |= PROT_WRITE;
}
return prot;
}
#endif
// maybe we should move this function to another .cpp file to avoid unwanted optimization?
void NPrivate::Precharge(const void* data, size_t dataSize, size_t off, size_t size) {
if (off > dataSize) {
assert(false);
return;
}
size_t endOff = (size == (size_t)-1 ? dataSize : off + size);
if (endOff > dataSize) {
assert(false);
endOff = dataSize;
}
size = endOff - off;
if (dataSize == 0 || size == 0) {
return;
}
volatile const char *c = (const char*)data + off, *e = c + size;
for (; c < e; c += 512) {
*c;
}
}
class TMemoryMap::TImpl: public TAtomicRefCount<TImpl> {
public:
inline void CreateMapping() {
#if defined(_win_)
Mapping_ = nullptr;
if (Length_) {
Mapping_ = CreateFileMapping(File_.GetHandle(), nullptr,
(Mode_ & oAccessMask) == TFileMap::oRdWr ? PAGE_READWRITE : PAGE_READONLY,
(DWORD)(Length_ >> 32), (DWORD)(Length_ & 0xFFFFFFFF), nullptr);
if (Mapping_ == nullptr) {
ythrow yexception() << "Can't create file mapping of '" << DbgName_ << "': " << LastSystemErrorText();
}
} else {
Mapping_ = MAP_FAILED;
}
#elif defined(_unix_)
if (!(Mode_ & oNotGreedy)) {
PtrStart_ = mmap((caddr_t) nullptr, Length_, ModeToMmapProt(Mode_), ModeToMmapFlags(Mode_), File_.GetHandle(), 0);
if ((MAP_FAILED == PtrStart_) && Length_) {
ythrow yexception() << "Can't map " << (unsigned long)Length_ << " bytes of file '" << DbgName_ << "' at offset 0: " << LastSystemErrorText();
}
} else {
PtrStart_ = nullptr;
}
#endif
}
void CheckFile() const {
if (!File_.IsOpen()) {
ythrow yexception() << "TMemoryMap: FILE '" << DbgName_ << "' is not open, " << LastSystemErrorText();
}
if (Length_ < 0) {
ythrow yexception() << "'" << DbgName_ << "' is not a regular file";
}
}
inline TImpl(FILE* f, EOpenMode om, TString dbgName)
: File_(Duplicate(f))
, DbgName_(std::move(dbgName))
, Length_(File_.GetLength())
, Mode_(om)
{
CheckFile();
CreateMapping();
}
inline TImpl(const TString& name, EOpenMode om)
: File_(name, (om & oRdWr) ? OpenExisting | RdWr : OpenExisting | RdOnly)
, DbgName_(name)
, Length_(File_.GetLength())
, Mode_(om)
{
CheckFile();
CreateMapping();
}
inline TImpl(const TString& name, i64 length, EOpenMode om)
: File_(name, (om & oRdWr) ? OpenExisting | RdWr : OpenExisting | RdOnly)
, DbgName_(name)
, Length_(length)
, Mode_(om)
{
CheckFile();
if (File_.GetLength() < Length_) {
File_.Resize(Length_);
}
CreateMapping();
}
inline TImpl(const TFile& file, EOpenMode om, const TString& dbgName)
: File_(file)
, DbgName_(File_.GetName() ? File_.GetName() : dbgName)
, Length_(File_.GetLength())
, Mode_(om)
{
CheckFile();
CreateMapping();
}
inline bool IsOpen() const noexcept {
return File_.IsOpen()
#if defined(_win_)
&& Mapping_ != nullptr
#endif
;
}
inline bool IsWritable() const noexcept {
return (Mode_ & oRdWr || Mode_ & oCopyOnWr);
}
inline TMapResult Map(i64 offset, size_t size) {
assert(File_.IsOpen());
if (offset > Length_) {
ythrow yexception() << "Can't map something at offset " << offset << " of '" << DbgName_ << "' with length " << Length_;
}
if (offset + (i64)size > Length_) {
ythrow yexception() << "Can't map " << (unsigned long)size << " bytes at offset " << offset << " of '" << DbgName_ << "' with length " << Length_;
}
TMapResult result;
i64 base = DownToGranularity(offset);
result.Head = (i32)(offset - base);
size += result.Head;
#if defined(_win_)
result.Ptr = MapViewOfFile(Mapping_,
(Mode_ & oAccessMask) == oRdOnly ? FILE_MAP_READ : (Mode_ & oAccessMask) == oCopyOnWr ? FILE_MAP_COPY
: FILE_MAP_WRITE,
Hi32(base), Lo32(base), size);
#else
#if defined(_unix_)
if (Mode_ & oNotGreedy) {
#endif
result.Ptr = mmap((caddr_t) nullptr, size, ModeToMmapProt(Mode_), ModeToMmapFlags(Mode_), File_.GetHandle(), base);
if (result.Ptr == (char*)(-1)) {
result.Ptr = nullptr;
}
#if defined(_unix_)
} else {
result.Ptr = PtrStart_ ? static_cast<caddr_t>(PtrStart_) + base : nullptr;
}
#endif
#endif
if (result.Ptr != nullptr || size == 0) { // allow map of size 0
result.Size = size;
} else {
ythrow yexception() << "Can't map " << (unsigned long)size << " bytes at offset " << offset << " of '" << DbgName_ << "': " << LastSystemErrorText();
}
NSan::Unpoison(result.Ptr, result.Size);
if (Mode_ & oPrecharge) {
NPrivate::Precharge(result.Ptr, result.Size, 0, result.Size);
}
return result;
}
#if defined(_win_)
inline bool Unmap(void* ptr, size_t) {
return ::UnmapViewOfFile(ptr) != FALSE;
}
#else
inline bool Unmap(void* ptr, size_t size) {
#if defined(_unix_)
if (Mode_ & oNotGreedy) {
#endif
return size == 0 || ::munmap(static_cast<caddr_t>(ptr), size) == 0;
#if defined(_unix_)
} else {
return true;
}
#endif
}
#endif
void SetSequential() {
#if defined(_unix_)
if (!(Mode_ & oNotGreedy) && Length_) {
MadviseSequentialAccess(PtrStart_, Length_);
}
#endif
}
void Evict(void* ptr, size_t len) {
MadviseEvict(ptr, len);
}
void Evict() {
#if defined(_unix_)
// Evict(PtrStart_, Length_);
#endif
}
inline ~TImpl() {
#if defined(_win_)
if (Mapping_) {
::CloseHandle(Mapping_); // != FALSE
Mapping_ = nullptr;
}
#elif defined(_unix_)
if (PtrStart_) {
munmap((caddr_t)PtrStart_, Length_);
}
#endif
}
inline i64 Length() const noexcept {
return Length_;
}
inline TFile GetFile() const noexcept {
return File_;
}
inline TString GetDbgName() const {
return DbgName_;
}
inline EOpenMode GetMode() const noexcept {
return Mode_;
}
private:
TFile File_;
TString DbgName_; // This string is never used to actually open a file, only in exceptions
i64 Length_;
EOpenMode Mode_;
#if defined(_win_)
void* Mapping_;
#elif defined(_unix_)
void* PtrStart_;
#endif
};
TMemoryMap::TMemoryMap(const TString& name)
: Impl_(new TImpl(name, EOpenModeFlag::oRdOnly))
{
}
TMemoryMap::TMemoryMap(const TString& name, EOpenMode om)
: Impl_(new TImpl(name, om))
{
}
TMemoryMap::TMemoryMap(const TString& name, i64 length, EOpenMode om)
: Impl_(new TImpl(name, length, om))
{
}
TMemoryMap::TMemoryMap(FILE* f, TString dbgName)
: Impl_(new TImpl(f, EOpenModeFlag::oRdOnly, std::move(dbgName)))
{
}
TMemoryMap::TMemoryMap(FILE* f, EOpenMode om, TString dbgName)
: Impl_(new TImpl(f, om, std::move(dbgName)))
{
}
TMemoryMap::TMemoryMap(const TFile& file, const TString& dbgName)
: Impl_(new TImpl(file, EOpenModeFlag::oRdOnly, dbgName))
{
}
TMemoryMap::TMemoryMap(const TFile& file, EOpenMode om, const TString& dbgName)
: Impl_(new TImpl(file, om, dbgName))
{
}
TMemoryMap::~TMemoryMap() = default;
TMemoryMap::TMapResult TMemoryMap::Map(i64 offset, size_t size) {
return Impl_->Map(offset, size);
}
bool TMemoryMap::Unmap(void* ptr, size_t size) {
return Impl_->Unmap(ptr, size);
}
bool TMemoryMap::Unmap(TMapResult region) {
return Unmap(region.Ptr, region.Size);
}
void TMemoryMap::ResizeAndReset(i64 size) {
EOpenMode om = Impl_->GetMode();
TFile file = GetFile();
file.Resize(size);
Impl_.Reset(new TImpl(file, om, Impl_->GetDbgName()));
}
TMemoryMap::TMapResult TMemoryMap::ResizeAndRemap(i64 offset, size_t size) {
ResizeAndReset(offset + (i64)size);
return Map(offset, size);
}
void TMemoryMap::SetSequential() {
Impl_->SetSequential();
}
void TMemoryMap::Evict(void* ptr, size_t len) {
Impl_->Evict(ptr, len);
}
void TMemoryMap::Evict() {
Impl_->Evict();
}
i64 TMemoryMap::Length() const noexcept {
return Impl_->Length();
}
bool TMemoryMap::IsOpen() const noexcept {
return Impl_->IsOpen();
}
bool TMemoryMap::IsWritable() const noexcept {
return Impl_->IsWritable();
}
TMemoryMap::EOpenMode TMemoryMap::GetMode() const noexcept {
return Impl_->GetMode();
}
TFile TMemoryMap::GetFile() const noexcept {
return Impl_->GetFile();
}
TFileMap::TFileMap(const TMemoryMap& map) noexcept
: Map_(map)
{
}
TFileMap::TFileMap(const TString& name)
: Map_(name)
{
}
TFileMap::TFileMap(const TString& name, EOpenMode om)
: Map_(name, om)
{
}
TFileMap::TFileMap(const TString& name, i64 length, EOpenMode om)
: Map_(name, length, om)
{
}
TFileMap::TFileMap(FILE* f, EOpenMode om, TString dbgName)
: Map_(f, om, std::move(dbgName))
{
}
TFileMap::TFileMap(const TFile& file, EOpenMode om, const TString& dbgName)
: Map_(file, om, dbgName)
{
}
TFileMap::TFileMap(const TFileMap& fm) noexcept
: Map_(fm.Map_)
{
}
void TFileMap::Flush(void* ptr, size_t size, bool sync) {
Y_ASSERT(ptr >= Ptr());
Y_ASSERT(static_cast<char*>(ptr) + size <= static_cast<char*>(Ptr()) + MappedSize());
if (!Region_.IsMapped()) {
return;
}
#if defined(_win_)
if (sync) {
FlushViewOfFile(ptr, size);
}
#else
msync(ptr, size, sync ? MS_SYNC : MS_ASYNC);
#endif
}
TFileMap::TMapResult TFileMap::Map(i64 offset, size_t size) {
Unmap();
Region_ = Map_.Map(offset, size);
return Region_;
}
TFileMap::TMapResult TFileMap::ResizeAndRemap(i64 offset, size_t size) {
// explicit Unmap() is required because in oNotGreedy mode the Map_ object doesn't own the mapped area
Unmap();
Region_ = Map_.ResizeAndRemap(offset, size);
return Region_;
}
void TFileMap::Unmap() {
if (!Region_.IsMapped()) {
return;
}
if (Map_.Unmap(Region_)) {
Region_.Reset();
} else {
ythrow yexception() << "can't unmap file";
}
}
TFileMap::~TFileMap() {
try {
// explicit Unmap() is required because in oNotGreedy mode the Map_ object doesn't own the mapped area
Unmap();
} catch (...) {
// ¯\_(ツ)_/¯
}
}
void TFileMap::Precharge(size_t pos, size_t size) const {
NPrivate::Precharge(Ptr(), MappedSize(), pos, size);
}
TMappedAllocation::TMappedAllocation(size_t size, bool shared, void* addr)
: Ptr_(nullptr)
, Size_(0)
, Shared_(shared)
#if defined(_win_)
, Mapping_(nullptr)
#endif
{
if (size != 0) {
Alloc(size, addr);
}
}
void* TMappedAllocation::Alloc(size_t size, void* addr) {
assert(Ptr_ == nullptr);
#if defined(_win_)
(void)addr;
Mapping_ = CreateFileMapping((HANDLE)-1, nullptr, PAGE_READWRITE, 0, size ? size : 1, nullptr);
Ptr_ = MapViewOfFile(Mapping_, FILE_MAP_WRITE, 0, 0, size ? size : 1);
#else
Ptr_ = mmap(addr, size, PROT_READ | PROT_WRITE, (Shared_ ? MAP_SHARED : MAP_PRIVATE) | MAP_ANON, -1, 0);
if (Ptr_ == (void*)MAP_FAILED) {
Ptr_ = nullptr;
}
#endif
if (Ptr_ != nullptr) {
Size_ = size;
}
return Ptr_;
}
void TMappedAllocation::Dealloc() {
if (Ptr_ == nullptr) {
return;
}
#if defined(_win_)
UnmapViewOfFile(Ptr_);
CloseHandle(Mapping_);
Mapping_ = nullptr;
#else
munmap((caddr_t)Ptr_, Size_);
#endif
Ptr_ = nullptr;
Size_ = 0;
}
void TMappedAllocation::swap(TMappedAllocation& with) noexcept {
DoSwap(Ptr_, with.Ptr_);
DoSwap(Size_, with.Size_);
#if defined(_win_)
DoSwap(Mapping_, with.Mapping_);
#endif
}