kraken_engine/kraken
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Asynchronous streaming and memory management improvements in progress.

Browse Source 
Note: This branch is unstable

--HG--
branch : async_streaming
This commit is contained in:
Kearwood Gilbert
2013-10-06 18:56:23 -07:00
parent de324ff279
commit 4dddec7f85
20 changed files with 488 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
KREngine/kraken/KRDataBlock.cpp
View File

@@ -30,12 +30,18 @@
//
#include "KRDataBlock.h"
#include "KREngine-common.h"
KRDataBlock::KRDataBlock() {
m_data = NULL;
m_data_size = 0;
m_data_offset = 0;
m_fdPackFile = 0;
m_mmapData = NULL;
m_fileOwnerDataBlock = NULL;
m_bMalloced = false;
m_lockCount = 0;
m_bReadOnly = false;
}
KRDataBlock::~KRDataBlock() {
@@ -45,10 +51,13 @@ KRDataBlock::~KRDataBlock() {
// Unload a file, releasing any mmap'ed file handles or malloc'ed ram that was in use
void KRDataBlock::unload()
{
assert(m_lockCount == 0);
if(m_fdPackFile) {
// Memory mapped file
munmap(m_data, m_data_size);
close(m_fdPackFile);
if(m_fileOwnerDataBlock == this) {
close(m_fdPackFile);
}
} else if(m_data != NULL && m_bMalloced) {
// Malloc'ed data
free(m_data);
@@ -57,7 +66,11 @@ void KRDataBlock::unload()
m_bMalloced = false;
m_data = NULL;
m_data_size = 0;
m_data_offset = 0;
m_fdPackFile = 0;
m_mmapData = NULL;
m_fileOwnerDataBlock = NULL;
m_bReadOnly = false;
}
// Encapsulate a pointer. Note - The pointer will not be free'ed
@@ -66,6 +79,8 @@ bool KRDataBlock::load(void *data, size_t size)
unload();
m_data = data;
m_data_size = size;
m_data_offset = 0;
m_bReadOnly = false;
return true;
}
@@ -76,14 +91,14 @@ bool KRDataBlock::load(const std::string &path)
unload();
struct stat statbuf;
m_bReadOnly = true;
m_fdPackFile = open(path.c_str(), O_RDONLY);
if(m_fdPackFile >= 0) {
m_fileOwnerDataBlock = this;
if(fstat(m_fdPackFile, &statbuf) >= 0) {
if ((m_data = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, m_fdPackFile, 0)) == (caddr_t) -1) {
} else {
m_data_size = statbuf.st_size;
success = true;
}
m_data_size = statbuf.st_size;
m_data_offset = 0;
success = true;
}
}
if(!success) {
@@ -93,13 +108,32 @@ bool KRDataBlock::load(const std::string &path)
return success;
}
// Create a KRDataBlock encapsulating a sub-region of this block. The caller is responsible to free the object.
KRDataBlock *KRDataBlock::getSubBlock(int start, int length)
{
KRDataBlock *new_block = new KRDataBlock();
new_block->m_data_size = length;
if(m_fdPackFile) {
new_block->m_fdPackFile = m_fdPackFile;
new_block->m_fileOwnerDataBlock = m_fileOwnerDataBlock;
new_block->m_data_offset = start;
} else if(m_bMalloced) {
new_block->m_data = (unsigned char *)m_data + start;
}
new_block->m_bReadOnly = true;
return new_block;
}
// Return a pointer to the start of the data block
void *KRDataBlock::getStart() {
assertLocked();
return m_data;
}
// Return a pointer to the byte after the end of the data block
void *KRDataBlock::getEnd() {
assertLocked();
return (unsigned char *)m_data + m_data_size;
}
@@ -111,28 +145,33 @@ size_t KRDataBlock::getSize() const {
// Expand the data block, and switch it to read-write mode. Note - this may result in a mmap'ed file being copied to malloc'ed ram and then closed
void KRDataBlock::expand(size_t size)
{
if(m_data == NULL) {
if(m_data == NULL && m_fdPackFile == 0) {
// Starting with an empty data block; allocate memory on the heap
m_data = malloc(size);
assert(m_data != NULL);
m_data_size = size;
m_data_offset = 0;
m_bMalloced = true;
} else if(m_bMalloced) {
// Starting with a malloc'ed data block; realloc it expand
m_data = realloc(m_data, m_data_size + size);
m_data_size += size;
} else {
// Starting with a mmap'ed data block; copy it to ram before expanding to avoid updating the original file until save() is called
// Starting with a mmap'ed data block, an encapsulated pointer, or a sub-block; copy it to ram before expanding to avoid updating the original file until save() is called
// ... Or starting with a pointer reference, we must make our own copy and must not free the pointer
void *pNewData = malloc(m_data_size + size);
assert(pNewData != NULL);
memcpy((unsigned char *)pNewData, m_data, m_data_size); // Copy exising data
// Copy exising data
copy(pNewData);
// Unload existing data allocation, which is now redundant
size_t new_size = m_data_size + size; // We need to store this before unload() as unload() will reset it
unload();
m_bMalloced = true;
m_data = pNewData;
m_data_size = new_size;
m_data_offset = 0;
}
}
@@ -142,12 +181,31 @@ void KRDataBlock::append(void *data, size_t size) {
expand(size);
// Fill the new space with the data to append
lock();
memcpy((unsigned char *)m_data + m_data_size - size, data, size);
unlock();
}
// Copy the entire data block to the destination pointer
void KRDataBlock::copy(void *dest) {
lock();
memcpy((unsigned char *)dest, m_data, m_data_size);
unlock();
}
// Copy a range of data to the destination pointer
void KRDataBlock::copy(void *dest, int start, int count) {
lock();
memcpy((unsigned char *)dest, (unsigned char *)m_data + start, count);
unlock();
}
// Append data to the end of the block, increasing the size of the block and making it read-write.
void KRDataBlock::append(KRDataBlock &data) {
data.lock();
append(data.getStart(), data.getSize());
data.unlock();
}
// Append string to the end of the block, increasing the size of the block and making it read-write. The null terminating character is included
@@ -157,7 +215,7 @@ void KRDataBlock::append(const std::string &s)
append((void *)szText, strlen(szText)+1);
}
// Save the data to a file, and switch to read-only mode. The data pointer will be replaced with a mmap'ed address of the file; the malloc'ed data will be freed
// Save the data to a file.
bool KRDataBlock::save(const std::string& path) {
int fdNewFile = open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600);
if(fdNewFile == -1) {
@@ -173,20 +231,16 @@ bool KRDataBlock::save(const std::string& path) {
close(fdNewFile);
return false;
} else if(m_data != NULL) {
// Copy data to new file
memcpy(pNewData, m_data, m_data_size);
copy(pNewData);
// Unload existing data allocation, which is now redundant
size_t new_size = m_data_size; // We need to store this, as unload() will reset it
unload();
// Protect new mmap'ed memory
mprotect(pNewData, m_data_size, PROT_READ);
// Unmap the new file
munmap(pNewData, m_data_size);
// Switch pointer to use new mmap'ed memory
m_data_size = new_size;
m_fdPackFile = fdNewFile;
m_data = pNewData;
// Close the new file
close(fdNewFile);
}
return true;
}
@@ -195,8 +249,57 @@ bool KRDataBlock::save(const std::string& path) {
// Get contents as a string
std::string KRDataBlock::getString()
{
lock();
KRDataBlock b;
b.append(*this);
b.append((void *)"\0", 1); // Ensure data is null terminated, to read as a string safely
return std::string((char *)b.getStart());
b.lock();
std::string ret = std::string((char *)b.getStart());
b.unlock();
unlock();
return ret;
}
// Lock the memory, forcing it to be loaded into a contiguous block of address space
void KRDataBlock::lock()
{
m_lockCount++;
if(m_lockCount == 1) {
// Memory mapped file; ensure data is mapped to ram
if(m_fdPackFile) {
// Round m_data_offset down to the next memory page, as required by mmap
size_t alignment_offset = m_data_offset & (KRAKEN_MEM_PAGE_SIZE - 1);
if ((m_mmapData = mmap(0, m_data_size + alignment_offset, m_bReadOnly ? PROT_READ : PROT_WRITE, MAP_SHARED, m_fdPackFile, m_data_offset - alignment_offset)) == (caddr_t) -1) {
assert(false); // mmap() failed.
}
m_data = (unsigned char *)m_mmapData + alignment_offset;
}
}
}
// Unlock the memory, releasing the address space for use by other allocations
void KRDataBlock::unlock()
{
// We expect that the data block was previously locked
assertLocked();
m_lockCount--;
if(m_lockCount == 0) {
// Memory mapped file; ensure data is unmapped from ram
if(m_fdPackFile) {
munmap(m_mmapData, m_data_size);
m_data = NULL;
m_mmapData = NULL;
}
}
}
// Assert if not locked
void KRDataBlock::assertLocked()
{
assert(m_lockCount > 0);
}