50#include "OgSparseDataReader.h"
98template <
class Data_T>
101 int64_t bytesFreed = 0;
116#if F3D_SHORT_MUTEX_ARRAY
117 boost::mutex::scoped_lock
147template <
class Data_T>
162 boost::mutex::scoped_lock lock_A(
m_mutex);
174 int64_t bytesFreed = 0;
179 if (bytesFreed > 0) {
185 if (bytesFreed > 0) {
191 if (bytesFreed > 0) {
197 if (bytesFreed > 0) {
203 if (bytesFreed > 0) {
209 if (bytesFreed > 0) {
225 boost::mutex::scoped_lock lock(
m_mutex);
261 int fileId,
int blockIdx)
295 long long int numLoads = 0;
305 for (
size_t i=0; i<
m_fileData.numRefs<
float>(); i++) {
313 for (
size_t i=0; i<
m_fileData.numRefs<
double>(); i++) {
328 long long int numBlocks = 0;
338 for (
size_t i=0; i<
m_fileData.numRefs<
float>(); i++) {
346 for (
size_t i=0; i<
m_fileData.numRefs<
double>(); i++) {
361 long long int numBlocks = 0;
371 for (
size_t i=0; i<
m_fileData.numRefs<
float>(); i++) {
379 for (
size_t i=0; i<
m_fileData.numRefs<
double>(); i++) {
423 for (
size_t i=0; i<
m_fileData.numRefs<
float>(); i++) {
431 for (
size_t i=0; i<
m_fileData.numRefs<
double>(); i++) {
444 boost::mutex::scoped_lock lock(
m_mutex);
454 Mutex::scoped_lock lock(
m_mutex);
456 long long int size = 0;
467 for (
size_t i = 0, end =
m_hRefs.size(); i < end; ++i) {
470 for (
size_t i = 0, end =
m_vhRefs.size(); i < end; ++i) {
473 for (
size_t i = 0, end =
m_fRefs.size(); i < end; ++i) {
476 for (
size_t i = 0, end =
m_vfRefs.size(); i < end; ++i) {
479 for (
size_t i = 0, end =
m_dRefs.size(); i < end; ++i) {
482 for (
size_t i = 0, end =
m_vdRefs.size(); i < end; ++i) {
497template <
class Data_T>
500 boost::mutex::scoped_lock lock(
m_mutex);
503#if F3D_NO_BLOCKS_ARRAY
505 assert(
blocks[blockIdx].data != NULL);
519 assert(
blocks[blockIdx]->data != NULL);
536template <
class Data_T>
542 boost::mutex::scoped_lock lock_A(
m_mutex);
579 "Couldn't find layer group " +
layerPath +
581 throw FileIntegrityException(
filename);
585 "Couldn't open HDF5 file ");
586 throw NoSuchFileException(
filename);
601#define FIELD3D_INSTANTIATION_LOADBLOCK(type) \
603 void Reference<type>::loadBlock(int blockIdx); \
614#define FIELD3D_INSTANTIATION_OPENFILE(type) \
616 void Reference<type>::openFile(); \
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
boost::recursive_mutex::scoped_lock GlobalLock
Contains the SparseField class.
#define FIELD3D_INSTANTIATION_OPENFILE(type)
#define FIELD3D_INSTANTIATION_LOADBLOCK(type)
FIELD3D_NAMESPACE_OPENtypedef ::half half
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
void resetCacheStatistics()
Resets block load.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
int64_t m_memUse
Current amount of memory in use in bytes.
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads....
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
long long int memSize() const
Returns the number of bytes used by the SparseFileManager itself.
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
float m_maxMemUse
Max amount om memory to use in megabytes.
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
SparseFileManager()
Private to prevent instantiation.
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
long long totalLoads()
Returns the total number of block loads in the cache.
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
long long int memSize() const
Returns the memory use for the refs.
std::deque< Reference< double >::Ptr > m_dRefs
std::deque< Reference< float >::Ptr > m_fRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
std::deque< Reference< half >::Ptr > m_hRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
std::vector< int > fileBlockIndices
Index in file for each block.
OgIGroupPtr m_ogRoot
Ogawa archive root.
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
IArchivePtr m_ogArchive
Ogawa archive.
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
size_t m_numActiveBlocks
Number of currently active blocks.
bool fileIsOpen()
Checks if the file used by this reference is open already.
Namespace for Exception objects.
Contains utility functions and classes for Hdf5 files.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
#define FIELD3D_NAMESPACE_HEADER_CLOSE