10#include "AggMemberDatasetDimensionCache.h"
11#include "AggMemberDataset.h"
17#include <libdap/util.h>
18#include "BESInternalError.h"
21#include "TheBESKeys.h"
24static const string BES_DATA_ROOT(
"BES.Data.RootDirectory");
25static const string BES_CATALOG_ROOT(
"BES.Catalog.catalog.RootDirectory");
31AggMemberDatasetDimensionCache *AggMemberDatasetDimensionCache::d_instance = 0;
32bool AggMemberDatasetDimensionCache::d_enabled =
true;
34const string AggMemberDatasetDimensionCache::CACHE_DIR_KEY =
"NCML.DimensionCache.directory";
35const string AggMemberDatasetDimensionCache::PREFIX_KEY =
"NCML.DimensionCache.prefix";
36const string AggMemberDatasetDimensionCache::SIZE_KEY =
"NCML.DimensionCache.size";
43unsigned long AggMemberDatasetDimensionCache::getCacheSizeFromConfig(){
47 unsigned long size_in_megabytes = 0;
50 std::istringstream iss(size);
51 iss >> size_in_megabytes;
54 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getCacheSize() - The BES Key " + SIZE_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
55 BESDEBUG(
"cache", msg << endl);
58 return size_in_megabytes;
65string AggMemberDatasetDimensionCache::getCacheDirFromConfig(){
71 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getSubDirFromConfig() - The BES Key " + CACHE_DIR_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
72 BESDEBUG(
"cache", msg << endl);
84string AggMemberDatasetDimensionCache::getDimCachePrefixFromConfig(){
92 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getResultPrefix() - The BES Key " + PREFIX_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
93 BESDEBUG(
"cache", msg << endl);
106string AggMemberDatasetDimensionCache::getBesDataRootDirFromConfig(){
108 string cacheDir =
"";
113 string msg = ((string)
"[ERROR] AggMemberDatasetDimensionCache::getStoredResultsDir() - Neither the BES Key ") + BES_CATALOG_ROOT +
114 "or the BES key " + BES_DATA_ROOT +
" have been set! One MUST be set to utilize the NcML Dimension Cache. ";
115 BESDEBUG(
"cache", msg << endl);
126AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache()
128 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
130 d_dimCacheDir = getCacheDirFromConfig();
131 d_dataRootDir = getBesDataRootDirFromConfig();
133 d_dimCacheFilePrefix = getDimCachePrefixFromConfig();
134 d_maxCacheSize = getCacheSizeFromConfig();
136 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache() - Stored results cache configuration params: " << d_dimCacheDir <<
", " << d_dimCacheFilePrefix <<
", " << d_maxCacheSize << endl);
139 initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
141 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
148AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache(
const string &data_root_dir,
const string &cache_dir,
const string &prefix,
unsigned long long size){
150 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
152 d_dataRootDir = data_root_dir;
153 d_dimCacheDir = cache_dir;
154 d_dimCacheFilePrefix = prefix;
155 d_maxCacheSize = size;
158 initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
160 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
168AggMemberDatasetDimensionCache *
171 if (d_enabled && d_instance == 0){
172 if (libdap::dir_exists(cache_dir)) {
173 d_instance =
new AggMemberDatasetDimensionCache(data_root_dir, cache_dir, result_file_prefix, max_cache_size);
178 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
179 "Cache is DISABLED"<< endl);
183 atexit(delete_instance);
185 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
186 "Cache is ENABLED"<< endl);
198AggMemberDatasetDimensionCache *
201 if (d_enabled && d_instance == 0) {
207 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
208 "Cache is DISABLED"<< endl);
212 atexit(delete_instance);
214 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
215 "Cache is ENABLED"<< endl);
226void AggMemberDatasetDimensionCache::delete_instance() {
227 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::delete_instance() - Deleting singleton BESStoredDapResultCache instance." << endl);
234AggMemberDatasetDimensionCache::~AggMemberDatasetDimensionCache()
249bool AggMemberDatasetDimensionCache::is_valid(
const string &cache_file_name,
const string &local_id)
255 off_t entry_size = 0;
256 time_t entry_time = 0;
258 if (stat(cache_file_name.c_str(), &buf) == 0) {
259 entry_size = buf.st_size;
260 entry_time = buf.st_mtime;
269 time_t dataset_time = entry_time;
270 if (stat(datasetFileName.c_str(), &buf) == 0) {
271 dataset_time = buf.st_mtime;
280 if (dataset_time > entry_time)
295 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - BEGIN" << endl );
299 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - local resource id: "<< local_id << endl );
301 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - cache_file_name: "<< cache_file_name << endl );
309 if (!is_valid(cache_file_name, local_id)){
310 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
315 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Dimension cache file exists. Loading dimension cache from file: " << cache_file_name << endl);
317 ifstream istrm(cache_file_name.c_str());
319 throw libdap::InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to read cached dimensions.");
339 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Created and locked cache file: " << cache_file_name << endl);
342 ofstream ostrm(cache_file_name.c_str());
344 throw libdap::InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
371 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Couldn't create and lock cache file, But I got a read lock. "
372 "Cache file may have been rebuilt by another process. "
373 "Cache file: " << cache_file_name << endl);
376 throw libdap::InternalErr(__FILE__, __LINE__,
"AggMemberDatasetDimensionCache::loadDimensionCache() - Cache error during function invocation.");
380 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - unlocking and closing cache file "<< cache_file_name << endl );
384 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - caught exception, unlocking cache and re-throw." << endl );
389 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - END (local_id=`"<< local_id <<
"')" << endl );
virtual void unlock_cache()
bool cache_enabled() const
void initialize(const std::string &cache_dir, const std::string &prefix, unsigned long long size)
Initialize an instance of FileLockingCache.
virtual void unlock_and_close(const std::string &target)
virtual unsigned long long update_cache_info(const std::string &target)
Update the cache info file to include 'target'.
virtual bool create_and_lock(const std::string &target, int &fd)
Create a file in the cache and lock it for write access.
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
virtual bool get_read_lock(const std::string &target, int &fd)
Get a read-only lock on the file if it exists.
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big.
virtual void update_and_purge(const std::string &new_file)
Purge files from the cache.
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
exception thrown if internal error encountered
static std::string lowercase(const std::string &s)
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
void loadDimensionCache(AggMemberDataset *amd)
static AggMemberDatasetDimensionCache * get_instance()
virtual void loadDimensionCache(std::istream &istr)=0
const std::string & getLocation() const
virtual void saveDimensionCache(std::ostream &ostr)=0
virtual void fillDimensionCacheByUsingDDS()=0
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...