38#include <libdap/DAS.h>
42#include <libdap/DapObj.h>
43#include <libdap/DDS.h>
44#include <libdap/DMR.h>
45#include <libdap/D4ParserSax2.h>
46#include <libdap/XMLWriter.h>
47#include <libdap/BaseTypeFactory.h>
48#include <libdap/D4BaseTypeFactory.h>
50#include "PicoSHA2/picosha2.h"
53#include "TheBESKeys.h"
56#include "BESContextManager.h"
58#include "BESRequestHandler.h"
59#include "BESRequestHandlerList.h"
60#include "BESNotFoundError.h"
62#include "BESInternalError.h"
63#include "BESInternalFatalError.h"
65#include "GlobalMetadataStore.h"
67#define DEBUG_KEY "metadata_store"
68#define MAINTAIN_STORE_SIZE_EVEN_WHEN_UNLIMITED 0
71#define AT_EXIT(x) atexit((x))
85#undef SYMETRIC_ADD_RESPONSES
87#define prolog std::string("GlobalMetadataStore::").append(__func__).append("() - ")
93static const unsigned int default_cache_size = 20;
94static const string default_cache_prefix =
"mds";
95static const string default_cache_dir =
"";
96static const string default_ledger_name =
"mds_ledger.txt";
98static const string PATH_KEY =
"DAP.GlobalMetadataStore.path";
99static const string PREFIX_KEY =
"DAP.GlobalMetadataStore.prefix";
100static const string SIZE_KEY =
"DAP.GlobalMetadataStore.size";
101static const string LEDGER_KEY =
"DAP.GlobalMetadataStore.ledger";
102static const string LOCAL_TIME_KEY =
"BES.LogTimeLocal";
105bool GlobalMetadataStore::d_enabled =
true;
123 static const int BUFFER_SIZE = 16*1024;
125#if _POSIX_C_SOURCE >= 200112L
127 int status = posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
129 ERROR_LOG(prolog <<
"Error calling posix_advise() in the GlobalMetadataStore: " << strerror(status) << endl);
132 char buf[BUFFER_SIZE + 1];
134 while(
int bytes_read = read(fd, buf, BUFFER_SIZE))
137 throw BESInternalError(
"Could not read dds from the metadata store.", __FILE__, __LINE__);
141 os.write(buf, bytes_read);
159 static const int BUFFER_SIZE = 1024;
161#if _POSIX_C_SOURCE >= 200112L
163 int status = posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
165 ERROR_LOG(prolog <<
"Error calling posix_advise() in the GlobalMetadataStore: " << strerror(status) << endl);
168 char buf[BUFFER_SIZE + 1];
169 size_t bytes_read = read(fd, buf, BUFFER_SIZE);
171 if(bytes_read == (
size_t)-1)
172 throw BESInternalError(
"Could not read dds from the metadata store.", __FILE__, __LINE__);
189 while (buf[i++] !=
'>')
196 char xml_base_literal[] =
"xml:base";
197 while (i < bytes_read) {
199 os.write(buf + s, i - s);
200 os <<
" xml:base=\"" << xml_base <<
"\"";
203 else if (j ==
sizeof(xml_base_literal) - 1) {
204 os.write(buf + s, i - s);
205 while (buf[i++] !=
'=')
207 while (buf[i++] !=
'"')
209 while (buf[i++] !=
'"')
211 os <<
"=\"" << xml_base <<
"\"";
214 else if (buf[i] == xml_base_literal[j]) {
225 os.write(buf + i, bytes_read - i);
231unsigned long GlobalMetadataStore::get_cache_size_from_config()
235 unsigned long size_in_megabytes = default_cache_size;
239 "GlobalMetadataStore::getCacheSizeFromConfig(): Located BES key " << SIZE_KEY <<
"=" << size << endl);
240 istringstream iss(size);
241 iss >> size_in_megabytes;
244 return size_in_megabytes;
247string GlobalMetadataStore::get_cache_prefix_from_config()
250 string prefix = default_cache_prefix;
254 "GlobalMetadataStore::getCachePrefixFromConfig(): Located BES key " << PREFIX_KEY <<
"=" << prefix << endl);
262string GlobalMetadataStore::get_cache_dir_from_config()
266 string cacheDir = default_cache_dir;
270 "GlobalMetadataStore::getCacheDirFromConfig(): Located BES key " << PATH_KEY<<
"=" << cacheDir << endl);
310 if (d_enabled && d_instance == 0) {
317 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::"<<__func__ <<
"() - " <<
"MDS is DISABLED"<< endl);
320 AT_EXIT(delete_instance);
322 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::"<<__func__ <<
"() - " <<
"MDS is ENABLED"<< endl);
326 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::get_instance(dir,prefix,size) - d_instance: " << d_instance << endl);
340 if (d_enabled && d_instance == 0) {
341 d_instance =
new GlobalMetadataStore(get_cache_dir_from_config(), get_cache_prefix_from_config(),
342 get_cache_size_from_config());
346 d_instance =
nullptr;
347 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::"<<__func__ <<
"() - " <<
"MDS is DISABLED"<< endl);
350 AT_EXIT(delete_instance);
352 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::"<<__func__ <<
"() - " <<
"MDS is ENABLED"<< endl);
356 BESDEBUG(DEBUG_KEY,
"GlobalMetadataStore::get_instance() - d_instance: " << (
void *) d_instance << endl);
372 BESDEBUG(DEBUG_KEY,
"Located BES key " << LEDGER_KEY <<
"=" << d_ledger_name << endl);
375 d_ledger_name = default_ledger_name;
378 ofstream of(d_ledger_name.c_str(), ios::app);
381 string local_time =
"no";
383 d_use_local_time = (local_time ==
"YES" || local_time ==
"Yes" || local_time ==
"yes");
402 :
BESFileLockingCache(get_cache_dir_from_config(), get_cache_prefix_from_config(), get_cache_size_from_config())
418static void dump_time(ostream &os,
bool use_local_time)
422 char buf[
sizeof "YYYY-MM-DDTHH:MM:SSzone"];
432 if (!use_local_time) {
433 gmtime_r(&now, &result);
434 status = strftime(buf,
sizeof buf,
"%FT%T%Z", &result);
437 localtime_r(&now, &result);
438 status = strftime(buf,
sizeof buf,
"%FT%T%Z", &result);
441 ERROR_LOG(prolog <<
"Error getting time for Metadata Store ledger.");
456 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Ledger " << d_ledger_name <<
" write locked." << endl);
459 dump_time(of, d_use_local_time);
460 of <<
" " << d_ledger_entry << endl;
461 VERBOSE(
"MDS Ledger name: '" << d_ledger_name <<
"', entry: '" << d_ledger_entry +
"'.");
470 ERROR_LOG(prolog <<
"Warning: Metadata store could not write to its ledger file.");
475 throw BESInternalError(
"Could not write lock '" + d_ledger_name, __FILE__, __LINE__);
489 throw BESInternalError(
"Empty name passed to the Metadata Store.", __FILE__, __LINE__);
491 return picosha2::hash256_hex_string(name[0] ==
'/' ? name :
"/" + name);
513 D4BaseTypeFactory factory;
514 DMR dmr(&factory, *d_dds);
521 d_dmr->print_dap4(xml);
534 d_dmr->getDDS()->print(os);
542 d_dds->print_das(os);
544 d_dmr->getDDS()->print_das(os);
565 const string &response_name)
567 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" BEGIN " << key << endl);
573 BESDEBUG(DEBUG_KEY,__FUNCTION__ <<
" Storing " << item_name << endl);
576 ofstream response(item_name.c_str(), ios::out|ios::app);
577 if (!response.is_open())
578 throw BESInternalError(
"Could not open '" + key +
"' to write the response.", __FILE__, __LINE__);
587 if (!
is_unlimited() || MAINTAIN_STORE_SIZE_EVEN_WHEN_UNLIMITED) {
605 VERBOSE(
"Metadata store: Wrote " << response_name <<
" response for '" << name <<
"'." << endl);
606 d_ledger_entry.append(
" ").append(key);
612 BESDEBUG(DEBUG_KEY,__FUNCTION__ <<
" Found " << item_name <<
" in the store already." << endl);
615 ERROR_LOG(prolog <<
"Metadata store: unable to store the " << response_name <<
" response for '" << name <<
"'." << endl);
620 throw BESInternalError(
"Could neither create or open '" + item_name +
"' in the metadata store.", __FILE__, __LINE__);
654 d_ledger_entry = string(
"add DDS ").append(name);
667#if SYMETRIC_ADD_RESPONSES
674#if SYMETRIC_ADD_RESPONSES
675 return (stored_dds && stored_das && stored_dmr);
677 return (stored_dds && stored_das);
696 d_ledger_entry = string(
"add DMR ").append(name);
703#if SYMETRIC_ADD_RESPONSES
716#if SYMETRIC_ADD_RESPONSES
717 return (stored_dds && stored_das && stored_dmr);
739 BESDEBUG(DEBUG_KEY, __func__ <<
"() MDS hashing name '" << name <<
"', '" << suffix <<
"'"<< endl);
743 "GlobalMetadataStore::get_read_lock_helper(). That should never happen.", __FILE__, __LINE__);
748 BESDEBUG(DEBUG_KEY, __func__ <<
"() MDS lock for " << item_name <<
": " << lock() << endl);
751 INFO_LOG(prolog <<
"MDS Cache hit for '" << name <<
"' and response " << object_name << endl);
753 INFO_LOG(prolog <<
"MDS Cache miss for '" << name <<
"' and response " << object_name << endl);
981 time_t file_time = besRH->
get_lmt(realName);
987 if (file_time > cache_time){
1006 struct stat statbuf;
1008 if (stat(item_name.c_str(), &statbuf) == -1){
1012 return statbuf.st_mtime;
1031 VERBOSE(
"Metadata store: Cache hit: read " << object_name <<
" response for '" << name <<
"'." << endl);
1032 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Found " << item_name <<
" in the store." << endl);
1043 throw BESInternalError(
"Could not open '" + item_name +
"' in the metadata store.", __FILE__, __LINE__);
1057 const string &object_name)
1062 VERBOSE(
"Metadata store: Cache hit: read " << object_name <<
" response for '" << name <<
"'." << endl);
1063 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Found " << item_name <<
" in the store." << endl);
1076 throw BESInternalError(
"Could not open '" + item_name +
"' in the metadata store.", __FILE__, __LINE__);
1115 string xml_base = BESContextManager::TheManager()->
get_context(
"xml:base", found);
1117#if XML_BASE_MISSING_MEANS_OMIT_ATTRIBUTE
1120 throw BESInternalError(
"Could not read the value of xml:base.", __FILE__, __LINE__);
1138 string xml_base = BESContextManager::TheManager()->
get_context(
"xml:base", found);
1140#if XML_BASE_MISSING_MEANS_OMIT_ATTRIBUTE
1143 throw BESInternalError(
"Could not read the value of xml:base.", __FILE__, __LINE__);
1161 string hash =
get_hash(name + suffix);
1163 VERBOSE(
"Metadata store: Removed " << object_name <<
" response for '" << hash <<
"'." << endl);
1164 d_ledger_entry.append(
" ").append(hash);
1168 ERROR_LOG(prolog <<
"Metadata store: unable to remove the " << object_name <<
" response for '" << name <<
"' (" << strerror(errno) <<
")."<< endl);
1184 d_ledger_entry = string(
"remove ").append(name);
1196#if SYMETRIC_ADD_RESPONSES
1197 return (removed_dds && removed_das && removed_dmr);
1199 return (removed_dds || removed_das || removed_dmr || removed_dmrpp);
1221 D4BaseTypeFactory d4_btf;
1222 unique_ptr<DMR> dmr(
new DMR(&d4_btf,
"mds"));
1224 D4ParserSax2 parser;
1225 parser.intern(oss.str(), dmr.get());
1227 dmr->set_factory(0);
1229 return dmr.release();
1259 fstream dds_fs(dds_tmp_name.c_str(), std::fstream::out);
1269 BaseTypeFactory btf;
1270 unique_ptr<DDS> dds(
new DDS(&btf));
1271 dds->parse(dds_tmp_name);
1275 fstream das_fs(das_tmp_name.c_str(), std::fstream::out);
1285 unique_ptr<DAS> das(
new DAS());
1286 das->parse(das_tmp_name);
1288 dds->transfer_attributes(das.get());
1289 dds->set_factory(
nullptr);
1291 return dds.release();
1296GlobalMetadataStore::parse_das_from_mds(libdap::DAS* das,
const std::string &name) {
1297 string suffix =
"das_r";
1301 VERBOSE(
"Metadata store: Cache hit: read " <<
" response for '" << name <<
"'." << endl);
1302 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Found " << item_name <<
" in the store." << endl);
1305 das->parse(item_name);
1314 throw BESInternalError(
"Could not open '" + item_name +
"' in the metadata store.", __FILE__, __LINE__);
A container is something that holds data. E.G., a netcdf file or a database entry.
std::string get_relative_name() const
Get the relative name of the object in this container.
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
std::string get_real_name() const
retrieve the real name for this container, such as a file name.
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
Implementation of a caching mechanism for compressed data.
bool cache_enabled() const
virtual void unlock_and_close(const std::string &target)
const std::string get_cache_directory()
bool is_unlimited() const
Is this cache allowed to store as much as it wants?
virtual unsigned long long update_cache_info(const std::string &target)
Update the cache info file to include 'target'.
virtual bool create_and_lock(const std::string &target, int &fd)
Create a file in the cache and lock it for write access.
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
virtual bool get_read_lock(const std::string &target, int &fd)
Get a read-only lock on the file if it exists.
virtual bool get_exclusive_lock(const std::string &target, int &fd)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big.
virtual void update_and_purge(const std::string &new_file)
Purge files from the cache.
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
error thrown if the resource requested cannot be found
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
virtual time_t get_lmt(const std::string &name)
Get the Last modified time for.
static std::string lowercase(const std::string &s)
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
Get a new temporary file.
std::string create(const std::string &dir_name="/tmp/hyrax_tmp", const std::string &path_template="opendap")
Create a new temporary file.