bes Updated for version 3.20.13
|
Store the DAP DMR++ metadata responses. More...
#include <DmrppMetadataStore.h>
Classes | |
struct | StreamDMRpp |
Hack use a DMR to write a DMR++ response. WIP. More... | |
Public Types | |
typedef struct MDSReadLock | MDSReadLock |
Public Member Functions | |
virtual bool | add_dmrpp_response (libdap::DMR *dmrpp, const std::string &name) |
virtual bool | add_responses (libdap::DMR *dmrpp, const std::string &name) |
Add the DAP4 metadata responses using a DMR. | |
bool | cache_enabled () const |
virtual bool | cache_too_big (unsigned long long current_size) const |
look at the cache size; is it too large? Look at the cache size and see if it is too big. | |
virtual bool | create_and_lock (const std::string &target, int &fd) |
Create a file in the cache and lock it for write access. | |
void | disable () |
Disable the cache. | |
virtual void | dump (std::ostream &strm) const |
dumps information about this object | |
void | enable () |
Enable the cache. | |
virtual void | exclusive_to_shared_lock (int fd) |
Transfer from an exclusive lock to a shared lock. | |
const std::string | get_cache_directory () |
virtual std::string | get_cache_file_name (const std::string &src, bool mangle=true) |
const std::string | get_cache_file_prefix () |
virtual time_t | get_cache_lmt (const std::string &fileName, const std::string &suffix) |
Get the last modified time for the cached object file. | |
virtual unsigned long long | get_cache_size () |
Get the cache size. | |
virtual libdap::DDS * | get_dds_object (const std::string &name) |
Build a DDS object from the cached Response. | |
virtual libdap::DMR * | get_dmr_object (const std::string &name) |
Build a DMR object from the cached Response. | |
virtual libdap::DMR * | get_dmr_object (const string &name) |
Use the DMR response to build a DMR with Dmrpp Types. | |
virtual dmrpp::DMRpp * | get_dmrpp_object (const std::string &name) |
Build a DMR++ object from the cached Response. | |
virtual bool | get_exclusive_lock (const std::string &target, int &fd) |
virtual bool | get_exclusive_lock_nb (const std::string &target, int &fd) |
virtual bool | get_read_lock (const std::string &target, int &fd) |
Get a read-only lock on the file if it exists. | |
void | initialize (const std::string &cache_dir, const std::string &prefix, unsigned long long size) |
Initialize an instance of FileLockingCache. | |
virtual bool | is_available_helper (const std::string &realName, const std::string &relativeName, const std::string &fileType, const std::string &suffix) |
helper function that checks if last modified time is greater than cached file | |
virtual MDSReadLock | is_das_available (const BESContainer &container) |
virtual MDSReadLock | is_das_available (const std::string &name) |
Is the DAS response for. | |
virtual MDSReadLock | is_dds_available (const BESContainer &container) |
Is the DDS response for. | |
virtual MDSReadLock | is_dds_available (const std::string &name) |
Is the DDS response for. | |
virtual MDSReadLock | is_dmr_available (const BESContainer &container) |
virtual MDSReadLock | is_dmr_available (const std::string &name) |
Is the DMR response for. | |
virtual MDSReadLock | is_dmr_available (const std::string &realName, const std::string &relativeName, const std::string &fileType) |
virtual MDSReadLock | is_dmrpp_available (const BESContainer &container) |
virtual MDSReadLock | is_dmrpp_available (const std::string &name) |
Is the DMR++ response for. | |
bool | is_unlimited () const |
Is this cache allowed to store as much as it wants? | |
virtual void | lock_cache_read () |
virtual void | lock_cache_write () |
virtual void | parse_das_from_mds (libdap::DAS *das, const std::string &name) |
virtual void | purge_file (const std::string &file) |
Purge a single file from the cache. | |
virtual bool | remove_responses (const std::string &name) |
Remove all cached responses and objects for a granule. | |
virtual void | unlock_and_close (const std::string &target) |
virtual void | unlock_cache () |
virtual void | update_and_purge (const std::string &new_file) |
Purge files from the cache. | |
virtual unsigned long long | update_cache_info (const std::string &target) |
Update the cache info file to include 'target'. | |
virtual void | write_das_response (const std::string &name, std::ostream &os) |
Write the stored DAS response to a stream. | |
virtual void | write_dds_response (const std::string &name, std::ostream &os) |
Write the stored DDS response to a stream. | |
virtual void | write_dmr_response (const std::string &name, std::ostream &os) |
Write the stored DMR response to a stream. | |
virtual void | write_dmrpp_response (const std::string &name, std::ostream &os) |
Write the stored DMR++ response to a stream. | |
Add responses to the GlobalMetadataStore | |
These methods use a DDS or DMR object to generate the DDS, DAS and DMR responses for DAP (2 and 4). They store those in the MDS and then update the MDS ledger file with the operation (add), the kind of object used to build the responses (DDS or DMR), name of the granule and hashes/names for each of the three files in the MDS that hold the responses. If verbose logging is on, the bes log also will hold information about the operation. If there is an error, that will always be recorded in the bes log. | |
virtual bool | add_responses (libdap::DDS *dds, const std::string &name) |
Add the DAP2 metadata responses using a DDS. | |
Static Public Member Functions | |
static bool | dir_exists (const std::string &dir) |
Get an instance of DmrppMetadataStore | |
There are two ways to get an instance of DmrppMetadataStore singleton.
| |
static DmrppMetadataStore * | get_instance (const std::string &cache_dir, const std::string &prefix, unsigned long long size) |
Get an instance of the DmrppMetadataStore object. | |
static DmrppMetadataStore * | get_instance () |
Protected Member Functions | |
DmrppMetadataStore (const DmrppMetadataStore &src) | |
DmrppMetadataStore (const std::string &cache_dir, const std::string &prefix, unsigned long long size) | |
std::string | get_hash (const std::string &name) |
MDSReadLock | get_read_lock_helper (const std::string &name, const std::string &suffix, const std::string &object_name) |
void | initialize () |
Configure the ledger using LEDGER_KEY and LOCAL_TIME_KEY. | |
bool | remove_response_helper (const std::string &name, const std::string &suffix, const std::string &object_name) |
bool | store_dap_response (StreamDAP &writer, const std::string &key, const std::string &name, const std::string &response_name) |
void | write_ledger () |
write_response_helper | |
void | write_response_helper (const std::string &name, std::ostream &os, const std::string &suffix, const std::string &object_name) |
void | write_response_helper (const std::string &name, std::ostream &os, const std::string &suffix, const std::string &xml_base, const std::string &object_name) |
This version looks at the first few bytes and substitutes a new value for xml:base. | |
Static Protected Member Functions | |
static std::string | get_cache_dir_from_config () |
static std::string | get_cache_prefix_from_config () |
static unsigned long | get_cache_size_from_config () |
static void | insert_xml_base (int fd, std::ostream &os, const std::string &xml_base) |
like transfer_bytes(), but adds the xml:base attribute to the DMR/++ | |
static void | transfer_bytes (int fd, std::ostream &os) |
Protected Attributes | |
std::string | d_ledger_entry |
Friends | |
class | DmrppMetadataStoreTest |
Store the DAP DMR++ metadata responses.
Provide a way to add DMR++ responses to the Global MDS (see GlobalMetadataStore). This specialization of the MDS adds the ability to add DMR++ responses and extract DMR++ objects. This code is accessible only in the DMR++ handler, while the Global MDS is available anywhere (it's defined in bes/dap). Because the BES framework needs to determine if DMR++ responses are included in the MDS, the Global MDS class has methods to test for that and to 'write' that response.
In order for the BES to work with a DMR++ object, however, it will need to transfer control to the DMR++ Handler and, if it is going to use the response in the MDS, use this specialization to read from the MDS.
The class is implemented as a singleton; use the get_instance() methods to get an instance of the DMR++ metadata store. It uses the same configuration parameters as the Global MDS, so if the defaults are uses, both the singleton Global MDS and this DMR++ MDS will be using the same directory, file prefix and size limit.
BES Keys used:
Definition at line 83 of file DmrppMetadataStore.h.
|
inherited |
Definition at line 213 of file GlobalMetadataStore.h.
|
inlineprotected |
Definition at line 105 of file DmrppMetadataStore.h.
|
inlineprotected |
Definition at line 108 of file DmrppMetadataStore.h.
|
inlineprotected |
Definition at line 109 of file DmrppMetadataStore.h.
|
inlinevirtual |
Definition at line 116 of file DmrppMetadataStore.h.
|
virtual |
Definition at line 213 of file DmrppMetadataStore.cc.
|
virtualinherited |
Add the DAP2 metadata responses using a DDS.
This method adds only the DDS and DAS unless the code was compiled with the symbol SYMETRIC_ADD_RESPONSES defined.
name | The granule name or identifier |
dds | A DDS built from the granule |
Definition at line 651 of file GlobalMetadataStore.cc.
|
virtual |
Add the DAP4 metadata responses using a DMR.
This method adds only the DMR unless the code was compiled with the symbol SYMETRIC_ADD_RESPONSES defined.
name | The granule name or identifier |
dmr | A DMR built from the granule |
Reimplemented from bes::GlobalMetadataStore.
Definition at line 192 of file DmrppMetadataStore.cc.
|
inlineinherited |
Definition at line 198 of file BESFileLockingCache.h.
|
virtualinherited |
look at the cache size; is it too large? Look at the cache size and see if it is too big.
Definition at line 772 of file BESFileLockingCache.cc.
Create a file in the cache and lock it for write access.
If the file does not exist, make it, open it for read-write access and get an exclusive lock on it. The locking operation blocks, although that should never happen.
target | The name of the file to make/open/lock |
fd | Value-result param that holds the file descriptor of the opened file |
BESBESInternalErroror | if any error except EEXIST is returned by open(2) or if fcntl(2) returns an error. |
Definition at line 592 of file BESFileLockingCache.cc.
|
staticinherited |
Does the directory exist?
dir | The pathname to test. |
Definition at line 1121 of file BESFileLockingCache.cc.
|
inlineinherited |
Disable the cache.
Definition at line 204 of file BESFileLockingCache.h.
|
virtualinherited |
dumps information about this object
Displays the pointer value of this instance along with information about this cache.
strm | C++ i/o stream to dump the information to |
Implements BESObj.
Definition at line 1136 of file BESFileLockingCache.cc.
|
inlineinherited |
Enable the cache.
Definition at line 210 of file BESFileLockingCache.h.
|
virtualinherited |
Transfer from an exclusive lock to a shared lock.
If the file has an exclusive write lock on it, change that to a shared read lock. This is an atomic operation. If the call to fcntl(2) is protected by locking the cache, a dead lock will result given typical use of this class. This method exists to help with the situation where one process has the cache locked and is blocking on a shared read lock for a file that a second process has locked exclusively (for writing). By changing the exclusive lock to a shared lock, the first process can get its shared lock and then release the cache.
fd | The file descriptor that is exclusively locked and which, on exit, will have a shared lock. |
Definition at line 622 of file BESFileLockingCache.cc.
|
staticprotectedinherited |
Definition at line 262 of file GlobalMetadataStore.cc.
|
inlineinherited |
Definition at line 188 of file BESFileLockingCache.h.
|
virtualinherited |
Returns the fully qualified file system path name for the cache file associated with this particular cache resource.
src | The source name to cache |
mangle | If True, assume the name is a file pathname and mangle it. If false, do not mangle the name (assume the caller has sent a suitable string) but do turn the string into a pathname located in the cache directory with the cache prefix. The 'mangle' param is true by default. |
Reimplemented in BESUncompressCache, and http::HttpCache.
Definition at line 444 of file BESFileLockingCache.cc.
|
inlineinherited |
Definition at line 182 of file BESFileLockingCache.h.
|
virtualinherited |
Get the last modified time for the cached object file.
name | - name of the object |
suffix | - suffix of the object |
Definition at line 1003 of file GlobalMetadataStore.cc.
|
staticprotectedinherited |
Definition at line 247 of file GlobalMetadataStore.cc.
|
virtualinherited |
Get the cache size.
Read the size information from the cache info file and return it. This methods locks the cache.
Definition at line 784 of file BESFileLockingCache.cc.
|
staticprotectedinherited |
Definition at line 231 of file GlobalMetadataStore.cc.
|
virtualinherited |
Build a DDS object from the cached Response.
Read the DDS and DAS responses, build a DDS using their information and return the binary DDS response. The variables are built using the default BaseTypeFactory but the DDS object has the factory set to null when it is returned. The DDS is 'loaded' with attribute information as well, so it can be used to return the DDX response.
name | Path to the dataset, relative to the BES data root directory. |
BESInternalError | is thrown if
|
Definition at line 1254 of file GlobalMetadataStore.cc.
|
virtualinherited |
Build a DMR object from the cached Response.
Read and parse a DMR response , building a binary DMR object. The object is returned with a null factory. The variables are built using the default DAP4 type factory.
name | Name of the dataset |
BESInternalError | is thrown if
|
Definition at line 1216 of file GlobalMetadataStore.cc.
|
virtual |
Use the DMR response to build a DMR with Dmrpp Types.
name | The pathname to the dataset, relative to the BES data root directory |
Definition at line 238 of file DmrppMetadataStore.cc.
|
virtual |
Build a DMR++ object from the cached Response.
Read and parse a DMR++ response , building a binary DMR++ object. The object is returned with a null factory. The variables are built using DmrppTypeFactory
name | Name of the dataset |
BESInternalError | is thrown if
|
Definition at line 270 of file DmrppMetadataStore.cc.
|
virtualinherited |
A blocking call to get an exclusive (write) lock on a file in the cache. Because this cache uses per-process advisory locking, it's possible to call this several times from within a single process and get a lock each time.
file_name | Name of the file to lock |
ref_fd | Return value parameter that holds the file descriptor that is locked. |
BESInternalError | is thrown on any condition other than the file not existing |
Definition at line 1024 of file BESFileLockingCache.cc.
|
virtualinherited |
A non-blocking call to get an exclusive (write) lock on a file in the cache. Because this cache uses per-process advisory locking, it's possible to call this several times from within a single process and get a lock each time.
file_name | Name of the file to lock |
ref_fd | Return value parameter that holds the file descriptor that is locked. |
BESInternalError | is thrown on any condition other than the file not existing or the lock being unavailable. |
Definition at line 870 of file BESFileLockingCache.cc.
|
protectedinherited |
Compute the SHA256 hash for the item name
name | The name to hash |
Definition at line 486 of file GlobalMetadataStore.cc.
|
static |
Get an instance of the DmrppMetadataStore using the default values for the cache directory, prefix and size.
Definition at line 138 of file DmrppMetadataStore.cc.
|
static |
Get an instance of the DmrppMetadataStore object.
This class is a singleton, so the first call to any of two 'get_instance()' methods makes an instance and subsequent calls return a pointer to that instance.
cache_dir_key | Key to use to get the value of the cache directory. If this is the empty string, return null right away. |
prefix_key | Key for the item/file prefix. Each item added to the cache uses this as a prefix so cached items can be easily identified when the same directory is used for several caches or /tmp is used for the cache. |
size_key | The maximum size of the data stored in the cache, in megabytes |
Definition at line 108 of file DmrppMetadataStore.cc.
Get a read-only lock on the file if it exists.
Try to get a read-only lock on the file, blocking until we can get it. If the file does not exist, return false.
target | The path of the cached file |
fd | A value-result parameter set to the locked cached file. Undefined if the file could not be locked for read access. |
Error | if the attempt to get the (shared) lock failed for any reason other than that the file does/did not exist. |
Definition at line 537 of file BESFileLockingCache.cc.
|
protectedinherited |
Common code to acquire a read lock on a MDS item. This method locks the response for reading. When the MDSReadLock goes out of scope, the response is unlocked.
This method logs (using LOG, not VERBOSE) cache hits and misses.
name | Granule name |
suffix | One of 'dds_r', 'das_r' or 'dmr_r' |
object_name | One of DDS, DAS or DMR (used for logging only) |
Definition at line 737 of file GlobalMetadataStore.cc.
|
protectedinherited |
Configure the ledger using LEDGER_KEY and LOCAL_TIME_KEY.
Definition at line 366 of file GlobalMetadataStore.cc.
|
inherited |
Initialize an instance of FileLockingCache.
Initialize and instance of FileLockingCache using the passed values for the cache directory, item prefix and max cache size. This will ignore the value of enable_cache() (but will correctly (re)set it based on the directory, ..., values). This provides a way for clients to re-initialize caches on the fly.
cache_dir | The directory into which the cache files will be written. |
prefix | The prefix that will be added to each cache file. |
size | The size of the cache in MBytes |
BESInternalError | If the cache_dir does not exist or is not writable. size is 0, or if cache dir does not exist. |
BESError | If the parameters (directory, ...) are invalid. |
Definition at line 116 of file BESFileLockingCache.cc.
|
staticprotectedinherited |
like transfer_bytes(), but adds the xml:base attribute to the DMR/++
fd | Open file descriptor to read from; assumed open and positioned at the start of the file. |
os | Write to this C++ stream |
xml_base | Value of the xml:base attribute. |
BESInternalError | Thrown if there's a problem reading or writing. |
Definition at line 157 of file GlobalMetadataStore.cc.
|
virtualinherited |
helper function that checks if last modified time is greater than cached file
realName | - complete path to file used to find actual file |
relativeName | - relative filename used to find cached file |
fileType | - used to retrieve correct BESRequestHandler from BESRequestHandlerList |
suffix | - One of 'dmr_r', 'dds_r', 'das_r' or 'dmrpp_r' |
Definition at line 975 of file GlobalMetadataStore.cc.
|
virtualinherited |
Definition at line 891 of file GlobalMetadataStore.cc.
|
virtualinherited |
Is the DAS response for.
@Deprecated - 6.25.19 SBL
name | Find the DAS response for |
Definition at line 885 of file GlobalMetadataStore.cc.
|
virtualinherited |
Is the DDS response for.
This is the preferred method since it checks the LMT of the underlying dataset.
container | Find the DDS response for the dataset in
|
Definition at line 854 of file GlobalMetadataStore.cc.
|
virtualinherited |
Is the DDS response for.
@Deprecated - 6.25.19 SBL
name | Find the DDS response for |
Definition at line 838 of file GlobalMetadataStore.cc.
|
virtualinherited |
Definition at line 783 of file GlobalMetadataStore.cc.
|
virtualinherited |
Is the DMR response for.
@Deprecated - 6.25.19 SBL
Look in the MDS to see if the DMR response has been stored/cached for
name | Find the DMR response for
|
Definition at line 777 of file GlobalMetadataStore.cc.
|
virtualinherited |
Definition at line 807 of file GlobalMetadataStore.cc.
|
virtualinherited |
Definition at line 940 of file GlobalMetadataStore.cc.
|
virtualinherited |
Is the DMR++ response for.
@Deprecated - 6.25.19 SBL
Look in the MDS to see if the DMR++ response has been stored/cached for
name | Find the DMR++ response for
|
Definition at line 934 of file GlobalMetadataStore.cc.
|
inlineinherited |
Is this cache allowed to store as much as it wants?
If the size of the cache is zero bytes, then it is allowed to grow with out bounds.
Definition at line 177 of file BESFileLockingCache.h.
|
virtualinherited |
Get a shared lock on the 'cache info' file.
Definition at line 661 of file BESFileLockingCache.cc.
|
virtualinherited |
Get an exclusive lock on the 'cache info' file. The 'cache info' file is used to control certain cache actions, ensuring that they are atomic. These include making sure that the create_and_lock() and read_and_lock() operations are atomic as well as the purge and related operations.
Definition at line 646 of file BESFileLockingCache.cc.
|
virtualinherited |
Definition at line 1296 of file GlobalMetadataStore.cc.
|
virtualinherited |
Purge a single file from the cache.
Purge a single file from the cache. The file might be old, etc., and need to be removed. Don't use this to shrink the cache when it gets too big, use update_and_purge() instead since that file optimizes accesses to the cache control file for several changes in a row.
file | The name of the file to purge. |
Definition at line 1070 of file BESFileLockingCache.cc.
|
protectedinherited |
Common code to remove a stored response.
name | Granule name |
suffix | One of 'dds_r', 'das_r' or 'dmr_r' |
object_name | One of DDS, DAS or DMR |
Definition at line 1159 of file GlobalMetadataStore.cc.
|
virtualinherited |
Remove all cached responses and objects for a granule.
name |
Definition at line 1181 of file GlobalMetadataStore.cc.
|
protectedinherited |
Store the DAP metadata responses
writer | A child instance of StreamDAP, instantiated using a DDS or DMR. An instance of StreamDDS will write a DDS response, StreamDAS writes a DAS response,and StreamDMR writes a DMR response. |
key | Unique Id for this response; used to store the response in the MDS. |
name | The granule/file name or pathname |
response_name | The name of the particular response (DDS, DAS, DMR). Used for log messages. |
BESInternalError | If ... |
Definition at line 564 of file GlobalMetadataStore.cc.
|
staticprotectedinherited |
Hacked from GNU wc (in coreutils). This was found to be faster than a memory mapped file read.
https://stackoverflow.com/questions/17925051/fast-textfile-reading-in-c
fd | Open file descriptor to read from; assumed open and positioned at the start of the file. |
os | Write to this C++ stream |
BESInternalError | Thrown if there's a problem reading or writing. |
Definition at line 121 of file GlobalMetadataStore.cc.
|
virtualinherited |
Unlock the named file.
This does not do any name mangling; it just closes and unlocks whatever is named (or throws BESBESInternalErroror if the file cannot be closed). If the file was opened more than once, all descriptors are closed. If you need to close a specific descriptor, use the other version of unlock_and_close().
file_name | The name of the file to unlock. |
BESBESInternalErroror |
Definition at line 705 of file BESFileLockingCache.cc.
|
virtualinherited |
Unlock the cache info file.
Definition at line 678 of file BESFileLockingCache.cc.
|
virtualinherited |
Purge files from the cache.
Purge files, oldest to newest, if the current size of the cache exceeds the size of the cache specified in the constructor. This method uses an exclusive lock on the cache for the duration of the purge process.
new_file | Do not delete this file. The name of a file this process just added to the cache. Using fcntl(2) locking there is no way this process can detect its own lock, so the shared read lock on the new file won't keep this process from deleting it (but will keep other processes from deleting it). |
Definition at line 927 of file BESFileLockingCache.cc.
|
virtualinherited |
Update the cache info file to include 'target'.
Add the size of the named file to the total cache size recorded in the cache info file. The cache info file is exclusively locked by this method for its duration. This updates the cache info file and returns the new size.
target | The name of the file |
Definition at line 729 of file BESFileLockingCache.cc.
|
virtualinherited |
Write the stored DAS response to a stream.
name | The (path)name of the granule |
os | Write to this stream |
Definition at line 1100 of file GlobalMetadataStore.cc.
|
virtualinherited |
Write the stored DDS response to a stream.
name | The (path)name of the granule |
os | Write to this stream |
Definition at line 1088 of file GlobalMetadataStore.cc.
|
virtualinherited |
Write the stored DMR response to a stream.
name | The (path)name of the granule |
os | Write to this stream |
Definition at line 1112 of file GlobalMetadataStore.cc.
|
virtualinherited |
Write the stored DMR++ response to a stream.
name | The (path)name of the granule |
os | Write to this stream |
Definition at line 1135 of file GlobalMetadataStore.cc.
|
protectedinherited |
Write the current text of d_ledger_entry to the metadata store ledger
Definition at line 450 of file GlobalMetadataStore.cc.
|
protectedinherited |
Common code to copy a response to an output stream.
name | Granule name |
os | Write the response to this stream |
suffix | One of 'dds_r', 'das_r' or 'dmr_r' |
object_name | One of DDS, DAS or DMR; used for error reporting. |
Definition at line 1026 of file GlobalMetadataStore.cc.
|
protectedinherited |
This version looks at the first few bytes and substitutes a new value for xml:base.
name | Granule name |
os | Write the response to this stream |
suffix | One of 'dds_r', 'das_r' or 'dmr_r' |
xml_base | Value of the xml:base attribute in the <Dataset...> element |
object_name | One of DDS, DAS or DMR; used for error reporting. |
Definition at line 1056 of file GlobalMetadataStore.cc.
|
friend |
Definition at line 94 of file DmrppMetadataStore.h.
|
protectedinherited |
Definition at line 111 of file GlobalMetadataStore.h.