44#include "BESCatalogUtils.h"
45#include "BESCatalogEntry.h"
47#include "CatalogNode.h"
48#include "CatalogItem.h"
51#include "BESContainerStorageList.h"
52#include "BESFileContainerStorage.h"
55#include "BESInternalError.h"
56#include "BESForbiddenError.h"
57#include "BESNotFoundError.h"
58#include "BESSyntaxUserError.h"
60#include "TheBESKeys.h"
65#include "CmrCatalog.h"
70#define prolog std::string("CmrCatalog::").append(__func__).append("() - ")
89 throw BESInternalError(
string(
"The CMR module must define at least one collection name using the key; '")+CMR_COLLECTIONS
90 +
"'", __FILE__, __LINE__);
96 throw BESInternalError(
string(
"The CMR module must define at least one facet name using the key; '")+CMR_COLLECTIONS
97 +
"'", __FILE__, __LINE__);
101CmrCatalog::~CmrCatalog()
105CmrCatalog::get_node(
const string &path)
const
107 return get_node_NEW(path);
112CmrCatalog::get_node_NEW(
const string &ppath)
const
116 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
123 if(path_elements.empty()){
125 node->set_lmt(epoch_time);
126 node->set_catalog_name(CMR_CATALOG_NAME);
127 for(
size_t i=0; i<d_collections.size() ; i++){
129 collection->
set_name(d_collections[i]);
130 collection->
set_type(CatalogItem::node);
131 node->add_node(collection);
135 for(
size_t i=0; i< path_elements.size() ;i++){
136 if(path_elements[i]==
"-")
137 path_elements[i] =
"";
140 string collection = path_elements[0];
141 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
142 bool valid_collection =
false;
143 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
144 if(collection == d_collections[i])
145 valid_collection =
true;
147 if(!valid_collection){
148 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
150 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
151 if(path_elements.size() >1){
152 string facet = path_elements[1];
153 bool valid_facet =
false;
154 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
155 if(facet == d_facets[i])
159 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
162 if(facet==
"temporal"){
163 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
165 node->set_lmt(epoch_time);
166 node->set_catalog_name(CMR_CATALOG_NAME);
169 switch( path_elements.size()){
173 vector<string> years;
175 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
176 cmrApi.get_years(collection, years);
177 for(
size_t i=0; i<years.size() ; i++){
179 collection->
set_type(CatalogItem::node);
182 collection->
set_lmt(epoch_time);
184 node->add_node(collection);
191 string year = path_elements[2];
194 vector<string> months;
196 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
197 cmrApi.get_months(collection, year, months);
198 for(
size_t i=0; i<months.size() ; i++){
200 collection->
set_type(CatalogItem::node);
203 collection->
set_lmt(epoch_time);
205 node->add_node(collection);
212 string year = path_elements[2];
213 string month = path_elements[3];
217 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
218 cmrApi.get_days(collection, year, month, days);
219 for(
size_t i=0; i<days.size() ; i++){
221 collection->
set_type(CatalogItem::node);
224 collection->
set_lmt(epoch_time);
226 node->add_node(collection);
233 string year = path_elements[2];
234 string month = path_elements[3];
235 string day = path_elements[4];
236 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
237 vector<Granule *> granules;
238 cmrApi.get_granules(collection, year, month, day, granules);
239 for(
size_t i=0; i<granules.size() ; i++){
247 string year = path_elements[2];
248 string month = path_elements[3];
249 string day = path_elements[4];
250 string granule_id = path_elements[5];
251 BESDEBUG(MODULE, prolog <<
"Request resolved to leaf granule/dataset name, collection: " << collection <<
" year: " << year
252 <<
" month: " << month <<
" day: " << day <<
" granule: " << granule_id << endl);
253 Granule *granule = cmrApi.get_granule(collection,year,month,day,granule_id);
256 granuleItem->
set_type(CatalogItem::leaf);
257 granuleItem->
set_name(granule->getName());
259 granuleItem->
set_lmt(granule->getLastModifiedStr());
260 granuleItem->
set_size(granule->getSize());
261 node->set_leaf(granuleItem);
271 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
278 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
282 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
284 node->set_lmt(epoch_time);
285 node->set_catalog_name(CMR_CATALOG_NAME);
286 for(
size_t i=0; i<d_facets.size() ; i++){
289 collection->
set_type(CatalogItem::node);
290 collection->
set_lmt(epoch_time);
291 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
292 node->add_node(collection);
326 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
333 if(path_elements.empty()){
335 node->set_lmt(epoch_time);
336 node->set_catalog_name(CMR_CATALOG_NAME);
337 for(
size_t i=0; i<d_collections.size() ; i++){
339 collection->
set_name(d_collections[i]);
340 collection->
set_type(CatalogItem::node);
341 node->add_node(collection);
345 string collection = path_elements[0];
346 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
347 bool valid_collection =
false;
348 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
349 if(collection == d_collections[i])
350 valid_collection =
true;
352 if(!valid_collection){
353 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
355 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
356 if(path_elements.size() >1){
357 string facet = path_elements[1];
358 bool valid_facet =
false;
359 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
360 if(facet == d_facets[i])
364 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
367 if(facet==
"temporal"){
368 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
370 node->set_lmt(epoch_time);
371 node->set_catalog_name(CMR_CATALOG_NAME);
374 switch( path_elements.size()){
377 vector<string> years;
379 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
381 for(
size_t i=0; i<years.size() ; i++){
383 collection->
set_type(CatalogItem::node);
386 collection->
set_lmt(epoch_time);
388 node->add_node(collection);
394 string year = path_elements[2];
397 vector<string> months;
399 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
401 for(
size_t i=0; i<months.size() ; i++){
403 collection->
set_type(CatalogItem::node);
406 collection->
set_lmt(epoch_time);
408 node->add_node(collection);
414 string year = path_elements[2];
415 string month = path_elements[3];
419 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
420 cmrApi.
get_days(collection, year, month, days);
421 for(
size_t i=0; i<days.size() ; i++){
423 collection->
set_type(CatalogItem::node);
426 collection->
set_lmt(epoch_time);
428 node->add_node(collection);
434 string year = path_elements[2];
435 string month = path_elements[3];
436 string day = path_elements[4];
437 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
438 vector<Granule *> granules;
439 cmrApi.
get_granules(collection, year, month, day, granules);
440 for(
size_t i=0; i<granules.size() ; i++){
446 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
451 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
455 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
457 node->set_lmt(epoch_time);
458 node->set_catalog_name(CMR_CATALOG_NAME);
459 for(
size_t i=0; i<d_facets.size() ; i++){
462 collection->
set_type(CatalogItem::node);
463 collection->
set_lmt(epoch_time);
464 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
465 node->add_node(collection);
474CmrCatalog::get_node(
const string &path)
const
477 string rootdir = d_utils->get_root_dir();
485 string fullpath = rootdir + path;
487 DIR *dip = opendir(fullpath.c_str());
490 "A CMRCatalog can only return nodes for directory. The path '" + path
491 +
"' is not a directory for BESCatalog '" +
get_catalog_name() +
"'.", __FILE__, __LINE__);
498 if (d_utils->exclude(path))
500 string(
"The path '") + path +
"' is not included in the catalog '" +
get_catalog_name() +
"'.",
507 int statret = stat(fullpath.c_str(), &buf);
509 node->set_lmt(get_time(buf.st_mtime));
512 while ((dit = readdir(dip)) != NULL) {
513 string item = dit->d_name;
514 if (item ==
"." || item ==
"..")
continue;
516 string item_path = fullpath +
"/" + item;
526 if (d_utils->follow_sym_links() ==
false) {
528 (void) lstat(item_path.c_str(), &lbuf);
529 if (S_ISLNK(lbuf.st_mode))
continue;
533 statret = stat(item_path.c_str(), &buf);
534 if (statret == 0 && S_ISDIR(buf.st_mode) && !d_utils->exclude(item)) {
537 node->add_item(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
539 node->add_node(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
541 else if (statret == 0 && S_ISREG(buf.st_mode) && d_utils->include(item)) {
544 node->add_item(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
545 d_utils->is_data(item), CatalogItem::leaf));
547 node->add_leaf(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
548 d_utils->is_data(item), CatalogItem::leaf));
551 VERBOSE(
"Excluded the item '" << item_path <<
"' from the catalog '" <<
get_catalog_name() <<
"' node listing." << endl);
559 sort(node->nodes_begin(), node->nodes_end(), ordering);
560 sort(node->leaves_begin(), node->leaves_end(), ordering);
581 strm << BESIndent::LMarg << prolog <<
"(" << (
void *)
this <<
")" << endl;
584 strm << BESIndent::LMarg <<
"catalog utilities: " << endl;
587 BESIndent::UnIndent();
588 BESIndent::UnIndent();
virtual void dump(std::ostream &strm) const
dump the contents of this object to the specified ostream
Catalogs provide a hierarchical organization for data.
virtual BESCatalogUtils * get_catalog_utils() const
Get a pointer to the utilities, customized for this catalog.
virtual std::string get_catalog_name() const
Get the name for this catalog.
error thrown if the BES is not allowed to access the resource requested
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
error thrown if there is a user syntax error in the request or any other user error
static std::vector< std::string > split(const std::string &s, char delim='/', bool skip_empty=true)
Splits the string s into the return vector of tokens using the delimiter delim and skipping empty val...
static void check_path(const std::string &path, const std::string &root, bool follow_sym_links)
Is the combination of root + path a pathname the BES can/should access?
static std::string normalize_path(const std::string &path, bool leading_separator, bool trailing_separator, std::string separator="/")
Removes duplicate separators and provides leading and trailing separators as directed.
static std::string get_time(bool use_local_time=false)
static TheBESKeys * TheKeys()
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
void set_name(std::string n)
Set the name of the item.
void set_size(size_t s)
Set the size of the item.
void set_is_data(bool id)
Is this item data that the BES should interpret?
void set_lmt(std::string lmt)
Set the LMT for this item.
void set_type(item_type t)
Set the type for this item.
void get_years(std::string collection_name, std::vector< std::string > &years_result)
void get_days(std::string collection_name, std::string r_year, std::string r_month, std::vector< std::string > &days_result)
void get_granules(std::string collection_name, std::string r_year, std::string r_month, std::string r_day, std::vector< cmr::Granule * > &granules)
void get_months(std::string collection_name, std::string year, std::vector< std::string > &months_result)
virtual bes::CatalogNode * get_node_OLD(const std::string &path) const
Get a CatalogNode for the given path in the current catalog.
CmrCatalog(const std::string &name=CMR_CATALOG_NAME)
A catalog based on NASA's CMR system.
virtual void dump(std::ostream &strm) const
dumps information about this object