45#include "BESCatalogUtils.h"
46#include "BESCatalogList.h"
47#include "TheBESKeys.h"
48#include "BESInternalError.h"
49#include "BESSyntaxUserError.h"
50#include "BESNotFoundError.h"
54#include "BESContainerStorageList.h"
55#include "BESContainerStorage.h"
56#include "BESCatalogEntry.h"
83BESCatalogUtils::BESCatalogUtils(
const string &n,
bool strict) :
84 d_name(n), d_follow_syms(false)
86 string key =
"BES.Catalog." + n +
".RootDirectory";
89 if (strict && (!found || d_root_dir ==
"")) {
90 string s = key +
" not defined in BES configuration file";
94 if(d_root_dir !=
"UNUSED"){
96 DIR *dip = opendir(d_root_dir.c_str());
98 string serr =
"BESCatalogDirectory - root directory " + d_root_dir +
" does not exist";
105 key = (string)
"BES.Catalog." + n +
".Exclude";
108 vector<string>::iterator ei = vals.begin();
109 vector<string>::iterator ee = vals.end();
110 for (; ei != ee; ei++) {
111 string e_str = (*ei);
112 if (!e_str.empty() && e_str !=
";")
BESUtil::explode(
';', e_str, d_exclude);
115 key = (string)
"BES.Catalog." + n +
".Include";
118 vector<string>::iterator ii = vals.begin();
119 vector<string>::iterator ie = vals.end();
120 for (; ii != ie; ii++) {
121 string i_str = (*ii);
122 if (!i_str.empty() && i_str !=
";")
BESUtil::explode(
';', i_str, d_include);
125 key =
"BES.Catalog." + n +
".TypeMatch";
126 list<string> match_list;
129 if (strict && (!found || vals.size() == 0)) {
130 string s = key +
" not defined in key file";
133 vector<string>::iterator vi = vals.begin();
134 vector<string>::iterator ve = vals.end();
135 for (; vi != ve; vi++) {
139 list<string>::iterator mli = match_list.begin();
140 list<string>::iterator mle = match_list.end();
141 for (; mli != mle; mli++) {
142 if (!((*mli).empty()) && *(mli) !=
";") {
145 if (amatch.size() != 2) {
146 string s = (string)
"Catalog type match malformed, " +
"looking for type:regexp;[type:regexp;]";
149 list<string>::iterator ami = amatch.begin();
150 handler_regex newval;
151 newval.handler = (*ami);
153 newval.regex = (*ami);
154 d_match_list.push_back(newval);
158 key = (string)
"BES.Catalog." + n +
".FollowSymLinks";
162 if (s_str ==
"yes" || s_str ==
"on" || s_str ==
"true") {
163 d_follow_syms =
true;
180 bool toInclude =
false;
185 if (d_include.size() == 0) {
189 list<string>::const_iterator i_iter = d_include.begin();
190 list<string>::const_iterator i_end = d_include.end();
191 for (; i_iter != i_end; i_iter++) {
192 string reg = *i_iter;
198 if (reg_expr.
match(inQuestion.c_str(), inQuestion.length())
199 ==
static_cast<int>(inQuestion.length())) {
204 string serr = (string)
"Unable to get catalog information, "
205 +
"malformed Catalog Include parameter " +
"in bes configuration file around " + reg +
": "
213 if (toInclude ==
true) {
231 list<string>::const_iterator e_iter = d_exclude.begin();
232 list<string>::const_iterator e_end = d_exclude.end();
233 for (; e_iter != e_end; e_iter++) {
234 string reg = *e_iter;
238 if (reg_expr.
match(inQuestion.c_str(), inQuestion.length()) ==
static_cast<int>(inQuestion.length())) {
243 string serr = (string)
"Unable to get catalog information, " +
"malformed Catalog Exclude parameter "
244 +
"in bes configuration file around " + reg +
": " + e.
get_message();
257BESCatalogUtils::match_citer BESCatalogUtils::match_list_begin()
const
259 return d_match_list.begin();
267BESCatalogUtils::match_citer BESCatalogUtils::match_list_end()
const
269 return d_match_list.end();
285 unsigned int cnt = 0;
288 int statret = stat(fullnode.c_str(), &cbuf);
290 if (errno == ENOENT) {
291 char *s_err = strerror(errno);
292 throw BESNotFoundError((s_err) ?
string(s_err) :
string(
"Node ") + use_node +
" does not exist", __FILE__,
297 char *s_err = strerror(errno);
298 throw BESNotFoundError((s_err) ?
string(s_err) :
string(
"Access denied for node ") + use_node, __FILE__,
304 while ((dit = readdir(dip)) != NULL) {
305 string dirEntry = dit->d_name;
306 if (dirEntry ==
"." || dirEntry ==
"..") {
310 string fullPath = fullnode +
"/" + dirEntry;
313 if (follow_sym_links() ==
false) {
315 (void) lstat(fullPath.c_str(), &lbuf);
316 if (S_ISLNK(lbuf.st_mode))
325 statret = stat(fullPath.c_str(), &buf);
326 if (statret == 0 && S_ISDIR(buf.st_mode)) {
327 if (
exclude(dirEntry) ==
false) {
330 bes_add_stat_info(curr_entry, buf);
332 entry->add_entry(curr_entry);
338 curr_entry->add_entry(blank_entry);
341 else if (statret == 0 && S_ISREG(buf.st_mode)) {
342 if (!dirs_only &&
include(dirEntry)) {
344 bes_add_stat_info(curr_entry, buf);
346 list<string> services;
348 isData(fullPath, d_name, services);
349 curr_entry->set_service_list(services);
351 bes_add_stat_info(curr_entry, buf);
353 entry->add_entry(curr_entry);
367 map<string, string> props;
368 if (entry->get_catalog() == defcatname) {
369 props[
"name"] = entry->get_name();
372 string name = entry->get_catalog() +
"/";
373 if (entry->get_name() !=
"/") {
374 name = name + entry->get_name();
376 props[
"name"] = name;
378 props[
"catalog"] = entry->get_catalog();
379 props[
"size"] = entry->get_size();
380 props[
"lastModified"] = entry->get_mod_date() +
"T" + entry->get_mod_time();
381 if (entry->is_collection()) {
382 props[
"node"] =
"true";
384 strm << entry->get_count();
385 props[
"count"] = strm.str();
388 props[
"node"] =
"false";
390 info->begin_tag(
"dataset", &props);
392 list<string> services = entry->get_service_list();
393 if (services.size()) {
394 list<string>::const_iterator si = services.begin();
395 list<string>::const_iterator se = services.end();
396 for (; si != se; si++) {
397 info->add_tag(
"serviceRef", (*si));
419 for (
auto i = match_list_begin(), e = match_list_end(); i != e; ++i) {
421 if (expr.
match(item.c_str(), item.length()) == (
int)item.length()) {
444 for (
auto i = match_list_begin(), e = match_list_end(); i != e; ++i) {
446 if (expr.
match(item.c_str(), item.length()) == (
int)item.length()) {
461void BESCatalogUtils::bes_add_stat_info(
BESCatalogEntry *entry,
const string &fullnode)
464 int statret = stat(fullnode.c_str(), &cbuf);
466 bes_add_stat_info(entry, cbuf);
470void BESCatalogUtils::bes_add_stat_info(
BESCatalogEntry *entry,
struct stat &buf)
472 off_t sz = buf.st_size;
477 time_t mod = buf.st_mtime;
479 gmtime_r(&mod, &stm);
481 strftime(mdate, 64,
"%Y-%m-%d", &stm);
483 strftime(mtime, 64,
"%T", &stm);
487 entry->set_mod_date(sdt.str());
491 entry->set_mod_time(stt.str());
494bool BESCatalogUtils::isData(
const string &inQuestion,
const string &catalog, list<string> &services)
497 if (!store)
return false;
499 return store->
isData(inQuestion, services);
504 strm << BESIndent::LMarg <<
"BESCatalogUtils::dump - (" << (
void *)
this <<
")" << endl;
507 strm << BESIndent::LMarg <<
"root directory: " << d_root_dir << endl;
509 if (d_include.size()) {
510 strm << BESIndent::LMarg <<
"include list:" << endl;
512 list<string>::const_iterator i_iter = d_include.begin();
513 list<string>::const_iterator i_end = d_include.end();
514 for (; i_iter != i_end; i_iter++) {
515 if (!(*i_iter).empty()) {
516 strm << BESIndent::LMarg << *i_iter << endl;
519 BESIndent::UnIndent();
522 strm << BESIndent::LMarg <<
"include list: empty" << endl;
525 if (d_exclude.size()) {
526 strm << BESIndent::LMarg <<
"exclude list:" << endl;
528 list<string>::const_iterator e_iter = d_exclude.begin();
529 list<string>::const_iterator e_end = d_exclude.end();
530 for (; e_iter != e_end; e_iter++) {
531 if (!(*e_iter).empty()) {
532 strm << BESIndent::LMarg << *e_iter << endl;
535 BESIndent::UnIndent();
538 strm << BESIndent::LMarg <<
"exclude list: empty" << endl;
541 if (d_match_list.size()) {
542 strm << BESIndent::LMarg <<
"type matches:" << endl;
544 BESCatalogUtils::match_citer i = d_match_list.begin();
545 BESCatalogUtils::match_citer ie = d_match_list.end();
546 for (; i != ie; i++) {
547 handler_regex match = (*i);
548 strm << BESIndent::LMarg << match.handler <<
" : " << match.regex << endl;
550 BESIndent::UnIndent();
553 strm << BESIndent::LMarg <<
" type matches: empty" << endl;
557 strm << BESIndent::LMarg <<
" follow symbolic links: on" << endl;
560 strm << BESIndent::LMarg <<
" follow symbolic links: off" << endl;
563 BESIndent::UnIndent();
568BESCatalogUtils::Utils(
const string &cat_name)
573 BESCatalogUtils::_instances[cat_name] = utils;
582void BESCatalogUtils::delete_all_catalogs()
584 map<string, BESCatalogUtils*>::iterator i = BESCatalogUtils::_instances.begin();
585 map<string, BESCatalogUtils*>::iterator e = BESCatalogUtils::_instances.end();
587 delete (*i++).second;
virtual std::string default_catalog_name() const
The name of the default catalog.
static BESCatalogList * TheCatalogList()
Get the singleton BESCatalogList instance.
virtual bool exclude(const std::string &inQuestion) const
Should this file/directory be excluded in the catalog?
virtual unsigned int get_entries(DIR *dip, const std::string &fullnode, const std::string &use_node, BESCatalogEntry *entry, bool dirs_only)
virtual bool include(const std::string &inQuestion) const
Should this file/directory be included in the catalog?
std::string get_handler_name(const std::string &item) const
Find the handler name that will process.
bool is_data(const std::string &item) const
is there a handler that can process this
virtual void dump(std::ostream &strm) const
dump the contents of this object to the specified ostream
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
provides persistent storage for data storage information represented by a container.
virtual bool isData(const std::string &inQuestion, std::list< std::string > &provides)=0
determine if the given container is data and what services are available for it
Base exception class for the BES with basic string message.
std::string get_message() const
get the error message for this exception
informational response object
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
Regular expression matching.
int match(const char *s, int len, int pos=0) const
Does the pattern match.
error thrown if there is a user syntax error in the request or any other user error
static void explode(char delim, const std::string &str, std::list< std::string > &values)
static std::string lowercase(const std::string &s)
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.