44#include <BESContextManager.h>
45#include "BESSyntaxUserError.h"
46#include "BESForbiddenError.h"
47#include "BESNotFoundError.h"
48#include "BESTimeoutError.h"
49#include "BESInternalError.h"
52#include "TheBESKeys.h"
55#include "BESStopWatch.h"
57#include "BESSyntaxUserError.h"
60#include "ProxyConfig.h"
61#include "AllowedHosts.h"
63#include "EffectiveUrlCache.h"
73using std::stringstream;
74using std::ostringstream;
77#define prolog std::string("CurlUtils::").append(__func__).append("() - ")
81static const unsigned int retry_limit = 10;
82static const useconds_t uone_second = 1000 * 1000;
85curl_slist *add_edl_auth_headers(
struct curl_slist *request_headers);
90#define CLIENT_ERR_MIN 400
91#define CLIENT_ERR_MAX 417
92const char *http_client_errors[CLIENT_ERR_MAX - CLIENT_ERR_MIN + 1] = {
94 "Unauthorized: Contact the server administrator.",
96 "Forbidden: Contact the server administrator.",
97 "Not Found: The underlying data source or server could not be found.",
98 "Method Not Allowed.",
100 "Proxy Authentication Required.",
105 "Precondition Failed.",
106 "Request Entity Too Large.",
107 "Request URI Too Large.",
108 "Unsupported Media Type.",
109 "Requested Range Not Satisfiable.",
110 "Expectation Failed."
113#define SERVER_ERR_MIN 500
114#define SERVER_ERR_MAX 505
115const char *http_server_errors[SERVER_ERR_MAX - SERVER_ERR_MIN + 1] =
117 "Internal Server Error.",
120 "Service Unavailable.",
122 "HTTP Version Not Supported."
133string http_status_to_string(
int status) {
134 if (status >= CLIENT_ERR_MIN && status <= CLIENT_ERR_MAX)
135 return string(http_client_errors[status - CLIENT_ERR_MIN]);
136 else if (status >= SERVER_ERR_MIN && status <= SERVER_ERR_MAX)
137 return string(http_server_errors[status - SERVER_ERR_MIN]);
140 msg <<
"Unknown HTTP Error: " << status;
150static string getCurlAuthTypeName(
const int auth_type) {
152 string authTypeString;
155 match = auth_type & CURLAUTH_BASIC;
157 authTypeString +=
"CURLAUTH_BASIC";
160 match = auth_type & CURLAUTH_DIGEST;
162 if (!authTypeString.empty())
163 authTypeString +=
" ";
164 authTypeString +=
"CURLAUTH_DIGEST";
167 match = auth_type & CURLAUTH_DIGEST_IE;
169 if (!authTypeString.empty())
170 authTypeString +=
" ";
171 authTypeString +=
"CURLAUTH_DIGEST_IE";
174 match = auth_type & CURLAUTH_GSSNEGOTIATE;
176 if (!authTypeString.empty())
177 authTypeString +=
" ";
178 authTypeString +=
"CURLAUTH_GSSNEGOTIATE";
181 match = auth_type & CURLAUTH_NTLM;
183 if (!authTypeString.empty())
184 authTypeString +=
" ";
185 authTypeString +=
"CURLAUTH_NTLM";
189 match = auth_type & CURLAUTH_ANY;
191 if(!authTypeString.empty())
192 authTypeString +=
" ";
193 authTypeString +=
"CURLAUTH_ANY";
197 match = auth_type & CURLAUTH_ANY;
199 if(!authTypeString.empty())
200 authTypeString +=
" ";
201 authTypeString +=
"CURLAUTH_ANYSAFE";
205 match = auth_type & CURLAUTH_ANY;
207 if(!authTypeString.empty())
208 authTypeString +=
" ";
209 authTypeString +=
"CURLAUTH_ONLY";
213 return authTypeString;
219static size_t writeNothing(
char *,
size_t ,
size_t nmemb,
void * ) {
227 static size_t writeToOpenFileDescriptor(
char *data,
size_t ,
size_t nmemb,
void *userdata) {
229 int *fd = (
int *) userdata;
231 BESDEBUG(MODULE, prolog <<
"Bytes received " << nmemb << endl);
232 int wrote = write(*fd, data, nmemb);
233 BESDEBUG(MODULE, prolog <<
"Bytes written " << wrote << endl);
261static size_t save_http_response_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs) {
262 BESDEBUG(MODULE, prolog <<
"Inside the header parser." << endl);
263 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
266 string complete_line;
267 if (nmemb > 1 && *(
static_cast<char *
>(ptr) + size * (nmemb - 2)) ==
'\r')
268 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 2));
270 complete_line.assign(
static_cast<char *
>(ptr), size * (nmemb - 1));
273 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
274 BESDEBUG(MODULE, prolog <<
"Header line: " << complete_line << endl);
275 hdrs->push_back(complete_line);
289static int curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *) {
290 string message(msg, size);
294 BESDEBUG(MODULE, prolog <<
"Text: " << message << endl);
296 case CURLINFO_HEADER_IN:
297 BESDEBUG(MODULE, prolog <<
"Header in: " << message << endl);
299 case CURLINFO_HEADER_OUT:
300 BESDEBUG(MODULE, prolog <<
"Header out: " << endl << message << endl);
302 case CURLINFO_DATA_IN:
303 BESDEBUG(MODULE, prolog <<
"Data in: " << message << endl);
305 case CURLINFO_DATA_OUT:
306 BESDEBUG(MODULE, prolog <<
"Data out: " << message << endl);
309 BESDEBUG(MODULE, prolog <<
"End: " << message << endl);
311#ifdef CURLINFO_SSL_DATA_IN
312 case CURLINFO_SSL_DATA_IN:
313 BESDEBUG(MODULE, prolog <<
"SSL Data in: " << message << endl );
break;
315#ifdef CURLINFO_SSL_DATA_OUT
316 case CURLINFO_SSL_DATA_OUT:
317 BESDEBUG(MODULE, prolog <<
"SSL Data out: " << message << endl );
break;
320 BESDEBUG(MODULE, prolog <<
"Curl info: " << message << endl);
332class BuildHeaders :
public std::unary_function<const string &, void> {
333 struct curl_slist *d_cl;
336 BuildHeaders() : d_cl(0) {}
338 void operator()(
const string &header) {
339 BESDEBUG(MODULE, prolog <<
"Adding '" << header.c_str() <<
"' to the header list." << endl);
340 d_cl = curl_slist_append(d_cl, header.c_str());
343 struct curl_slist *get_headers() {
368 bool configure_curl_handle_for_proxy(CURL *ceh,
const string &target_url) {
369 BESDEBUG(MODULE, prolog <<
"BEGIN." << endl);
371 bool using_proxy = http::ProxyConfig::theOne()->is_configured();
374 BESDEBUG(MODULE, prolog <<
"Proxy has been configured..." << endl);
379 string proxyHost = proxy->host();
380 int proxyPort = proxy->port();
381 string proxyPassword = proxy->proxy_password();
382 string proxyUser = proxy->user();
383 string proxyUserPW = proxy->password();
384 int proxyAuthType = proxy->auth_type();
385 string no_proxy_regex = proxy->no_proxy_regex();
392 if (!no_proxy_regex.empty()) {
393 BESDEBUG(MODULE, prolog <<
"Found NoProxyRegex." << endl);
395 if (r.match(target_url.c_str(), target_url.length()) != -1) {
397 prolog <<
"Found NoProxy match. BESRegex: " << no_proxy_regex <<
"; Url: " << target_url
405 char error_buffer[CURL_ERROR_SIZE];
407 BESDEBUG(MODULE, prolog <<
"Setting up a proxy server." << endl);
408 BESDEBUG(MODULE, prolog <<
"Proxy host: " << proxyHost << endl);
409 BESDEBUG(MODULE, prolog <<
"Proxy port: " << proxyPort << endl);
411 set_error_buffer(ceh, error_buffer);
413 res = curl_easy_setopt(ceh, CURLOPT_PROXY, proxyHost.data());
414 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXY", error_buffer, __FILE__, __LINE__);
416 res = curl_easy_setopt(ceh, CURLOPT_PROXYPORT, proxyPort);
417 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYPORT", error_buffer, __FILE__, __LINE__);
426 res = curl_easy_setopt(ceh, CURLOPT_PROXYAUTH, proxyAuthType);
427 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYAUTH", error_buffer, __FILE__, __LINE__);
428 BESDEBUG(MODULE, prolog <<
"Using CURLOPT_PROXYAUTH = " << getCurlAuthTypeName(proxyAuthType) << endl);
430 if (!proxyUser.empty()) {
431 res = curl_easy_setopt(ceh, CURLOPT_PROXYUSERNAME, proxyUser.data());
432 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYUSERNAME", error_buffer, __FILE__,
434 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSERNAME : " << proxyUser << endl);
436 if (!proxyPassword.empty()) {
437 res = curl_easy_setopt(ceh, CURLOPT_PROXYPASSWORD, proxyPassword.data());
438 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYPASSWORD", error_buffer, __FILE__,
440 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYPASSWORD: " << proxyPassword << endl);
442 }
else if (!proxyUserPW.empty()) {
443 res = curl_easy_setopt(ceh, CURLOPT_PROXYUSERPWD, proxyUserPW.data());
444 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYUSERPWD", error_buffer, __FILE__, __LINE__);
445 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSERPWD : " << proxyUserPW << endl);
447 unset_error_buffer(ceh);
450 BESDEBUG(MODULE, prolog <<
"END. using_proxy: " << (using_proxy ?
"true" :
"false") << endl);
455 bool configure_curl_handle_for_proxy(CURL *ceh,
const string &target_url) {
456 BESDEBUG(MODULE, prolog <<
"BEGIN." << endl);
458 bool using_proxy =
false;
463 string proxyHost = proxy->host();
464 int proxyPort = proxy->port();
465 string proxyPassword = proxy->proxy_password();
466 string proxyUser = proxy->user();
467 string proxyUserPW = proxy->password();
468 int proxyAuthType = proxy->auth_type();
469 string no_proxy_regex = proxy->no_proxy_regex();
471 if (!proxyHost.empty()) {
482 BESDEBUG(MODULE, prolog <<
"Found proxy configuration." << endl);
488 if (!no_proxy_regex.empty()) {
489 BESDEBUG(MODULE, prolog <<
"Found NoProxyRegex." << endl);
491 if (r.match(target_url.c_str(), target_url.length()) != -1) {
493 prolog <<
"Found NoProxy match. BESRegex: " << no_proxy_regex <<
"; Url: " << target_url
501 char error_buffer[CURL_ERROR_SIZE];
503 BESDEBUG(MODULE, prolog <<
"Setting up a proxy server." << endl);
504 BESDEBUG(MODULE, prolog <<
"Proxy host: " << proxyHost << endl);
505 BESDEBUG(MODULE, prolog <<
"Proxy port: " << proxyPort << endl);
507 set_error_buffer(ceh, error_buffer);
509 res = curl_easy_setopt(ceh, CURLOPT_PROXY, proxyHost.data());
510 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXY", error_buffer, __FILE__, __LINE__);
512 res = curl_easy_setopt(ceh, CURLOPT_PROXYPORT, proxyPort);
513 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYPORT", error_buffer, __FILE__, __LINE__);
522 res = curl_easy_setopt(ceh, CURLOPT_PROXYAUTH, proxyAuthType);
523 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYAUTH", error_buffer, __FILE__, __LINE__);
524 BESDEBUG(MODULE, prolog <<
"Using CURLOPT_PROXYAUTH = " << getCurlAuthTypeName(proxyAuthType) << endl);
526 if (!proxyUser.empty()) {
527 res = curl_easy_setopt(ceh, CURLOPT_PROXYUSERNAME, proxyUser.data());
528 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYUSERNAME", error_buffer, __FILE__, __LINE__);
529 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSERNAME : " << proxyUser << endl);
531 if (!proxyPassword.empty()) {
532 res = curl_easy_setopt(ceh, CURLOPT_PROXYPASSWORD, proxyPassword.data());
533 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYPASSWORD", error_buffer, __FILE__,
535 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYPASSWORD: " << proxyPassword << endl);
538 else if (!proxyUserPW.empty()) {
539 res = curl_easy_setopt(ceh, CURLOPT_PROXYUSERPWD, proxyUserPW.data());
540 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PROXYUSERPWD", error_buffer, __FILE__, __LINE__);
541 BESDEBUG(MODULE, prolog <<
"CURLOPT_PROXYUSERPWD : " << proxyUserPW << endl);
543 unset_error_buffer(ceh);
546 BESDEBUG(MODULE, prolog <<
"END." << endl);
552CURL *init(
const string &target_url,
553 const struct curl_slist *http_request_headers,
554 vector<string> *http_response_hdrs) {
555 CURL *swanky_new_curl_easy_handle = curl_easy_init();
556 return init(swanky_new_curl_easy_handle, target_url, http_request_headers, http_response_hdrs);
574 const string &target_url,
575 const struct curl_slist *http_request_headers,
576 vector<string> *http_response_hdrs
578 char error_buffer[CURL_ERROR_SIZE];
583 throw BESInternalError(
"Could not initialize cURL easy handle.", __FILE__, __LINE__);
586 set_error_buffer(ceh, error_buffer);
589 res = curl_easy_setopt(ceh, CURLOPT_URL, target_url.c_str());
590 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_URL", error_buffer, __FILE__, __LINE__);
600 if (http_request_headers) {
602 res = curl_easy_setopt(ceh, CURLOPT_HTTPHEADER, http_request_headers);
603 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPHEADER", error_buffer, __FILE__, __LINE__);
607 if (http_response_hdrs) {
608 res = curl_easy_setopt(ceh, CURLOPT_HEADERFUNCTION, save_http_response_headers);
609 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERFUNCTION", error_buffer, __FILE__, __LINE__);
614 res = curl_easy_setopt(ceh, CURLOPT_WRITEHEADER, http_response_hdrs);
615 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEHEADER", error_buffer, __FILE__, __LINE__);
619#ifndef CURLOPT_ACCEPT_ENCODING
620 res = curl_easy_setopt(ceh, CURLOPT_ENCODING,
"");
621 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_ENCODING", error_buffer, __FILE__, __LINE__);
623 res = curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING,
"");
624 check_setopt_result(res, prolog,
"CURLOPT_ACCEPT_ENCODING", error_buffer, __FILE__,__LINE__);
627 res = curl_easy_setopt(ceh, CURLOPT_NOPROGRESS, 1L);
628 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NOPROGRESS", error_buffer, __FILE__, __LINE__);
631 res = curl_easy_setopt(ceh, CURLOPT_NOSIGNAL, 1L);
632 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NOSIGNAL", error_buffer, __FILE__, __LINE__);
641 res = curl_easy_setopt(ceh, CURLOPT_FAILONERROR, 0L);
642 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FAILONERROR", error_buffer, __FILE__, __LINE__);
648 res = curl_easy_setopt(ceh, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
649 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", error_buffer, __FILE__, __LINE__);
655 res = curl_easy_setopt(ceh, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
656 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC", error_buffer, __FILE__, __LINE__);
659 string netrc_file = get_netrc_filename();
660 if (!netrc_file.empty()) {
661 res = curl_easy_setopt(ceh, CURLOPT_NETRC_FILE, netrc_file.c_str());
662 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", error_buffer, __FILE__, __LINE__);
665 VERBOSE(prolog <<
" is using the netrc file '"
666 << ((!netrc_file.empty()) ? netrc_file :
"~/.netrc") <<
"'" << endl);
672 res = curl_easy_setopt(ceh, CURLOPT_COOKIEFILE, curl::get_cookie_filename().c_str());
673 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", error_buffer, __FILE__, __LINE__);
675 res = curl_easy_setopt(ceh, CURLOPT_COOKIEJAR, curl::get_cookie_filename().c_str());
676 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", error_buffer, __FILE__, __LINE__);
681 res = curl_easy_setopt(ceh, CURLOPT_FOLLOWLOCATION, 1L);
682 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", error_buffer, __FILE__, __LINE__);
684 res = curl_easy_setopt(ceh, CURLOPT_MAXREDIRS, max_redirects());
685 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_MAXREDIRS", error_buffer, __FILE__, __LINE__);
688 res = curl_easy_setopt(ceh, CURLOPT_USERAGENT, hyrax_user_agent().c_str());
689 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_USERAGENT", error_buffer, __FILE__, __LINE__);
693if (!d_rcr->get_validate_ssl() == 0) {
694 res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
695 check_setopt_result(res, prolog,
"CURLOPT_SSL_VERIFYPEER", error_buffer, __FILE__, __LINE__);
696 res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
697 check_setopt_result(res, prolog,
"CURLOPT_SSL_VERIFYHOST", error_buffer, __FILE__, __LINE__);
702 BESDEBUG(MODULE, prolog <<
"Curl version: " << curl_version() << endl);
703 res = curl_easy_setopt(ceh, CURLOPT_VERBOSE, 1L);
704 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_VERBOSE", error_buffer, __FILE__, __LINE__);
705 BESDEBUG(MODULE, prolog <<
"Curl in verbose mode." << endl);
707 res = curl_easy_setopt(ceh, CURLOPT_DEBUGFUNCTION, curl_debug);
708 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_DEBUGFUNCTION", error_buffer, __FILE__, __LINE__);
709 BESDEBUG(MODULE, prolog <<
"Curl debugging function installed." << endl);
713 unset_error_buffer(ceh);
715 curl::configure_curl_handle_for_proxy(ceh, target_url);
717 BESDEBUG(MODULE, prolog <<
"curl: " << (
void *) ceh << endl);
721string get_range_arg_string(
const unsigned long long &offset,
const unsigned long long &size) {
723 range << offset <<
"-" << offset + size - 1;
724 BESDEBUG(MODULE, prolog <<
" range: " << range.str() << endl);
743CURL *init_effective_url_retriever_handle(
const string &target_url,
struct curl_slist *req_headers,
744 vector<string> &resp_hdrs) {
745 char error_buffer[CURL_ERROR_SIZE];
751 ceh = curl::init(target_url, req_headers, &resp_hdrs);
753 set_error_buffer(ceh, error_buffer);
756 res = curl_easy_setopt(ceh, CURLOPT_RANGE, get_range_arg_string(0, 4).c_str());
757 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_RANGE", error_buffer, __FILE__, __LINE__);
759 res = curl_easy_setopt(ceh, CURLOPT_WRITEFUNCTION, writeNothing);
760 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", error_buffer, __FILE__, __LINE__);
765 res = curl_easy_setopt(ceh, CURLOPT_WRITEHEADER, &resp_hdrs);
766 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEHEADER", error_buffer, __FILE__, __LINE__);
768 unset_error_buffer(ceh);
790void http_get_and_write_resource(
const std::shared_ptr<http::url>& target_url,
792 vector<string> *http_response_headers) {
794 char error_buffer[CURL_ERROR_SIZE];
797 curl_slist *req_headers =
nullptr;
798 BuildHeaders header_builder;
800 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
803 string err = (string)
"The specified URL " + target_url->str()
804 +
" does not match any of the accessible services in"
805 +
" the allowed hosts list.";
806 BESDEBUG(MODULE, prolog << err << endl);
811 req_headers = add_edl_auth_headers(req_headers);
815 ceh = init(target_url->str(), req_headers, http_response_headers);
817 set_error_buffer(ceh, error_buffer);
819 res = curl_easy_setopt(ceh, CURLOPT_WRITEFUNCTION, writeToOpenFileDescriptor);
820 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", error_buffer, __FILE__, __LINE__);
822#ifdef CURLOPT_WRITEDATA
823 res = curl_easy_setopt(ceh, CURLOPT_WRITEDATA, &fd);
824 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", error_buffer, __FILE__, __LINE__);
826 res = curl_easy_setopt(ceh, CURLOPT_FILE, &fd);
827 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FILE", error_buffer, __FILE__, __LINE__);
829 unset_error_buffer(ceh);
831 super_easy_perform(ceh, fd);
835 curl_slist_free_all(req_headers);
837 curl_easy_cleanup(ceh);
838 BESDEBUG(MODULE, prolog <<
"Called curl_easy_cleanup()." << endl);
842 curl_slist_free_all(req_headers);
844 curl_easy_cleanup(ceh);
847 BESDEBUG(MODULE, prolog <<
"END" << endl);
857string error_message(
const CURLcode response_code,
char *error_buffer) {
858 std::ostringstream oss;
859 size_t len = strlen(error_buffer);
861 oss <<
"cURL_error_buffer: '" << error_buffer;
863 oss <<
"' cURL_message: '" << curl_easy_strerror(response_code);
864 oss <<
"' (code: " << (
int) response_code <<
")";
881size_t c_write_data(
void *buffer,
size_t size,
size_t nmemb,
void *data) {
882 size_t nbytes = size * nmemb;
884 memcpy(data, buffer, nbytes);
894std::string http_get_as_string(
const std::string &target_url) {
901 char response_buf[1024 * 1024];
903 http_get(target_url, response_buf);
904 string response(response_buf);
915rapidjson::Document http_get_as_json(
const std::string &target_url) {
923 char response_buf[1024 * 1024];
925 curl::http_get(target_url, response_buf);
926 rapidjson::Document d;
927 d.Parse(response_buf);
936void http_get(
const std::string &target_url,
char *response_buf) {
938 char errbuf[CURL_ERROR_SIZE];
942 curl_slist *request_headers =
nullptr;
944 request_headers = add_edl_auth_headers(request_headers);
948 ceh = curl::init(target_url, request_headers,
nullptr);
950 throw BESInternalError(
string(
"ERROR! Failed to acquire cURL Easy Handle! "), __FILE__, __LINE__);
953 set_error_buffer(ceh, errbuf);
956 res = curl_easy_setopt(ceh, CURLOPT_WRITEFUNCTION, c_write_data);
957 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", errbuf, __FILE__, __LINE__);
960 res = curl_easy_setopt(ceh, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(response_buf));
961 eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", errbuf, __FILE__, __LINE__);
963 unset_error_buffer(ceh);
965 super_easy_perform(ceh);
968 curl_slist_free_all(request_headers);
970 curl_easy_cleanup(ceh);
974 curl_slist_free_all(request_headers);
976 curl_easy_cleanup(ceh);
988CURL *set_up_easy_handle(
const string &target_url,
struct curl_slist *request_headers,
char *response_buff) {
989 char errbuf[CURL_ERROR_SIZE];
993 d_handle = curl::init(target_url,request_headers,NULL);
995 throw BESInternalError(
string(
"ERROR! Failed to acquire cURL Easy Handle! "), __FILE__, __LINE__);
998 set_error_buffer(d_handle,errbuf);
1001 res = curl_easy_setopt(d_handle, CURLOPT_WRITEFUNCTION, c_write_data);
1002 check_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", errbuf, __FILE__, __LINE__);
1005 res = curl_easy_setopt(d_handle, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(response_buff));
1006 check_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", errbuf, __FILE__, __LINE__);
1011 res = curl_easy_setopt(d_handle, CURLOPT_FOLLOWLOCATION, 1L);
1012 check_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", errbuf, __FILE__, __LINE__);
1015 res = curl_easy_setopt(d_handle, CURLOPT_COOKIEFILE, cookies_file.c_str());
1016 check_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", errbuf, __FILE__, __LINE__);
1018 res = curl_easy_setopt(d_handle, CURLOPT_COOKIEJAR, cookies_file.c_str());
1019 check_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", errbuf, __FILE__, __LINE__);
1022 res = curl_easy_setopt(d_handle, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
1023 check_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", errbuf, __FILE__, __LINE__);
1026 res = curl_easy_setopt(d_handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1027 check_setopt_result(res, prolog,
"CURLOPT_NETRC", errbuf, __FILE__, __LINE__);
1030 string netrc_file = get_netrc_filename();
1031 if (!netrc_file.empty()) {
1032 res = curl_easy_setopt(d_handle, CURLOPT_NETRC_FILE, netrc_file.c_str());
1033 check_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", errbuf, __FILE__, __LINE__);
1036 VERBOSE(__FILE__ <<
"::get_easy_handle() is using the netrc file '"
1037 << ((!netrc_file.empty()) ? netrc_file :
"~/.netrc") <<
"'" << endl);
1040 unset_error_buffer(d_handle);
1065void super_easy_perform(CURL *c_handle){
1067 super_easy_perform(c_handle, fd);
1070void super_easy_perform(CURL *c_handle,
const int fd)
1072 unsigned int attempts = 0;
1073 useconds_t retry_time = uone_second / 4;
1076 char curlErrorBuf[CURL_ERROR_SIZE];
1080 target_url = get_effective_url(c_handle, empty_str);
1082 if (target_url.empty())
1086 set_error_buffer(c_handle, curlErrorBuf);
1088 curlErrorBuf[0] = 0;
1090 BESDEBUG(MODULE, prolog <<
"Requesting URL: " << target_url <<
" attempt: " << attempts << endl);
1092 curl_code = curl_easy_perform(c_handle);
1093 success = eval_curl_easy_perform_code(c_handle, target_url, curl_code, curlErrorBuf, attempts);
1096 success = eval_http_get_response(c_handle, curlErrorBuf, target_url);
1101 if (attempts == retry_limit) {
1103 msg << prolog <<
"ERROR - Made " << retry_limit <<
" failed attempts to retrieve the URL " << target_url;
1104 msg <<
" The retry limit has been exceeded. Giving up!";
1105 ERROR_LOG(msg.str() << endl);
1109 ERROR_LOG(prolog <<
"ERROR - Problem with data transfer. Will retry (url: " << target_url <<
1110 " attempt: " << attempts <<
")." << endl);
1118 int val = fcntl(fd, F_GETFL, 0);
1121 ss << prolog <<
"Encountered fcntl error " << val <<
" for fd: " << fd << endl;
1122 BESDEBUG(MODULE, ss.str());
1123 ERROR_LOG(ss.str());
1126 int accmode = val & O_ACCMODE;
1129 if (accmode == O_RDONLY) {
1130 BESDEBUG(MODULE, prolog <<
" FILE " << fd <<
" is open and read only" << endl);
1132 else if (accmode == O_WRONLY) {
1133 BESDEBUG(MODULE, prolog <<
" FILE " << fd <<
" is open and write only" << endl);
1135 else if (accmode == O_RDWR) {
1136 BESDEBUG(MODULE, prolog <<
" FILE " << fd <<
" is open for read and write" << endl);
1140 ss << prolog <<
"ERROR Unknown access mode mode for FILE '" << fd <<
"'" << endl;
1141 BESDEBUG(MODULE, ss.str());
1142 ERROR_LOG(ss.str());
1147 if (accmode == O_WRONLY || accmode == O_RDWR){
1148 int status = ftruncate(fd, 0);
1150 throw BESInternalError(
"Could not truncate the file prior to retrying from remote. ", __FILE__, __LINE__);
1151 BESDEBUG(MODULE, prolog <<
"Truncated file, length is zero." << endl);
1163 unset_error_buffer(c_handle);
1172void read_data(CURL *c_handle) {
1174 unsigned int attempts = 0;
1175 useconds_t retry_time = uone_second / 4;
1178 char curlErrorBuf[CURL_ERROR_SIZE];
1181 curl_easy_getinfo(c_handle, CURLINFO_EFFECTIVE_URL, &urlp);
1190 set_error_buffer(c_handle, curlErrorBuf);
1195 BESDEBUG(MODULE, prolog <<
"Requesting URL: " << urlp <<
" attempt: " << attempts << endl);
1197 curl_code = curl_easy_perform(c_handle);
1198 success = eval_curl_easy_perform_code(c_handle, urlp, curl_code, curlErrorBuf, attempts);
1201 success = eval_http_get_response(c_handle, urlp);
1206 if (attempts == retry_limit) {
1207 string msg = prolog +
"ERROR - Problem with data transfer. Number of re-tries exceeded. Giving up.";
1212 LOG(prolog <<
"ERROR - Problem with data transfer. Will retry (url: " << urlp <<
1213 " attempt: " << attempts <<
")." << endl);
1223 curlErrorBuf[0] = 0;
1224 curl_code = curl_easy_perform(c_handle);
1227 if (CURLE_OK != curl_code) {
1229 string(
"read_data() - ERROR! Message: ").append(error_message(curl_code, curlErrorBuf)),
1230 __FILE__, __LINE__);
1233 success = eval_get_response(c_handle, urlp);
1236 if (tries == retry_limit) {
1237 string msg = prolog +
"Data transfer error: Number of re-tries exceeded: "+ error_message(curl_code, curlErrorBuf);
1244 ss <<
"HTTP transfer 500 error, will retry (trial " << tries <<
" for: " << urlp <<
").";
1245 BESDEBUG(MODULE, ss.str());
1254 unset_error_buffer(c_handle);
1258string get_cookie_file_base() {
1260 string cookie_filename;
1263 cookie_filename = HTTP_DEFAULT_COOKIES_FILE;
1265 return cookie_filename;
1268string get_cookie_filename() {
1269 string cookie_file_base = get_cookie_file_base();
1270 stringstream cf_with_pid;
1271 cf_with_pid << cookie_file_base <<
"-" << getpid();
1272 return cf_with_pid.str();
1275void clear_cookies() {
1276 string cf = get_cookie_filename();
1277 int ret = unlink(cf.c_str());
1279 string msg = prolog +
"Failed to unlink the cookie file: " + cf;
1280 ERROR_LOG(msg << endl);
1281 BESDEBUG(MODULE, prolog << msg << endl);
1295bool is_retryable(std::string target_url) {
1296 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
1297 bool retryable =
true;
1299 vector<string> nr_regexs;
1303 vector<string>::iterator it;
1304 for (it = nr_regexs.begin(); it != nr_regexs.end() && retryable; it++) {
1305 BESRegex no_retry_regex((*it).c_str(), (*it).size());
1306 size_t match_length;
1307 match_length = no_retry_regex.match(target_url.c_str(), target_url.size(), 0);
1308 if (match_length == target_url.size()) {
1309 BESDEBUG(MODULE, prolog <<
"The url: '" << target_url <<
"' fully matched the "
1310 << HTTP_NO_RETRY_URL_REGEX_KEY <<
": '" << *it <<
"'" << endl);
1315 BESDEBUG(MODULE, prolog <<
"END retryable: " << (retryable ?
"true" :
"false") << endl);
1353bool eval_http_get_response(CURL *ceh,
char *error_buffer,
const string &requested_url) {
1354 BESDEBUG(MODULE, prolog <<
"Requested URL: " << requested_url << endl);
1356 string last_accessed_url = get_effective_url(ceh, requested_url);
1357 BESDEBUG(MODULE, prolog <<
"Last Accessed URL(CURLINFO_EFFECTIVE_URL): " << last_accessed_url << endl);
1361 curl_code = curl_easy_getinfo(ceh, CURLINFO_RESPONSE_CODE, &http_code);
1362 if (curl_code == CURLE_GOT_NOTHING) {
1367 msg << prolog <<
"ERROR - cURL returned CURLE_GOT_NOTHING. Message: '";
1368 msg << error_message(curl_code, error_buffer) <<
"' ";
1369 msg <<
"CURLINFO_EFFECTIVE_URL: " << last_accessed_url <<
" ";
1370 msg <<
"A retry may be possible for: " << requested_url <<
")." << endl;
1371 BESDEBUG(MODULE, msg.str());
1372 ERROR_LOG(msg.str());
1375 else if (curl_code != CURLE_OK) {
1378 string(
"Error acquiring HTTP response code: ").append(curl::error_message(curl_code, error_buffer)),
1379 __FILE__, __LINE__);
1384 curl_easy_getinfo(ceh, CURLINFO_REDIRECT_COUNT, &redirects);
1385 BESDEBUG(MODULE, prolog <<
"CURLINFO_REDIRECT_COUNT: " << redirects << endl);
1387 char *redirect_url = NULL;
1388 curl_easy_getinfo(ceh, CURLINFO_REDIRECT_URL, &redirect_url);
1390 BESDEBUG(MODULE, prolog <<
"CURLINFO_REDIRECT_URL: " << redirect_url << endl);
1394 if (http_code >= 400) {
1395 msg <<
"ERROR - The HTTP GET request for the source URL: " << requested_url <<
" FAILED. ";
1396 msg <<
"CURLINFO_EFFECTIVE_URL: " << last_accessed_url <<
" ";
1397 BESDEBUG(MODULE, prolog << msg.str() << endl);
1399 msg <<
"The response had an HTTP status of " << http_code;
1400 msg <<
" which means '" << http_status_to_string(http_code) <<
"'";
1403 switch (http_code) {
1406 if(requested_url.find(FILE_PROTOCOL)!=0){
1407 ERROR_LOG(msg.str() << endl);
1422 ERROR_LOG(msg.str() << endl);
1428 ERROR_LOG(msg.str() << endl);
1432 ERROR_LOG(msg.str() << endl);
1436 ERROR_LOG(msg.str() << endl);
1445 if (!is_retryable(last_accessed_url)) {
1446 msg <<
" The semantics of this particular last accessed URL indicate that it should not be retried.";
1447 ERROR_LOG(msg.str() << endl);
1454 ERROR_LOG(msg.str() << endl);
1481bool eval_curl_easy_perform_code(
1483 const string requested_url,
1486 const unsigned int attempt
1488 bool success =
true;
1489 string last_accessed_url = get_effective_url(ceh, requested_url);
1490 if (curl_code == CURLE_SSL_CONNECT_ERROR) {
1492 msg << prolog <<
"ERROR - cURL experienced a CURLE_SSL_CONNECT_ERROR error. Message: '";
1493 msg << error_message(curl_code, error_buffer) <<
"' ";
1494 msg <<
"CURLINFO_EFFECTIVE_URL: " << last_accessed_url <<
" ";
1495 msg <<
"A retry may be possible for: " << requested_url <<
" (attempt: " << attempt <<
")." << endl;
1496 BESDEBUG(MODULE, msg.str());
1497 ERROR_LOG(msg.str());
1500 else if (curl_code == CURLE_SSL_CACERT_BADFILE) {
1502 msg << prolog <<
"ERROR - cURL experienced a CURLE_SSL_CACERT_BADFILE error. Message: '";
1503 msg << error_message(curl_code, error_buffer) <<
"' ";
1504 msg <<
"CURLINFO_EFFECTIVE_URL: " << last_accessed_url <<
" ";
1505 msg <<
"A retry may be possible for: " << requested_url <<
" (attempt: " << attempt <<
")." << endl;
1506 BESDEBUG(MODULE, msg.str());
1507 ERROR_LOG(msg.str());
1510 else if (curl_code == CURLE_GOT_NOTHING) {
1515 msg << prolog <<
"ERROR - cURL returned CURLE_GOT_NOTHING. Message: ";
1516 msg << error_message(curl_code, error_buffer) <<
"' ";
1517 msg <<
"CURLINFO_EFFECTIVE_URL: " << last_accessed_url <<
" ";
1518 msg <<
"A retry may be possible for: " << requested_url <<
" (attempt: " << attempt <<
")." << endl;
1519 BESDEBUG(MODULE, msg.str());
1520 ERROR_LOG(msg.str());
1523 else if (CURLE_OK != curl_code) {
1525 msg <<
"ERROR - Problem with data transfer. Message: " << error_message(curl_code, error_buffer);
1526 string effective_url = get_effective_url(ceh, requested_url);
1527 msg <<
" CURLINFO_EFFECTIVE_URL: " << effective_url;
1528 BESDEBUG(MODULE, prolog << msg.str() << endl);
1529 ERROR_LOG(msg.str() << endl);
1543 void retrieve_effective_url(
const string &target_url,
string &last_accessed_url) {
1544 vector<string> resp_hdrs;
1547 curl_slist *request_headers = NULL;
1549 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
1552 request_headers = add_auth_headers(request_headers);
1556 prolog <<
"BESDebug::IsSet(" << MODULE <<
"): " << (
BESDebug::IsSet(MODULE) ?
"true" :
"false")
1558 BESDEBUG(MODULE, prolog <<
"BESDebug::IsSet(" << TIMING_LOG_KEY <<
"): "
1561 prolog <<
"BESLog::TheLog()->is_verbose(): " << (BESLog::TheLog()->is_verbose() ?
"true" :
"false")
1564 ceh = init_effective_url_retriever_handle(target_url, request_headers, resp_hdrs);
1569 BESLog::TheLog()->is_verbose()) {
1570 sw.
start(prolog +
" Following Redirects Starting With: " + target_url);
1572 super_easy_perform(ceh);
1576 last_accessed_url = get_effective_url(ceh, target_url);
1577 BESDEBUG(MODULE, prolog <<
"Last Accessed URL(CURLINFO_EFFECTIVE_URL): " << last_accessed_url << endl);
1579 prolog <<
"Source URL: '" << target_url <<
"' CURLINFO_EFFECTIVE_URL: '" << last_accessed_url <<
"'"
1582 if (request_headers)
1583 curl_slist_free_all(request_headers);
1585 curl_easy_cleanup(ceh);
1588 if (request_headers)
1589 curl_slist_free_all(request_headers);
1591 curl_easy_cleanup(ceh);
1604 std::shared_ptr<http::EffectiveUrl> retrieve_effective_url(
const std::shared_ptr<http::url> &starting_point_url) {
1606 vector<string> resp_hdrs;
1607 CURL *ceh =
nullptr;
1609 curl_slist *request_headers =
nullptr;
1611 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
1614 request_headers = add_edl_auth_headers(request_headers);
1618 prolog <<
"BESDebug::IsSet(" << MODULE <<
"): " << (
BESDebug::IsSet(MODULE) ?
"true" :
"false")
1620 BESDEBUG(MODULE, prolog <<
"BESDebug::IsSet(" << TIMING_LOG_KEY <<
"): "
1622 BESDEBUG(MODULE, prolog <<
"BESLog::TheLog()->is_verbose(): "
1623 << (BESLog::TheLog()->is_verbose() ?
"true" :
"false") << endl);
1625 ceh = init_effective_url_retriever_handle(starting_point_url->str(), request_headers, resp_hdrs);
1630 BESLog::TheLog()->is_verbose()) {
1631 sw.
start(prolog +
" Following Redirects Starting With: " + starting_point_url->str());
1633 super_easy_perform(ceh);
1637 string e_url_str = get_effective_url(ceh, starting_point_url->str());
1638 std::shared_ptr<http::EffectiveUrl> eurl(
new EffectiveUrl(e_url_str, resp_hdrs, starting_point_url->is_trusted()));
1640 BESDEBUG(MODULE, prolog <<
"Last Accessed URL(CURLINFO_EFFECTIVE_URL): " << eurl->str() <<
1641 "(" << (eurl->is_trusted()?
"":
"NOT ") <<
"trusted)" << endl);
1643 INFO_LOG(prolog <<
"Source URL: '" << starting_point_url->str() <<
"(" << (starting_point_url->is_trusted() ?
"" :
"NOT ") <<
"trusted)" <<
1644 "' CURLINFO_EFFECTIVE_URL: '" << eurl->str() <<
"'" <<
"(" << (eurl->is_trusted()?
"":
"NOT ") <<
"trusted)" << endl);
1647 if (request_headers)
1648 curl_slist_free_all(request_headers);
1650 curl_easy_cleanup(ceh);
1655 if (request_headers)
1656 curl_slist_free_all(request_headers);
1658 curl_easy_cleanup(ceh);
1664 unsigned int attempts = 0;
1665 bool success =
true;
1666 useconds_t retry_time = uone_second / 4;
1668 char error_buffer[CURL_ERROR_SIZE];
1669 vector<string> resp_hdrs;
1673 struct curl_slist *request_headers = NULL;
1675 request_headers = get_auth_headers(request_headers);
1678 ceh = init_effective_url_retriever_handle(
url, request_headers, resp_hdrs);
1679 set_error_buffer(ceh, error_buffer);
1682 error_buffer[0] = 0;
1684 BESDEBUG(MODULE, prolog <<
"Requesting URL: " << starting_point_url <<
" attempt: " << attempts << endl);
1686 curl_code = curl_easy_perform(ceh);
1687 success = eval_curl_easy_perform_code(ceh, starting_point_url, curl_code, error_buffer, attempts);
1690 success = eval_http_get_response(ceh, starting_point_url);
1692 if (attempts == retry_limit) {
1693 string msg = prolog +
1694 "ERROR - Problem with data transfer. Number of re-tries exceeded. Giving up.";
1698 LOG(prolog <<
"ERROR - Problem with data transfer. Will retry (url: " << starting_point_url <<
1699 " attempt: " << attempts <<
")." << endl);
1710 char *effective_url = 0;
1711 curl_easy_getinfo(ceh, CURLINFO_EFFECTIVE_URL, &effective_url);
1712 BESDEBUG(MODULE, prolog <<
" CURLINFO_EFFECTIVE_URL: " << effective_url << endl);
1713 last_accessed_url = effective_url;
1715 LOG(prolog <<
"Source URL: '" << starting_point_url <<
"' Last Accessed URL: '" << last_accessed_url <<
"'" << endl);
1717 unset_error_buffer(ceh);
1720 curl_slist_free_all(request_headers);
1721 curl_easy_cleanup(ceh);
1726 if (request_headers)
1727 curl_slist_free_all(request_headers);
1729 curl_easy_cleanup(ceh);
1747string get_netrc_filename() {
1748 string netrc_filename;
1752 BESDEBUG(MODULE, prolog <<
"Using netrc file: " << netrc_filename << endl);
1755 BESDEBUG(MODULE, prolog <<
"Using default netrc file. (~/.netrc)" << endl);
1757 return netrc_filename;
1765void set_error_buffer(CURL *ceh,
char *error_buffer) {
1767 res = curl_easy_setopt(ceh, CURLOPT_ERRORBUFFER, error_buffer);
1768 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_ERRORBUFFER", error_buffer, __FILE__, __LINE__);
1776void unset_error_buffer(CURL *ceh) {
1777 set_error_buffer(ceh, NULL);
1785string hyrax_user_agent() {
1789 if(!found || user_agent.empty()){
1790 user_agent = HTTP_DEFAULT_USER_AGENT;
1792 BESDEBUG(MODULE, prolog <<
"User-Agent: "<< user_agent << endl);
1811void eval_curl_easy_setopt_result(
1817 unsigned int line) {
1818 if (curl_code != CURLE_OK) {
1820 msg << msg_base <<
"ERROR - cURL failed to set " << opt_name <<
" Message: " << curl::error_message(curl_code, ebuf);
1825unsigned long max_redirects() {
1840curl_slist *append_http_header(curl_slist *slist,
const string &header_name,
const string &value)
1843 string full_header = header_name;
1844 full_header.append(
": ").append(value);
1846 BESDEBUG(MODULE, prolog << full_header << endl);
1849 struct curl_slist *temp = curl_slist_append(slist, full_header.c_str());
1852 msg << prolog <<
"Encountered cURL Error setting the " << header_name <<
" header. full_header: " << full_header;
1889curl_slist *add_edl_auth_headers(curl_slist *request_headers) {
1893 s = BESContextManager::TheManager()->
get_context(EDL_UID_KEY, found);
1894 if (found && !s.empty()) {
1895 request_headers = append_http_header(request_headers,
"User-Id",s);
1898 s = BESContextManager::TheManager()->
get_context(EDL_AUTH_TOKEN_KEY, found);
1899 if (found && !s.empty()) {
1900 request_headers = append_http_header(request_headers,
"Authorization",s);
1903 s = BESContextManager::TheManager()->
get_context(EDL_ECHO_TOKEN_KEY, found);
1904 if (found && !s.empty()) {
1905 request_headers = append_http_header(request_headers,
"Echo-Token",s);
1908 return request_headers;
1918string get_effective_url(CURL *ceh,
string requested_url) {
1919 char *effectve_url =
nullptr;
1920 CURLcode curl_code = curl_easy_getinfo(ceh, CURLINFO_EFFECTIVE_URL, &effectve_url);
1921 if (curl_code != CURLE_OK) {
1923 msg << prolog <<
"Unable to determine CURLINFO_EFFECTIVE_URL! Requested URL: " << requested_url;
1924 BESDEBUG(MODULE, msg.str() << endl);
1927 return effectve_url;
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
error thrown if the BES is not allowed to access the resource requested
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
Regular expression matching.
virtual bool start(std::string name)
error thrown if there is a user syntax error in the request or any other user error
error thrown if there is a user syntax error in the request or any other user error
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
static AllowedHosts * theHosts()
Static accessor for the singleton.
utility class for the HTTP catalog module
size_t load_max_redirects_from_keys()