40#include <libdap/util.h>
44#include "BESInternalError.h"
45#include "BESForbiddenError.h"
46#include "AllowedHosts.h"
48#include "DmrppCommon.h"
49#include "DmrppNames.h"
51#include "CurlHandlePool.h"
53#include "CredentialsManager.h"
54#include "AccessCredentials.h"
59#define prolog std::string("CurlHandlePool::").append(__func__).append("() - ")
64string pthread_error(
unsigned int err){
68 error_msg =
"The mutex was either created with the "
69 "protocol attribute having the value "
70 "PTHREAD_PRIO_PROTECT and the calling "
71 "thread's priority is higher than the "
72 "mutex's current priority ceiling."
73 "OR The value specified by mutex does not "
74 "refer to an initialized mutex object.";
78 error_msg =
"The mutex could not be acquired "
79 "because it was already locked.";
83 error_msg =
"The mutex could not be acquired because "
84 "the maximum number of recursive locks "
85 "for mutex has been exceeded.";
89 error_msg =
"The current thread already owns the mutex";
93 error_msg =
"The current thread does not own the mutex.";
97 error_msg =
"Unknown pthread error type.";
111string dump(
const char *text,
unsigned char *ptr,
size_t size)
115 unsigned int width=0x10;
118 oss << text <<
", " << std::setw(10) << (long)size << std::setbase(16) << (long)size << endl;
120 for(i=0; i<size; i+= width) {
121 oss << std::setw(4) << (long)i;
125 for(c = 0; c < width; c++) {
127 oss << std::setw(2) << ptr[i+c];
137 for(c = 0; (c < width) && (i+c < size); c++) {
138 char x = (ptr[i+c] >= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] :
'.';
140 oss << std::setw(1) << x;
158int curl_trace(CURL *, curl_infotype type,
char *data,
size_t ,
void *)
164 case CURLINFO_HEADER_OUT:
165 case CURLINFO_HEADER_IN: {
168 while ((pos = text.find(
'\n')) != string::npos)
169 text = text.substr(0, pos);
174 case CURLINFO_DATA_OUT:
175 case CURLINFO_SSL_DATA_OUT:
176 case CURLINFO_DATA_IN:
177 case CURLINFO_SSL_DATA_IN:
185 LOG(
"libcurl == Info: " << text << endl);
188 case CURLINFO_HEADER_OUT:
189 LOG(
"libcurl == Send header: " << text << endl);
191 case CURLINFO_HEADER_IN:
192 LOG(
"libcurl == Recv header: " << text << endl);
196 case CURLINFO_DATA_OUT:
197 case CURLINFO_SSL_DATA_OUT:
198 case CURLINFO_DATA_IN:
199 case CURLINFO_SSL_DATA_IN:
213 d_handle = curl_easy_init();
214 if (!d_handle)
throw BESInternalError(
"Could not allocate CURL handle", __FILE__, __LINE__);
216 curl::set_error_buffer(d_handle, d_errbuf);
218 res = curl_easy_setopt(d_handle, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
219 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_SSLVERSION", d_errbuf, __FILE__, __LINE__);
223 res = curl_easy_setopt(d_handle, CURLOPT_DEBUGFUNCTION, curl_trace);
224 curl::check_setopt_result(res, prolog,
"CURLOPT_DEBUGFUNCTION", d_errbuf, __FILE__, __LINE__);
227 res = curl_easy_setopt(d_handle, CURLOPT_VERBOSE, 1L);
228 curl::check_setopt_result(res, prolog,
"CURLOPT_VERBOSE", d_errbuf, __FILE__, __LINE__);
231 res = curl_easy_setopt(d_handle, CURLOPT_HEADERFUNCTION, chunk_header_callback);
232 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERFUNCTION", d_errbuf, __FILE__, __LINE__);
235 res = curl_easy_setopt(d_handle, CURLOPT_WRITEFUNCTION, chunk_write_data);
236 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", d_errbuf, __FILE__, __LINE__);
238#ifdef CURLOPT_TCP_KEEPALIVE
240 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPALIVE, 1L);
241 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPALIVE", d_errbuf, __FILE__, __LINE__);
244#ifdef CURLOPT_TCP_KEEPIDLE
246 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPIDLE, 120L);
247 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPIDLE", d_errbuf, __FILE__, __LINE__);
250#ifdef CURLOPT_TCP_KEEPINTVL
252 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPINTVL, 120L)
253 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPINTVL", d_errbuf, __FILE__, __LINE__);
260dmrpp_easy_handle::~dmrpp_easy_handle() {
261 if (d_handle) curl_easy_cleanup(d_handle);
262 if (d_request_headers) curl_slist_free_all(d_request_headers);
279 if (d_url->protocol() == HTTPS_PROTOCOL || d_url->protocol() == HTTP_PROTOCOL) {
280 curl::super_easy_perform(d_handle);
283 CURLcode curl_code = curl_easy_perform(d_handle);
284 if (CURLE_OK != curl_code) {
285 string msg = prolog +
"ERROR - Data transfer error: ";
286 throw BESInternalError(msg.append(curl::error_message(curl_code, d_errbuf)), __FILE__, __LINE__);
290 d_chunk->set_is_read(
true);
293CurlHandlePool::CurlHandlePool(
unsigned int max_handles) : d_max_easy_handles(max_handles) {
294 for (
unsigned int i = 0; i < d_max_easy_handles; ++i) {
318 string reason =
"The requested resource does not match any of the AllowedHost rules.";
321 ss <<
"ERROR! The chunk url "<< chunk->
get_data_url()->str() <<
" was rejected because: " << reason;
325 std::lock_guard<std::recursive_mutex> lock_me(d_get_easy_handle_mutex);
328 for (
auto i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
329 if (!(*i)->d_in_use) {
337 handle->d_in_use =
true;
340 handle->d_chunk = chunk;
342 CURLcode res = curl_easy_setopt(handle->d_handle, CURLOPT_URL, chunk->
get_data_url()->str().c_str());
343 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_URL", handle->d_errbuf, __FILE__, __LINE__);
347 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_RANGE", handle->d_errbuf, __FILE__, __LINE__);
350 res = curl_easy_setopt(handle->d_handle, CURLOPT_HEADERDATA,
reinterpret_cast<void *
>(chunk));
351 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERDATA", handle->d_errbuf, __FILE__, __LINE__);
354 res = curl_easy_setopt(handle->d_handle, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(chunk));
355 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", handle->d_errbuf, __FILE__, __LINE__);
358 res = curl_easy_setopt(handle->d_handle, CURLOPT_PRIVATE,
reinterpret_cast<void *
>(handle));
359 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PRIVATE", handle->d_errbuf, __FILE__, __LINE__);
362 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEFILE, curl::get_cookie_filename().c_str());
363 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", handle->d_errbuf, __FILE__, __LINE__);
365 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEJAR, curl::get_cookie_filename().c_str());
366 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", handle->d_errbuf, __FILE__, __LINE__);
369 res = curl_easy_setopt(handle->d_handle, CURLOPT_FOLLOWLOCATION, 1);
370 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", handle->d_errbuf, __FILE__, __LINE__);
372 res = curl_easy_setopt(handle->d_handle, CURLOPT_MAXREDIRS, curl::max_redirects());
373 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_MAXREDIRS", handle->d_errbuf, __FILE__, __LINE__);
376 res = curl_easy_setopt(handle->d_handle, CURLOPT_USERAGENT, curl::hyrax_user_agent().c_str());
377 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_USERAGENT", handle->d_errbuf, __FILE__, __LINE__);
382 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
383 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", handle->d_errbuf, __FILE__, __LINE__);
386 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
387 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC", handle->d_errbuf, __FILE__, __LINE__);
391 string netrc_file = curl::get_netrc_filename();
392 if (!netrc_file.empty()) {
393 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC_FILE, netrc_file.c_str());
394 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", handle->d_errbuf, __FILE__, __LINE__);
398 if (credentials && credentials->
is_s3_cred()) {
400 prolog <<
"Got AccessCredentials instance: " << endl << credentials->to_json() << endl);
403 const std::time_t request_time = std::time(0);
405 const std::string auth_header =
406 AWSV4::compute_awsv4_signature(
409 credentials->
get(AccessCredentials::ID_KEY),
410 credentials->
get(AccessCredentials::KEY_KEY),
411 credentials->
get(AccessCredentials::REGION_KEY),
415 handle->d_request_headers = curl::append_http_header((curl_slist *)0,
"Authorization", auth_header);
416 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-content-sha256",
417 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
418 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-date", AWSV4::ISO8601_date(request_time));
420 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPHEADER, handle->d_request_headers);
421 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPHEADER", handle->d_errbuf, __FILE__, __LINE__);
442 std::lock_guard<std::recursive_mutex> lock_me(d_get_easy_handle_mutex);
447 handle->d_url =
nullptr;
449 handle->d_in_use =
false;
453 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
455 BESDEBUG(
"dmrpp:5",
"Found a handle match for the " << i - d_easy_handles.begin() <<
"th easy handle." << endl);
470 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
471 if ((*i)->d_chunk == chunk) {
486 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
virtual std::string get(const std::string &key)
virtual bool is_s3_cred()
Do the URL, ID, Key amd Region items make up an S3 Credential?
error thrown if the BES is not allowed to access the resource requested
exception thrown if internal error encountered
AccessCredentials * get(std::shared_ptr< http::url > &url)
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
virtual std::shared_ptr< http::url > get_data_url() const
void release_handle(dmrpp_easy_handle *h)
void release_all_handles()
dmrpp_easy_handle * get_easy_handle(Chunk *chunk)
Bundle a libcurl easy handle with other information.
void read_data()
This is the read_data() method for all transfers.
dmrpp_easy_handle()
Build a string with hex info about stuff libcurl gets.
static AllowedHosts * theHosts()
Static accessor for the singleton.