39#include "rapidjson/writer.h"
41#include "rapidjson/stringbuffer.h"
42#include "rapidjson/filereadstream.h"
45#include "BESNotFoundError.h"
46#include "BESSyntaxUserError.h"
47#include "BESInternalError.h"
51#include "BESStopWatch.h"
53#include "TheBESKeys.h"
56#include "RemoteResource.h"
64#define prolog string("NgapApi::").append(__func__).append("() - ")
68const unsigned int REFRESH_THRESHOLD = 3600;
71NgapApi::NgapApi() : d_cmr_hostname(DEFAULT_CMR_ENDPOINT_URL), d_cmr_search_endpoint_path(DEFAULT_CMR_SEARCH_ENDPOINT_PATH) {
76 d_cmr_hostname = cmr_hostname;
79 string cmr_search_endpoint_path;
82 d_cmr_search_endpoint_path = cmr_search_endpoint_path;
88std::string NgapApi::get_cmr_search_endpoint_url(){
101std::string NgapApi::build_cmr_query_url_old_rpath_format(
const std::string &restified_path) {
104 string r_path = ( restified_path[0] !=
'/' ?
"/" :
"") + restified_path;
106 size_t provider_index = r_path.find(NGAP_PROVIDERS_KEY);
107 if(provider_index == string::npos){
109 msg << prolog <<
"The specified path '" << r_path <<
"'";
110 msg <<
" does not contain the required path element '" << NGAP_PROVIDERS_KEY <<
"'";
113 if(provider_index != 0){
115 msg << prolog <<
"The specified path '" << r_path <<
"'";
116 msg <<
" has the path element '" << NGAP_PROVIDERS_KEY <<
"' located in the incorrect position (";
117 msg << provider_index <<
") expected 0.";
120 provider_index += string(NGAP_PROVIDERS_KEY).length();
122 bool use_collection_concept_id =
false;
123 size_t collection_index = r_path.find(NGAP_COLLECTIONS_KEY);
124 if(collection_index == string::npos) {
125 size_t concepts_index = r_path.find(NGAP_CONCEPTS_KEY);
126 if (concepts_index == string::npos) {
128 msg << prolog <<
"The specified path '" << r_path <<
"'";
129 msg <<
" contains neither the '" << NGAP_COLLECTIONS_KEY <<
"'";
130 msg <<
" nor the '" << NGAP_CONCEPTS_KEY <<
"'";
131 msg <<
" key, one must be provided.";
134 collection_index = concepts_index;
135 use_collection_concept_id =
true;
137 if(collection_index <= provider_index+1){
139 msg << prolog <<
"The specified path '" << r_path <<
"'";
140 msg <<
" has the path element '" << (use_collection_concept_id?NGAP_CONCEPTS_KEY:NGAP_COLLECTIONS_KEY) <<
"' located in the incorrect position (";
141 msg << collection_index <<
") expected at least " << provider_index+1;
144 string provider = r_path.substr(provider_index,collection_index - provider_index);
145 collection_index += use_collection_concept_id?string(NGAP_CONCEPTS_KEY).length():string(NGAP_COLLECTIONS_KEY).length();
148 size_t granule_index = r_path.find(NGAP_GRANULES_KEY);
149 if(granule_index == string::npos){
151 msg << prolog <<
"The specified path '" << r_path <<
"'";
152 msg <<
" does not contain the required path element '" << NGAP_GRANULES_KEY <<
"'";
155 if(granule_index <= collection_index+1){
157 msg << prolog <<
"The specified path '" << r_path <<
"'";
158 msg <<
" has the path element '" << NGAP_GRANULES_KEY <<
"' located in the incorrect position (";
159 msg << granule_index <<
") expected at least " << collection_index+1;
162 string collection = r_path.substr(collection_index,granule_index - collection_index);
163 granule_index += string(NGAP_GRANULES_KEY).length();
166 string granule = r_path.substr(granule_index);
169 string cmr_url = get_cmr_search_endpoint_url() +
"?";
172 CURL *ceh = curl_easy_init();
173 char *esc_url_content;
176 esc_url_content = curl_easy_escape(ceh, provider.c_str(), provider.size());
177 cmr_url += string(CMR_PROVIDER).append(
"=").append(esc_url_content).append(
"&");
178 curl_free(esc_url_content);
180 esc_url_content = curl_easy_escape(ceh, collection.c_str(), collection.size());
181 if(use_collection_concept_id){
183 cmr_url += string(CMR_COLLECTION_CONCEPT_ID).append(
"=").append(esc_url_content).append(
"&");
187 cmr_url += string(CMR_ENTRY_TITLE).append(
"=").append(esc_url_content).append(
"&");
190 curl_free(esc_url_content);
192 esc_url_content = curl_easy_escape(ceh, granule.c_str(), granule.size());
193 cmr_url += string(CMR_GRANULE_UR).append(
"=").append(esc_url_content);
194 curl_free(esc_url_content);
196 curl_easy_cleanup(ceh);
217std::string NgapApi::build_cmr_query_url(
const std::string &restified_path) {
220 string r_path = ( restified_path[0] !=
'/' ?
"/" :
"") + restified_path;
222 size_t provider_index = r_path.find(NGAP_PROVIDERS_KEY);
223 if(provider_index != string::npos){
224 return build_cmr_query_url_old_rpath_format(restified_path);
227 size_t collections_key_index = r_path.find(NGAP_COLLECTIONS_KEY);
228 if(collections_key_index == string::npos) {
230 msg << prolog <<
"The specified path '" << r_path <<
"'";
231 msg <<
" contains neither the '" << NGAP_COLLECTIONS_KEY <<
"'";
232 msg <<
" nor the '" << NGAP_CONCEPTS_KEY <<
"'";
233 msg <<
" one must be provided.";
236 if(collections_key_index != 0){
238 msg << prolog <<
"The specified path '" << r_path <<
"'";
239 msg <<
" has the path element '" << NGAP_COLLECTIONS_KEY <<
"' located in the incorrect position (";
240 msg << collections_key_index <<
") expected at least " << provider_index + 1;
244 size_t collections_index = collections_key_index + string(NGAP_COLLECTIONS_KEY).length();
246 size_t granules_key_index = r_path.find(NGAP_GRANULES_KEY);
247 if(granules_key_index == string::npos){
249 msg << prolog <<
"The specified path '" << r_path <<
"'";
250 msg <<
" does not contain the required path element '" << NGAP_GRANULES_KEY <<
"'";
256 if(granules_key_index <= collections_index + 1){
258 msg << prolog <<
"The specified path '" << r_path <<
"'";
259 msg <<
" has the path element '" << NGAP_GRANULES_KEY <<
"' located in the incorrect position (";
260 msg << granules_key_index <<
") expected at least " << collections_index + 1;
263 size_t granules_index = granules_key_index + string(NGAP_GRANULES_KEY).length();
265 string granule_name = r_path.substr(granules_index);
269 string collection_name = r_path.substr(collections_index, granules_key_index - collections_index);
275 string optional_part;
276 size_t slash_pos = collection_name.find(
'/');
277 if(slash_pos != string::npos){
278 optional_part = collection_name.substr(slash_pos);
279 BESDEBUG(MODULE, prolog <<
"Found optional collections name component: " << optional_part << endl);
280 collection_name = collection_name.substr(0,slash_pos);
282 BESDEBUG(MODULE, prolog <<
"Found collection_name (aka collection_concept_id): " << collection_name << endl);
285 string cmr_url = get_cmr_search_endpoint_url() +
"?";
288 CURL *ceh = curl_easy_init();
289 char *esc_url_content;
291 esc_url_content = curl_easy_escape(ceh, collection_name.c_str(), collection_name.size());
292 cmr_url += string(CMR_COLLECTION_CONCEPT_ID).append(
"=").append(esc_url_content).append(
"&");
293 curl_free(esc_url_content);
295 esc_url_content = curl_easy_escape(ceh, granule_name.c_str(), granule_name.size());
296 cmr_url += string(CMR_GRANULE_UR).append(
"=").append(esc_url_content);
297 curl_free(esc_url_content);
299 curl_easy_cleanup(ceh);
314std::string NgapApi::find_get_data_url_in_granules_umm_json_v1_4(
const std::string &restified_path, rapidjson::Document &cmr_granule_response)
317 string data_access_url;
319 rapidjson::Value &val = cmr_granule_response[
"hits"];
320 int hits = val.GetInt();
322 throw BESNotFoundError(
string(
"The specified path '").append(restified_path).append(
323 "' does not identify a granule in CMR."), __FILE__, __LINE__);
326 rapidjson::Value &items = cmr_granule_response[
"items"];
327 if (items.IsArray()) {
330 const string RJ_TYPE_NAMES[] = {string(
"kNullType"),string(
"kFalseType"),string(
"kTrueType"),
331 string(
"kObjectType"),string(
"kArrayType"),string(
"kStringType"),string(
"kNumberType")};
332 for (rapidjson::SizeType i = 0; i < items.Size(); i++)
333 ss <<
"items[" << i <<
"]: " << RJ_TYPE_NAMES[items[i].GetType()] << endl;
334 BESDEBUG(MODULE, prolog <<
"items size: " << items.Size() << endl << ss.str() << endl);
337 rapidjson::Value &items_obj = items[0];
339 auto mitr = items_obj.FindMember(
"umm");
341 rapidjson::Value &umm = mitr->value;
342 mitr = umm.FindMember(
"RelatedUrls");
343 if (mitr == umm.MemberEnd()) {
344 throw BESInternalError(
"Error! The umm/RelatedUrls object was not located!", __FILE__, __LINE__);
346 rapidjson::Value &related_urls = mitr->value;
348 if (!related_urls.IsArray()) {
349 throw BESNotFoundError(
"Error! The RelatedUrls object in the CMR response is not an array!", __FILE__,
353 BESDEBUG(MODULE, prolog <<
" Found RelatedUrls array in CMR response." << endl);
356 for (rapidjson::SizeType i = 0; i < related_urls.Size() && data_access_url.empty(); i++) {
357 rapidjson::Value &obj = related_urls[i];
358 mitr = obj.FindMember(
"URL");
359 if (mitr == obj.MemberEnd()) {
361 err <<
"Error! The umm/RelatedUrls[" << i <<
"] does not contain the URL object";
364 rapidjson::Value &r_url = mitr->value;
366 mitr = obj.FindMember(
"Type");
367 if (mitr == obj.MemberEnd()) {
369 err <<
"Error! The umm/RelatedUrls[" << i <<
"] does not contain the Type object";
372 rapidjson::Value &r_type = mitr->value;
374 noSubtype = obj.FindMember(
"Subtype") == obj.MemberEnd();
376 BESDEBUG(MODULE, prolog <<
"RelatedUrl Object:" <<
377 " URL: '" << r_url.GetString() <<
"'" <<
378 " Type: '" << r_type.GetString() <<
"'" <<
379 " SubType: '" << (noSubtype ?
"Absent" :
"Present") <<
"'" << endl);
381 if ((r_type.GetString() ==
string(CMR_URL_TYPE_GET_DATA)) && noSubtype) {
385 string candidate_url = r_url.GetString();
386 if(candidate_url.substr(0,8) ==
"https://" || candidate_url.substr(0,7) ==
"http://"){
387 data_access_url = candidate_url;
393 if (data_access_url.empty()) {
394 throw BESInternalError(
string(
"ERROR! Failed to locate a data access URL for the path: ") + restified_path,
398 return data_access_url;
425 string NgapApi::convert_ngap_resty_path_to_data_access_url(
426 const std::string &restified_path,
427 const std::string &uid
429 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
430 string data_access_url;
432 string cmr_query_url = build_cmr_query_url(restified_path);
434 BESDEBUG(MODULE, prolog <<
"CMR Request URL: " << cmr_query_url << endl);
436 BESDEBUG(MODULE, prolog <<
"Building new RemoteResource." << endl);
437 std::shared_ptr<http::url> cmr_query_url_ptr(
new http::url(cmr_query_url));
441 if (BESISDEBUG(MODULE) ||
BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()){
442 besTimer.
start(
"CMR Query: " + cmr_query_url);
446 rapidjson::Document cmr_response = cmr_query.
get_as_json();
448 data_access_url = find_get_data_url_in_granules_umm_json_v1_4(restified_path, cmr_response);
450 BESDEBUG(MODULE, prolog <<
"END (data_access_url: "<< data_access_url <<
")" << endl);
452 return data_access_url;
458 bool NgapApi::signed_url_is_expired(
const http::url &signed_url)
463 BESDEBUG(MODULE, prolog <<
"now: " << now << endl);
465 time_t expires = now;
468 time_t ingest_time = signed_url.ingest_time();
470 if(!cf_expires.empty()){
471 expires = stoll(cf_expires);
472 BESDEBUG(MODULE, prolog <<
"Using "<< CLOUDFRONT_EXPIRES_HEADER_KEY <<
": " << expires << endl);
474 else if(!aws_expires.empty()){
478 time_t start_time = ingest_time;
482 if(!aws_date.empty()){
483 string date = aws_date;
484 string year = date.substr(0,4);
485 string month = date.substr(4,2);
486 string day = date.substr(6,2);
487 string hour = date.substr(9,2);
488 string minute = date.substr(11,2);
489 string second = date.substr(13,2);
491 BESDEBUG(MODULE, prolog <<
"date: "<< date <<
492 " year: " << year <<
" month: " << month <<
" day: " << day <<
493 " hour: " << hour <<
" minute: " << minute <<
" second: " << second << endl);
495 struct tm *ti = gmtime(&now);
496 ti->tm_year = stoll(year) - 1900;
497 ti->tm_mon = stoll(month) - 1;
498 ti->tm_mday = stoll(day);
499 ti->tm_hour = stoll(hour);
500 ti->tm_min = stoll(minute);
501 ti->tm_sec = stoll(second);
503 BESDEBUG(MODULE, prolog <<
"ti->tm_year: "<< ti->tm_year <<
504 " ti->tm_mon: " << ti->tm_mon <<
505 " ti->tm_mday: " << ti->tm_mday <<
506 " ti->tm_hour: " << ti->tm_hour <<
507 " ti->tm_min: " << ti->tm_min <<
508 " ti->tm_sec: " << ti->tm_sec << endl);
511 start_time = mktime(ti);
512 BESDEBUG(MODULE, prolog <<
"AWS (computed) start_time: "<< start_time << endl);
514 expires = start_time + stoll(aws_expires);
515 BESDEBUG(MODULE, prolog <<
"Using "<< AMS_EXPIRES_HEADER_KEY <<
": " << aws_expires <<
516 " (expires: " << expires <<
")" << endl);
518 time_t remaining = expires - now;
519 BESDEBUG(MODULE, prolog <<
"expires_time: " << expires <<
520 " remaining_time: " << remaining <<
521 " refresh_threshold: " << REFRESH_THRESHOLD << endl);
523 is_expired = remaining < REFRESH_THRESHOLD;
524 BESDEBUG(MODULE, prolog <<
"is_expired: " << (is_expired?
"true":
"false") << endl);
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
virtual bool start(std::string name)
error thrown if there is a user syntax error in the request or any other user error
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
rapidjson::Document get_as_json()
get_as_json() This function returns the cached resource parsed into a JSON document.
virtual std::string query_parameter_value(const std::string &key) const