31#include <libdap/Ancillary.h>
32#include <libdap/DMR.h>
33#include <libdap/D4Group.h>
34#include <libdap/DAS.h>
36#include <libdap/InternalErr.h>
37#include <libdap/mime_util.h>
39#include <BESResponseHandler.h>
40#include <BESResponseNames.h>
41#include <BESDapNames.h>
42#include <BESDataNames.h>
43#include <BESDASResponse.h>
44#include <BESDDSResponse.h>
45#include <BESDataDDSResponse.h>
46#include <BESVersionInfo.h>
47#include <BESContainer.h>
48#include <ObjMemCache.h>
50#include <BESDMRResponse.h>
52#include <BESConstraintFuncs.h>
53#include <BESServiceRegistry.h>
56#include <TheBESKeys.h>
58#include <BESDapError.h>
59#include <BESInternalFatalError.h>
61#include <BESStopWatch.h>
64#define PUGIXML_NO_XPATH
65#define PUGIXML_HEADER_ONLY
69#include "DmrppNames.h"
70#include "DmrppTypeFactory.h"
71#include "DmrppRequestHandler.h"
72#include "CurlHandlePool.h"
73#include "CredentialsManager.h"
79#define MODULE_NAME "dmrpp_module"
81#define MODULE_VERSION "unset"
84#define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
86#define USE_DMZ_TO_MANAGE_XML 1
94shared_ptr<DMZ> DmrppRequestHandler::dmz(
nullptr);
98CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
100bool DmrppRequestHandler::d_use_transfer_threads =
true;
101unsigned int DmrppRequestHandler::d_max_transfer_threads = 8;
103bool DmrppRequestHandler::d_use_compute_threads =
true;
104unsigned int DmrppRequestHandler::d_max_compute_threads = 8;
107unsigned long long DmrppRequestHandler::d_contiguous_concurrent_threshold = DMRPP_DEFAULT_CONTIGUOUS_CONCURRENT_THRESHOLD;
112bool DmrppRequestHandler::d_require_chunks =
false;
115bool DmrppRequestHandler::d_emulate_original_filter_order_behavior =
false;
117static void read_key_value(
const std::string &key_name,
bool &key_value)
119 bool key_found =
false;
124 key_value = (value ==
"true" || value ==
"yes");
128static void read_key_value(
const std::string &key_name,
unsigned int &key_value)
130 bool key_found =
false;
134 istringstream iss(value);
138static void read_key_value(
const std::string &key_name,
unsigned long long &key_value)
140 bool key_found =
false;
144 istringstream iss(value);
166 read_key_value(DMRPP_USE_TRANSFER_THREADS_KEY, d_use_transfer_threads);
167 read_key_value(DMRPP_MAX_TRANSFER_THREADS_KEY, d_max_transfer_threads);
168 msg << prolog <<
"Concurrent Transfer Threads: ";
169 if(DmrppRequestHandler::d_use_transfer_threads){
170 msg <<
"Enabled. max_transfer_threads: " << DmrppRequestHandler::d_max_transfer_threads << endl;
173 msg <<
"Disabled." << endl;
176 INFO_LOG(msg.str() );
177 msg.str(std::string());
179 read_key_value(DMRPP_USE_COMPUTE_THREADS_KEY, d_use_compute_threads);
180 read_key_value(DMRPP_MAX_COMPUTE_THREADS_KEY, d_max_compute_threads);
181 msg << prolog <<
"Concurrent Compute Threads: ";
182 if(DmrppRequestHandler::d_use_compute_threads){
183 msg <<
"Enabled. max_compute_threads: " << DmrppRequestHandler::d_max_compute_threads << endl;
186 msg <<
"Disabled." << endl;
189 INFO_LOG(msg.str() );
190 msg.str(std::string());
193 read_key_value(DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY, d_contiguous_concurrent_threshold);
194 msg << prolog <<
"Contiguous Concurrency Threshold: " << d_contiguous_concurrent_threshold <<
" bytes." << endl;
195 INFO_LOG(msg.str() );
197#if !HAVE_CURL_MULTI_API
198 if (DmrppRequestHandler::d_use_transfer_threads)
199 ERROR_LOG(
"The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
204 if (!curl_handle_pool)
212 curl_global_init(CURL_GLOBAL_DEFAULT);
215DmrppRequestHandler::~DmrppRequestHandler()
217 delete curl_handle_pool;
218 curl_global_cleanup();
227handle_exception(
const string &file,
int line)
234 catch (
const InternalErr &e) {
235 throw BESDapError(e.get_error_message(),
true, e.get_error_code(), file, line);
237 catch (
const Error &e) {
238 throw BESDapError(e.get_error_message(),
false, e.get_error_code(), file, line);
240 catch (
const std::exception &e) {
248void DmrppRequestHandler::build_dmr_from_file(
BESContainer *container, DMR* dmr)
250 string data_pathname = container->
access();
252 dmr->set_filename(data_pathname);
253 dmr->set_name(name_path(data_pathname));
255#if USE_DMZ_TO_MANAGE_XML
256 dmz = shared_ptr<DMZ>(
new DMZ);
259 DmrppTypeFactory BaseFactory(dmz);
260 dmr->set_factory(&BaseFactory);
262 dmz->parse_xml_doc(data_pathname);
263 dmz->build_thin_dmr(dmr);
265 dmz->load_all_attributes(dmr);
267 DmrppTypeFactory BaseFactory;
268 dmr->set_factory(&BaseFactory);
270 DmrppParserSax2 parser;
271 ifstream in(data_pathname.c_str(), ios::in);
272 parser.intern(in, dmr);
292 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
296 if (!bdmr)
throw BESInternalError(
"Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
299 build_dmr_from_file(dhi.
container, bdmr->get_dmr());
304 handle_exception(__FILE__, __LINE__);
307 BESDEBUG(MODULE, prolog <<
"END" << endl);
322 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
326 if (!bdmr)
throw BESInternalError(
"Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
329 build_dmr_from_file(dhi.
container, bdmr->get_dmr());
334 bdmr->set_dap4_constraint(dhi);
335 bdmr->set_dap4_function(dhi);
338 handle_exception(__FILE__, __LINE__);
341 BESDEBUG(MODULE, prolog <<
"END" << endl);
356 DDS *dds = bdds->get_dds();
357 if (!container_name_str.empty()) dds->container_name(container_name_str);
361 DDS *cached_dds_ptr = 0;
362 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(accessed)))) {
363 BESDEBUG(MODULE, prolog <<
"DDS Cached hit for : " << accessed << endl);
364 *dds = *cached_dds_ptr;
368 build_dmr_from_file(dhi.
container, &dmr);
378 dds_cache->
add(
new DDS(*dds), accessed);
391 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
395 if (!bdds)
throw BESInternalError(
"Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
398 get_dds_from_dmr_or_cache<BESDataDDSResponse>(dhi, bdds);
403 handle_exception(__FILE__, __LINE__);
406 BESDEBUG(MODULE, prolog <<
"END" << endl);
419 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
423 if (!bdds)
throw BESInternalError(
"Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
426 get_dds_from_dmr_or_cache<BESDDSResponse>(dhi, bdds);
432 handle_exception(__FILE__, __LINE__);
435 BESDEBUG(MODULE, prolog <<
"END" << endl);
450 if (!bdas)
throw BESInternalError(
"Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
455 DAS *das = bdas->get_das();
456 if (!container_name_str.empty()) das->container_name(container_name_str);
460 DAS *cached_das_ptr = 0;
461 if (das_cache && (cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(accessed)))) {
463 *das = *cached_das_ptr;
467 build_dmr_from_file(dhi.
container, &dmr);
474 unique_ptr<DDS> dds(dmr.getDDS());
478 Ancillary::read_ancillary_das(*das, accessed);
484 das_cache->
add(
new DAS(*das), accessed);
491 handle_exception(__FILE__, __LINE__);
494 BESDEBUG(MODULE, prolog <<
"END" << endl);
504 info->add_module(MODULE_NAME, MODULE_VERSION);
515 map<string, string> attrs;
516 attrs[
"name"] = MODULE_NAME;
517 attrs[
"version"] = MODULE_VERSION;
518 list<string> services;
520 if (services.size() > 0) {
522 attrs[
"handles"] = handles;
524 info->begin_tag(
"module", &attrs);
525 info->end_tag(
"module");
532 strm << BESIndent::LMarg <<
"DmrppRequestHandler::dump - (" << (
void *)
this <<
")" << endl;
535 BESIndent::UnIndent();
A container is something that holds data. E.G., a netcdf file or a database entry.
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Base exception class for the BES with basic string message.
informational response object
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
static bool dap_build_dds(BESDataHandlerInterface &dhi)
void dump(std::ostream &strm) const override
dumps information about this object
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
static bool dap_build_dap4data(BESDataHandlerInterface &dhi)
Build a DAP4 data response. Adds timing to dap_build_dmr()
DmrppRequestHandler(const std::string &name)