bes Updated for version 3.20.13
DmrppRequestHandler.cc
1// DmrppRequestHandler.cc
2
3// Copyright (c) 2016 OPeNDAP, Inc. Author: James Gallagher
4// <jgallagher@opendap.org>, Patrick West <pwest@opendap.org>
5// Nathan Potter <npotter@opendap.org>
6//
7// modify it under the terms of the GNU Lesser General Public License
8// as published by the Free Software Foundation; either version 2.1 of
9// the License, or (at your option) any later version.
10//
11// This library is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14// Lesser General Public License for more details.
15//
16// License along with this library; if not, write to the Free Software
17// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18// 02110-1301 U\ SA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI.
21// 02874-0112.
22
23#include "config.h"
24
25#include <string>
26#include <memory>
27#include <sstream>
28
29#include <curl/curl.h>
30
31#include <libdap/Ancillary.h>
32#include <libdap/DMR.h>
33#include <libdap/D4Group.h>
34#include <libdap/DAS.h>
35
36#include <libdap/InternalErr.h>
37#include <libdap/mime_util.h> // for name_path
38
39#include <BESResponseHandler.h>
40#include <BESResponseNames.h>
41#include <BESDapNames.h>
42#include <BESDataNames.h>
43#include <BESDASResponse.h>
44#include <BESDDSResponse.h>
45#include <BESDataDDSResponse.h>
46#include <BESVersionInfo.h>
47#include <BESContainer.h>
48#include <ObjMemCache.h>
49
50#include <BESDMRResponse.h>
51
52#include <BESConstraintFuncs.h>
53#include <BESServiceRegistry.h>
54#include <BESUtil.h>
55#include <BESLog.h>
56#include <TheBESKeys.h>
57
58#include <BESDapError.h>
59#include <BESInternalFatalError.h>
60#include <BESDebug.h>
61#include <BESStopWatch.h>
62
63#if 1
64#define PUGIXML_NO_XPATH
65#define PUGIXML_HEADER_ONLY
66#include <pugixml.hpp>
67#endif
68
69#include "DmrppNames.h"
70#include "DmrppTypeFactory.h"
71#include "DmrppRequestHandler.h"
72#include "CurlHandlePool.h"
73#include "CredentialsManager.h"
74
75using namespace bes;
76using namespace libdap;
77using namespace std;
78
79#define MODULE_NAME "dmrpp_module"
80#ifndef MODULE_VERSION
81#define MODULE_VERSION "unset" // Set this in the Makefile.am
82#endif
83
84#define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
85
86#define USE_DMZ_TO_MANAGE_XML 1
87
88namespace dmrpp {
89
90ObjMemCache *DmrppRequestHandler::das_cache = 0;
91ObjMemCache *DmrppRequestHandler::dds_cache = 0;
92ObjMemCache *DmrppRequestHandler::dmr_cache = 0;
93
94shared_ptr<DMZ> DmrppRequestHandler::dmz(nullptr);
95
96// This is used to maintain a pool of reusable curl handles that enable connection
97// reuse. jhrg
98CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
99
100bool DmrppRequestHandler::d_use_transfer_threads = true;
101unsigned int DmrppRequestHandler::d_max_transfer_threads = 8;
102
103bool DmrppRequestHandler::d_use_compute_threads = true;
104unsigned int DmrppRequestHandler::d_max_compute_threads = 8;
105
106// Default minimum value is 2MB: 2 * (1024*1024)
107unsigned long long DmrppRequestHandler::d_contiguous_concurrent_threshold = DMRPP_DEFAULT_CONTIGUOUS_CONCURRENT_THRESHOLD;
108
109// This behavior mirrors the SAX2 parser behavior where the software doesn't require that
110// a variable actually have chunks (that is, some variable might not have any data).
111// We could make this a run-time option if needed. jhrg 11/4/21
112bool DmrppRequestHandler::d_require_chunks = false;
113
114// See the comment in the header for more about this kludge. jhrg 11/9/21
115bool DmrppRequestHandler::d_emulate_original_filter_order_behavior = false;
116
117static void read_key_value(const std::string &key_name, bool &key_value)
118{
119 bool key_found = false;
120 string value;
121 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
122 if (key_found) {
123 value = BESUtil::lowercase(value);
124 key_value = (value == "true" || value == "yes");
125 }
126}
127
128static void read_key_value(const std::string &key_name, unsigned int &key_value)
129{
130 bool key_found = false;
131 string value;
132 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
133 if (key_found) {
134 istringstream iss(value);
135 iss >> key_value;
136 }
137}
138static void read_key_value(const std::string &key_name, unsigned long long &key_value)
139{
140 bool key_found = false;
141 string value;
142 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
143 if (key_found) {
144 istringstream iss(value);
145 iss >> key_value;
146 }
147}
148
155{
156 add_method(DMR_RESPONSE, dap_build_dmr);
157 add_method(DAP4DATA_RESPONSE, dap_build_dap4data);
158 add_method(DAS_RESPONSE, dap_build_das);
159 add_method(DDS_RESPONSE, dap_build_dds);
160 add_method(DATA_RESPONSE, dap_build_dap2data);
161
162 add_method(VERS_RESPONSE, dap_build_vers);
163 add_method(HELP_RESPONSE, dap_build_help);
164
165 stringstream msg;
166 read_key_value(DMRPP_USE_TRANSFER_THREADS_KEY, d_use_transfer_threads);
167 read_key_value(DMRPP_MAX_TRANSFER_THREADS_KEY, d_max_transfer_threads);
168 msg << prolog << "Concurrent Transfer Threads: ";
169 if(DmrppRequestHandler::d_use_transfer_threads){
170 msg << "Enabled. max_transfer_threads: " << DmrppRequestHandler::d_max_transfer_threads << endl;
171 }
172 else{
173 msg << "Disabled." << endl;
174 }
175
176 INFO_LOG(msg.str() );
177 msg.str(std::string());
178
179 read_key_value(DMRPP_USE_COMPUTE_THREADS_KEY, d_use_compute_threads);
180 read_key_value(DMRPP_MAX_COMPUTE_THREADS_KEY, d_max_compute_threads);
181 msg << prolog << "Concurrent Compute Threads: ";
182 if(DmrppRequestHandler::d_use_compute_threads){
183 msg << "Enabled. max_compute_threads: " << DmrppRequestHandler::d_max_compute_threads << endl;
184 }
185 else{
186 msg << "Disabled." << endl;
187 }
188
189 INFO_LOG(msg.str() );
190 msg.str(std::string());
191
192 // DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY
193 read_key_value(DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY, d_contiguous_concurrent_threshold);
194 msg << prolog << "Contiguous Concurrency Threshold: " << d_contiguous_concurrent_threshold << " bytes." << endl;
195 INFO_LOG(msg.str() );
196
197#if !HAVE_CURL_MULTI_API
198 if (DmrppRequestHandler::d_use_transfer_threads)
199 ERROR_LOG("The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
200#endif
201
203
204 if (!curl_handle_pool)
205 curl_handle_pool = new CurlHandlePool(d_max_transfer_threads);
206
207 // This and the matching cleanup function can be called many times as long as
208 // they are called in balanced pairs. jhrg 9/3/20
209 // TODO 10/8/21 move this into the http at the top level of the BES. That is, all
210 // calls to this should be moved out of handlers and handlers can/should
211 // assume that curl is present and init'd. jhrg
212 curl_global_init(CURL_GLOBAL_DEFAULT);
213}
214
215DmrppRequestHandler::~DmrppRequestHandler()
216{
217 delete curl_handle_pool;
218 curl_global_cleanup();
219}
220
226void
227handle_exception(const string &file, int line)
228 try {
229 throw;
230 }
231 catch (const BESError &e) {
232 throw;
233 }
234 catch (const InternalErr &e) {
235 throw BESDapError(e.get_error_message(), true, e.get_error_code(), file, line);
236 }
237 catch (const Error &e) {
238 throw BESDapError(e.get_error_message(), false, e.get_error_code(), file, line);
239 }
240 catch (const std::exception &e) {
241 throw BESInternalFatalError(string("C++ exception: ").append(e.what()), file, line);
242 }
243 catch (...) {
244 throw BESInternalFatalError("Unknown exception caught building DAP4 Data response", file, line);
245 }
246
247
248void DmrppRequestHandler::build_dmr_from_file(BESContainer *container, DMR* dmr)
249{
250 string data_pathname = container->access();
251
252 dmr->set_filename(data_pathname);
253 dmr->set_name(name_path(data_pathname));
254
255#if USE_DMZ_TO_MANAGE_XML
256 dmz = shared_ptr<DMZ>(new DMZ);
257
258 // Enable adding the DMZ to the BaseTypes built by the factory
259 DmrppTypeFactory BaseFactory(dmz);
260 dmr->set_factory(&BaseFactory);
261
262 dmz->parse_xml_doc(data_pathname);
263 dmz->build_thin_dmr(dmr);
264
265 dmz->load_all_attributes(dmr);
266#else
267 DmrppTypeFactory BaseFactory; // Use the factory for this handler's types
268 dmr->set_factory(&BaseFactory);
269
270 DmrppParserSax2 parser;
271 ifstream in(data_pathname.c_str(), ios::in);
272 parser.intern(in, dmr);
273
274 dmr->set_factory(0);
275#endif
276}
277
291{
292 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
293
294 BESResponseObject *response = dhi.response_handler->get_response_object();
295 BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
296 if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
297
298 try {
299 build_dmr_from_file(dhi.container, bdmr->get_dmr());
300 bdmr->set_dap4_constraint(dhi);
301 bdmr->set_dap4_function(dhi);
302 }
303 catch (...) {
304 handle_exception(__FILE__, __LINE__);
305 }
306
307 BESDEBUG(MODULE, prolog << "END" << endl);
308
309 return true;
310}
311
318{
319 BESStopWatch sw;
320 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
321
322 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
323
324 BESResponseObject *response = dhi.response_handler->get_response_object();
325 auto *bdmr = dynamic_cast<BESDMRResponse *>(response);
326 if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
327
328 try {
329 build_dmr_from_file(dhi.container, bdmr->get_dmr());
330
331 // We don't need all the attributes, so use the lazy-load feature implemented
332 // using overloads of the BaseType::set_send_p() method.
333
334 bdmr->set_dap4_constraint(dhi);
335 bdmr->set_dap4_function(dhi);
336 }
337 catch (...) {
338 handle_exception(__FILE__, __LINE__);
339 }
340
341 BESDEBUG(MODULE, prolog << "END" << endl);
342
343 return true;
344}
345
352template <class T>
353void DmrppRequestHandler::get_dds_from_dmr_or_cache(BESDataHandlerInterface &dhi, T *bdds) {
354 string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
355
356 DDS *dds = bdds->get_dds();
357 if (!container_name_str.empty()) dds->container_name(container_name_str);
358 string accessed = dhi.container->access();
359
360 // Look in memory cache, if it's initialized
361 DDS *cached_dds_ptr = 0;
362 if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
363 BESDEBUG(MODULE, prolog << "DDS Cached hit for : " << accessed << endl);
364 *dds = *cached_dds_ptr;
365 }
366 else {
367 DMR dmr;
368 build_dmr_from_file(dhi.container, &dmr);
369
370 delete dds; // delete the current one;
371 dds = dmr.getDDS(); // assign the new one.
372
373 // Stuff it into the response.
374 bdds->set_dds(dds);
375
376 // Cache it, if the cache is active.
377 if (dds_cache) {
378 dds_cache->add(new DDS(*dds), accessed);
379 }
380 }
381}
382
387{
388 BESStopWatch sw;
389 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
390
391 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
392
393 BESResponseObject *response = dhi.response_handler->get_response_object();
394 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
395 if (!bdds) throw BESInternalError("Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
396
397 try {
398 get_dds_from_dmr_or_cache<BESDataDDSResponse>(dhi, bdds);
399 bdds->set_constraint(dhi);
400 bdds->clear_container();
401 }
402 catch (...) {
403 handle_exception(__FILE__, __LINE__);
404 }
405
406 BESDEBUG(MODULE, prolog << "END" << endl);
407 return true;
408}
409
410
415{
416 BESStopWatch sw;
417 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
418
419 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
420
421 BESResponseObject *response = dhi.response_handler->get_response_object();
422 BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(response);
423 if (!bdds) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
424
425 try {
426 get_dds_from_dmr_or_cache<BESDDSResponse>(dhi, bdds);
427
428 bdds->set_constraint(dhi);
429 bdds->clear_container();
430 }
431 catch (...) {
432 handle_exception(__FILE__, __LINE__);
433 }
434
435 BESDEBUG(MODULE, prolog << "END" << endl);
436 return true;
437}
438
444{
445 BESStopWatch sw;
446 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
447
448 BESResponseObject *response = dhi.response_handler->get_response_object();
449 BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(response);
450 if (!bdas) throw BESInternalError("Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
451
452 try {
453 string container_name_str = bdas->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
454
455 DAS *das = bdas->get_das();
456 if (!container_name_str.empty()) das->container_name(container_name_str);
457 string accessed = dhi.container->access();
458
459 // Look in memory cache (if it's initialized)
460 DAS *cached_das_ptr = 0;
461 if (das_cache && (cached_das_ptr = static_cast<DAS*>(das_cache->get(accessed)))) {
462 // copy the cached DAS into the BES response object
463 *das = *cached_das_ptr;
464 }
465 else {
466 DMR dmr;
467 build_dmr_from_file(dhi.container, &dmr);
468
469 // Get a DDS from the DMR, getDDS() allocates all new objects. Use unique_ptr
470 // to ensure this is deleted. jhrg 11/12/21
471 // TODO Add a getDAS() method to DMR so we don't have to go the long way?
472 // Or not and drop the DAP2 stuff until the code is higher up the chain?
473 // jhrg 11/12/21
474 unique_ptr<DDS> dds(dmr.getDDS());
475
476 // Load the BESDASResponse DAS from the DDS
477 dds->get_das(das);
478 Ancillary::read_ancillary_das(*das, accessed);
479
480 // Add to cache if cache is active
481 if (das_cache) {
482 // copy because the BES deletes the DAS held by the DHI.
483 // TODO Change the DHI to use shared_ptr objects. I think ... jhrg 11/12/21
484 das_cache->add(new DAS(*das), accessed);
485 }
486 }
487
488 bdas->clear_container();
489 }
490 catch (...) {
491 handle_exception(__FILE__, __LINE__);
492 }
493
494 BESDEBUG(MODULE, prolog << "END" << endl);
495 return true;
496}
497
498
499bool DmrppRequestHandler::dap_build_vers(BESDataHandlerInterface &dhi)
500{
501 BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
502 if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
503
504 info->add_module(MODULE_NAME, MODULE_VERSION);
505 return true;
506}
507
508bool DmrppRequestHandler::dap_build_help(BESDataHandlerInterface &dhi)
509{
510 BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
511 if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
512
513 // This is an example. If you had a help file you could load it like
514 // this and if your handler handled the following responses.
515 map<string, string> attrs;
516 attrs["name"] = MODULE_NAME;
517 attrs["version"] = MODULE_VERSION;
518 list<string> services;
519 BESServiceRegistry::TheRegistry()->services_handled(MODULE, services);
520 if (services.size() > 0) {
521 string handles = BESUtil::implode(services, ',');
522 attrs["handles"] = handles;
523 }
524 info->begin_tag("module", &attrs);
525 info->end_tag("module");
526
527 return true;
528}
529
530void DmrppRequestHandler::dump(ostream &strm) const
531{
532 strm << BESIndent::LMarg << "DmrppRequestHandler::dump - (" << (void *) this << ")" << endl;
533 BESIndent::Indent();
535 BESIndent::UnIndent();
536}
537
538} // namespace dmrpp
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Base exception class for the BES with basic string message.
Definition: BESError.h:59
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:254
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
static bool dap_build_dds(BESDataHandlerInterface &dhi)
void dump(std::ostream &strm) const override
dumps information about this object
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
static bool dap_build_dap4data(BESDataHandlerInterface &dhi)
Build a DAP4 data response. Adds timing to dap_build_dmr()
DmrppRequestHandler(const std::string &name)