bes Updated for version 3.20.13
DDSLoader.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2009 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29
30#include "config.h"
31
32#include <sstream>
33#include <algorithm>
34
35#include <libdap/DataDDS.h>
36
37#include <BESConstraintFuncs.h>
38#include <BESContainerStorage.h>
39#include <BESContainerStorageList.h>
40#include <BESDapNames.h>
41#include <BESDapResponse.h>
42#include <BESDataDDSResponse.h>
43#include <BESDataHandlerInterface.h>
44#include <BESDDSResponse.h>
45#include <BESStopWatch.h>
46#include <BESInternalError.h>
47#include <BESResponseHandler.h>
48#include <BESResponseNames.h>
49#include <BESRequestHandlerList.h>
50#include <BESServiceRegistry.h>
51#include <BESTextInfo.h>
52#include <BESUtil.h>
53#include <BESVersionInfo.h>
54
55#include <BESDebug.h>
56#include <BESLog.h>
57
58#include "DDSLoader.h"
59#include "NCMLDebug.h"
60#include "NCMLUtil.h"
61
62using namespace std;
63using namespace agg_util;
64using namespace libdap;
65
66// Rep Init
67
68/* static */
69long DDSLoader::_gensymID = 0L;
70
71// Impl
72
74 _dhi(dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(""), _origActionName(
75 ""), _origContainer(0), _origResponse(0)
76{
77}
78
79// WE ONLY COPY THE DHI! I got forced to impl this.
80DDSLoader::DDSLoader(const DDSLoader& proto) :
81 _dhi(proto._dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(
82 ""), _origActionName(""), _origContainer(0), _origResponse(0)
83{
84}
85
87DDSLoader::operator=(const DDSLoader& rhs)
88{
89 BESDEBUG("ncml", "DDSLoader::operator=: " << endl);
90
91 if (&rhs == this) {
92 return *this;
93 }
94
95 // First cleanup any state
96
97 // Old comment, written in the midst of fixing bug #2176...
98 // I removed this call because ensureClean() will call restoreDHI()
99 // and then we will call the BESDataHandlerInterface::clone() method
100 // which will take the DHI from the DDSLoader passed in and clone it.
101 // ...no sense setting it twice.
102 //
103 // After the fix...
104 // Added it back in because I think it might be used after all. In many
105 // (all?) cases, the rhs._dhi is the same object as this->_dhi, so the
106 // clone() method will never get called. However, the 'saved state' of
107 // the DHI instance might be needed. It's not needed for the current
108 // operations (no tests fail when it is removed), but future versions
109 // might make use of the saved state.
110 // jhrg 4/18/14
111 ensureClean();
112
113 // Now copy the dhi only, since
114 // we assume we'll be using this fresh.
115 // Normally we don't want to copy these
116 // but I got forced into it.
117 //
118 // Update. Fix for bug #2176. With clang-503.0.40 calling
119 // BESDataHandlerInterface::make_copy() was inexplicably nulling the 'data'
120 // map member of the DHI. This was happening because the two maps were the same
121 // because the two DHI instances were the same - that is the 'rhs._dhi' field
122 // and this' _dhi field were/are one and the same. I'm going to leave this
123 // test here even though the code in BESDataHandlerInterface has been fixed to
124 // test for this case - and new copy ctor and operator=() methods added.
125 // jhrg 4/18/14
126 if (&_dhi != &rhs._dhi) _dhi.make_copy(rhs._dhi);
127
128 return *this;
129}
130
132{
133 ensureClean();
134}
135
136#if 0
137// Never used. 10/16/15 jhrg
138unique_ptr<BESDapResponse> DDSLoader::load(const string& location, ResponseType type)
139{
140 // We need to make the proper response object as well, since in this call the dhi is coming in with the
141 // response object for the original ncml request.
142 std::unique_ptr<BESDapResponse> response = makeResponseForType(type);
143 loadInto(location, type, response.get());
144 return response; // relinquish
145}
146#endif
147
148void DDSLoader::loadInto(const std::string& location, ResponseType type, BESDapResponse* pResponse)
149{
150 VALID_PTR(pResponse);
151 VALID_PTR(_dhi.response_handler);
152
153 // Just be sure we're cleaned up before doing anything, in case the caller calls load again after exception
154 // and before dtor.
155 ensureClean();
156
157 _filename = location;
158
159 // Remember current state of dhi before we touch it -- _hijacked is now true!!
160 snapshotDHI();
161
162#if 0
163 BESContainer *container = nullptr;
164
165 try {
166 // Add a new symbol to the storage list and return container for it.
167 // We will remove this new container on the way out.
168 //
169 // HK-474: If there is no handler configured to read the dataset (based on the values
170 // of the various TypeMatch parameters), code nested inside this call with throw a
171 // BESInternalError and the 'unwinding' of the containers will fail with some odd
172 // ramifications, including segfaults and no message to the user/client. jhrg 11/13/19
173 container = addNewContainerToStorage();
174 }
175 catch (BESError &e) {
176 *(BESLog::TheLog()) << "WARNING - " << string(__PRETTY_FUNCTION__) << ": " << e.get_file() << ":" << e.get_line() << ": "
177 << e.get_message() << " (the exception was re-thrown)."<< endl;
178
179 // Get rid of the container we added.
180 removeContainerFromStorage();
181
182 // Put back the dhi state we hijacked
183 restoreDHI();
184
185 throw e;
186 }
187#endif
188
189 try {
190 // Add a new symbol to the storage list and return container for it.
191 // We will remove this new container on the way out.
192 //
193 // HK-474: If there is no handler configured to read the dataset (based on the values
194 // of the various TypeMatch parameters), code nested inside this call with throw a
195 // BESInternalError and the 'unwinding' of the containers will fail with some odd
196 // ramifications, including segfaults and no message to the user/client. jhrg 11/13/19
197 BESContainer *container = addNewContainerToStorage();
198
199 // Take over the dhi
200 // Container is allocated using ptr_duplicate. Must free existing container. See RestoreDHI. jhrg 6/19/19
201 _dhi.container = container;
202 _dhi.response_handler->set_response_object(pResponse);
203
204 // Choose the proper request type...
205 _dhi.action = getActionForType(type);
206 _dhi.action_name = getActionNameForType(type);
207
208 // Figure out which underlying type of response it is to get the DDS (or DataDDS via DDS super).
210 if (!pDDS) {
211 THROW_NCML_INTERNAL_ERROR("DDSLoader::load expected BESDDSResponse or BESDataDDSResponse but got neither!");
212 }
213 pDDS->set_request_xml_base(pResponse->get_request_xml_base());
214
215 // DO IT!
216
217 BESDEBUG("ncml", "Before BESRequestHandlerList::TheList()->execute_current" << endl);
218 BESDEBUG("ncml", "Handler name: " << BESRequestHandlerList::TheList()->get_handler_names() << endl);
219
220 BESRequestHandlerList::TheList()->execute_current(_dhi);
221
222 // Some NcML operations like rename/add attributes need to have attributes in the data access.
223 // So we need to check if the attributes are added by the underneath handlers.
224 // If not, it will be added here. KY 10/30/19
225 if(type == eRT_RequestDataDDS) {
226
227 BESResponseObject *response = _dhi.response_handler->get_response_object();
228 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
229 if (!bdds)
230 throw BESInternalError("cast error", __FILE__, __LINE__);
231
232 if(bdds->get_ia_flag() == false) {
233 BESDEBUG("ncml", "Underneath handler "<< _dhi.container->get_container_type() << " call add_attributes() " << endl);
234 BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(_dhi.container->get_container_type());
235 besRH->add_attributes(_dhi);
236 }
237 }
238
239 BESDEBUG("ncml", "After BESRequestHandlerList::TheList()->execute_current" << endl);
240
241 _filename = "";
242
243 ensureClean();
244 }
245 catch (BESError &e) {
246 ERROR_LOG("WARNING - " << string(__PRETTY_FUNCTION__) << ": " << e.get_file() << ":" << e.get_line() << ": "
247 << e.get_message() << " (the exception was re-thrown)."<< endl);
248
249 // We should be clean here too.
250 ensureClean();
251
252 throw e;
253 }
254}
255
257{
258 ensureClean();
259}
260
261bool is_url(const std::string &location) {
262 std::string http("http://");
263 std::string https("https://");
264
265 // case insensitive check
266 std::string tip = location.substr(0,http.size());
267 std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
268 bool result = http == tip; // http.compare(tip)==0;
269
270 // case insensitive check
271 tip = location.substr(0,https.size());
272 std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
273
274 result = result || http == tip; //http.compare(tip)==0;
275
276 return result;
277}
278
280DDSLoader::addNewContainerToStorage()
281{
282 // Make sure we can find the storage
283 BESContainerStorageList *store_list = BESContainerStorageList::TheList();
284 VALID_PTR(store_list);
285
286 // Check for URL in the _filename, 'cause that means gateway!
287 BESContainerStorage* store;
288 if(is_url(_filename)){
289 BESDEBUG("ncml", __func__ << "() - GATEWAY CONTAINER!" << endl);
290 store = store_list->find_persistence("gateway");
291 }
292 else {
293 store = store_list->find_persistence("catalog");
294 }
295 if (!store) {
296 throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
297 }
298
299 // Make a new symbol from the ncml filename
300 string newSymbol = getNextContainerName() + "__" + _filename;
301
302 // this will throw an exception if the location isn't found in the
303 // catalog. Might want to catch this. Wish the add returned the
304 // container object created. Might want to change it.
305 //
306 // HK-474 This is the crux of the problem.
307 store->add_container(newSymbol, _filename, "");
308
309 // If we were successful, note the store location and symbol we added for removal later.
310 _store = store;
311 _containerSymbol = newSymbol;
312
313 // Now look up the symbol we added
314 BESContainer *container = store->look_for(_containerSymbol);
315 if (!container) {
316 throw BESInternalError("couldn't find the container we just added:" + newSymbol, __FILE__, __LINE__);
317 }
318
319 return container;
320}
321
322void DDSLoader::removeContainerFromStorage()
323{
324 // If we have non-null _store, we added the container symbol,
325 // so get rid of it
326 if (_store) {
327 try {
328 // This should go through, but if there's an exception, we could unwind through the dtor,
329 // so make sure we don't.
330 _store->del_container(_containerSymbol);
331 }
332 catch (BESError& besErr) {
333 ERROR_LOG("WARNING: tried to remove symbol " << _containerSymbol
334 << " from singleton but unexpectedly it was not there." << endl);
335 }
336 _containerSymbol = "";
337 _store = 0;
338 }
339}
340
341void DDSLoader::snapshotDHI()
342{
343 VALID_PTR(_dhi.response_handler);
344
345 BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Taking snapshot of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
346 BESDEBUG( "ncml_verbose", "original dhi = " << _dhi << endl );
347
348 // Store off the container for the original ncml file call and replace with the new one
349 _origContainer = _dhi.container;
350 _dhi.container = 0; // Mark the container so that we do not call release() more than once. jhrg 11/14/19
351 _origAction = _dhi.action;
352 _origActionName = _dhi.action_name;
353
354 _origResponse = _dhi.response_handler->get_response_object();
355
356 BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Replaced with DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
357
358 _hijacked = true;
359}
360
361void DDSLoader::restoreDHI()
362{
363 VALID_PTR(_dhi.response_handler);
364
365 // Make sure we have state before we go mucking
366 if (!_hijacked) {
367 return;
368 }
369
370 // Before we overwrite the 'high jacked' DHI's container, call
371 // release(). If this is a simple file, it's no big deal to
372 // skip this (because the file is closed elsewhere). But, if the
373 // DHI is working with a _compressed_ file, it is a big deal
374 // because this is the call that closes the cached uncompressed
375 // file and frees the lock. This was the bug associated with
376 // ticket HR-64. jhrg 10/16/15
377 // Only release the container if it is not null. This was happening
378 // because DDSLoader:loadInto() was failing in mid-process and the
379 // DHI had been hijacked but the new container was not set. See HK-474.
380 // jhrg 11/14/19
381 if (_dhi.container) _dhi.container->release();
382 // Leak. Allocated locally by addNewContainerToStorage() in loadInto(). jhrg 6/19/19
383 delete _dhi.container;
384
385 // Restore saved state
386 _dhi.container = _origContainer;
387 _dhi.action = _origAction;
388 _dhi.action_name = _origActionName;
389
390 _dhi.response_handler->set_response_object(_origResponse);
391
392 BESDEBUG( "ncml", "DDSLoader::restoreDHI() - Restored of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
393
394 BESDEBUG( "ncml_verbose", "restored dhi = " << _dhi << endl );
395
396 // clear our copy of saved state
397 _origAction = "";
398 _origActionName = "";
399 _origResponse = 0;
400 _origContainer = 0;
401 _filename = "";
402
403 _hijacked = false;
404}
405
409void DDSLoader::ensureClean()
410{
411 // If we're still hijacked here, there was an exception in load, so clean
412 // up if needed.
413 if (_hijacked) {
414 restoreDHI();
415 }
416
417 // Make sure we've removed the new symbol from the container list as well.
418 removeContainerFromStorage();
419}
420
421/* static */
422std::string DDSLoader::getNextContainerName()
423{
424 static const string _sPrefix = "__DDSLoader_Container_ID_";
425 _gensymID++;
426 std::ostringstream oss;
427 oss << _sPrefix << (_gensymID);
428 return oss.str();
429}
430
431unique_ptr<BESDapResponse> DDSLoader::makeResponseForType(ResponseType type)
432{
433 if (type == eRT_RequestDDX) {
434 // The BaseTypeFactory is leaked. jhrg 6/19/19
435 return unique_ptr<BESDapResponse>(new BESDDSResponse(new DDS(nullptr /*new BaseTypeFactory()*/, "virtual")));
436 }
437 else if (type == eRT_RequestDataDDS) {
438 // Leak fix jhrg 6/19/19
439 return unique_ptr<BESDapResponse>(new BESDataDDSResponse(new DDS(nullptr /*new BaseTypeFactory()*/, "virtual")));
440 }
441 else {
442 THROW_NCML_INTERNAL_ERROR("DDSLoader::makeResponseForType() got unknown type!");
443 }
444}
445
447{
448 if (type == eRT_RequestDDX) {
449 return DDS_RESPONSE;
450 }
451 else if (type == eRT_RequestDataDDS) {
452 return DATA_RESPONSE;
453 }
454
455 THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionForType(): unknown type!");
456}
457
459{
460 if (type == eRT_RequestDDX) {
461 return DDX_RESPONSE_STR;
462 }
463 else if (type == eRT_RequestDataDDS) {
464 return DATA_RESPONSE_STR;
465 }
466
467 THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionNameForType(): unknown type!");
468}
469
471{
472 if (type == eRT_RequestDDX) {
473 return dynamic_cast<BESDDSResponse*>(pResponse);
474 }
475 else if (type == eRT_RequestDataDDS) {
476 return dynamic_cast<BESDataDDSResponse*>(pResponse);
477 }
478 else {
479 return false;
480 }
481}
482
Provides a mechanism for accessing container information from different container stores registered w...
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
provides persistent storage for data storage information represented by a container.
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Holds a DDS object within the BES.
Represents an OPeNDAP DAP response object within the BES.
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
void make_copy(const BESDataHandlerInterface &copy_from)
deprecated
std::string action
the response object requested, e.g. das, dds
BESContainer * container
pointer to current container in this interface
Base exception class for the BES with basic string message.
Definition: BESError.h:59
unsigned int get_line() const
get the line number where the exception was thrown
Definition: BESError.h:129
std::string get_file() const
get the file name where the exception was thrown
Definition: BESError.h:120
std::string get_message() const
get the error message for this exception
Definition: BESError.h:111
exception thrown if internal error encountered
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
virtual BESResponseObject * set_response_object(BESResponseObject *o)
replaces the current response object with the specified one, returning the current response object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
static std::unique_ptr< BESDapResponse > makeResponseForType(ResponseType type)
Definition: DDSLoader.cc:431
void loadInto(const std::string &location, ResponseType type, BESDapResponse *pResponse)
Load a DDX or DataDDS response into the given pResponse object, which must be non-null.
Definition: DDSLoader.cc:148
DDSLoader(BESDataHandlerInterface &dhi)
Create a loader that will hijack dhi on a load call, then restore it's state.
Definition: DDSLoader.cc:73
static bool checkResponseIsValidType(ResponseType type, BESDapResponse *pResponse)
Definition: DDSLoader.cc:470
void cleanup()
restore dhi to clean state
Definition: DDSLoader.cc:256
static std::string getActionForType(ResponseType type)
Definition: DDSLoader.cc:446
static std::string getActionNameForType(ResponseType type)
Definition: DDSLoader.cc:458
virtual ~DDSLoader()
Dtor restores the state of dhi Restores the state of the dhi to what it was when object if it is stil...
Definition: DDSLoader.cc:131
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:356
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
utility class for the HTTP catalog module
Definition: AllowedHosts.cc:55