bes Updated for version 3.20.13
NCMLRequestHandler.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2009 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29
30#include "config.h"
31
32#include <memory>
33
34#include <libdap/DMR.h>
35#include <libdap/DataDDS.h>
36
37#include <libdap/mime_util.h>
38#include <libdap/D4BaseTypeFactory.h>
39
40#include "NCMLRequestHandler.h"
41
42#include <BESConstraintFuncs.h>
43#include <BESContainerStorage.h>
44#include <BESContainerStorageList.h>
45#include <BESDapNames.h>
46#include "BESDataDDSResponse.h"
47#include <BESDataNames.h>
48#include <BESDASResponse.h>
49#include <BESDDSResponse.h>
50#include <BESDMRResponse.h>
51
52#include <BESDebug.h>
53#include "BESStopWatch.h"
54#include <BESInternalError.h>
55#include <BESDapError.h>
56#include <BESError.h>
57#include <BESRequestHandlerList.h>
58#include <BESResponseHandler.h>
59#include <BESResponseNames.h>
60#include <BESServiceRegistry.h>
61#include <BESTextInfo.h>
62#include <BESUtil.h>
63#include <BESVersionInfo.h>
64#include <TheBESKeys.h>
65
66#include "DDSLoader.h"
67
68#include "NCMLDebug.h"
69#include "NCMLUtil.h"
70#include "NCMLParser.h"
71#include "NCMLResponseNames.h"
72#include "SimpleLocationParser.h"
73
74using namespace agg_util;
75using namespace ncml_module;
76using namespace libdap;
77
78bool NCMLRequestHandler::_global_attributes_container_name_set = false;
79string NCMLRequestHandler::_global_attributes_container_name;
80
81NCMLRequestHandler::NCMLRequestHandler(const string &name) :
83{
84 add_method(DAS_RESPONSE, NCMLRequestHandler::ncml_build_das);
85 add_method(DDS_RESPONSE, NCMLRequestHandler::ncml_build_dds);
86 add_method(DATA_RESPONSE, NCMLRequestHandler::ncml_build_data);
87
88 add_method(DMR_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
89 add_method(DAP4DATA_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
90
91 add_method(VERS_RESPONSE, NCMLRequestHandler::ncml_build_vers);
92 add_method(HELP_RESPONSE, NCMLRequestHandler::ncml_build_help);
93
94 if (NCMLRequestHandler::_global_attributes_container_name_set == false) {
95 bool key_found = false;
96 string value;
97 TheBESKeys::TheKeys()->get_value("NCML.GlobalAttributesContainerName", value, key_found);
98 if (key_found) {
99 // It was set in the conf file
100 NCMLRequestHandler::_global_attributes_container_name_set = true;
101
102 NCMLRequestHandler::_global_attributes_container_name = value;
103 }
104 }
105}
106
107NCMLRequestHandler::~NCMLRequestHandler()
108{
109}
110
111#if 0
112// Not used. jhrg 4/16/14
113
114// This is the original example from Patrick or James for loading local file within the BES...
115// Still used by DataDDS call, but the other callbacks use DDSLoader
116// to get a brandy new DDX response.
117// @see DDSLoader
118bool NCMLRequestHandler::ncml_build_redirect(BESDataHandlerInterface &dhi, const string& location)
119{
120 // The current container in dhi is a reference to the ncml file.
121 // Need to parse the ncml file here and get the list of locations
122 // that we will be using. Any constraints defined?
123
124 // do this for each of the locations retrieved from the ncml file.
125 // If there are more than one locations in the ncml then we can't
126 // set the context for dap_format to dap2. This will create a
127 // structure for each of the locations in the resulting dap object.
128 string sym_name = dhi.container->get_symbolic_name();
129 BESContainerStorageList *store_list = BESContainerStorageList::TheList();
130 BESContainerStorage *store = store_list->find_persistence("catalog");
131 if (!store) {
132 throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
133 }
134 // this will throw an exception if the location isn't found in the
135 // catalog. Might want to catch this. Wish the add returned the
136 // container object created. Might want to change it.
137 string new_sym = sym_name + "_location1";
138 store->add_container(new_sym, location, "");
139
140 BESContainer *container = store->look_for(new_sym);
141 if (!container) {
142 throw BESInternalError("couldn't find the container" + sym_name, __FILE__, __LINE__);
143 }
144 BESContainer *ncml_container = dhi.container;
145 dhi.container = container;
146
147 // this will throw an exception if there is a problem building the
148 // response for this container. Might want to catch this
149 BESRequestHandlerList::TheList()->execute_current(dhi);
150
151 // clean up
152 dhi.container = ncml_container;
153 store->del_container(new_sym);
154
155 return true;
156}
157#endif
158
159// Here we load the DDX response with by hijacking the current dhi via DDSLoader
160// and hand it to our parser to load the ncml, load the DDX for the location,
161// apply ncml transformations to it, then return the modified DDS.
162bool NCMLRequestHandler::ncml_build_das(BESDataHandlerInterface &dhi)
163{
164 BESStopWatch sw;
165 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_das", dhi.data[REQUEST_ID]);
166
167 string filename = dhi.container->access();
168
169 // Any exceptions winding through here will cause the loader and parser dtors
170 // to clean up dhi state, etc.
171 DDSLoader loader(dhi);
172 NCMLParser parser(loader);
173 unique_ptr<BESDapResponse> loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
174
175 // Now fill in the desired DAS response object from the DDS
176 DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
177 VALID_PTR(dds);
178
179 BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(dhi.response_handler->get_response_object());
180 VALID_PTR(bdas);
181
182 // Copy the modified DDS attributes into the DAS response object!
183 DAS *das = bdas->get_das();
184
185 if (dds->get_dap_major() < 4)
186 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
187 NCMLRequestHandler::get_global_attributes_container_name());
188
190
191 // loaded_bdds destroys itself.
192 return true;
193}
194
195bool NCMLRequestHandler::ncml_build_dds(BESDataHandlerInterface &dhi)
196{
197#if 0
198 // original version 8/13/15
199 BESStopWatch sw;
200 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
201
202 string filename = dhi.container->access();
203
204 // Any exceptions winding through here will cause the loader and parser dtors
205 // to clean up dhi state, etc.
206 unique_ptr<BESDapResponse> loaded_bdds(0);
207 {
208 DDSLoader loader(dhi);
209 NCMLParser parser(loader);
210 loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
211 }
212 if (!loaded_bdds.get()) {
213 throw BESInternalError("Null BESDDSResonse in ncml DDS handler.", __FILE__, __LINE__);
214 }
215
216 // Poke the handed down original response object with the loaded and modified one.
217 DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
218 VALID_PTR(dds);
219 BESResponseObject *response = dhi.response_handler->get_response_object();
220 BESDDSResponse *bdds_out = dynamic_cast<BESDDSResponse *>(response);
221 VALID_PTR(bdds_out);
222 DDS *dds = bdds_out->get_dds();
223 VALID_PTR(dds);
224
225 if (dds->get_dap_major() < 4)
226 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
227 NCMLRequestHandler::get_global_attributes_container_name());
228
229 // If we just use DDS::operator=, we get into trouble with copied
230 // pointers, bashing of the dataset name, etc etc so I specialize the copy for now.
232
233 // Apply constraints to the result
234 // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
235 bdds_out->set_constraint(dhi);
236
237 // Also copy in the name of the original ncml request
238 // TODO @HACK Not sure I want just the basename for the filename,
239 // but since the DDS/DataDDS response fills the dataset name with it,
240 // Our bes-testsuite fails since we get local path info in the dataset name.
241 dds->filename(name_path(filename));
242 dds->set_dataset_name(name_path(filename));
243
244 return true;
245#endif
246
247 BESStopWatch sw;
248 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
249
250 string filename = dhi.container->access();
251
252 // it better be a data response!
253 BESDDSResponse* ddsResponse = dynamic_cast<BESDDSResponse *>(dhi.response_handler->get_response_object());
254 NCML_ASSERT_MSG(ddsResponse,
255 "NCMLRequestHandler::ncml_build_data(): expected BESDDSResponse* but didn't get it!!");
256
257 // Block it up to force cleanup of DHI.
258 {
259 DDSLoader loader(dhi);
260 NCMLParser parser(loader);
261 parser.parseInto(filename, DDSLoader::eRT_RequestDDX, ddsResponse);
262 }
263
264 DDS *dds = ddsResponse->get_dds();
265 VALID_PTR(dds);
266
267 if (dds->get_dap_major() < 4)
268 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
269 NCMLRequestHandler::get_global_attributes_container_name());
270
271 // Apply constraints to the result
272 // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
273 ddsResponse->set_constraint(dhi);
274
275 // Also copy in the name of the original ncml request
276 dds->filename(name_path(filename));
277 dds->set_dataset_name(name_path(filename));
278
279 return true;
280}
281
282bool NCMLRequestHandler::ncml_build_data(BESDataHandlerInterface &dhi)
283{
284 BESStopWatch sw;
285 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_data", dhi.data[REQUEST_ID]);
286
287 string filename = dhi.container->access();
288
289 // it better be a data response!
290 BESDataDDSResponse* dataResponse = dynamic_cast<BESDataDDSResponse *>(dhi.response_handler->get_response_object());
291 NCML_ASSERT_MSG(dataResponse,
292 "NCMLRequestHandler::ncml_build_data(): expected BESDataDDSResponse* but didn't get it!!");
293
294 // Block it up to force cleanup of DHI.
295 {
296 DDSLoader loader(dhi);
297 NCMLParser parser(loader);
298 parser.parseInto(filename, DDSLoader::eRT_RequestDataDDS, dataResponse);
299 }
300
301 // Apply constraints to the result
302
303 // dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
304 // Replaced the above with the code below. P West said, a while ago, that using set_constraint
305 // was better because BES containers would be supported. Not sure if that's a factor in this
306 // code... jhrg 8/12/15
307 dataResponse->set_constraint(dhi);
308
309 // Also copy in the name of the original ncml request
310 DDS* dds = NCMLUtil::getDDSFromEitherResponse(dataResponse);
311 VALID_PTR(dds);
312
313 dds->filename(name_path(filename));
314 dds->set_dataset_name(name_path(filename));
315
316 return true;
317}
318
319bool NCMLRequestHandler::ncml_build_dmr(BESDataHandlerInterface &dhi)
320{
321 BESStopWatch sw;
322 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dmr", dhi.data[REQUEST_ID]);
323
324 // Because this code does not yet know how to build a DMR directly, use
325 // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
326 // First step, build the 'full DDS'
327 string data_path = dhi.container->access();
328
329 DDS *dds = 0; // This will be deleted when loaded_bdds goes out of scope.
330 unique_ptr<BESDapResponse> loaded_bdds;
331 try {
332 DDSLoader loader(dhi);
333 NCMLParser parser(loader);
334 loaded_bdds = parser.parse(data_path, DDSLoader::eRT_RequestDDX);
335 if (!loaded_bdds.get()) throw BESInternalError("Null BESDDSResonse in the NCML DDS handler.", __FILE__, __LINE__);
336 dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
337 VALID_PTR(dds);
338 dds->filename(data_path);
339 dds->set_dataset_name(data_path);
340 }
341 catch (InternalErr &e) {
342 throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
343 }
344 catch (Error &e) {
345 throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
346 }
347 catch (BESError &e){
348 throw;
349 }
350 catch (...) {
351 throw BESDapError("Caught unknown error build ** DMR response", true, unknown_error, __FILE__, __LINE__);
352 }
353
354 // Extract the DMR Response object - this holds the DMR used by the
355 // other parts of the framework.
356 BESResponseObject *response = dhi.response_handler->get_response_object();
357 BESDMRResponse &bdmr = dynamic_cast<BESDMRResponse &>(*response);
358
359 // Get the DMR made by the BES in the BES/dap/BESDMRResponseHandler, make sure there's a
360 // factory we can use and then dump the DAP2 variables and attributes in using the
361 // BaseType::transform_to_dap4() method that transforms individual variables
362 DMR *dmr = bdmr.get_dmr();
363 dmr->set_factory(new D4BaseTypeFactory);
364 dmr->build_using_dds(*dds);
365
366 // Instead of fiddling with the internal storage of the DHI object,
367 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
368 // methods to set the constraints. But, why? Ans: from Patrick is that
369 // in the 'container' mode of BES each container can have a different
370 // CE.
371 bdmr.set_dap4_constraint(dhi);
372 bdmr.set_dap4_function(dhi);
373
374 return true;
375}
376
377bool NCMLRequestHandler::ncml_build_vers(BESDataHandlerInterface &dhi)
378{
379 BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
380 if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
381
382 info->add_module(MODULE_NAME, MODULE_VERSION);
383 return true;
384}
385
386bool NCMLRequestHandler::ncml_build_help(BESDataHandlerInterface &dhi)
387{
388 BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
389 if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
390
391 // This is an example. If you had a help file you could load it like
392 // this and if your handler handled the following responses.
393 map<string, string> attrs;
394 attrs["name"] = MODULE_NAME;
395 attrs["version"] = MODULE_VERSION;
396
397 list<string> services;
398 BESServiceRegistry::TheRegistry()->services_handled(ncml_module::ModuleConstants::NCML_NAME, services);
399 if (services.size() > 0) {
400 string handles = BESUtil::implode(services, ',');
401 attrs["handles"] = handles;
402 }
403 info->begin_tag("module", &attrs);
404 //info->add_data_from_file( "NCML.Help", "NCML Help" ) ;
405 info->add_data("Please consult the online documentation at " + ncml_module::ModuleConstants::DOC_WIKI_URL);
406 info->end_tag("module");
407
408 return true;
409}
410
411void NCMLRequestHandler::dump(ostream &strm) const
412{
413 strm << BESIndent::LMarg << "NCMLRequestHandler::dump - (" << (void *) this << ")" << endl;
414 BESIndent::Indent();
416 BESIndent::UnIndent();
417}
418
Provides a mechanism for accessing container information from different container stores registered w...
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
provides persistent storage for data storage information represented by a container.
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
Holds a DDS object within the BES.
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Base exception class for the BES with basic string message.
Definition: BESError.h:59
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
Represents a specific data type request handler.
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
virtual void dump(std::ostream &strm) const
dumps information about this object
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:356
static void populateDASFromDDS(libdap::DAS *das, const libdap::DDS &dds_const)
Definition: NCMLUtil.cc:276
static void copyVariablesAndAttributesInto(libdap::DDS *dds_out, const libdap::DDS &dds_in)
Definition: NCMLUtil.cc:332
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...
static const std::string DOC_WIKI_URL
static const std::string NCML_NAME