bes Updated for version 3.20.13
FONcTransform.cc
1// FONcTransform.cc
2
3// This file is part of BES Netcdf File Out Module
4
5// Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact University Corporation for Atmospheric Research at
23// 3080 Center Green Drive, Boulder, CO 80301
24
25// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26// Please read the full copyright statement in the file COPYRIGHT_UCAR.
27//
28// Authors:
29// pwest Patrick West <pwest@ucar.edu>
30// jgarcia Jose Garcia <jgarcia@ucar.edu>
31// kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32// slloyd Samuel Lloyd <slloyd@opendap.org> (netCDF file streaming)
33
34#include "config.h"
35
36#include <sstream>
37
38#include <netcdf.h>
39
40#include <libdap/DDS.h>
41#include <libdap/DMR.h>
42#include <libdap/D4Group.h>
43#include <libdap/D4Attributes.h>
44#include <libdap/Structure.h>
45#include <libdap/Array.h>
46#include <libdap/Grid.h>
47#include <libdap/Sequence.h>
48
49#include <BESResponseObject.h>
50#include <BESDapResponseBuilder.h>
51#include <BESDataHandlerInterface.h>
52#include <BESUtil.h>
53#include <TempFile.h>
54#include <BESDapNames.h>
55#include <BESDataNames.h>
56#include <BESDataDDSResponse.h>
57#include <BESDMRResponse.h>
58#include <BESRequestHandlerList.h>
59#include <BESDapFunctionResponseCache.h>
60#include <BESDebug.h>
61#include <BESInternalError.h>
62#include <BESInternalFatalError.h>
63#include "BESSyntaxUserError.h"
64#include "RequestServiceTimer.h"
65
66#include "DapFunctionUtils.h"
67
68#include "FONcRequestHandler.h" // for the keys
69
70#include "FONcTransform.h"
71#include "FONcUtils.h"
72#include "FONcBaseType.h"
73#include "FONcAttributes.h"
74#include "FONcTransmitter.h"
75#include "history_utils.h"
76#include "FONcNames.h"
77
78
79using namespace libdap;
80using namespace std;
81
82#define MODULE "fonc"
83#define prolog std::string("FONcTransform::").append(__func__).append("() - ")
84
85#define FOUR_GB_IN_KB (4294967296/1024)
86#define TWO_GB_IN_KB (2147483648/1024)
87#define MSG_LABEL_CLASSIC_MODEL " (classic model)"
88#define MSG_LABEL_SIXTYFOUR_BIT_MODEL " (64-bit offset model)"
89
101FONcTransform::FONcTransform(DDS *dds, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
102 _dds(dds), _localfile(localfile), _returnAs(ncVersion) {
103 if (!_dds) {
104 throw BESInternalError("File out netcdf, null DDS passed to constructor", __FILE__, __LINE__);
105 }
106 if (_localfile.empty()) {
107 throw BESInternalError("File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
108 }
109#if 0
110 _localfile = localfile;
111 _dds = dds;
112 _returnAs = ncVersion;
113#endif
114
115 // if there is a variable, attribute, dimension name that is not
116 // compliant with netcdf naming conventions then we will create
117 // a new name. If the new name does not begin with an alpha
118 // character then we will prefix it with name_prefix. We will
119 // get this prefix from the type of data that we are reading in,
120 // such as nc, h4, h5, ff, jg, etc...
121 dhi.first_container();
122 if (dhi.container) {
124 }
125 else {
127 }
128}
129
141FONcTransform::FONcTransform(DMR *dmr, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
142 _dmr(dmr), _localfile(localfile), _returnAs(ncVersion) {
143 if (!_dmr) {
144 throw BESInternalError("File out netcdf, null DMR passed to constructor", __FILE__, __LINE__);
145 }
146 if (_localfile.empty()) {
147 throw BESInternalError("File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
148 }
149#if 0
150 _localfile = localfile;
151 _dmr = dmr;
152 _returnAs = ncVersion;
153#endif
154
155 // if there is a variable, attribute, dimension name that is not
156 // compliant with netcdf naming conventions then we will create
157 // a new name. If the new name does not begin with an alpha
158 // character then we will prefix it with name_prefix. We will
159 // get this prefix from the type of data that we are reading in,
160 // such as nc, h4, h5, ff, jg, etc...
161 dhi.first_container();
162 if (dhi.container) {
164 }
165 else {
167 }
168}
169
182 const string &ncVersion)
183 : d_obj(obj), d_dhi(dhi), _localfile(localfile), _returnAs(ncVersion) {
184 if (!d_obj) {
185 throw BESInternalError("File out netcdf, null BESResponseObject passed to constructor", __FILE__, __LINE__);
186 }
187 if (_localfile.empty()) {
188 throw BESInternalError("File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
189 }
190
191 // if there is a variable, attribute, dimension name that is not
192 // compliant with netcdf naming conventions then we will create
193 // a new name. If the new name does not begin with an alpha
194 // character then we will prefix it with name_prefix. We will
195 // get this prefix from the type of data that we are reading in,
196 // such as nc, h4, h5, ff, jg, etc...
197 dhi->first_container();
198 if (dhi->container) {
200 }
201 else {
203 }
204}
205
211 for (auto &b: _fonc_vars) {
212 delete b;
213 }
214 for (auto &b: _total_fonc_vars_in_grp) {
215 delete b;
216 }
217 // _dmr is not managed by the BESDMRResponse class in this code. However,
218 // _dds still is. jhrg 8/13/21
219 delete _dmr;
220}
221
222
223
233string FONcTransform::too_big_error_msg(
234 const unsigned dap_version,
235 const string &return_encoding,
236 const unsigned long long dap2_response_size_kb,
237 const unsigned long long contextual_max_response_size_kb,
238 const string &ce
239){
240
241 stringstream msg;
242
243 msg << "Your request was for a (DAP"<< dap_version << " data model response) to be encoded as ";
244 msg << return_encoding << ". ";
245 msg << "The response to your specific request will produce a " << dap2_response_size_kb;
246 msg << " kilobyte response. On this server the response size for your request is limited to ";
247 msg << contextual_max_response_size_kb << " kilobytes. ";
248
249 msg << "The server is configured to allow ";
250 auto conf_max_request_size_kb =FONcRequestHandler::get_request_max_size_kb();
251 if(conf_max_request_size_kb==0){
252 msg << " responses of unlimited size. ";
253 }
254 else {
255 msg << "responses as large as: " << conf_max_request_size_kb <<" kilobytes. ";
256 }
257
258 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF3) {
259 msg << "Additionally, the requested response encoding " << return_encoding << " is structurally limited to ";
260 if (FONcRequestHandler::nc3_classic_format) {
261 msg << TWO_GB_IN_KB << " kilobytes" << MSG_LABEL_CLASSIC_MODEL << ".";
262 }
263 else {
264 msg << FOUR_GB_IN_KB << " kilobytes" << MSG_LABEL_SIXTYFOUR_BIT_MODEL << ".";
265 }
266 msg << "One thing to try would be to reissue the the request, but change the requested response encoding ";
267 msg << "to NetCDF-4. This can be accomplished with the buttons in the Data Request Form, or by modifying ";
268 msg << "the request URL by changing the terminal path suffix from \".nc\" to \".nc4\". ";
269 }
270
271 if(ce.empty()){
272 msg << "I've noticed that no constraint expression accompanied your request. ";
273 } else {
274 msg << "Your request employed the constraint expression: \"" << ce << "\" ";
275 }
276 msg << "You may also reduce the size of the request by choosing just the variable(s) you need and/or by ";
277 msg << "using the DAP index based array sub-setting syntax to additionally limit the amount of data requested.";
278 return msg.str();
279}
280
281
291void FONcTransform::set_max_size_and_encoding(unsigned long long &max_request_size_kb, string &return_encoding){
292
293 return_encoding.clear();
294
295 // The following conditional accomplishes two things:
296 // 1) It correctly controls the values of "max_request_size_kb" so that even if it's
297 // set to unlimited (aka 0) rational limits will be enforced based on the type of
298 // response coding that was requested.
299 // 2) It constructs the string "return_encoding" for use in debugging and as
300 // a component of the "it's too big" error message.
301 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF3) {
302 return_encoding = string(FONC_RETURN_AS_NETCDF3).append("-3 ");
303 if (FONcRequestHandler::nc3_classic_format) {
304 return_encoding += MSG_LABEL_CLASSIC_MODEL;
305 if (max_request_size_kb == 0 || max_request_size_kb >= TWO_GB_IN_KB) {
306 max_request_size_kb = TWO_GB_IN_KB - 1 /* kb */;
307 BESDEBUG(MODULE, prolog << "Configured max request size was incompatible with NetCDF-3 classic format. " <<
308 "Reset to: " << max_request_size_kb << endl);
309 }
310 }
311 else {
312 return_encoding += MSG_LABEL_SIXTYFOUR_BIT_MODEL;
313 if (max_request_size_kb == 0 || max_request_size_kb >= FOUR_GB_IN_KB) {
314 max_request_size_kb = FOUR_GB_IN_KB - 1 /* kb */;
315 BESDEBUG(MODULE, prolog << "Configured max request size was incompatible with NetCDF-3 w/64-bit offset format. " <<
316 "Reset to: " << max_request_size_kb << endl);
317 }
318 }
319 }
320 else {
321 return_encoding = FONC_RETURN_AS_NETCDF4;
322 if (FONcRequestHandler::nc3_classic_format) {
323 return_encoding += MSG_LABEL_CLASSIC_MODEL;
324 }
325 }
326 BESDEBUG(MODULE, prolog << "return_encoding: " << return_encoding << endl);
327 BESDEBUG(MODULE, prolog << "max_request_size_kb: " << max_request_size_kb << endl);
328}
329
330
337void FONcTransform::throw_if_dap2_response_too_big(DDS *dds, const string &dap2_ce)
338{
339 string return_encoding;
340
341 unsigned long long max_response_size_kb = FONcRequestHandler::get_request_max_size_kb();
342 BESDEBUG(MODULE, prolog << "Configured max_request_size_kb: " << max_response_size_kb << endl);
343
344 unsigned long long dap2_response_size_kb = dds->get_request_size_kb(true);
345 BESDEBUG(MODULE, prolog << "dds->get_request_size_kb(): " << dap2_response_size_kb << endl);
346
347 set_max_size_and_encoding(max_response_size_kb, return_encoding);
348
349 // set the max request size in kilobytes for testing if the request is too large
350 dds->set_response_limit_kb(max_response_size_kb);
351
352 if (dds->too_big()) {
353 string err_msg = too_big_error_msg(2,return_encoding,dap2_response_size_kb, max_response_size_kb, dap2_ce);
354 throw BESSyntaxUserError(err_msg,__FILE__,__LINE__);
355 }
356}
357
366void FONcTransform::transform_dap2(ostream &strm) {
367
368 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
369 BESDEBUG(MODULE, prolog << "Reading data into DataDDS" << endl);
370
372
373 d_dhi->first_container();
374
375 auto bdds = dynamic_cast<BESDataDDSResponse *>(d_obj);
376 if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
377
378 _dds = bdds->get_dds();
379
381
382 besDRB.set_dataset_name(_dds->filename());
383 besDRB.set_ce(d_dhi->data[POST_CONSTRAINT]);
384 besDRB.set_async_accepted(d_dhi->data[ASYNC]);
385 besDRB.set_store_result(d_dhi->data[STORE_RESULT]);
386
387
388 // This function is used by all fileout modules, and they need to include the attributes in data access.
389 // So obtain the attributes if necessary. KY 2019-10-30
390 if (bdds->get_ia_flag() == false) {
391 BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(
392 d_dhi->container->get_container_type());
393 besRH->add_attributes(*d_dhi);
394 }
395
396 ConstraintEvaluator &eval = bdds->get_ce();
397
398 // Split constraint into two halves; stores the function and non-function parts in this instance.
399 besDRB.split_ce(eval);
400 // If there are functions, parse them and eval.
401 // Use that DDS and parse the non-function ce
402 // Serialize using the second ce and the second dds
403 if (!besDRB.get_btp_func_ce().empty()) {
404 BESDEBUG(MODULE, prolog << "Found function(s) in CE: " << besDRB.get_btp_func_ce() << endl);
405
406 BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
407
408 ConstraintEvaluator func_eval;
409 DDS *fdds = nullptr;
410 if (responseCache && responseCache->can_be_cached(_dds, besDRB.get_btp_func_ce())) {
411 fdds = responseCache->get_or_cache_dataset(_dds, besDRB.get_btp_func_ce());
412 }
413 else {
414 func_eval.parse_constraint(besDRB.get_btp_func_ce(), *_dds);
415 fdds = func_eval.eval_function_clauses(*_dds);
416 }
417
418 delete _dds; // Delete so that we can ...
419 bdds->set_dds(fdds); // Transfer management responsibility
420 _dds = fdds;
421
422 // Server functions might mark (i.e. setting send_p) so variables will use their read()
423 // methods. Clear that so the CE in d_dap2ce will control what is
424 // sent. If that is empty (there was only a function call), all
425 // variables in the intermediate DDS (i.e., the function
426 // result) will be sent.
427 _dds->mark_all(false);
428
429 // Look for one or more top level Structures whose name indicates (by way of ending with
430 // "_uwrap") that their contents should be moved to the top level.
431 //
432 // This is in support of a hack around the current API where server side functions
433 // may only return a single DAP object and not a collection of objects. The name suffix
434 // "_unwrap" is used as a signal from the function to the the various response
435 // builders and transmitters that the representation needs to be altered before
436 // transmission, and that in fact is what happens in our friend
437 // promote_function_output_structures()
438 promote_function_output_structures(_dds);
439 }
440
441
442 // evaluate the rest of the CE - the part that follows the function calls.
443 eval.parse_constraint(besDRB.get_ce(), *_dds);
444
445 _dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
446
447 throw_if_dap2_response_too_big(_dds, besDRB.get_ce());
448
449 // Convert the DDS into an internal format to keep track of
450 // variables, arrays, shared dimensions, grids, common maps,
451 // embedded structures. It only grabs the variables that are to be
452 // sent.
453 for (auto vi = _dds->var_begin(), ve = _dds->var_end(); vi != ve; vi++) {
454 if ((*vi)->send_p()) {
455 BESDEBUG(MODULE, prolog << "Converting variable '" << (*vi)->name() << "'" << endl);
456
457 // This is a factory class call, and 'fg' is specialized for '*vi'
458 FONcBaseType *fb = FONcUtils::convert((*vi), FONcTransform::_returnAs, FONcRequestHandler::classic_model);
459
460 _fonc_vars.push_back(fb);
461 vector <string> embed;
462 fb->convert(embed);
463 }
464 }
465
466 fonc_history_util::updateHistoryAttributes(_dds, d_dhi->data[POST_CONSTRAINT]);
467
468 // Open the file for writing
469 int stax;
470 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
471 if (FONcRequestHandler::classic_model) {
472 BESDEBUG(MODULE, prolog << "Opening NetCDF-4 cache file in classic mode. fileName: "
473 << _localfile << endl);
474 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
475 }
476 else {
477 BESDEBUG(MODULE, prolog << "Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
478 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
479 }
480 }
481 else {
482 BESDEBUG(MODULE, prolog << "Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
483 if (FONcRequestHandler::nc3_classic_format)
484 stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
485 else
486 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_64BIT_OFFSET, &_ncid);
487 }
488
489 if (stax != NC_NOERR) {
490 FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
491 }
492
493 int current_fill_prop_vaule;
494
495 stax = nc_set_fill(_ncid, NC_NOFILL, &current_fill_prop_vaule);
496 if (stax != NC_NOERR) {
497 FONcUtils::handle_error(stax, "File out netcdf, unable to set fill to NC_NOFILL: " + _localfile, __FILE__,
498 __LINE__);
499 }
500
501 try {
502 // Here we will be defining the variables of the netcdf and
503 // adding attributes. To do this we must be in define mode.
504 nc_redef(_ncid);
505
506 // For each converted FONc object, call define on it to define
507 // that object to the netcdf file. This also adds the attributes
508 // for the variables to the netcdf file
509 for (FONcBaseType *fbt: _fonc_vars) {
510 BESDEBUG(MODULE, prolog << "Defining variable: " << fbt->name() << endl);
511 fbt->define(_ncid);
512 }
513
514 if (FONcRequestHandler::no_global_attrs == false) {
515 // Add any global attributes to the netcdf file
516 AttrTable &globals = _dds->get_attr_table();
517 BESDEBUG(MODULE, prolog << "Adding Global Attributes" << endl << globals << endl);
518 bool is_netCDF_enhanced = false;
519 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
520 is_netCDF_enhanced = true;
521 FONcAttributes::add_attributes(_ncid, NC_GLOBAL, globals, "", "", is_netCDF_enhanced);
522 // We could add the json history directly to the netcdf file here. For now,
523 // this code, which adds it to the global attribute table and then moves
524 // those into the netcdf file, is working. There are two other places in the
525 // file where this is true. Search for '***' jhrg 2/28/22
526 }
527
528 // We are done defining the variables, dimensions, and
529 // attributes of the netcdf file. End the define mode.
530 int stax = nc_enddef(_ncid);
531
532 // Check error for nc_enddef. Handling of HDF failures
533 // can be detected here rather than later. KY 2012-10-25
534 if (stax != NC_NOERR) {
535 FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__,
536 __LINE__);
537 }
538 // write file data
539 uint64_t byteCount = 0;
540
541 if (is_streamable()) {
542 // Verify the request hasn't exceeded bes_timeout.
543 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
544
545 // Now that we are ready to start streaming the response data we
546 // cancel any pending timeout alarm according to the configuration.
548
549 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
550 BESDEBUG(MODULE, prolog << "First write data to stream, count: " << byteCount << endl);
551 }
552
553 for (FONcBaseType *fbt: _fonc_vars) {
554 BESDEBUG(MODULE, prolog << "Writing data for variable: " << fbt->name() << endl);
555
556 fbt->set_dds(_dds);
557 fbt->set_eval(&eval);
558
559 fbt->write(_ncid);
560 nc_sync(_ncid);
561
562 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog + "ERROR: bes-timeout expired before transmit " + fbt->name() , __FILE__, __LINE__);
563
564 if (is_streamable()) {
565 // write the what's been written
566 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
567 BESDEBUG(MODULE, prolog << "Writing data to stream, count: " << byteCount << endl);
568 }
569 }
570
571 stax = nc_close(_ncid);
572 if (stax != NC_NOERR)
573 FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
574
575 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog + "ERROR: bes-timeout expired before transmit" , __FILE__, __LINE__);
576
577 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
578 BESDEBUG(MODULE, prolog << "After nc_close() count: " << byteCount << endl);
579 }
580 catch (BESError &e) {
581 (void) nc_close(_ncid); // ignore the error at this point
582 throw;
583 }
584}
585
593bool FONcTransform::is_streamable() {
594 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
595 return false;
596 }
597
598 if (_dds != nullptr) {
599 return is_dds_streamable();
600 }
601 else {
602 return is_dmr_streamable(_dmr->root());
603 }
604}
605
611bool FONcTransform::is_dds_streamable() {
612 for (auto var = _dds->var_begin(), varEnd = _dds->var_end(); var != varEnd; ++var) {
613 if ((*var)->type() == dods_structure_c) {
614 return false; // cannot be streamed
615 }
616 }
617 return true;
618}
619
626bool FONcTransform::is_dmr_streamable(D4Group *group) {
627 for (auto var = group->var_begin(), varEnd = group->var_end(); var != varEnd; ++var) {
628 if ((*var)->type() == dods_structure_c)
629 return false; // cannot be streamed
630
631 if ((*var)->type() == dods_group_c) {
632 D4Group *g = dynamic_cast<D4Group *>(*var);
633 if (g != nullptr && !is_dmr_streamable(g)) {
634 return false;
635 }
636 }
637 }
638 return true;
639}
640
648void FONcTransform::throw_if_dap4_response_too_big(DMR *dmr, const string &dap4_ce)
649{
650 unsigned long long max_response_size_kb = FONcRequestHandler::get_request_max_size_kb();
651 BESDEBUG(MODULE, prolog << "Configured max_request_size_kb: " << max_response_size_kb << endl);
652
653 unsigned long long req_size_kb = dmr->request_size_kb(true);
654 BESDEBUG(MODULE, prolog << "dmr->get_request_size_kb(): " << req_size_kb << endl);
655
656 string return_encoding;
657 set_max_size_and_encoding(max_response_size_kb, return_encoding);
658
659 // set the max request size in kilobytes for testing if the request is too large
660 dmr->set_response_limit_kb(max_response_size_kb);
661
662 if (dmr->too_big()) {
663 string err_msg = too_big_error_msg(4,return_encoding,req_size_kb, max_response_size_kb, dap4_ce);
664 throw BESSyntaxUserError(err_msg,__FILE__,__LINE__);
665 }
666
667
668
669
670}
671
681 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
682
683 BESDEBUG(MODULE, prolog << "Reading data into DataDMR" << endl);
684
686
687 d_dhi->first_container();
688
689 BESDapResponseBuilder responseBuilder;
690 _dmr = responseBuilder.setup_dap4_intern_data(d_obj, *d_dhi).release();
691
692
693 _dmr->set_response_limit_kb(FONcRequestHandler::get_request_max_size_kb());
694 throw_if_dap4_response_too_big(_dmr,responseBuilder.get_dap4ce() );
695
697
698 besDRB.set_dataset_name(_dmr->filename());
699
700 // Added set of DAP4 fields. jhrg 5/30/21
701 besDRB.set_dap4ce(d_dhi->data[DAP4_CONSTRAINT]);
702 besDRB.set_dap4function(d_dhi->data[DAP4_FUNCTION]);
703
704 besDRB.set_async_accepted(d_dhi->data[ASYNC]);
705 besDRB.set_store_result(d_dhi->data[STORE_RESULT]);
706
707 // Convert the DMR into an internal format to keep track of
708 // variables, arrays, shared dimensions, grids, common maps,
709 // embedded structures. It only grabs the variables that are to be
710 // sent.
711
712
713 // First check if this DMR has groups etc.
714 bool support_group = check_group_support();
715
716 if (true == support_group) {
717
718 int stax = -1;
719 BESDEBUG(MODULE, prolog << "Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
720 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
721 if (stax != NC_NOERR)
722 FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
723
724 D4Group *root_grp = _dmr->root();
725
726 // Declare the dimname to dimid map to handle netCDF-4 dimensions
727 map<string, int> fdimname_to_id;
728
729 // Generate a list of the groups in the final netCDF file.
730 // The attributes of these groups should be included.
731 gen_included_grp_list(root_grp);
732
733#if !NDEBUG
734 for (std::set<string>::iterator it = _included_grp_names.begin(); it != _included_grp_names.end(); ++it)
735 BESDEBUG(MODULE, prolog << "Included group list name is: " << *it << endl);
736#endif
737 // Build a global dimension name table for all variables if
738 // the constraint is not empty!
739 check_and_obtain_dimensions(root_grp, true);
740
741 // Don't remove the following code, they are for debugging.
742#if !NDEBUG
743 map<string, unsigned long>::iterator it;
744
745 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
746 BESDEBUG(MODULE, prolog << "Final GFQN dim name is: " << it->first << endl);
747 BESDEBUG(MODULE, prolog << "Final GFQN dim size is: " << it->second << endl);
748 }
749
750 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
751 BESDEBUG(MODULE, prolog << "Final VFQN dim name is: " << it->first << endl);
752 BESDEBUG(MODULE, prolog << "Final VFQN dim size is: " << it->second << endl);
753 }
754#endif
755
756 // DAP4 requires the DAP4 dimension sizes defined in the group should be changed
757 // according to the corresponding variable sizes. Check section 8.6.2 at
758 // https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1
759 //
760 map<string, unsigned long>::iterator git, vit;
761 for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
762 for (vit = VFQN_dimname_to_dimsize.begin(); vit != VFQN_dimname_to_dimsize.end(); ++vit) {
763 if (git->first == vit->first) {
764 if (git->second != vit->second)
765 git->second = vit->second;
766 break;
767 }
768 }
769 }
770
771 // This part of code is to address the possible dimension name conflict
772 // when variables in the constraint don't have dimension names. Fileout netCDF
773 // adds the fake dimensions such as dim1, dim2...to these variables.
774 // If these dimension names are used by
775 // the file to be handled, the dimension conflict will corrupt the final output.
776 // The idea is to find if there are any dimension names like dim1, dim2 ...
777 // under the root group.
778 // We will remember them and not use these names as fake dimension names.
779 //
780 // Obtain the dim. names under the root group
781 vector <string> root_d4_dimname_list;
782 for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
783 string d4_temp_dimname = git->first.substr(1);
784 //BESDEBUG(MODULE, prolog << "d4_temp_dimname: "<<d4_temp_dimname<<endl);
785 if (d4_temp_dimname.find('/') == string::npos)
786 root_d4_dimname_list.push_back(d4_temp_dimname);
787 }
788
789#if !NDEBUG
790 for (unsigned int i = 0; i < root_d4_dimname_list.size(); i++)
791 BESDEBUG(MODULE, prolog << "root_d4 dim name is: " << root_d4_dimname_list[i] << endl);
792#endif
793
794 // Only remember the root dimension names that are like "dim1,dim2,..."
795 vector<int> root_dim_suffix_nums;
796 for (unsigned int i = 0; i < root_d4_dimname_list.size(); i++) {
797 if (root_d4_dimname_list[i].size() < 4)
798 continue;
799 else if (root_d4_dimname_list[i].substr(0, 3) != "dim")
800 continue;
801 else {
802 string temp_suffix = root_d4_dimname_list[i].substr(3);
803 //BESDEBUG(MODULE, prolog << "temp_suffix: "<<temp_suffix<<endl);
804 bool ignored_suffix = false;
805 for (unsigned int j = 0; j < temp_suffix.size(); j++) {
806 if (!isdigit(temp_suffix[j])) {
807 ignored_suffix = true;
808 break;
809 }
810 }
811 if (ignored_suffix == true)
812 continue;
813 else
814 root_dim_suffix_nums.push_back(atoi(temp_suffix.c_str()));
815 }
816 }
817
818#if !NDEBUG
819 for (unsigned int i = 0; i < root_dim_suffix_nums.size(); i++)
820 BESDEBUG(MODULE, prolog << "root_dim_suffix_nums: " << root_dim_suffix_nums[i] << endl);
821
822
823 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
824 BESDEBUG(MODULE, prolog << "RFinal GFQN dim name is: " << it->first << endl);
825 BESDEBUG(MODULE, prolog << "RFinal GFQN dim size is: " << it->second << endl);
826 }
827
828 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
829 BESDEBUG(MODULE, prolog << "RFinal VFQN dim name is: " << it->first << endl);
830 BESDEBUG(MODULE, prolog << "RFinal VFQN dim size is: " << it->second << endl);
831 }
832#endif
833
834 // Now we transform all the objects(including groups) to netCDF-4
835 transform_dap4_group(root_grp, true, _ncid, fdimname_to_id, root_dim_suffix_nums);
836 stax = nc_close(_ncid);
837 if (stax != NC_NOERR)
838 FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
839
840 }
841 else // No group, handle as the classic way
842 transform_dap4_no_group();
843
844 BESDEBUG(MODULE, prolog << "END" << endl);
845
846 return;
847}
848
854void FONcTransform::transform_dap4_no_group() {
855
856 D4Group *root_grp = _dmr->root();
857#if !NDEBUG
858 D4Dimensions *root_dims = root_grp->dims();
859 for (D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
860 BESDEBUG(MODULE, prolog << "transform_dap4() - check dimensions" << endl);
861 BESDEBUG(MODULE, prolog << "transform_dap4() - dim name is: " << (*di)->name() << endl);
862 BESDEBUG(MODULE, prolog << "transform_dap4() - dim size is: " << (*di)->size() << endl);
863 BESDEBUG(MODULE, prolog << "transform_dap4() - fully_qualfied_dim name is: " << (*di)->fully_qualified_name() << endl);
864 }
865#endif
866 Constructor::Vars_iter vi = root_grp->var_begin();
867 Constructor::Vars_iter ve = root_grp->var_end();
868
869 for (; vi != ve; vi++) {
870 if ((*vi)->send_p()) {
871 BaseType *v = *vi;
872
873 BESDEBUG(MODULE, prolog << "Converting variable '" << v->name() << "'" << endl);
874
875 // This is a factory class call, and 'fg' is specialized for 'v'
876 FONcBaseType *fb = FONcUtils::convert(v, FONcTransform::_returnAs, FONcRequestHandler::classic_model);
877 _fonc_vars.push_back(fb);
878
879 vector <string> embed;
880 fb->convert(embed, true, false);
881 }
882 }
883
884#if !NDEBUG
885 if (root_grp->grp_begin() == root_grp->grp_end())
886 BESDEBUG(MODULE, prolog << "No group " << endl);
887 else
888 BESDEBUG(MODULE, prolog << "Has group " << endl);
889 for (D4Group::groupsIter gi = root_grp->grp_begin(), ge = root_grp->grp_end(); gi != ge; ++gi)
890 BESDEBUG(MODULE, prolog << "Group name: " << (*gi)->name() << endl);
891#endif
892
893 fonc_history_util::updateHistoryAttributes(_dmr, d_dhi->data[POST_CONSTRAINT]);
894
895 // Open the file for writing
896 int stax = -1;
897 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
898 if (FONcRequestHandler::classic_model) {
899 BESDEBUG(MODULE, prolog << "Opening NetCDF-4 cache file in classic mode. fileName: "
900 << _localfile << endl);
901 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
902 }
903 else {
904 BESDEBUG(MODULE, prolog << "Opening NetCDF-4 cache file. fileName: " << _localfile
905 << endl);
906 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
907 }
908 }
909 else {
910 BESDEBUG(MODULE, prolog << "Opening NetCDF-3 cache file. fileName: " << _localfile
911 << endl);
912 if (FONcRequestHandler::nc3_classic_format)
913 stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
914 else
915 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_64BIT_OFFSET, &_ncid);
916 }
917
918 if (stax != NC_NOERR) {
919 FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
920 }
921
922 try {
923 // Here we will be defining the variables of the netcdf and
924 // adding attributes. To do this we must be in define mode.
925 nc_redef(_ncid);
926
927 // For each converted FONc object, call define on it to define
928 // that object to the netcdf file. This also adds the attributes
929 // for the variables to the netcdf file
930 vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
931 vector<FONcBaseType *>::iterator e = _fonc_vars.end();
932 for (; i != e; i++) {
933 FONcBaseType *fbt = *i;
934 BESDEBUG(MODULE, prolog << "Defining variable: " << fbt->name() << endl);
935 //fbt->set_is_dap4(true);
936 fbt->define(_ncid);
937 }
938
939 if (FONcRequestHandler::no_global_attrs == false) {
940
941 // Add any global attributes to the netcdf file
942 D4Group *root_grp = _dmr->root();
943 D4Attributes *d4_attrs = root_grp->attributes();
944
945 BESDEBUG(MODULE, prolog << "Handle GLOBAL DAP4 attributes " << d4_attrs << endl);
946#if !NDEBUG
947 for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end();
948 ii != ee; ++ii) {
949 string name = (*ii)->name();
950 BESDEBUG(MODULE, prolog << "GLOBAL attribute name is " << name << endl);
951 }
952#endif
953 bool is_netCDF_enhanced = false;
954 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
955 is_netCDF_enhanced = true;
956 FONcAttributes::add_dap4_attributes(_ncid, NC_GLOBAL, d4_attrs, "", "", is_netCDF_enhanced);
957 // *** Add the json history here
958 }
959
960 // We are done defining the variables, dimensions, and
961 // attributes of the netcdf file. End the define mode.
962 int stax = nc_enddef(_ncid);
963
964 // Check error for nc_enddef. Handling of HDF failures
965 // can be detected here rather than later. KY 2012-10-25
966 if (stax != NC_NOERR) {
967 FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__,
968 __LINE__);
969 }
970
971 // Write everything out
972 i = _fonc_vars.begin();
973 e = _fonc_vars.end();
974 for (; i != e; i++) {
975 FONcBaseType *fbt = *i;
976 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog + "ERROR: bes-timeout expired before transmit " + fbt->name() , __FILE__, __LINE__);
977 BESDEBUG(MODULE, prolog << "Writing data for variable: " << fbt->name() << endl);
978 fbt->write(_ncid);
979 }
980
981 stax = nc_close(_ncid);
982 if (stax != NC_NOERR)
983 FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
984 }
985 catch (BESError &e) {
986 (void) nc_close(_ncid); // ignore the error at this point
987 throw;
988 }
989
990}
991
992// Transform the DMR to a netCDF-4 file when there are DAP4 groups.
993void FONcTransform::transform_dap4_group(D4Group *grp,
994 bool is_root_grp,
995 int par_grp_id, map<string, int> &fdimname_to_id,
996 vector<int> &root_dim_suffix_nums) {
997
998 bool included_grp = false;
999
1000 if (_dmr->get_ce_empty()) {
1001 BESDEBUG(MODULE, prolog << "In group - group name: " << grp->FQN() << endl);
1002 included_grp = true;
1003 }
1004 // Always include the root and its attributes.
1005 else if (is_root_grp == true)
1006 included_grp = true;
1007 else {
1008 // Check if this group is in the group list kept in the file.
1009 set<string>::iterator iset;
1010 if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
1011 included_grp = true;
1012 }
1013
1014 // Call the internal routine to transform the DMR that has groups if this group is in the group list..
1015 // If this group is not in the group list, we know all its subgroups are also not in the list, just stop and return.
1016 if (included_grp == true)
1017 transform_dap4_group_internal(grp, is_root_grp, par_grp_id, fdimname_to_id, root_dim_suffix_nums);
1018 return;
1019}
1020
1029void FONcTransform::transform_dap4_group_internal(D4Group *grp,
1030 bool is_root_grp,
1031 int par_grp_id, map<string, int> &fdimname_to_id,
1032 vector<int> &rds_nums) {
1033
1034 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
1035 int grp_id = -1;
1036 int stax = -1;
1037
1038 fonc_history_util::updateHistoryAttributes(_dmr, d_dhi->data[POST_CONSTRAINT]);
1039
1040 if (is_root_grp == true)
1041 grp_id = _ncid;
1042 else {
1043 stax = nc_def_grp(par_grp_id, (*grp).name().c_str(), &grp_id);
1044 BESDEBUG(MODULE, prolog << "Group name is " << (*grp).name() << endl);
1045 if (stax != NC_NOERR)
1046 FONcUtils::handle_error(stax, "File out netcdf, unable to define group: " + _localfile, __FILE__, __LINE__);
1047
1048 }
1049
1050 D4Dimensions *grp_dims = grp->dims();
1051 for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1052
1053#if !NDEBUG
1054 BESDEBUG(MODULE, prolog << "Check dimensions" << endl);
1055 BESDEBUG(MODULE, prolog << "Dim name is: " << (*di)->name() << endl);
1056 BESDEBUG(MODULE, prolog << "Dim size is: " << (*di)->size() << endl);
1057 BESDEBUG(MODULE, prolog << "Fully_qualfied_dim name is: " << (*di)->fully_qualified_name() << endl);
1058#endif
1059
1060 unsigned long dimsize = (*di)->size();
1061
1062 // The dimension size may need to be updated because of the expression constraint.
1063 map<string, unsigned long>::iterator it;
1064 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
1065 if (it->first == (*di)->fully_qualified_name())
1066 dimsize = it->second;
1067 }
1068
1069 // Define dimension.
1070 int g_dimid = -1;
1071 stax = nc_def_dim(grp_id, (*di)->name().c_str(), dimsize, &g_dimid);
1072 if (stax != NC_NOERR)
1073 FONcUtils::handle_error(stax, "File out netcdf, unable to define dimension: " + _localfile, __FILE__,
1074 __LINE__);
1075 // Save this dimension ID in a map.
1076 fdimname_to_id[(*di)->fully_qualified_name()] = g_dimid;
1077 }
1078
1079 Constructor::Vars_iter vi = grp->var_begin();
1080 Constructor::Vars_iter ve = grp->var_end();
1081
1082 vector < FONcBaseType * > fonc_vars_in_grp;
1083
1084 for (; vi != ve; vi++) {
1085 if ((*vi)->send_p()) {
1086 BaseType *v = *vi;
1087
1088 BESDEBUG(MODULE, prolog << "Converting variable '" << v->name() << "'" << endl);
1089
1090 // This is a factory class call, and 'fg' is specialized for 'v'
1091 //FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
1092 FONcBaseType *fb = FONcUtils::convert(v, FONC_RETURN_AS_NETCDF4, false, fdimname_to_id, rds_nums);
1093
1094 fonc_vars_in_grp.push_back(fb);
1095
1096 // This is needed to avoid the memory leak.
1097 _total_fonc_vars_in_grp.push_back(fb);
1098
1099 vector <string> embed;
1100 fb->convert(embed, true, true);
1101 }
1102 }
1103
1104#if !NDEBUG
1105 if (grp->grp_begin() == grp->grp_end())
1106 BESDEBUG(MODULE, prolog << "No group" << endl);
1107 else
1108 BESDEBUG(MODULE, prolog << "Has group" << endl);
1109#endif
1110
1111
1112 try {
1113 // Here we will be defining the variables of the netcdf and
1114 // adding attributes. To do this we must be in define mode.
1115 //nc_redef(_ncid);
1116
1117 vector<FONcBaseType *>::iterator i = fonc_vars_in_grp.begin();
1118 vector<FONcBaseType *>::iterator e = fonc_vars_in_grp.end();
1119 for (; i != e; i++) {
1120 FONcBaseType *fbt = *i;
1121 BESDEBUG(MODULE, prolog << "Defining variable: " << fbt->name() << endl);
1122 //fbt->set_is_dap4(true);
1123 fbt->define(grp_id);
1124 }
1125
1126 bool is_netCDF_enhanced = false;
1127 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
1128 is_netCDF_enhanced = true;
1129
1130
1131 bool add_attr = true;
1132
1133 // Only the root attribute may be ignored.
1134 if (FONcRequestHandler::no_global_attrs == true && is_root_grp == true)
1135 add_attr = false;
1136
1137 if (true == add_attr) {
1138 D4Attributes *d4_attrs = grp->attributes();
1139 BESDEBUG(MODULE, prolog << "Adding Group Attributes" << endl);
1140 // add dap4 group attributes.
1141 FONcAttributes::add_dap4_attributes(grp_id, NC_GLOBAL, d4_attrs, "", "", is_netCDF_enhanced);
1142 // *** Add the json history here
1143 }
1144
1145 // Write every variable in this group.
1146 i = fonc_vars_in_grp.begin();
1147 e = fonc_vars_in_grp.end();
1148 for (; i != e; i++) {
1149 FONcBaseType *fbt = *i;
1150 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog + "ERROR: bes-timeout expired before transmit " + fbt->name() , __FILE__, __LINE__);
1151 BESDEBUG(MODULE, prolog << "Writing data for variable: " << fbt->name() << endl);
1152 //fbt->write(_ncid);
1153 fbt->write(grp_id);
1154 }
1155
1156 // Now handle all the child groups.
1157 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1158 BESDEBUG(MODULE, prolog << "In group: " << (*gi)->name() << endl);
1159 transform_dap4_group(*gi, false, grp_id, fdimname_to_id, rds_nums);
1160 }
1161
1162 }
1163 catch (BESError &e) {
1164 (void) nc_close(_ncid); // ignore the error at this point
1165 throw;
1166 }
1167 BESDEBUG(MODULE, prolog << "END" << endl);
1168}
1169
1170
1171// Group support is only on when netCDF-4 is in enhanced model and there are groups in the DMR.
1172bool FONcTransform::check_group_support() {
1173 if (FONC_RETURN_AS_NETCDF4 == FONcTransform::_returnAs && false == FONcRequestHandler::classic_model &&
1174 (_dmr->root()->grp_begin() != _dmr->root()->grp_end()))
1175 return true;
1176 else
1177 return false;
1178}
1179
1180// Generate the final group list in the netCDF-4 file. Empty groups and their attributes will be removed.
1181void FONcTransform::gen_included_grp_list(D4Group *grp) {
1182 bool grp_has_var = false;
1183 if (grp) {
1184 BESDEBUG(MODULE, prolog << "Processing D4 Group: " << grp->name() << " fullpath: " << grp->FQN() << endl);
1185
1186 if (grp->var_begin() != grp->var_end()) {
1187
1188 BESDEBUG(MODULE, prolog << "Has child variables" << endl);
1189 Constructor::Vars_iter vi = grp->var_begin();
1190 Constructor::Vars_iter ve = grp->var_end();
1191
1192 for (; vi != ve; vi++) {
1193
1194 // This variable is selected(in the local constraints).
1195 if ((*vi)->send_p()) {
1196 grp_has_var = true;
1197
1198 //If a var in this group is selected, we need to include this group in the netcdf-4 file.
1199 //We always include root attributes, so no need to obtain grp_names for the root.
1200 if (grp->FQN() != "/")
1201 _included_grp_names.insert(grp->FQN());
1202 break;
1203 }
1204 }
1205 }
1206 // Loop through the subgroups to build up the list.
1207 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1208 BESDEBUG(MODULE, prolog << "Obtain included groups - group name: " << (*gi)->name() << endl);
1209 gen_included_grp_list(*gi);
1210 }
1211 }
1212
1213 // If this group is in the final list, all its ancestors(except root, since it is always selected),should also be included.
1214 if (grp_has_var == true) {
1215 D4Group *temp_grp = grp;
1216 while (temp_grp) {
1217 if (temp_grp->get_parent()) {
1218 temp_grp = static_cast<D4Group *>(temp_grp->get_parent());
1219 if (temp_grp->FQN() != "/")
1220 _included_grp_names.insert(temp_grp->FQN());
1221 }
1222 else
1223 temp_grp = 0;
1224 }
1225 }
1226
1227}
1228
1229void FONcTransform::check_and_obtain_dimensions(D4Group *grp, bool is_root_grp) {
1230
1231 // We may not need to do this way,it may overkill.
1232 bool included_grp = false;
1233
1234 if (_dmr->get_ce_empty())
1235 included_grp = true;
1236 // Always include the root attributes.
1237 else if (is_root_grp == true)
1238 included_grp = true;
1239 else {
1240 // Check if this group is in the group list kept in the file.
1241 set<string>::iterator iset;
1242 if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
1243 included_grp = true;
1244 }
1245
1246 if (included_grp == true)
1247 check_and_obtain_dimensions_internal(grp);
1248}
1249
1250void FONcTransform::check_and_obtain_dimensions_internal(D4Group *grp) {
1251
1252 // Remember the Group Fully Qualified dimension Name and the corresponding dimension size.
1253 D4Dimensions *grp_dims = grp->dims();
1254 if (grp_dims) {
1255 for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1256
1257#if !NDEBUG
1258 BESDEBUG(MODULE, prolog << "Check dimensions" << endl);
1259 BESDEBUG(MODULE, prolog << "Dim name is: " << (*di)->name() << endl);
1260 BESDEBUG(MODULE, prolog << "Dim size is: " << (*di)->size() << endl);
1261 BESDEBUG(MODULE, prolog << "Fully qualfied dim name: " << (*di)->fully_qualified_name() << endl);
1262#endif
1263 unsigned long dimsize = (*di)->size();
1264 if ((*di)->constrained()) {
1265 dimsize = ((*di)->c_stop() - (*di)->c_start()) / (*di)->c_stride() + 1;
1266
1267 }
1268 GFQN_dimname_to_dimsize[(*di)->fully_qualified_name()] = dimsize;
1269 }
1270 }
1271
1272 // The size of DAP4 dimension needs to be updated if the dimension size of a variable with the same dimension is
1273 // different. So we also need to remember the Variable FQN dimension name and size.
1274 // Check section 8.6.2 of DAP4 specification(https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1)
1275 Constructor::Vars_iter vi = grp->var_begin();
1276 Constructor::Vars_iter ve = grp->var_end();
1277 for (; vi != ve; vi++) {
1278 if ((*vi)->send_p()) {
1279 if ((*vi)->is_vector_type()) {
1280 Array *t_a = dynamic_cast<Array *>(*vi);
1281 Array::Dim_iter dim_i = t_a->dim_begin();
1282 Array::Dim_iter dim_e = t_a->dim_end();
1283 for (; dim_i != dim_e; dim_i++) {
1284 if ((*dim_i).name != "") {
1285 D4Dimension *d4dim = t_a->dimension_D4dim(dim_i);
1286 if (d4dim) {
1287 BESDEBUG(MODULE, prolog << "Check dim- dim name is: " << d4dim->name() << endl);
1288 BESDEBUG(MODULE, prolog << "Check dim- dim size is: " << d4dim->size() << endl);
1289 BESDEBUG(MODULE, prolog << "Check dim- fully_qualfied_dim name is: "
1290 << d4dim->fully_qualified_name() << endl);
1291
1292 unsigned long dimsize = t_a->dimension_size(dim_i, true);
1293#if !NDEBUG
1294 BESDEBUG(MODULE, prolog << "Check dim- final dim size is: " << dimsize << endl);
1295#endif
1296 pair<map<string, unsigned long>::iterator, bool> ret_it;
1297 ret_it = VFQN_dimname_to_dimsize.insert(
1298 pair<string, unsigned long>(d4dim->fully_qualified_name(), dimsize));
1299 if (ret_it.second == false && ret_it.first->second != dimsize) {
1300 string err = "fileout_netcdf-4: dimension found with the same name, but different size";
1301 throw BESInternalError(err, __FILE__, __LINE__);
1302 }
1303 //VFQN_dimname_to_dimsize[d4dim->fully_qualified_name()] = dimsize;
1304 }
1305 else
1306 throw BESInternalError("Has dimension name but D4 dimension is NULL", __FILE__, __LINE__);
1307 }
1308 // No need to handle the case when the dimension name doesn't exist. This will be handled in FONcArray.cc.
1309 // else { }
1310 }
1311 }
1312 }
1313 }
1314
1315#if !NDEBUG
1316 map<string, unsigned long>::iterator it;
1317 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
1318 BESDEBUG(MODULE, prolog << "GFQN dim name is: " << it->first << endl);
1319 BESDEBUG(MODULE, prolog << "GFQN dim size is: " << it->second << endl);
1320 }
1321
1322 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
1323 BESDEBUG(MODULE, prolog << "VFQN dim name is: " << it->first << endl);
1324 BESDEBUG(MODULE, prolog << "VFQN dim size is: " << it->second << endl);
1325 }
1326
1327#endif
1328
1329 // Go through all the descendent groups.
1330 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1331 BESDEBUG(MODULE,prolog << "In group: " << (*gi)->name() << endl);
1332 check_and_obtain_dimensions(*gi, false);
1333 }
1334
1335}
1336
1346void FONcTransform::dump(ostream &strm) const {
1347 strm << BESIndent::LMarg << "FONcTransform::dump - (" << (void *) this << ")" << endl;
1348 BESIndent::Indent();
1349 strm << BESIndent::LMarg << "ncid = " << _ncid << endl;
1350 strm << BESIndent::LMarg << "temporary file = " << _localfile << endl;
1351 BESIndent::Indent();
1352 vector<FONcBaseType *>::const_iterator i = _fonc_vars.begin();
1353 vector<FONcBaseType *>::const_iterator e = _fonc_vars.end();
1354 for (; i != e; i++) {
1355 FONcBaseType *fbt = *i;
1356 fbt->dump(strm);
1357 }
1358 BESIndent::UnIndent();
1359 BESIndent::UnIndent();
1360}
1361
1362
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void set_dap4function(std::string _func)
virtual void set_ce(std::string _ce)
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
Base exception class for the BES with basic string message.
Definition: BESError.h:59
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
Abstract base class representing a specific set of information in response to a request to the BES.
error thrown if there is a user syntax error in the request or any other user error
static void conditional_timeout_cancel()
Checks if the timeout alarm should be canceled based on the value of the BES key BES....
Definition: BESUtil.cc:895
static void add_dap4_attributes(int ncid, int varid, D4Attributes *d4_attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
add_dap4_attributes
static void add_attributes(int ncid, int varid, AttrTable &attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
helper function for add_attributes
A DAP BaseType with file out netcdf information included.
Definition: FONcBaseType.h:60
virtual void define(int ncid)
Define the variable in the netcdf file.
Definition: FONcBaseType.cc:60
virtual void dump(std::ostream &strm) const =0
dump the contents of this object to the specified ostream
virtual void dump(ostream &strm) const
dumps information about this transformation object for debugging purposes
virtual void transform_dap2(ostream &strm)
Transforms each of the variables of the DataDDS to the NetCDF file.
FONcTransform(libdap::DDS *dds, BESDataHandlerInterface &dhi, const std::string &localfile, const std::string &netcdfVersion="netcdf")
Constructor that creates transformation object from the specified DataDDS object to the specified fil...
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
virtual ~FONcTransform()
Destructor.
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
Definition: FONcUtils.cc:424
static string name_prefix
If a variable name, dimension name, or attribute name begins with a character that is not supported b...
Definition: FONcUtils.h:60
static void reset()
Resets the FONc transformation for a new input and out file.
Definition: FONcUtils.cc:68
static RequestServiceTimer * TheTimer()
Return a pointer to a singleton timer instance. If an instance does not exist it will create and init...
void throw_if_timeout_expired(const std::string &message, const std::string &file, const int line)
Checks the RequestServiceTimer to determine if the time spent servicing the request at this point has...