40#include <libdap/DDS.h>
41#include <libdap/DMR.h>
42#include <libdap/D4Group.h>
43#include <libdap/D4Attributes.h>
44#include <libdap/Structure.h>
45#include <libdap/Array.h>
46#include <libdap/Grid.h>
47#include <libdap/Sequence.h>
49#include <BESResponseObject.h>
50#include <BESDapResponseBuilder.h>
51#include <BESDataHandlerInterface.h>
54#include <BESDapNames.h>
55#include <BESDataNames.h>
56#include <BESDataDDSResponse.h>
57#include <BESDMRResponse.h>
58#include <BESRequestHandlerList.h>
59#include <BESDapFunctionResponseCache.h>
61#include <BESInternalError.h>
62#include <BESInternalFatalError.h>
63#include "BESSyntaxUserError.h"
64#include "RequestServiceTimer.h"
66#include "DapFunctionUtils.h"
68#include "FONcRequestHandler.h"
70#include "FONcTransform.h"
72#include "FONcBaseType.h"
73#include "FONcAttributes.h"
74#include "FONcTransmitter.h"
75#include "history_utils.h"
83#define prolog std::string("FONcTransform::").append(__func__).append("() - ")
85#define FOUR_GB_IN_KB (4294967296/1024)
86#define TWO_GB_IN_KB (2147483648/1024)
87#define MSG_LABEL_CLASSIC_MODEL " (classic model)"
88#define MSG_LABEL_SIXTYFOUR_BIT_MODEL " (64-bit offset model)"
102 _dds(dds), _localfile(localfile), _returnAs(ncVersion) {
104 throw BESInternalError(
"File out netcdf, null DDS passed to constructor", __FILE__, __LINE__);
106 if (_localfile.empty()) {
107 throw BESInternalError(
"File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
110 _localfile = localfile;
112 _returnAs = ncVersion;
142 _dmr(dmr), _localfile(localfile), _returnAs(ncVersion) {
144 throw BESInternalError(
"File out netcdf, null DMR passed to constructor", __FILE__, __LINE__);
146 if (_localfile.empty()) {
147 throw BESInternalError(
"File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
150 _localfile = localfile;
152 _returnAs = ncVersion;
182 const string &ncVersion)
183 : d_obj(obj), d_dhi(dhi), _localfile(localfile), _returnAs(ncVersion) {
185 throw BESInternalError(
"File out netcdf, null BESResponseObject passed to constructor", __FILE__, __LINE__);
187 if (_localfile.empty()) {
188 throw BESInternalError(
"File out netcdf, empty local file name passed to constructor", __FILE__, __LINE__);
211 for (
auto &b: _fonc_vars) {
214 for (
auto &b: _total_fonc_vars_in_grp) {
233string FONcTransform::too_big_error_msg(
234 const unsigned dap_version,
235 const string &return_encoding,
236 const unsigned long long dap2_response_size_kb,
237 const unsigned long long contextual_max_response_size_kb,
243 msg <<
"Your request was for a (DAP"<< dap_version <<
" data model response) to be encoded as ";
244 msg << return_encoding <<
". ";
245 msg <<
"The response to your specific request will produce a " << dap2_response_size_kb;
246 msg <<
" kilobyte response. On this server the response size for your request is limited to ";
247 msg << contextual_max_response_size_kb <<
" kilobytes. ";
249 msg <<
"The server is configured to allow ";
250 auto conf_max_request_size_kb =FONcRequestHandler::get_request_max_size_kb();
251 if(conf_max_request_size_kb==0){
252 msg <<
" responses of unlimited size. ";
255 msg <<
"responses as large as: " << conf_max_request_size_kb <<
" kilobytes. ";
258 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF3) {
259 msg <<
"Additionally, the requested response encoding " << return_encoding <<
" is structurally limited to ";
260 if (FONcRequestHandler::nc3_classic_format) {
261 msg << TWO_GB_IN_KB <<
" kilobytes" << MSG_LABEL_CLASSIC_MODEL <<
".";
264 msg << FOUR_GB_IN_KB <<
" kilobytes" << MSG_LABEL_SIXTYFOUR_BIT_MODEL <<
".";
266 msg <<
"One thing to try would be to reissue the the request, but change the requested response encoding ";
267 msg <<
"to NetCDF-4. This can be accomplished with the buttons in the Data Request Form, or by modifying ";
268 msg <<
"the request URL by changing the terminal path suffix from \".nc\" to \".nc4\". ";
272 msg <<
"I've noticed that no constraint expression accompanied your request. ";
274 msg <<
"Your request employed the constraint expression: \"" << ce <<
"\" ";
276 msg <<
"You may also reduce the size of the request by choosing just the variable(s) you need and/or by ";
277 msg <<
"using the DAP index based array sub-setting syntax to additionally limit the amount of data requested.";
291void FONcTransform::set_max_size_and_encoding(
unsigned long long &max_request_size_kb,
string &return_encoding){
293 return_encoding.clear();
301 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF3) {
302 return_encoding = string(FONC_RETURN_AS_NETCDF3).append(
"-3 ");
303 if (FONcRequestHandler::nc3_classic_format) {
304 return_encoding += MSG_LABEL_CLASSIC_MODEL;
305 if (max_request_size_kb == 0 || max_request_size_kb >= TWO_GB_IN_KB) {
306 max_request_size_kb = TWO_GB_IN_KB - 1 ;
307 BESDEBUG(MODULE, prolog <<
"Configured max request size was incompatible with NetCDF-3 classic format. " <<
308 "Reset to: " << max_request_size_kb << endl);
312 return_encoding += MSG_LABEL_SIXTYFOUR_BIT_MODEL;
313 if (max_request_size_kb == 0 || max_request_size_kb >= FOUR_GB_IN_KB) {
314 max_request_size_kb = FOUR_GB_IN_KB - 1 ;
315 BESDEBUG(MODULE, prolog <<
"Configured max request size was incompatible with NetCDF-3 w/64-bit offset format. " <<
316 "Reset to: " << max_request_size_kb << endl);
321 return_encoding = FONC_RETURN_AS_NETCDF4;
322 if (FONcRequestHandler::nc3_classic_format) {
323 return_encoding += MSG_LABEL_CLASSIC_MODEL;
326 BESDEBUG(MODULE, prolog <<
"return_encoding: " << return_encoding << endl);
327 BESDEBUG(MODULE, prolog <<
"max_request_size_kb: " << max_request_size_kb << endl);
337void FONcTransform::throw_if_dap2_response_too_big(DDS *dds,
const string &dap2_ce)
339 string return_encoding;
341 unsigned long long max_response_size_kb = FONcRequestHandler::get_request_max_size_kb();
342 BESDEBUG(MODULE, prolog <<
"Configured max_request_size_kb: " << max_response_size_kb << endl);
344 unsigned long long dap2_response_size_kb = dds->get_request_size_kb(
true);
345 BESDEBUG(MODULE, prolog <<
"dds->get_request_size_kb(): " << dap2_response_size_kb << endl);
347 set_max_size_and_encoding(max_response_size_kb, return_encoding);
350 dds->set_response_limit_kb(max_response_size_kb);
352 if (dds->too_big()) {
353 string err_msg = too_big_error_msg(2,return_encoding,dap2_response_size_kb, max_response_size_kb, dap2_ce);
368 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
369 BESDEBUG(MODULE, prolog <<
"Reading data into DataDDS" << endl);
376 if (!bdds)
throw BESInternalFatalError(
"Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
378 _dds = bdds->get_dds();
384 besDRB.set_async_accepted(d_dhi->
data[ASYNC]);
385 besDRB.set_store_result(d_dhi->
data[STORE_RESULT]);
390 if (bdds->get_ia_flag() ==
false) {
393 besRH->add_attributes(*d_dhi);
396 ConstraintEvaluator &eval = bdds->get_ce();
403 if (!besDRB.get_btp_func_ce().empty()) {
404 BESDEBUG(MODULE, prolog <<
"Found function(s) in CE: " << besDRB.get_btp_func_ce() << endl);
408 ConstraintEvaluator func_eval;
410 if (responseCache && responseCache->can_be_cached(_dds, besDRB.get_btp_func_ce())) {
414 func_eval.parse_constraint(besDRB.get_btp_func_ce(), *_dds);
415 fdds = func_eval.eval_function_clauses(*_dds);
427 _dds->mark_all(
false);
438 promote_function_output_structures(_dds);
443 eval.parse_constraint(besDRB.
get_ce(), *_dds);
445 _dds->tag_nested_sequences();
447 throw_if_dap2_response_too_big(_dds, besDRB.
get_ce());
453 for (
auto vi = _dds->var_begin(), ve = _dds->var_end(); vi != ve; vi++) {
454 if ((*vi)->send_p()) {
455 BESDEBUG(MODULE, prolog <<
"Converting variable '" << (*vi)->name() <<
"'" << endl);
458 FONcBaseType *fb = FONcUtils::convert((*vi), FONcTransform::_returnAs, FONcRequestHandler::classic_model);
460 _fonc_vars.push_back(fb);
461 vector <string> embed;
466 fonc_history_util::updateHistoryAttributes(_dds, d_dhi->
data[POST_CONSTRAINT]);
470 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
471 if (FONcRequestHandler::classic_model) {
472 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-4 cache file in classic mode. fileName: "
473 << _localfile << endl);
474 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
477 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
478 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
482 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
483 if (FONcRequestHandler::nc3_classic_format)
484 stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
486 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_64BIT_OFFSET, &_ncid);
489 if (stax != NC_NOERR) {
493 int current_fill_prop_vaule;
495 stax = nc_set_fill(_ncid, NC_NOFILL, ¤t_fill_prop_vaule);
496 if (stax != NC_NOERR) {
510 BESDEBUG(MODULE, prolog <<
"Defining variable: " << fbt->name() << endl);
514 if (FONcRequestHandler::no_global_attrs ==
false) {
516 AttrTable &globals = _dds->get_attr_table();
517 BESDEBUG(MODULE, prolog <<
"Adding Global Attributes" << endl << globals << endl);
518 bool is_netCDF_enhanced =
false;
519 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model ==
false)
520 is_netCDF_enhanced =
true;
530 int stax = nc_enddef(_ncid);
534 if (stax != NC_NOERR) {
539 uint64_t byteCount = 0;
541 if (is_streamable()) {
549 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
550 BESDEBUG(MODULE, prolog <<
"First write data to stream, count: " << byteCount << endl);
554 BESDEBUG(MODULE, prolog <<
"Writing data for variable: " << fbt->name() << endl);
557 fbt->set_eval(&eval);
564 if (is_streamable()) {
566 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
567 BESDEBUG(MODULE, prolog <<
"Writing data to stream, count: " << byteCount << endl);
571 stax = nc_close(_ncid);
572 if (stax != NC_NOERR)
577 byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
578 BESDEBUG(MODULE, prolog <<
"After nc_close() count: " << byteCount << endl);
581 (void) nc_close(_ncid);
593bool FONcTransform::is_streamable() {
594 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
598 if (_dds !=
nullptr) {
599 return is_dds_streamable();
602 return is_dmr_streamable(_dmr->root());
611bool FONcTransform::is_dds_streamable() {
612 for (
auto var = _dds->var_begin(), varEnd = _dds->var_end(); var != varEnd; ++var) {
613 if ((*var)->type() == dods_structure_c) {
626bool FONcTransform::is_dmr_streamable(D4Group *group) {
627 for (
auto var = group->var_begin(), varEnd = group->var_end(); var != varEnd; ++var) {
628 if ((*var)->type() == dods_structure_c)
631 if ((*var)->type() == dods_group_c) {
632 D4Group *g =
dynamic_cast<D4Group *
>(*var);
633 if (g !=
nullptr && !is_dmr_streamable(g)) {
648void FONcTransform::throw_if_dap4_response_too_big(DMR *dmr,
const string &dap4_ce)
650 unsigned long long max_response_size_kb = FONcRequestHandler::get_request_max_size_kb();
651 BESDEBUG(MODULE, prolog <<
"Configured max_request_size_kb: " << max_response_size_kb << endl);
653 unsigned long long req_size_kb = dmr->request_size_kb(
true);
654 BESDEBUG(MODULE, prolog <<
"dmr->get_request_size_kb(): " << req_size_kb << endl);
656 string return_encoding;
657 set_max_size_and_encoding(max_response_size_kb, return_encoding);
660 dmr->set_response_limit_kb(max_response_size_kb);
662 if (dmr->too_big()) {
663 string err_msg = too_big_error_msg(4,return_encoding,req_size_kb, max_response_size_kb, dap4_ce);
681 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
683 BESDEBUG(MODULE, prolog <<
"Reading data into DataDMR" << endl);
690 _dmr = responseBuilder.setup_dap4_intern_data(d_obj, *d_dhi).release();
693 _dmr->set_response_limit_kb(FONcRequestHandler::get_request_max_size_kb());
694 throw_if_dap4_response_too_big(_dmr,responseBuilder.
get_dap4ce() );
704 besDRB.set_async_accepted(d_dhi->
data[ASYNC]);
705 besDRB.set_store_result(d_dhi->
data[STORE_RESULT]);
714 bool support_group = check_group_support();
716 if (
true == support_group) {
719 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
720 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
721 if (stax != NC_NOERR)
724 D4Group *root_grp = _dmr->root();
727 map<string, int> fdimname_to_id;
731 gen_included_grp_list(root_grp);
734 for (std::set<string>::iterator it = _included_grp_names.begin(); it != _included_grp_names.end(); ++it)
735 BESDEBUG(MODULE, prolog <<
"Included group list name is: " << *it << endl);
739 check_and_obtain_dimensions(root_grp,
true);
743 map<string, unsigned long>::iterator it;
745 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
746 BESDEBUG(MODULE, prolog <<
"Final GFQN dim name is: " << it->first << endl);
747 BESDEBUG(MODULE, prolog <<
"Final GFQN dim size is: " << it->second << endl);
750 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
751 BESDEBUG(MODULE, prolog <<
"Final VFQN dim name is: " << it->first << endl);
752 BESDEBUG(MODULE, prolog <<
"Final VFQN dim size is: " << it->second << endl);
760 map<string, unsigned long>::iterator git, vit;
761 for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
762 for (vit = VFQN_dimname_to_dimsize.begin(); vit != VFQN_dimname_to_dimsize.end(); ++vit) {
763 if (git->first == vit->first) {
764 if (git->second != vit->second)
765 git->second = vit->second;
781 vector <string> root_d4_dimname_list;
782 for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
783 string d4_temp_dimname = git->first.substr(1);
785 if (d4_temp_dimname.find(
'/') == string::npos)
786 root_d4_dimname_list.push_back(d4_temp_dimname);
790 for (
unsigned int i = 0; i < root_d4_dimname_list.size(); i++)
791 BESDEBUG(MODULE, prolog <<
"root_d4 dim name is: " << root_d4_dimname_list[i] << endl);
795 vector<int> root_dim_suffix_nums;
796 for (
unsigned int i = 0; i < root_d4_dimname_list.size(); i++) {
797 if (root_d4_dimname_list[i].size() < 4)
799 else if (root_d4_dimname_list[i].substr(0, 3) !=
"dim")
802 string temp_suffix = root_d4_dimname_list[i].substr(3);
804 bool ignored_suffix =
false;
805 for (
unsigned int j = 0; j < temp_suffix.size(); j++) {
806 if (!isdigit(temp_suffix[j])) {
807 ignored_suffix =
true;
811 if (ignored_suffix ==
true)
814 root_dim_suffix_nums.push_back(atoi(temp_suffix.c_str()));
819 for (
unsigned int i = 0; i < root_dim_suffix_nums.size(); i++)
820 BESDEBUG(MODULE, prolog <<
"root_dim_suffix_nums: " << root_dim_suffix_nums[i] << endl);
823 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
824 BESDEBUG(MODULE, prolog <<
"RFinal GFQN dim name is: " << it->first << endl);
825 BESDEBUG(MODULE, prolog <<
"RFinal GFQN dim size is: " << it->second << endl);
828 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
829 BESDEBUG(MODULE, prolog <<
"RFinal VFQN dim name is: " << it->first << endl);
830 BESDEBUG(MODULE, prolog <<
"RFinal VFQN dim size is: " << it->second << endl);
835 transform_dap4_group(root_grp,
true, _ncid, fdimname_to_id, root_dim_suffix_nums);
836 stax = nc_close(_ncid);
837 if (stax != NC_NOERR)
842 transform_dap4_no_group();
844 BESDEBUG(MODULE, prolog <<
"END" << endl);
854void FONcTransform::transform_dap4_no_group() {
856 D4Group *root_grp = _dmr->root();
858 D4Dimensions *root_dims = root_grp->dims();
859 for (D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
860 BESDEBUG(MODULE, prolog <<
"transform_dap4() - check dimensions" << endl);
861 BESDEBUG(MODULE, prolog <<
"transform_dap4() - dim name is: " << (*di)->name() << endl);
862 BESDEBUG(MODULE, prolog <<
"transform_dap4() - dim size is: " << (*di)->size() << endl);
863 BESDEBUG(MODULE, prolog <<
"transform_dap4() - fully_qualfied_dim name is: " << (*di)->fully_qualified_name() << endl);
866 Constructor::Vars_iter vi = root_grp->var_begin();
867 Constructor::Vars_iter ve = root_grp->var_end();
869 for (; vi != ve; vi++) {
870 if ((*vi)->send_p()) {
873 BESDEBUG(MODULE, prolog <<
"Converting variable '" << v->name() <<
"'" << endl);
876 FONcBaseType *fb = FONcUtils::convert(v, FONcTransform::_returnAs, FONcRequestHandler::classic_model);
877 _fonc_vars.push_back(fb);
879 vector <string> embed;
880 fb->convert(embed,
true,
false);
885 if (root_grp->grp_begin() == root_grp->grp_end())
886 BESDEBUG(MODULE, prolog <<
"No group " << endl);
888 BESDEBUG(MODULE, prolog <<
"Has group " << endl);
889 for (D4Group::groupsIter gi = root_grp->grp_begin(), ge = root_grp->grp_end(); gi != ge; ++gi)
890 BESDEBUG(MODULE, prolog <<
"Group name: " << (*gi)->name() << endl);
893 fonc_history_util::updateHistoryAttributes(_dmr, d_dhi->
data[POST_CONSTRAINT]);
897 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4) {
898 if (FONcRequestHandler::classic_model) {
899 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-4 cache file in classic mode. fileName: "
900 << _localfile << endl);
901 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
904 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-4 cache file. fileName: " << _localfile
906 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
910 BESDEBUG(MODULE, prolog <<
"Opening NetCDF-3 cache file. fileName: " << _localfile
912 if (FONcRequestHandler::nc3_classic_format)
913 stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
915 stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_64BIT_OFFSET, &_ncid);
918 if (stax != NC_NOERR) {
930 vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
931 vector<FONcBaseType *>::iterator e = _fonc_vars.end();
932 for (; i != e; i++) {
934 BESDEBUG(MODULE, prolog <<
"Defining variable: " << fbt->name() << endl);
939 if (FONcRequestHandler::no_global_attrs ==
false) {
942 D4Group *root_grp = _dmr->root();
943 D4Attributes *d4_attrs = root_grp->attributes();
945 BESDEBUG(MODULE, prolog <<
"Handle GLOBAL DAP4 attributes " << d4_attrs << endl);
947 for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end();
949 string name = (*ii)->name();
950 BESDEBUG(MODULE, prolog <<
"GLOBAL attribute name is " << name << endl);
953 bool is_netCDF_enhanced =
false;
954 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model ==
false)
955 is_netCDF_enhanced =
true;
962 int stax = nc_enddef(_ncid);
966 if (stax != NC_NOERR) {
972 i = _fonc_vars.begin();
973 e = _fonc_vars.end();
974 for (; i != e; i++) {
977 BESDEBUG(MODULE, prolog <<
"Writing data for variable: " << fbt->name() << endl);
981 stax = nc_close(_ncid);
982 if (stax != NC_NOERR)
986 (void) nc_close(_ncid);
993void FONcTransform::transform_dap4_group(D4Group *grp,
995 int par_grp_id, map<string, int> &fdimname_to_id,
996 vector<int> &root_dim_suffix_nums) {
998 bool included_grp =
false;
1000 if (_dmr->get_ce_empty()) {
1001 BESDEBUG(MODULE, prolog <<
"In group - group name: " << grp->FQN() << endl);
1002 included_grp =
true;
1005 else if (is_root_grp ==
true)
1006 included_grp =
true;
1009 set<string>::iterator iset;
1010 if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
1011 included_grp =
true;
1016 if (included_grp ==
true)
1017 transform_dap4_group_internal(grp, is_root_grp, par_grp_id, fdimname_to_id, root_dim_suffix_nums);
1029void FONcTransform::transform_dap4_group_internal(D4Group *grp,
1031 int par_grp_id, map<string, int> &fdimname_to_id,
1032 vector<int> &rds_nums) {
1034 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
1038 fonc_history_util::updateHistoryAttributes(_dmr, d_dhi->
data[POST_CONSTRAINT]);
1040 if (is_root_grp ==
true)
1043 stax = nc_def_grp(par_grp_id, (*grp).name().c_str(), &grp_id);
1044 BESDEBUG(MODULE, prolog <<
"Group name is " << (*grp).name() << endl);
1045 if (stax != NC_NOERR)
1050 D4Dimensions *grp_dims = grp->dims();
1051 for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1054 BESDEBUG(MODULE, prolog <<
"Check dimensions" << endl);
1055 BESDEBUG(MODULE, prolog <<
"Dim name is: " << (*di)->name() << endl);
1056 BESDEBUG(MODULE, prolog <<
"Dim size is: " << (*di)->size() << endl);
1057 BESDEBUG(MODULE, prolog <<
"Fully_qualfied_dim name is: " << (*di)->fully_qualified_name() << endl);
1060 unsigned long dimsize = (*di)->size();
1063 map<string, unsigned long>::iterator it;
1064 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
1065 if (it->first == (*di)->fully_qualified_name())
1066 dimsize = it->second;
1071 stax = nc_def_dim(grp_id, (*di)->name().c_str(), dimsize, &g_dimid);
1072 if (stax != NC_NOERR)
1076 fdimname_to_id[(*di)->fully_qualified_name()] = g_dimid;
1079 Constructor::Vars_iter vi = grp->var_begin();
1080 Constructor::Vars_iter ve = grp->var_end();
1082 vector < FONcBaseType * > fonc_vars_in_grp;
1084 for (; vi != ve; vi++) {
1085 if ((*vi)->send_p()) {
1088 BESDEBUG(MODULE, prolog <<
"Converting variable '" << v->name() <<
"'" << endl);
1092 FONcBaseType *fb = FONcUtils::convert(v, FONC_RETURN_AS_NETCDF4,
false, fdimname_to_id, rds_nums);
1094 fonc_vars_in_grp.push_back(fb);
1097 _total_fonc_vars_in_grp.push_back(fb);
1099 vector <string> embed;
1100 fb->convert(embed,
true,
true);
1105 if (grp->grp_begin() == grp->grp_end())
1106 BESDEBUG(MODULE, prolog <<
"No group" << endl);
1108 BESDEBUG(MODULE, prolog <<
"Has group" << endl);
1117 vector<FONcBaseType *>::iterator i = fonc_vars_in_grp.begin();
1118 vector<FONcBaseType *>::iterator e = fonc_vars_in_grp.end();
1119 for (; i != e; i++) {
1121 BESDEBUG(MODULE, prolog <<
"Defining variable: " << fbt->name() << endl);
1126 bool is_netCDF_enhanced =
false;
1127 if (FONcTransform::_returnAs == FONC_RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model ==
false)
1128 is_netCDF_enhanced =
true;
1131 bool add_attr =
true;
1134 if (FONcRequestHandler::no_global_attrs ==
true && is_root_grp ==
true)
1137 if (
true == add_attr) {
1138 D4Attributes *d4_attrs = grp->attributes();
1139 BESDEBUG(MODULE, prolog <<
"Adding Group Attributes" << endl);
1146 i = fonc_vars_in_grp.begin();
1147 e = fonc_vars_in_grp.end();
1148 for (; i != e; i++) {
1151 BESDEBUG(MODULE, prolog <<
"Writing data for variable: " << fbt->name() << endl);
1157 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1158 BESDEBUG(MODULE, prolog <<
"In group: " << (*gi)->name() << endl);
1159 transform_dap4_group(*gi,
false, grp_id, fdimname_to_id, rds_nums);
1164 (void) nc_close(_ncid);
1167 BESDEBUG(MODULE, prolog <<
"END" << endl);
1172bool FONcTransform::check_group_support() {
1173 if (FONC_RETURN_AS_NETCDF4 == FONcTransform::_returnAs &&
false == FONcRequestHandler::classic_model &&
1174 (_dmr->root()->grp_begin() != _dmr->root()->grp_end()))
1181void FONcTransform::gen_included_grp_list(D4Group *grp) {
1182 bool grp_has_var =
false;
1184 BESDEBUG(MODULE, prolog <<
"Processing D4 Group: " << grp->name() <<
" fullpath: " << grp->FQN() << endl);
1186 if (grp->var_begin() != grp->var_end()) {
1188 BESDEBUG(MODULE, prolog <<
"Has child variables" << endl);
1189 Constructor::Vars_iter vi = grp->var_begin();
1190 Constructor::Vars_iter ve = grp->var_end();
1192 for (; vi != ve; vi++) {
1195 if ((*vi)->send_p()) {
1200 if (grp->FQN() !=
"/")
1201 _included_grp_names.insert(grp->FQN());
1207 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1208 BESDEBUG(MODULE, prolog <<
"Obtain included groups - group name: " << (*gi)->name() << endl);
1209 gen_included_grp_list(*gi);
1214 if (grp_has_var ==
true) {
1215 D4Group *temp_grp = grp;
1217 if (temp_grp->get_parent()) {
1218 temp_grp =
static_cast<D4Group *
>(temp_grp->get_parent());
1219 if (temp_grp->FQN() !=
"/")
1220 _included_grp_names.insert(temp_grp->FQN());
1229void FONcTransform::check_and_obtain_dimensions(D4Group *grp,
bool is_root_grp) {
1232 bool included_grp =
false;
1234 if (_dmr->get_ce_empty())
1235 included_grp =
true;
1237 else if (is_root_grp ==
true)
1238 included_grp =
true;
1241 set<string>::iterator iset;
1242 if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
1243 included_grp =
true;
1246 if (included_grp ==
true)
1247 check_and_obtain_dimensions_internal(grp);
1250void FONcTransform::check_and_obtain_dimensions_internal(D4Group *grp) {
1253 D4Dimensions *grp_dims = grp->dims();
1255 for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1258 BESDEBUG(MODULE, prolog <<
"Check dimensions" << endl);
1259 BESDEBUG(MODULE, prolog <<
"Dim name is: " << (*di)->name() << endl);
1260 BESDEBUG(MODULE, prolog <<
"Dim size is: " << (*di)->size() << endl);
1261 BESDEBUG(MODULE, prolog <<
"Fully qualfied dim name: " << (*di)->fully_qualified_name() << endl);
1263 unsigned long dimsize = (*di)->size();
1264 if ((*di)->constrained()) {
1265 dimsize = ((*di)->c_stop() - (*di)->c_start()) / (*di)->c_stride() + 1;
1268 GFQN_dimname_to_dimsize[(*di)->fully_qualified_name()] = dimsize;
1275 Constructor::Vars_iter vi = grp->var_begin();
1276 Constructor::Vars_iter ve = grp->var_end();
1277 for (; vi != ve; vi++) {
1278 if ((*vi)->send_p()) {
1279 if ((*vi)->is_vector_type()) {
1280 Array *t_a =
dynamic_cast<Array *
>(*vi);
1281 Array::Dim_iter dim_i = t_a->dim_begin();
1282 Array::Dim_iter dim_e = t_a->dim_end();
1283 for (; dim_i != dim_e; dim_i++) {
1284 if ((*dim_i).name !=
"") {
1285 D4Dimension *d4dim = t_a->dimension_D4dim(dim_i);
1287 BESDEBUG(MODULE, prolog <<
"Check dim- dim name is: " << d4dim->name() << endl);
1288 BESDEBUG(MODULE, prolog <<
"Check dim- dim size is: " << d4dim->size() << endl);
1289 BESDEBUG(MODULE, prolog <<
"Check dim- fully_qualfied_dim name is: "
1290 << d4dim->fully_qualified_name() << endl);
1292 unsigned long dimsize = t_a->dimension_size(dim_i,
true);
1294 BESDEBUG(MODULE, prolog <<
"Check dim- final dim size is: " << dimsize << endl);
1296 pair<map<string, unsigned long>::iterator,
bool> ret_it;
1297 ret_it = VFQN_dimname_to_dimsize.insert(
1298 pair<string, unsigned long>(d4dim->fully_qualified_name(), dimsize));
1299 if (ret_it.second ==
false && ret_it.first->second != dimsize) {
1300 string err =
"fileout_netcdf-4: dimension found with the same name, but different size";
1306 throw BESInternalError(
"Has dimension name but D4 dimension is NULL", __FILE__, __LINE__);
1316 map<string, unsigned long>::iterator it;
1317 for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
1318 BESDEBUG(MODULE, prolog <<
"GFQN dim name is: " << it->first << endl);
1319 BESDEBUG(MODULE, prolog <<
"GFQN dim size is: " << it->second << endl);
1322 for (it = VFQN_dimname_to_dimsize.begin(); it != VFQN_dimname_to_dimsize.end(); ++it) {
1323 BESDEBUG(MODULE, prolog <<
"VFQN dim name is: " << it->first << endl);
1324 BESDEBUG(MODULE, prolog <<
"VFQN dim size is: " << it->second << endl);
1330 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1331 BESDEBUG(MODULE,prolog <<
"In group: " << (*gi)->name() << endl);
1332 check_and_obtain_dimensions(*gi,
false);
1347 strm << BESIndent::LMarg <<
"FONcTransform::dump - (" << (
void *)
this <<
")" << endl;
1348 BESIndent::Indent();
1349 strm << BESIndent::LMarg <<
"ncid = " << _ncid << endl;
1350 strm << BESIndent::LMarg <<
"temporary file = " << _localfile << endl;
1351 BESIndent::Indent();
1352 vector<FONcBaseType *>::const_iterator i = _fonc_vars.begin();
1353 vector<FONcBaseType *>::const_iterator e = _fonc_vars.end();
1354 for (; i != e; i++) {
1358 BESIndent::UnIndent();
1359 BESIndent::UnIndent();
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void set_dap4function(std::string _func)
virtual void set_ce(std::string _ce)
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
Base exception class for the BES with basic string message.
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
Abstract base class representing a specific set of information in response to a request to the BES.
error thrown if there is a user syntax error in the request or any other user error
static void conditional_timeout_cancel()
Checks if the timeout alarm should be canceled based on the value of the BES key BES....
static void add_dap4_attributes(int ncid, int varid, D4Attributes *d4_attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
add_dap4_attributes
static void add_attributes(int ncid, int varid, AttrTable &attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
helper function for add_attributes
A DAP BaseType with file out netcdf information included.
virtual void define(int ncid)
Define the variable in the netcdf file.
virtual void dump(std::ostream &strm) const =0
dump the contents of this object to the specified ostream
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
static string name_prefix
If a variable name, dimension name, or attribute name begins with a character that is not supported b...
static void reset()
Resets the FONc transformation for a new input and out file.
static RequestServiceTimer * TheTimer()
Return a pointer to a singleton timer instance. If an instance does not exist it will create and init...
void throw_if_timeout_expired(const std::string &message, const std::string &file, const int line)
Checks the RequestServiceTimer to determine if the time spent servicing the request at this point has...