36#include <libdap/Array.h>
38#include <BESInternalError.h>
39#include <BESSyntaxUserError.h>
42#include "FONcRequestHandler.h"
48#include "FONcAttributes.h"
54#define CLEAR_LOCAL_DATA 1
56vector<FONcDim *> FONcArray::Dimensions;
58const int MAX_CHUNK_SIZE = 1024;
68FONcArray::FONcArray(BaseType *b) :
69 FONcBaseType(), d_a(0), d_array_type(NC_NAT), d_ndims(0), d_actual_ndims(0), d_nelements(1), d4_dim_ids(0),
70 d_dim_ids(0), d_dim_sizes(0), d_dont_use_it(false), d_chunksizes(0),
71 d_grid_maps(0), d4_def_dim(false) {
72 d_a =
dynamic_cast<Array *
>(b);
74 string s =
"File out netcdf, FONcArray was passed a variable that is not a DAP Array";
78 for (
unsigned int i = 0; i < d_a->dimensions(); i++)
79 use_d4_dim_ids.push_back(
false);
82FONcArray::FONcArray(BaseType *b,
const vector<int> &fd4_dim_ids,
const vector<bool> &fuse_d4_dim_ids,
83 const vector<int> &rds_nums) :
84 FONcBaseType(), d_a(0), d_array_type(NC_NAT), d_ndims(0), d_actual_ndims(0), d_nelements(1), d_dim_ids(0),
85 d_dim_sizes(0), d_dont_use_it(false), d_chunksizes(0), d_grid_maps(0) {
86 d_a =
dynamic_cast<Array *
>(b);
88 string s =
"File out netcdf, FONcArray was passed a variable that is not a DAP Array";
92 BESDEBUG(
"fonc",
"FONcArray() - constructor is dap4 " << endl);
93 d4_dim_ids = fd4_dim_ids;
94 use_d4_dim_ids = fuse_d4_dim_ids;
96 d4_rds_nums = rds_nums;
112 vector<FONcDim *>::iterator d = d_dims.begin();
113 while (d != d_dims.end()) {
119 vector<FONcMap *>::iterator i = d_grid_maps.begin();
120 while (i != d_grid_maps.end()) {
141 FONcBaseType::convert(embed, _dap4, is_dap4_group);
145 BESDEBUG(
"fonc",
"FONcArray::convert() - converting array " << _varname << endl);
149 if(d_array_type == NC_NAT) {
151 string err =
"fileout_netcdf: The datatype of this variable '" + _varname;
152 err +=
"' is not supported. It is very possible that you try to obtain ";
153 err +=
"a netCDF file that follows the netCDF classic model. ";
154 err +=
"The unsigned 32-bit integer and signed/unsigned 64-bit integer ";
155 err +=
"are not supported by the netCDF classic model. Downloading this file as the netCDF-4 file that ";
156 err +=
"follows the netCDF enhanced model should solve the problem.";
162 if (d4_dim_ids.size() > 0) {
163 BESDEBUG(
"fonc",
"FONcArray::convert() - d4_dim_ids size is " << d4_dim_ids.size() << endl);
167 d_ndims = d_a->dimensions();
168 d_actual_ndims = d_ndims;
169 if (d_array_type == NC_CHAR) {
178 d_dim_ids.resize(d_ndims);
179 d_dim_sizes.resize(d_ndims);
181 Array::Dim_iter di = d_a->dim_begin();
182 Array::Dim_iter de = d_a->dim_end();
184 for (; di != de; di++) {
185 int size = d_a->dimension_size(di,
true);
186 d_dim_sizes[dimnum] = size;
190 d_chunksizes.push_back(size <= MAX_CHUNK_SIZE ? size : MAX_CHUNK_SIZE);
192 BESDEBUG(
"fonc",
"FONcArray::convert() - dim num: " << dimnum <<
", dim size: " << size <<
", chunk size: "
193 << d_chunksizes[dimnum] << endl);
194 BESDEBUG(
"fonc",
"FONcArray::convert() - dim name: " << d_a->dimension_name(di) << endl);
197 if (
true == d4_def_dim && use_d4_dim_ids[dimnum] ==
true) {
198 d_dim_ids[dimnum] = d4_dim_ids[dimnum];
199 BESDEBUG(
"fonc",
"FONcArray::convert() - has dap4 group" << endl);
206 int ds_num = FONcDim::DimNameNum + 1;
207 while (find(d4_rds_nums.begin(), d4_rds_nums.end(), ds_num) != d4_rds_nums.end()) {
218 FONcDim::DimNameNum = ds_num - 1;
220 FONcDim *use_dim = find_dim(embed, d_a->dimension_name(di), size);
221 d_dims.push_back(use_dim);
228 if (d_array_type == NC_CHAR) {
243 d_a->intern_data(*get_eval(), *get_dds());
246 int array_length = d_a->length();
248 d_str_data.reserve(array_length);
249 d_a->value(d_str_data);
252 size_t max_length = 0;
253 for (
int i = 0; i < array_length; i++) {
254 if (d_str_data[i].length() > max_length) {
255 max_length = d_str_data[i].length();
260 size_t max_length = 0;
261 for (
int i = 0; i < array_length; i++) {
262 if (d_a->get_str()[i].length() > max_length) {
263 max_length = d_a->get_str()[i].length();
268 vector<string> empty_embed;
270 if (is_dap4_group ==
true) {
274 ostringstream dim_suffix_strm;
275 dim_suffix_strm <<
"_len" << FONcDim::DimNameNum + 1;
276 FONcDim::DimNameNum++;
277 lendim_name = _varname + dim_suffix_strm.str();
281 lendim_name = _varname +
"_len";
284 FONcDim *use_dim = find_dim(empty_embed, lendim_name, max_length,
true);
286 if (use_dim->size() <
static_cast<int>(max_length)) {
287 use_dim->update_size(max_length);
290 d_dim_sizes[d_ndims - 1] = use_dim->size();
291 d_dim_ids[d_ndims - 1] = use_dim->dimid();
294 use_d4_dim_ids.push_back(
false);
295 d_dims.push_back(use_dim);
304 d_chunksizes.push_back(max_length <= MAX_CHUNK_SIZE ? max_length : MAX_CHUNK_SIZE);
313 if(is_dap4 ==
false) {
314 if (!
FONcGrid::InGrid && d_actual_ndims == 1 && d_a->name() == d_a->dimension_name(d_a->dim_begin())) {
316 FONcMap *map = FONcGrid::InMaps(d_a);
320 d_grid_maps.push_back(new_map);
324 d_dont_use_it =
true;
329 BESDEBUG(
"fonc",
"FONcArray::convert() - done converting array " << _varname << endl);
346FONcArray::find_dim(vector<string> &embed,
const string &name,
int size,
bool ignore_size) {
350 vector<FONcDim *>::iterator i = FONcArray::Dimensions.begin();
351 vector<FONcDim *>::iterator e = FONcArray::Dimensions.end();
352 for (; i != e && !ret_dim; i++) {
353 if (!((*i)->name().empty()) && ((*i)->name() ==
name)) {
357 else if ((*i)->size() == size) {
361 if (embed.size() > 0) {
363 return find_dim(tmp, ename, size);
365 string err =
"fileout_netcdf: dimension found with the same name, but different size";
372 FONcArray::Dimensions.push_back(ret_dim);
395 BESDEBUG(
"fonc",
"FONcArray::define() - defining array '" << _varname <<
"'" << endl);
397 if (!_defined && !d_dont_use_it) {
399 BESDEBUG(
"fonc",
"FONcArray::define() - defining array ' defined already: " << _varname <<
"'" << endl);
405 if(d4_dim_ids.size() >0) {
406 if(d_array_type == NC_CHAR) {
407 if(d_dims.size() == 1) {
408 FONcDim *fd = *(d_dims.begin());
410 d_dim_ids[d_ndims-1] = fd->dimid();
421 if (
false == d4_def_dim) {
422 vector<FONcDim *>::iterator i = d_dims.begin();
423 vector<FONcDim *>::iterator e = d_dims.end();
425 for (; i != e; i++) {
429 d_dim_ids[dimnum] = fd->dimid();
430 BESDEBUG(
"fonc",
"FONcArray::define() - dim_id: " << fd->dimid() <<
" size:" << fd->size() << endl);
436 for (
unsigned int i = 0; i < use_d4_dim_ids.size(); i++) {
437 if (use_d4_dim_ids[i] ==
false) {
440 d_dim_ids[i] = fd->dimid();
446 int stax = nc_def_var(ncid, _varname.c_str(), d_array_type, d_ndims, d_dim_ids.data(), &_varid);
447 if (stax != NC_NOERR) {
448 string err = (string)
"fileout.netcdf - Failed to define variable " + _varname;
452 stax = nc_def_var_fill(ncid, _varid, NC_NOFILL, NULL );
453 if (stax != NC_NOERR) {
454 string err = (string)
"fileout.netcdf - " +
"Failed to clear fill value for " + _varname;
458 BESDEBUG(
"fonc",
"FONcArray::define() netcdf-4 version is " << _ncVersion << endl);
460 BESDEBUG(
"fonc",
"FONcArray::define() Working netcdf-4 branch " << endl);
461 if (FONcRequestHandler::chunk_size == 0)
463 stax = nc_def_var_chunking(ncid, _varid, NC_CONTIGUOUS, d_chunksizes.data());
465 stax = nc_def_var_chunking(ncid, _varid, NC_CHUNKED, d_chunksizes.data());
467 if (stax != NC_NOERR) {
468 string err =
"fileout.netcdf - Failed to define chunking for variable " + _varname;
474 if (FONcRequestHandler::use_compression) {
479 if (NC_SHORT == d_array_type || NC_USHORT == d_array_type || NC_INT == d_array_type ||
480 NC_UINT == d_array_type || NC_INT64 == d_array_type || NC_UINT64 == d_array_type ||
481 FONcRequestHandler::use_shuffle)
485 int deflate_level = 4;
486 stax = nc_def_var_deflate(ncid, _varid, shuffle, deflate, deflate_level);
488 if (stax != NC_NOERR) {
489 string err = (string)
"fileout.netcdf - Failed to define compression (deflate) level for variable "
498 D4Attributes *d4_attrs = d_a->attributes();
499 updateD4AttrType(d4_attrs, d_array_type);
502 AttrTable &attrs = d_a->get_attr_table();
503 updateAttrType(attrs, d_array_type);
506 BESDEBUG(
"fonc",
"FONcArray::define() - Adding attributes " << endl);
513 BESDEBUG(
"fonc",
"FONcArray::define() - variable " << _varname <<
" is already defined" << endl);
516 BESDEBUG(
"fonc",
"FONcArray::define() - variable " << _varname <<
" is not being used" << endl);
520 BESDEBUG(
"fonc",
"FONcArray::define() - done defining array '" << _varname <<
"'" << endl);
528void FONcArray::write_nc_variable(
int ncid, nc_type var_type) {
532 d_a->intern_data(*get_eval(), *get_dds());
538 stax = nc_put_var_uchar(ncid, _varid,
reinterpret_cast<unsigned char *
>(d_a->get_buf()));
541 stax = nc_put_var_schar(ncid, _varid,
reinterpret_cast<signed char *
>(d_a->get_buf()));
544 stax = nc_put_var_short(ncid, _varid,
reinterpret_cast<short *
>(d_a->get_buf()));
547 stax = nc_put_var_int(ncid, _varid,
reinterpret_cast<int *
>(d_a->get_buf()));
550 stax = nc_put_var_longlong(ncid, _varid,
reinterpret_cast<long long *
>(d_a->get_buf()));
553 stax = nc_put_var_float(ncid, _varid,
reinterpret_cast<float *
>(d_a->get_buf()));
556 stax = nc_put_var_double(ncid, _varid,
reinterpret_cast<double *
>(d_a->get_buf()));
559 stax = nc_put_var_ushort(ncid, _varid,
reinterpret_cast<unsigned short *
>(d_a->get_buf()));
562 stax = nc_put_var_uint(ncid, _varid,
reinterpret_cast<unsigned int *
>(d_a->get_buf()));
565 stax = nc_put_var_ulonglong(ncid, _varid,
reinterpret_cast<unsigned long long *
>(d_a->get_buf()));
569 throw BESInternalError(
"Failed to transform array of unknown type in file out netcdf (1)",
573 if (stax != NC_NOERR) {
574 string err =
"fileout.netcdf - Failed to create array of " + d_a->var()->type_name() +
" for " + _varname;
580 if (!FONcGrid::InMaps(d_a))
581 d_a->clear_local_data();
595 BESDEBUG(
"fonc",
"FONcArray::write() BEGIN var: " << _varname <<
"[" << d_nelements <<
"]" << endl);
596 BESDEBUG(
"fonc",
"FONcArray::write() BEGIN var type: " << d_array_type <<
" " << endl);
599 BESDEBUG(
"fonc",
"FONcTransform::write not using variable " << _varname << endl);
608 if (d_array_type == NC_CHAR) {
613 vector<size_t> var_count(d_ndims);
614 vector<size_t> var_start(d_ndims);
616 for (dim = 0; dim < d_ndims; dim++) {
626 for (
int element = 0; element < d_nelements; element++) {
627 var_count[d_ndims - 1] = d_a->get_str()[element].size() + 1;
628 var_start[d_ndims - 1] = 0;
631 int stax = nc_put_vara_text(ncid, _varid, var_start.data(), var_count.data(), d_a->get_str()[element].c_str());
632 if (stax != NC_NOERR) {
633 string err = (string)
"fileout.netcdf - Failed to create array of strings for " + _varname;
638 if (element + 1 < d_nelements) {
642 var_start[dim] = var_start[dim] + 1;
643 if (var_start[dim] == d_dim_sizes[dim]) {
654 d_a->get_str().clear();
658 else if (isNetCDF4_ENHANCED()) {
659 write_for_nc4_types(ncid);
662 libdap::Type element_type = d_a->var()->type();
664 switch (d_array_type) {
668 write_nc_variable(ncid, d_array_type);
676 if (element_type == libdap::dods_byte_c || element_type == libdap::dods_uint8_c) {
680 d_a->intern_data(*get_eval(), *get_dds());
684 vector<short> data(d_nelements);
685 for (
int d_i = 0; d_i < d_nelements; d_i++)
686 data[d_i] = *(
reinterpret_cast<unsigned char *
>(d_a->get_buf()) + d_i);
688 int stax = nc_put_var_short(ncid, _varid, data.data());
689 if (stax != NC_NOERR) {
690 string err = (string)
"fileout.netcdf - Failed to create array of shorts for " + _varname;
697 if (!FONcGrid::InMaps(d_a))
698 d_a->clear_local_data();
701 write_nc_variable(ncid, NC_SHORT);
708 if (element_type == libdap::dods_int64_c || element_type == libdap::dods_uint64_c) {
713 if (FONcRequestHandler::classic_model ==
false) {
714 msg =
"You asked for one or more 64-bit integer values returned using a netCDF3 file. "
715 "Try asking for netCDF4 enhanced and/or contact the server administrator.";
718 msg =
"You asked for one or more 64-bit integer values, but either returned using a netCDF3 file or "
719 "from a server that is configured to use the 'classic' netCDF data model with netCDF4. "
720 "Try netCDF4 and/or contact the server administrator.";
725 if (element_type == libdap::dods_uint16_c) {
729 d_a->intern_data(*get_eval(), *get_dds());
731 vector<int> data(d_nelements);
732 for (
int d_i = 0; d_i < d_nelements; d_i++)
733 data[d_i] = *(
reinterpret_cast<unsigned short *
>(d_a->get_buf()) + d_i);
735 int stax = nc_put_var_int(ncid, _varid, data.data());
736 if (stax != NC_NOERR) {
737 string err = (string)
"fileout.netcdf - Failed to create array of ints for " + _varname;
741 if (!FONcGrid::InMaps(d_a))
742 d_a->clear_local_data();
745 write_nc_variable(ncid, NC_INT);
750 throw BESInternalError(
"Failed to transform array of unknown type in file out netcdf (2)",
755 BESDEBUG(
"fonc",
"FONcArray::write() END var: " << _varname <<
"[" << d_nelements <<
"]" << endl);
775 strm << BESIndent::LMarg <<
"FONcArray::dump - (" << (
void *)
this <<
")" << endl;
777 strm << BESIndent::LMarg <<
"name = " << _varname << endl;
778 strm << BESIndent::LMarg <<
"ndims = " << d_ndims << endl;
779 strm << BESIndent::LMarg <<
"actual ndims = " << d_actual_ndims << endl;
780 strm << BESIndent::LMarg <<
"nelements = " << d_nelements << endl;
782 strm << BESIndent::LMarg <<
"dimensions:" << endl;
784 vector<FONcDim *>::const_iterator i = d_dims.begin();
785 vector<FONcDim *>::const_iterator e = d_dims.end();
786 for (; i != e; i++) {
789 BESIndent::UnIndent();
792 strm << BESIndent::LMarg <<
"dimensions: none" << endl;
794 BESIndent::UnIndent();
806void FONcArray::write_for_nc4_types(
int ncid) {
815 switch (d_array_type) {
826 write_nc_variable(ncid, d_array_type);
830 string err = (string)
"Failed to transform array of unknown type in file out netcdf";
exception thrown if internal error encountered
virtual void dump(std::ostream &strm) const override
dumps information about this object for debugging purposes
virtual void define(int ncid) override
define the DAP Array in the netcdf file
virtual ~FONcArray() override
Destructor that cleans up the array.
virtual void convert(std::vector< std::string > embed, bool _dap4=false, bool is_dap4_group=false) override
Converts the DAP Array to a FONcArray.
virtual void write(int ncid) override
Write the array out to the netcdf file.
std::string name() override
returns the name of the DAP Array
static void add_original_name(int ncid, int varid, const string &var_name, const string &orig)
Adds an attribute for the variable if the variable name had to be modified in any way.
static void add_variable_attributes(int ncid, int varid, BaseType *b, bool is_netCDF_enhanced, bool is_dap4)
Add the attributes for an OPeNDAP variable to the netcdf file.
A DAP BaseType with file out netcdf information included.
virtual bool isNetCDF4()
Returns true if NetCDF4 features will be required.
A class that represents the dimension of an array.
virtual void define(int ncid)
define the DAP dimension in the netcdf file
static vector< FONcMap * > Maps
global list of maps that could be shared amongst the different grids
static bool InGrid
tells whether we are converting or defining a grid.
A map of a DAP Grid with file out netcdf information included.
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
static nc_type get_nc_type(libdap::BaseType *element, bool isNC4_ENHANCED)
translate the OPeNDAP data type to a netcdf data type
static string gen_name(const vector< string > &embed, const string &name, string &original)
generate a new name for the embedded variable