bes Updated for version 3.20.13
ncdas.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of nc_handler, a data handler for the OPeNDAP data
4// server.
5
6// Copyright (c) 2002,2003 OPeNDAP, Inc.
7// Author: James Gallagher <jgallagher@opendap.org>
8//
9// This is free software; you can redistribute it and/or modify it under the
10// terms of the GNU Lesser General Public License as published by the Free
11// Software Foundation; either version 2.1 of the License, or (at your
12// option) any later version.
13//
14// This software is distributed in the hope that it will be useful, but
15// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
16// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
17// License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24
25// (c) COPYRIGHT URI/MIT 1994-1996
26// Please read the full copyright statement in the file COPYRIGHT.
27//
28// Authors:
29// reza Reza Nekovei (reza@intcomm.net)
30
31// This file contains functions which read the variables and their attributes
32// from a netcdf file and build the in-memory DAS. These functions form the
33// core of the server-side software necessary to extract the DAS from a
34// netcdf data file.
35//
36// It also contains test code which will print the in-memory DAS to
37// stdout. It uses both the DAS class and the netcdf library.
38// In addition, parts of these functions were taken from the netcdf program
39// ncdump, from the netcdf standard distribution (ver 2.3.2)
40//
41// jhrg 9/23/94
42
43#include "config_nc.h"
44
45#include <iostream>
46#include <string>
47#include <sstream>
48#include <iomanip>
49#include <vector>
50
51#include <cmath>
52
53#include <netcdf.h>
54
55#include <libdap/util.h>
56#include <libdap/escaping.h>
57#include <libdap/DAS.h>
58
59#include <BESDebug.h>
60
61#include "NCRequestHandler.h"
62#include "nc_util.h"
63
64#define ATTR_STRING_QUOTE_FIX 1
65#define STOP_ESCAPING_STRING_ATTRS 0 // Never do this. jhrg 2/28/22
66
67#define MODULE "nc"
68#define prolog std::string("ncdas::").append(__func__).append("() - ")
69
70using namespace libdap;
71
82static string print_attr(nc_type type, int loc, void *vals)
83{
84 ostringstream rep;
85 union {
86 char *cp;
87 char **stringp;
88 int16_t *sp;
89 uint16_t *usp;
90 int32_t *i;
91 uint32_t *ui;
92 float *fp;
93 double *dp;
94 } gp;
95
96 switch (type) {
97 case NC_UBYTE:
98 unsigned char uc;
99 gp.cp = (char *) vals;
100
101 uc = *(gp.cp + loc);
102 rep << (int) uc;
103 return rep.str();
104
105 case NC_BYTE:
106 if (NCRequestHandler::get_promote_byte_to_short()) {
107 signed char sc;
108 gp.cp = (char *) vals;
109
110 sc = *(gp.cp + loc);
111 rep << (int) sc;
112 return rep.str();
113 }
114 else {
115 unsigned char uc;
116 gp.cp = (char *) vals;
117
118 uc = *(gp.cp + loc);
119 rep << (int) uc;
120 return rep.str();
121 }
122
123 case NC_CHAR:
124 return escattr(static_cast<const char*>(vals));
125
126 case NC_STRING:
127 gp.stringp = (char **) vals;
128 rep << *(gp.stringp + loc);
129 return rep.str();
130
131 case NC_SHORT:
132 gp.sp = (short *) vals;
133 rep << *(gp.sp + loc);
134 return rep.str();
135
136 case NC_USHORT:
137 gp.usp = (uint16_t *) vals;
138 rep << *(gp.usp + loc);
139 return rep.str();
140
141 case NC_INT:
142 gp.i = (int32_t *) vals; // warning: long int format, int arg (arg 3)
143 rep << *(gp.i + loc);
144 return rep.str();
145
146 case NC_UINT:
147 gp.ui = (uint32_t *) vals;
148 rep << *(gp.ui + loc);
149 return rep.str();
150
151 case NC_FLOAT: {
152 gp.fp = (float *) vals;
153 float valAtLoc = *(gp.fp + loc);
154
155 rep << std::showpoint;
156 rep << std::setprecision(9);
157
158 if (isnan(valAtLoc)) {
159 rep << "NaN";
160 }
161 else {
162 rep << valAtLoc;
163 }
164 // If there's no decimal point and the rep does not use scientific
165 // notation, add a decimal point. This little jaunt was taken because
166 // this code is modeled after older code and that's what it did. I'm
167 // trying to keep the same behavior as the old code without its
168 // problems. jhrg 8/11/2006
169 string tmp_value = rep.str();
170 if (tmp_value.find('.') == string::npos && tmp_value.find('e') == string::npos && tmp_value.find('E') == string::npos
171 && tmp_value.find("nan") == string::npos && tmp_value.find("NaN") == string::npos && tmp_value.find("NAN") == string::npos) rep << ".";
172 return rep.str();
173 }
174
175 case NC_DOUBLE: {
176 gp.dp = (double *) vals;
177 double valAtLoc = *(gp.dp + loc);
178
179 rep << std::showpoint;
180 rep << std::setprecision(16);
181
182 if (std::isnan(valAtLoc)) {
183 rep << "NaN";
184 }
185 else {
186 rep << valAtLoc;
187 }
188 string tmp_value = rep.str();
189 if (tmp_value.find('.') == string::npos && tmp_value.find('e') == string::npos && tmp_value.find('E') == string::npos
190 && tmp_value.find("nan") == string::npos && tmp_value.find("NaN") == string::npos && tmp_value.find("NAN") == string::npos) rep << ".";
191 return rep.str();
192 }
193
194 default:
195 if (NCRequestHandler::get_ignore_unknown_types())
196 cerr << "The netcdf handler tried to print an attribute that has an unrecognized type. (1)" << endl;
197 else
198 throw InternalErr(__FILE__, __LINE__, "The netcdf handler tried to print an attribute that has an unrecognized type. (1)");
199 break;
200 }
201
202 return "";
203}
204
211static string print_type(nc_type datatype)
212{
213 switch (datatype) {
214 case NC_STRING:
215 case NC_CHAR:
216 return "String";
217
218 case NC_UBYTE:
219 return "Byte";
220
221 case NC_BYTE:
222 if (NCRequestHandler::get_promote_byte_to_short()) {
223 return "Int16";
224 }
225 else {
226 return "Byte";
227 }
228
229 case NC_SHORT:
230 return "Int16";
231
232 case NC_INT:
233 return "Int32";
234
235 case NC_USHORT:
236 return "UInt16";
237
238 case NC_UINT:
239 return "UInt32";
240
241 case NC_FLOAT:
242 return "Float32";
243
244 case NC_DOUBLE:
245 return "Float64";
246
247 case NC_COMPOUND:
248 return "NC_COMPOUND";
249
250 // These are all new netcdf 4 types that we don't support yet
251 // as attributes. It's useful to have a print representation for
252 // them so that we can return useful information about why some
253 // information was elided or an exception thrown.
254 case NC_INT64:
255 return "NC_INT64";
256
257 case NC_UINT64:
258 return "NC_UINT64";
259
260 case NC_VLEN:
261 return "NC_VLEN";
262 case NC_OPAQUE:
263 return "NC_OPAQUE";
264 case NC_ENUM:
265 return "NC_ENUM";
266
267 default:
268 if (NCRequestHandler::get_ignore_unknown_types())
269 cerr << "The netcdf handler tried to print an attribute that has an unrecognized type. (2)" << endl;
270 else
271 throw InternalErr(__FILE__, __LINE__, "The netcdf handler tried to print an attribute that has an unrecognized type. (2)");
272 break;
273 }
274
275 return "";
276}
277
282static void append_values(int ncid, int v, int len, nc_type datatype, char *attrname, AttrTable *at)
283{
284 size_t size;
285 int errstat = nc_inq_type(ncid, datatype, 0, &size);
286 if (errstat != NC_NOERR) throw Error(errstat, "Could not get the size for the type.");
287
288 vector<char> value((len + 1) * size);
289 errstat = nc_get_att(ncid, v, attrname, value.data());
290 if (errstat != NC_NOERR) {
291 throw Error(errstat, string("Could not get the value for attribute '") + attrname + string("'"));
292 }
293
294 // If the datatype is NC_CHAR then we have a string. netCDF 3
295 // represents strings as arrays of char, but we represent them as X
296 // strings. So... Add the null and set the length to 1
297 if (datatype == NC_CHAR) {
298 value[len] = '\0';
299 len = 1;
300 }
301
302 // add all the attributes in the array
303 for (int loc = 0; loc < len; loc++) {
304 string print_rep = print_attr(datatype, loc, value.data());
305 at->append_attr(attrname, print_type(datatype), print_rep);
306 }
307}
308
320static void read_attributes_netcdf4(int ncid, int varid, int natts, AttrTable *at)
321{
322 BESDEBUG(MODULE, prolog << "In read_attributes_netcdf4" << endl);
323
324 for (int attr_num = 0; attr_num < natts; ++attr_num) {
325 int errstat = NC_NOERR;
326 // Get the attribute name
327 char attrname[MAX_NC_NAME];
328 errstat = nc_inq_attname(ncid, varid, attr_num, attrname);
329 if (errstat != NC_NOERR) throw Error(errstat, "Could not get the name for attribute " + long_to_string(attr_num));
330
331 // Get datatype and len; len is the number of values.
332 nc_type datatype;
333 size_t len;
334 errstat = nc_inq_att(ncid, varid, attrname, &datatype, &len);
335 if (errstat != NC_NOERR) throw Error(errstat, "Could not get the name for attribute '" + string(attrname) + "'");
336
337 BESDEBUG(MODULE, prolog << "nc_inq_att returned datatype = " << datatype << " for '" << attrname << "'" << endl);
338
339 if (datatype >= NC_FIRSTUSERTYPEID) {
340 char type_name[NC_MAX_NAME + 1];
341 size_t size;
342 nc_type base_type;
343 size_t nfields;
344 int class_type;
345 errstat = nc_inq_user_type(ncid, datatype, type_name, &size, &base_type, &nfields, &class_type);
346 if (errstat != NC_NOERR)
347 throw(InternalErr(__FILE__, __LINE__, "Could not get information about a user-defined type (" + long_to_string(errstat) + ")."));
348
349 BESDEBUG(MODULE, prolog << "Before switch(class_type)" << endl);
350 switch (class_type) {
351 case NC_COMPOUND: {
352 // Make recursive attrs work?
353 vector<unsigned char> values((len + 1) * size);
354
355 int errstat = nc_get_att(ncid, varid, attrname, values.data());
356 if (errstat != NC_NOERR) throw Error(errstat, string("Could not get the value for attribute '") + attrname + string("'"));
357
358 for (size_t i = 0; i < nfields; ++i) {
359 char field_name[NC_MAX_NAME + 1];
360 nc_type field_typeid;
361 size_t field_offset;
362 nc_inq_compound_field(ncid, datatype, i, field_name, &field_offset, &field_typeid, 0, 0);
363
364 at->append_attr(field_name, print_type(field_typeid), print_attr(field_typeid, 0, values.data() + field_offset));
365 }
366 break;
367 }
368
369 case NC_VLEN:
370 if (NCRequestHandler::get_ignore_unknown_types())
371 cerr << "in build_user_defined; found a vlen." << endl;
372 else
373 throw Error("The netCDF handler does not yet support the NC_VLEN type.");
374 break;
375
376 case NC_OPAQUE: {
377 vector<unsigned char> values((len + 1) * size);
378
379 int errstat = nc_get_att(ncid, varid, attrname, values.data());
380 if (errstat != NC_NOERR) throw Error(errstat, string("Could not get the value for attribute '") + attrname + string("'"));
381
382 for (size_t i = 0; i < size; ++i)
383 at->append_attr(attrname, print_type(NC_BYTE), print_attr(NC_BYTE, i, values.data()));
384
385 break;
386 }
387
388 case NC_ENUM: {
389#if 0
390 nc_type basetype;
391 size_t base_size, num_members;
392 errstat = nc_inq_enum(ncid, datatype, 0/*char *name*/, &basetype, &base_size, &num_members);
393 if (errstat != NC_NOERR) throw Error(errstat, string("Could not get the size of the enum base type for '") + attrname + string("'"));
394#endif
395 vector<unsigned char> values((len + 1) * size);
396
397 int errstat = nc_get_att(ncid, varid, attrname, values.data());
398 if (errstat != NC_NOERR) throw Error(errstat, string("Could not get the value for attribute '") + attrname + string("'"));
399
400 for (size_t i = 0; i < len; ++i)
401 at->append_attr(attrname, print_type(base_type), print_attr(base_type, i, values.data()));
402
403 break;
404 }
405
406 default:
407 throw InternalErr(__FILE__, __LINE__, "Expected one of NC_COMPOUND, NC_VLEN, NC_OPAQUE or NC_ENUM");
408 }
409
410 BESDEBUG(MODULE, prolog << "After switch(class-type)" << endl);
411 }
412 else {
413 switch (datatype) {
414 case NC_STRING:
415 case NC_BYTE:
416 case NC_CHAR:
417 case NC_SHORT:
418 case NC_INT:
419 case NC_FLOAT:
420 case NC_DOUBLE:
421 case NC_UBYTE:
422 case NC_USHORT:
423 case NC_UINT:
424 BESDEBUG(MODULE, prolog << "Before append_values ..." << endl);
425 append_values(ncid, varid, len, datatype, attrname, at);
426 BESDEBUG(MODULE, prolog << "After append_values ..." << endl);
427 break;
428
429 case NC_INT64:
430 case NC_UINT64: {
431 string note = "Attribute elided: Unsupported attribute type ";
432 note += "(" + print_type(datatype) + ")";
433 at->append_attr(attrname, "String", note);
434 break;
435 }
436
437 case NC_COMPOUND:
438 case NC_VLEN:
439 case NC_OPAQUE:
440 case NC_ENUM:
441 throw InternalErr(__FILE__, __LINE__, "user-defined attribute type not recognized as such!");
442
443 default:
444 throw InternalErr(__FILE__, __LINE__, "Unrecognized attribute type.");
445 }
446 }
447 }
448 BESDEBUG(MODULE, prolog << "Exiting read_attributes_netcdf4" << endl);
449}
450
460void nc_read_dataset_attributes(DAS &das, const string &filename)
461{
462 BESDEBUG(MODULE, prolog << "In nc_read_dataset_attributes" << endl);
463
464 int ncid, errstat;
465 errstat = nc_open(filename.c_str(), NC_NOWRITE, &ncid);
466 if (errstat != NC_NOERR) throw Error(errstat, "NetCDF handler: Could not open " + filename + ".");
467
468 // how many variables? how many global attributes?
469 int nvars, ngatts;
470 errstat = nc_inq(ncid, (int *) 0, &nvars, &ngatts, (int *) 0);
471 if (errstat != NC_NOERR) throw Error(errstat, "NetCDF handler: Could not inquire about netcdf file: " + path_to_filename(filename) + ".");
472
473 // for each variable
474 char varname[MAX_NC_NAME];
475 int natts = 0;
476 nc_type var_type;
477 for (int varid = 0; varid < nvars; ++varid) {
478 BESDEBUG(MODULE, prolog << "Top of for loop; for each var..." << endl);
479
480 errstat = nc_inq_var(ncid, varid, varname, &var_type, (int*) 0, (int*) 0, &natts);
481 if (errstat != NC_NOERR) throw Error(errstat, "Could not get information for variable: " + long_to_string(varid));
482
483 AttrTable *attr_table_ptr = das.get_table(varname);
484 if (!attr_table_ptr) attr_table_ptr = das.add_table(varname, new AttrTable);
485
486 read_attributes_netcdf4(ncid, varid, natts, attr_table_ptr);
487
488 // Add a special attribute for string lengths
489 if (var_type == NC_CHAR) {
490 // number of dimensions and size of Nth dimension
491 int num_dim;
492 int vdimids[MAX_VAR_DIMS]; // variable dimension ids
493 errstat = nc_inq_var(ncid, varid, (char *) 0, (nc_type *) 0, &num_dim, vdimids, (int *) 0);
494 if (errstat != NC_NOERR)
495 throw Error(errstat, string("NetCDF handler: Could not read information about a NC_CHAR variable while building the DAS."));
496
497 if (num_dim == 0) {
498 // a scalar NC_CHAR is stuffed into a string of length 1
499 int size = 1;
500 string print_rep = print_attr(NC_INT, 0, (void *) &size);
501 attr_table_ptr->append_attr("string_length", print_type(NC_INT), print_rep);
502 }
503 else {
504 // size_t *dim_sizes = new size_t[num_dim];
505 vector<size_t> dim_sizes(num_dim);
506 for (int i = 0; i < num_dim; ++i) {
507 if ((errstat = nc_inq_dimlen(ncid, vdimids[i], &dim_sizes[i])) != NC_NOERR) {
508 throw Error(errstat,
509 string("NetCDF handler: Could not read dimension information about the variable `") + varname + string("'."));
510 }
511 }
512
513 // add attribute
514 string print_rep = print_attr(NC_INT, 0, (void *) (&dim_sizes[num_dim - 1]));
515 attr_table_ptr->append_attr("string_length", print_type(NC_INT), print_rep);
516 }
517 }
518 else if (is_user_defined_type(ncid, var_type)) {
519 //var_type >= NC_FIRSTUSERTYPEID) {
520 vector<char> name(MAX_NC_NAME + 1);
521 int class_type;
522 errstat = nc_inq_user_type(ncid, var_type, name.data(), 0, 0, 0, &class_type);
523 if (errstat != NC_NOERR)
524 throw(InternalErr(__FILE__, __LINE__, "Could not get information about a user-defined type (" + long_to_string(errstat) + ")."));
525
526 switch (class_type) {
527 case NC_OPAQUE: {
528 attr_table_ptr->append_attr("DAP2_OriginalNetCDFBaseType", print_type(NC_STRING), "NC_OPAQUE");
529 attr_table_ptr->append_attr("DAP2_OriginalNetCDFTypeName", print_type(NC_STRING), name.data());
530 break;
531 }
532
533 case NC_ENUM: {
534 //vector<char> name(MAX_NC_NAME + 1);
535 nc_type base_nc_type;
536 size_t base_size, num_members;
537 errstat = nc_inq_enum(ncid, var_type, 0/*name.data()*/, &base_nc_type, &base_size, &num_members);
538 if (errstat != NC_NOERR)
539 throw(InternalErr(__FILE__, __LINE__, "Could not get information about an enum(" + long_to_string(errstat) + ")."));
540
541 // If the base type is a 64-bit int, bail with an error or
542 // a message about unsupported types
543 if (base_nc_type == NC_INT64 || base_nc_type == NC_UINT64) {
544 if (NCRequestHandler::get_ignore_unknown_types())
545 cerr << "An Enum uses 64-bit integers, but this handler does not support that type." << endl;
546 else
547 throw Error("An Enum uses 64-bit integers, but this handler does not support that type.");
548 break;
549 }
550
551 for (size_t i = 0; i < num_members; ++i) {
552 vector<char> member_name(MAX_NC_NAME + 1);
553 vector<char> member_value(base_size);
554 errstat = nc_inq_enum_member(ncid, var_type, i, member_name.data(), member_value.data());
555 if (errstat != NC_NOERR)
556 throw(InternalErr(__FILE__, __LINE__, "Could not get information about an enum value (" + long_to_string(errstat) + ")."));
557 attr_table_ptr->append_attr("DAP2_EnumValues", print_type(base_nc_type), print_attr(base_nc_type, 0, member_value.data()));
558 attr_table_ptr->append_attr("DAP2_EnumNames", print_type(NC_STRING), member_name.data());
559 }
560
561 attr_table_ptr->append_attr("DAP2_OriginalNetCDFBaseType", print_type(NC_STRING), "NC_ENUM");
562 attr_table_ptr->append_attr("DAP2_OriginalNetCDFTypeName", print_type(NC_STRING), name.data());
563
564 break;
565 }
566
567 default:
568 break;
569 }
570 }
571 }
572
573 BESDEBUG(MODULE, prolog << "Starting global attributes" << endl);
574
575 // global attributes
576 if (ngatts > 0) {
577 AttrTable *attr_table_ptr = das.add_table("NC_GLOBAL", new AttrTable);
578 read_attributes_netcdf4(ncid, NC_GLOBAL, ngatts, attr_table_ptr);
579 }
580
581 // Add unlimited dimension name in DODS_EXTRA attribute table
582 int xdimid;
583 char dimname[MAX_NC_NAME];
584 nc_type datatype = NC_CHAR;
585 if ((errstat = nc_inq(ncid, (int *) 0, (int *) 0, (int *) 0, &xdimid)) != NC_NOERR)
586 throw InternalErr(__FILE__, __LINE__, string("NetCDF handler: Could not access variable information: ") + nc_strerror(errstat));
587 if (xdimid != -1) {
588 if ((errstat = nc_inq_dim(ncid, xdimid, dimname, (size_t *) 0)) != NC_NOERR)
589 throw InternalErr(__FILE__, __LINE__, string("NetCDF handler: Could not access dimension information: ") + nc_strerror(errstat));
590 string print_rep = print_attr(datatype, 0, dimname);
591 AttrTable *attr_table_ptr = das.add_table("DODS_EXTRA", new AttrTable);
592 attr_table_ptr->append_attr("Unlimited_Dimension", print_type(datatype), print_rep);
593 }
594
595 if (nc_close(ncid) != NC_NOERR) throw InternalErr(__FILE__, __LINE__, "NetCDF handler: Could not close the dataset!");
596
597 BESDEBUG(MODULE, prolog << "Exiting nc_read_dataset_attributes" << endl);
598}
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:868