bes Updated for version 3.20.13
DmrppCommon.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2016 OPeNDAP, Inc.
6// Author: James Gallagher <jgallagher@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#include "config.h"
25
26#include <string>
27#include <sstream>
28#include <vector>
29#include <iterator>
30#include <cstdlib>
31#include <cstring>
32
33#include <curl/curl.h>
34
35#include <libdap/BaseType.h>
36#include <libdap/D4Attributes.h>
37#include <libdap/XMLWriter.h>
38#include <libdap/util.h>
39
40#define PUGIXML_NO_XPATH
41#define PUGIXML_HEADER_ONLY
42#include <pugixml.hpp>
43
44#include "url_impl.h"
45#include "BESIndent.h"
46#include "BESDebug.h"
47#include "BESUtil.h"
48#include "BESInternalError.h"
49
50#include "DmrppRequestHandler.h"
51#include "DmrppCommon.h"
52#include "Chunk.h"
53#include "byteswap_compat.h"
54
55using namespace std;
56using namespace libdap;
57
58#define prolog std::string("DmrppCommon::").append(__func__).append("() - ")
59
60namespace dmrpp {
61
62// Used with BESDEBUG
63static const string dmrpp_3 = "dmrpp:3";
64static const string dmrpp_4 = "dmrpp:4";
65
67string DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#";
68string DmrppCommon::d_ns_prefix = "dmrpp";
69
81void join_threads(pthread_t threads[], unsigned int num_threads)
82{
83 int status;
84 for (unsigned int i = 0; i < num_threads; ++i) {
85 if (threads[i]) {
86 BESDEBUG(dmrpp_3, "Join thread " << i << " after an exception was caught." << endl);
87 string *error = NULL;
88 if ((status = pthread_join(threads[i], (void **) &error)) < 0) {
89 BESDEBUG(dmrpp_3, "Could not join thread " << i << ", " << strerror(status)<< endl);
90 }
91 else if (error != NULL) {
92 BESDEBUG(dmrpp_3, "Joined thread " << i << ", error exit: " << *error << endl);
93 }
94 else {
95 BESDEBUG(dmrpp_3, "Joined thread " << i << ", successful exit." << endl);
96 }
97 }
98 }
99}
100
102void DmrppCommon::set_filter(const string &value) {
103 if (DmrppRequestHandler::d_emulate_original_filter_order_behavior) {
104 d_filters = "";
105 if (value.find("shuffle") != string::npos)
106 d_filters.append(" shuffle");
107 if (value.find("deflate") != string::npos)
108 d_filters.append(" deflate");
109 if (value.find("fletcher32") != string::npos)
110 d_filters.append(" fletcher32");
111
113 }
114 else {
115 d_filters = value;
116 }
117}
118
128void DmrppCommon::parse_chunk_dimension_sizes(const string &chunk_dims_string)
129{
130 d_chunk_dimension_sizes.clear();
131
132 if (chunk_dims_string.empty()) return;
133
134 string chunk_dims = chunk_dims_string;
135 // If the input is anything other than integers and spaces, throw
136 if (chunk_dims.find_first_not_of("1234567890 ") != string::npos)
137 throw BESInternalError("while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
138
139 // TODO Rewrite this to use split. jhrg 5/2/22
140 string space(" ");
141 size_t strPos = 0;
142 string strVal;
143
144 // Are there spaces or multiple values?
145 if (chunk_dims.find(space) != string::npos) {
146 // Process space delimited content
147 while ((strPos = chunk_dims.find(space)) != string::npos) {
148 strVal = chunk_dims.substr(0, strPos);
149 // TODO stoull (CDS uses uint64_t) jhrg 5/2/22
150 d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(), nullptr, 10));
151 chunk_dims.erase(0, strPos + space.length());
152 }
153 }
154
155 // If it's multivalued there's still one more value left to process
156 // If it's single valued the same is true, so let's ingest that.
157 d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(), nullptr, 10));
158}
159
166void DmrppCommon::ingest_compression_type(const string &compression_type_string)
167{
168 if (compression_type_string.empty()) return;
169 set_filter(compression_type_string);
170}
171
177void DmrppCommon::ingest_byte_order(const string &byte_order_string) {
178
179 if (byte_order_string.empty()) return;
180
181 // Process content
182 if (byte_order_string.compare("LE") == 0) {
183 d_byte_order = "LE";
184 d_twiddle_bytes = is_host_big_endian();
185 } else {
186 if (byte_order_string.compare("BE") == 0) {
187 d_byte_order = "BE";
188 d_twiddle_bytes = !(is_host_big_endian());
189 } else {
190 throw BESInternalError("Did not recognize byteOrder.", __FILE__, __LINE__);
191 }
192 }
193}
194
195
209 shared_ptr<http::url> data_url,
210 const string &byte_order,
211 unsigned long long size,
212 unsigned long long offset,
213 const string &position_in_array)
214{
215 vector<unsigned long long> cpia_vector;
216 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
217 return add_chunk(move(data_url), byte_order, size, offset, cpia_vector);
218}
219
220
233 shared_ptr<http::url> data_url,
234 const string &byte_order,
235 unsigned long long size,
236 unsigned long long offset,
237 const vector<unsigned long long> &position_in_array)
238{
239 std::shared_ptr<Chunk> chunk(new Chunk(move(data_url), byte_order, size, offset, position_in_array));
240
241 d_chunks.push_back(chunk);
242 return d_chunks.size();
243}
244
261 const string &byte_order,
262 unsigned long long size,
263 unsigned long long offset,
264 const string &position_in_array)
265{
266 vector<unsigned long long> cpia_vector;
267 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
268 return add_chunk(byte_order, size, offset, cpia_vector);
269}
270
286 const string &byte_order,
287 unsigned long long size,
288 unsigned long long offset,
289 const vector<unsigned long long> &position_in_array)
290{
291 shared_ptr<Chunk> chunk(new Chunk( byte_order, size, offset, position_in_array));
292
293 d_chunks.push_back(chunk);
294 return d_chunks.size();
295}
296
297unsigned long DmrppCommon::add_chunk(
298 const string &byte_order,
299 const string &fill_value,
300 libdap::Type fv_type,
301 unsigned long long chunk_size,
302 const vector<unsigned long long> &position_in_array)
303{
304 shared_ptr<Chunk> chunk(new Chunk(byte_order, fill_value, fv_type, chunk_size, position_in_array));
305
306 d_chunks.push_back(chunk);
307 return d_chunks.size();
308}
309
327char *
328DmrppCommon::read_atomic(const string &name)
329{
330 if (get_chunks_size() != 1)
331 throw BESInternalError(string("Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
332
333 auto chunk = get_immutable_chunks()[0];
334
335 chunk->read_chunk();
336
337 return chunk->get_rbuf();
338}
339
353void
354DmrppCommon::print_chunks_element(XMLWriter &xml, const string &name_space)
355{
356 // Start element "chunks" with dmrpp namespace and attributes:
357 if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunks", NULL) < 0)
358 throw BESInternalError("Could not start chunks element.", __FILE__, __LINE__);
359
360 if (!d_filters.empty())
361 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "compressionType", (const xmlChar*) d_filters.c_str()) < 0)
362 throw BESInternalError("Could not write compression attribute.", __FILE__, __LINE__);
363
364 if (d_uses_fill_value && !d_fill_value_str.empty()) {
365 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "fillValue", (const xmlChar*) d_fill_value_str.c_str()) < 0)
366 throw BESInternalError("Could not write fillValue attribute.", __FILE__, __LINE__);
367 }
368
369 if(!d_chunks.empty()) {
370 auto first_chunk = get_immutable_chunks().front();
371 if (!first_chunk->get_byte_order().empty()) {
372 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "byteOrder",
373 (const xmlChar *) first_chunk->get_byte_order().c_str()) < 0)
374 throw BESInternalError("Could not write attribute byteOrder", __FILE__, __LINE__);
375 }
376 }
377
378 if (!d_chunk_dimension_sizes.empty()) { //d_chunk_dimension_sizes.size() > 0) {
379 // Write element "chunkDimensionSizes" with dmrpp namespace:
380 ostringstream oss;
381 copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss, " "));
382 string sizes = oss.str();
383 sizes.erase(sizes.size() - 1, 1); // trim the trailing space
384
385 if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar*) name_space.c_str(), (const xmlChar*) "chunkDimensionSizes", NULL,
386 (const xmlChar*) sizes.c_str()) < 0) throw BESInternalError("Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
387 }
388
389 // Start elements "chunk" with dmrpp namespace and attributes:
390 for(auto chunk: get_immutable_chunks()) {
391
392 if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunk", NULL) < 0)
393 throw BESInternalError("Could not start element chunk", __FILE__, __LINE__);
394
395 // Get offset string:
396 ostringstream offset;
397 offset << chunk->get_offset();
398 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "offset",
399 (const xmlChar *) offset.str().c_str()) < 0)
400 throw BESInternalError("Could not write attribute offset", __FILE__, __LINE__);
401
402 // Get nBytes string:
403 ostringstream nBytes;
404 nBytes << chunk->get_size();
405 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "nBytes",
406 (const xmlChar *) nBytes.str().c_str()) < 0)
407 throw BESInternalError("Could not write attribute nBytes", __FILE__, __LINE__);
408
409 if (chunk->get_position_in_array().size() > 0) {
410 // Get position in array string:
411 vector<unsigned long long> pia = chunk->get_position_in_array();
412 ostringstream oss;
413 oss << "[";
414 copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss, ","));
415 string pia_str = oss.str();
416 if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1, "]"); // replace the trailing ',' with ']'
417 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "chunkPositionInArray", (const xmlChar*) pia_str.c_str()) < 0)
418 throw BESInternalError("Could not write attribute position in array", __FILE__, __LINE__);
419 }
420
421 // End element "chunk":
422 if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunk element", __FILE__, __LINE__);
423 }
424
425 if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunks element", __FILE__, __LINE__);
426}
427
431void
432DmrppCommon::print_compact_element(XMLWriter &xml, const string &name_space, const std::string &encoded)
433{
434 // Write element "compact" with dmrpp namespace:
435 ostringstream oss;
436 copy(encoded.begin(), encoded.end(), ostream_iterator<char>(oss, ""));
437 string sizes = oss.str();
438
439 if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar *) name_space.c_str(),
440 (const xmlChar *) "compact", NULL,
441 (const xmlChar *) sizes.c_str()) < 0)
442 throw BESInternalError("Could not write compact element.", __FILE__, __LINE__);
443}
444
455void DmrppCommon::print_dmrpp(XMLWriter &xml, bool constrained /*false*/)
456{
457 BaseType &bt = dynamic_cast<BaseType&>(*this);
458 if (constrained && !bt.send_p())
459 return;
460
461 if (xmlTextWriterStartElement(xml.get_writer(), (const xmlChar*)bt.type_name().c_str()) < 0)
462 throw InternalErr(__FILE__, __LINE__, "Could not write " + bt.type_name() + " element");
463
464 if (!bt.name().empty())
465 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "name", (const xmlChar*)bt.name().c_str()) < 0)
466 throw InternalErr(__FILE__, __LINE__, "Could not write attribute for name");
467
468 if (bt.is_dap4())
469 bt.attributes()->print_dap4(xml);
470
471 if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
472 bt.get_attr_table().print_xml_writer(xml);
473
474 // This is the code added to libdap::BaseType::print_dap4(). jhrg 5/10/18
477
478 if (xmlTextWriterEndElement(xml.get_writer()) < 0)
479 throw InternalErr(__FILE__, __LINE__, "Could not end " + bt.type_name() + " element");
480}
481
482void DmrppCommon::dump(ostream & strm) const
483{
484 strm << BESIndent::LMarg << "is_filters_empty: " << (is_filters_empty() ? "true" : "false") << endl;
485 strm << BESIndent::LMarg << "filters: " << (d_filters.c_str()) << endl;
486
487 const vector<unsigned long long> &chunk_dim_sizes = get_chunk_dimension_sizes();
488
489 strm << BESIndent::LMarg << "chunk dimension sizes: [";
490 for (unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
491 strm << (i ? "][" : "") << chunk_dim_sizes[i];
492 }
493 strm << "]" << endl;
494
495 auto chunk_refs = get_immutable_chunks();
496 strm << BESIndent::LMarg << "Chunks (aka chunks):" << (chunk_refs.size() ? "" : "None Found.") << endl;
497 BESIndent::Indent();
498 for (auto & chunk_ref : chunk_refs) {
499 strm << BESIndent::LMarg;
500 chunk_ref->dump(strm);
501 strm << endl;
502 }
503}
504
509void
511 d_dmz->load_chunks(btp);
512}
513
518void
519DmrppCommon::load_attributes(libdap::BaseType *btp)
520{
521 d_dmz->load_attributes(btp);
522}
523
524} // namespace dmrpp
525
exception thrown if internal error encountered
static void removeLeadingAndTrailingBlanks(std::string &key)
Definition: BESUtil.cc:445
static std::string d_ns_prefix
The XML namespace prefix to use.
Definition: DmrppCommon.h:136
static bool d_print_chunks
if true, print_dap4() prints chunk elements
Definition: DmrppCommon.h:134
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
Definition: DmrppCommon.cc:166
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
Definition: DmrppCommon.cc:519
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
Definition: DmrppCommon.cc:432
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
Definition: DmrppCommon.h:191
static std::string d_dmrpp_ns
The DMR++ XML namespace.
Definition: DmrppCommon.h:135
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
Definition: DmrppCommon.cc:354
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
Definition: DmrppCommon.cc:128
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
Definition: DmrppCommon.h:185
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal ve...
Definition: DmrppCommon.cc:208
void set_filter(const std::string &value)
Set the value of the filters property.
Definition: DmrppCommon.cc:102
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
Definition: DmrppCommon.cc:177
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
Definition: DmrppCommon.h:195
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
Definition: DmrppCommon.cc:455
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
Definition: DmrppCommon.cc:510
virtual char * read_atomic(const std::string &name)
read method for the atomic types
Definition: DmrppCommon.cc:328