35#include <libdap/BaseType.h>
36#include <libdap/D4Attributes.h>
37#include <libdap/XMLWriter.h>
38#include <libdap/util.h>
40#define PUGIXML_NO_XPATH
41#define PUGIXML_HEADER_ONLY
48#include "BESInternalError.h"
50#include "DmrppRequestHandler.h"
51#include "DmrppCommon.h"
53#include "byteswap_compat.h"
58#define prolog std::string("DmrppCommon::").append(__func__).append("() - ")
63static const string dmrpp_3 =
"dmrpp:3";
64static const string dmrpp_4 =
"dmrpp:4";
81void join_threads(pthread_t threads[],
unsigned int num_threads)
84 for (
unsigned int i = 0; i < num_threads; ++i) {
86 BESDEBUG(dmrpp_3,
"Join thread " << i <<
" after an exception was caught." << endl);
88 if ((status = pthread_join(threads[i], (
void **) &error)) < 0) {
89 BESDEBUG(dmrpp_3,
"Could not join thread " << i <<
", " << strerror(status)<< endl);
91 else if (error != NULL) {
92 BESDEBUG(dmrpp_3,
"Joined thread " << i <<
", error exit: " << *error << endl);
95 BESDEBUG(dmrpp_3,
"Joined thread " << i <<
", successful exit." << endl);
103 if (DmrppRequestHandler::d_emulate_original_filter_order_behavior) {
105 if (value.find(
"shuffle") != string::npos)
106 d_filters.append(
" shuffle");
107 if (value.find(
"deflate") != string::npos)
108 d_filters.append(
" deflate");
109 if (value.find(
"fletcher32") != string::npos)
110 d_filters.append(
" fletcher32");
130 d_chunk_dimension_sizes.clear();
132 if (chunk_dims_string.empty())
return;
134 string chunk_dims = chunk_dims_string;
136 if (chunk_dims.find_first_not_of(
"1234567890 ") != string::npos)
137 throw BESInternalError(
"while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
145 if (chunk_dims.find(space) != string::npos) {
147 while ((strPos = chunk_dims.find(space)) != string::npos) {
148 strVal = chunk_dims.substr(0, strPos);
150 d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(),
nullptr, 10));
151 chunk_dims.erase(0, strPos + space.length());
157 d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(),
nullptr, 10));
168 if (compression_type_string.empty())
return;
179 if (byte_order_string.empty())
return;
182 if (byte_order_string.compare(
"LE") == 0) {
184 d_twiddle_bytes = is_host_big_endian();
186 if (byte_order_string.compare(
"BE") == 0) {
188 d_twiddle_bytes = !(is_host_big_endian());
209 shared_ptr<http::url> data_url,
210 const string &byte_order,
211 unsigned long long size,
212 unsigned long long offset,
213 const string &position_in_array)
215 vector<unsigned long long> cpia_vector;
216 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
217 return add_chunk(move(data_url), byte_order, size, offset, cpia_vector);
233 shared_ptr<http::url> data_url,
234 const string &byte_order,
235 unsigned long long size,
236 unsigned long long offset,
237 const vector<unsigned long long> &position_in_array)
239 std::shared_ptr<Chunk> chunk(
new Chunk(move(data_url), byte_order, size, offset, position_in_array));
241 d_chunks.push_back(chunk);
242 return d_chunks.size();
261 const string &byte_order,
262 unsigned long long size,
263 unsigned long long offset,
264 const string &position_in_array)
266 vector<unsigned long long> cpia_vector;
267 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
268 return add_chunk(byte_order, size, offset, cpia_vector);
286 const string &byte_order,
287 unsigned long long size,
288 unsigned long long offset,
289 const vector<unsigned long long> &position_in_array)
291 shared_ptr<Chunk> chunk(
new Chunk( byte_order, size, offset, position_in_array));
293 d_chunks.push_back(chunk);
294 return d_chunks.size();
298 const string &byte_order,
300 libdap::Type fv_type,
301 unsigned long long chunk_size,
302 const vector<unsigned long long> &position_in_array)
304 shared_ptr<Chunk> chunk(
new Chunk(byte_order,
fill_value, fv_type, chunk_size, position_in_array));
306 d_chunks.push_back(chunk);
307 return d_chunks.size();
331 throw BESInternalError(
string(
"Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
337 return chunk->get_rbuf();
357 if (xmlTextWriterStartElementNS(xml.get_writer(), (
const xmlChar*)name_space.c_str(), (
const xmlChar*)
"chunks", NULL) < 0)
358 throw BESInternalError(
"Could not start chunks element.", __FILE__, __LINE__);
360 if (!d_filters.empty())
361 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar*)
"compressionType", (
const xmlChar*) d_filters.c_str()) < 0)
362 throw BESInternalError(
"Could not write compression attribute.", __FILE__, __LINE__);
364 if (d_uses_fill_value && !d_fill_value_str.empty()) {
365 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar*)
"fillValue", (
const xmlChar*) d_fill_value_str.c_str()) < 0)
366 throw BESInternalError(
"Could not write fillValue attribute.", __FILE__, __LINE__);
369 if(!d_chunks.empty()) {
371 if (!first_chunk->get_byte_order().empty()) {
372 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar *)
"byteOrder",
373 (
const xmlChar *) first_chunk->get_byte_order().c_str()) < 0)
374 throw BESInternalError(
"Could not write attribute byteOrder", __FILE__, __LINE__);
378 if (!d_chunk_dimension_sizes.empty()) {
381 copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss,
" "));
382 string sizes = oss.str();
383 sizes.erase(sizes.size() - 1, 1);
385 if (xmlTextWriterWriteElementNS(xml.get_writer(), (
const xmlChar*) name_space.c_str(), (
const xmlChar*)
"chunkDimensionSizes", NULL,
386 (
const xmlChar*) sizes.c_str()) < 0)
throw BESInternalError(
"Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
392 if (xmlTextWriterStartElementNS(xml.get_writer(), (
const xmlChar*)name_space.c_str(), (
const xmlChar*)
"chunk", NULL) < 0)
393 throw BESInternalError(
"Could not start element chunk", __FILE__, __LINE__);
396 ostringstream offset;
397 offset << chunk->get_offset();
398 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar *)
"offset",
399 (
const xmlChar *) offset.str().c_str()) < 0)
400 throw BESInternalError(
"Could not write attribute offset", __FILE__, __LINE__);
403 ostringstream nBytes;
404 nBytes << chunk->get_size();
405 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar *)
"nBytes",
406 (
const xmlChar *) nBytes.str().c_str()) < 0)
407 throw BESInternalError(
"Could not write attribute nBytes", __FILE__, __LINE__);
409 if (chunk->get_position_in_array().size() > 0) {
411 vector<unsigned long long> pia = chunk->get_position_in_array();
414 copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss,
","));
415 string pia_str = oss.str();
416 if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1,
"]");
417 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar*)
"chunkPositionInArray", (
const xmlChar*) pia_str.c_str()) < 0)
418 throw BESInternalError(
"Could not write attribute position in array", __FILE__, __LINE__);
422 if (xmlTextWriterEndElement(xml.get_writer()) < 0)
throw BESInternalError(
"Could not end chunk element", __FILE__, __LINE__);
425 if (xmlTextWriterEndElement(xml.get_writer()) < 0)
throw BESInternalError(
"Could not end chunks element", __FILE__, __LINE__);
436 copy(encoded.begin(), encoded.end(), ostream_iterator<char>(oss,
""));
437 string sizes = oss.str();
439 if (xmlTextWriterWriteElementNS(xml.get_writer(), (
const xmlChar *) name_space.c_str(),
440 (
const xmlChar *)
"compact", NULL,
441 (
const xmlChar *) sizes.c_str()) < 0)
442 throw BESInternalError(
"Could not write compact element.", __FILE__, __LINE__);
457 BaseType &bt =
dynamic_cast<BaseType&
>(*this);
458 if (constrained && !bt.send_p())
461 if (xmlTextWriterStartElement(xml.get_writer(), (
const xmlChar*)bt.type_name().c_str()) < 0)
462 throw InternalErr(__FILE__, __LINE__,
"Could not write " + bt.type_name() +
" element");
464 if (!bt.name().empty())
465 if (xmlTextWriterWriteAttribute(xml.get_writer(), (
const xmlChar*)
"name", (
const xmlChar*)bt.name().c_str()) < 0)
466 throw InternalErr(__FILE__, __LINE__,
"Could not write attribute for name");
469 bt.attributes()->print_dap4(xml);
471 if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
472 bt.get_attr_table().print_xml_writer(xml);
478 if (xmlTextWriterEndElement(xml.get_writer()) < 0)
479 throw InternalErr(__FILE__, __LINE__,
"Could not end " + bt.type_name() +
" element");
482void DmrppCommon::dump(ostream & strm)
const
484 strm << BESIndent::LMarg <<
"is_filters_empty: " << (is_filters_empty() ?
"true" :
"false") << endl;
485 strm << BESIndent::LMarg <<
"filters: " << (d_filters.c_str()) << endl;
489 strm << BESIndent::LMarg <<
"chunk dimension sizes: [";
490 for (
unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
491 strm << (i ?
"][" :
"") << chunk_dim_sizes[i];
496 strm << BESIndent::LMarg <<
"Chunks (aka chunks):" << (chunk_refs.size() ?
"" :
"None Found.") << endl;
498 for (
auto & chunk_ref : chunk_refs) {
499 strm << BESIndent::LMarg;
500 chunk_ref->dump(strm);
511 d_dmz->load_chunks(btp);
521 d_dmz->load_attributes(btp);
exception thrown if internal error encountered
static void removeLeadingAndTrailingBlanks(std::string &key)
static std::string d_ns_prefix
The XML namespace prefix to use.
static bool d_print_chunks
if true, print_dap4() prints chunk elements
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
static std::string d_dmrpp_ns
The DMR++ XML namespace.
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal ve...
void set_filter(const std::string &value)
Set the value of the filters property.
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
virtual char * read_atomic(const std::string &name)
read method for the atomic types