bes Updated for version 3.20.13
DmrppCommon.h
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of the BES
5
6// Copyright (c) 2016 OPeNDAP, Inc.
7// Author: James Gallagher <jgallagher@opendap.org>
8//
9// This library is free software; you can redistribute it and/or
10// modify it under the terms of the GNU Lesser General Public
11// License as published by the Free Software Foundation; either
12// version 2.1 of the License, or (at your option) any later version.
13//
14// This library is distributed in the hope that it will be useful,
15// but WITHOUT ANY WARRANTY; without even the implied warranty of
16// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17// Lesser General Public License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24
25#ifndef _dmrpp_common_h
26#define _dmrpp_common_h 1
27
28#include <string>
29#include <vector>
30#include <memory>
31
32#define PUGIXML_NO_XPATH
33#define PUGIXML_HEADER_ONLY
34#include <pugixml.hpp>
35
36namespace libdap {
37class DMR;
38class BaseType;
39class D4BaseTypeFactory;
40class D4Group;
41class D4Attributes;
42class D4EnumDef;
43class D4Dimension;
44class XMLWriter;
45}
46
47namespace http {
48class url;
49}
50
51namespace pugi {
52class xml_node;
53}
54
55namespace dmrpp {
56
57class DMZ;
58class Chunk;
59class DmrppArray;
60class SuperChunk;
61
62void join_threads(pthread_t threads[], unsigned int num_threads);
63
68 int8_t int8;
69 int16_t int16;
70 int32_t int32;
71 int64_t int64;
72
73 uint8_t uint8;
74 uint16_t uint16;
75 uint32_t uint32;
76 uint64_t uint64;
77
78 float f;
79 double d;
80};
81
95
96 friend class DmrppCommonTest;
97 friend class DmrppParserTest;
98 friend class DMZTest;
99
100 bool d_compact = false;
101 std::string d_filters;
102 std::string d_byte_order;
103 std::vector<unsigned long long> d_chunk_dimension_sizes;
104 std::vector<std::shared_ptr<Chunk>> d_chunks;
105 bool d_twiddle_bytes = false;
106
107 // These indicate that the chunks or attributes have been loaded into the
108 // variable when the DMR++ handler is using lazy-loading of this data.
109 bool d_chunks_loaded = false;
110 bool d_attributes_loaded = false;
111
112 bool d_uses_fill_value {false};
113 // Convert fill_value to the correct numeric datatype at the time of use. jhrg 4/24/22
114 std::string d_fill_value_str;
115 libdap::Type d_fill_value_type{libdap::dods_null_c};
116 fill_value_union d_fill_value;
117
118 // Each instance of DmrppByte, ..., holds a shared pointer to the DMZ so that
119 // it can fetch more information from the XML if needed - this is how the lazy-load
120 // feature is implemented. The xml_node object is used to simplify finding where
121 // in the XML information about a variable is stored - to limit searching the
122 // document, the code caches the XML node.
123 std::shared_ptr<DMZ> d_dmz;
124 pugi::xml_node d_xml_node;
125
126protected:
127 virtual char *read_atomic(const std::string &name);
128
129 // This declaration allows code in the SuperChunky program to use the protected method.
130 // jhrg 10/25/21
131 friend void compute_super_chunks(dmrpp::DmrppArray *array, bool only_constrained, std::vector<dmrpp::SuperChunk *> &super_chunks);
132
133public:
134 static bool d_print_chunks;
135 static std::string d_dmrpp_ns;
136 static std::string d_ns_prefix;
137
138 DmrppCommon() = default;
139
140 explicit DmrppCommon(std::shared_ptr<DMZ> dmz) : d_dmz(std::move(dmz)) { }
141
142 DmrppCommon(const DmrppCommon &) = default;
143
144 virtual ~DmrppCommon()= default;
145
147 virtual std::string get_filters() const {
148 return d_filters;
149 }
150
151 void set_filter(const std::string &value);
152
153 virtual bool is_filters_empty() const {
154 return d_filters.empty();
155 }
156
158 virtual bool is_compact_layout() const {
159 return d_compact;
160 }
161
163 void set_compact(bool value) {
164 d_compact = value;
165 }
166
168 virtual bool twiddle_bytes() const { return d_twiddle_bytes; }
169
170 // @brief Provide access to the DMZ instance bound to this variable
171 // virtual const std::shared_ptr<DMZ> &get_dmz() const { return d_dmz; }
172
174 virtual bool get_chunks_loaded() const { return d_chunks_loaded; }
175 virtual void set_chunks_loaded(bool state) { d_chunks_loaded = state; }
176
178 virtual bool get_attributes_loaded() const { return d_attributes_loaded; }
179 virtual void set_attributes_loaded(bool state) { d_attributes_loaded = state; }
180
181 virtual const pugi::xml_node &get_xml_node() const { return d_xml_node; }
182 virtual void set_xml_node(pugi::xml_node node) { d_xml_node = node; }
183
185 virtual const std::vector<std::shared_ptr<Chunk>> &get_immutable_chunks() const {
186 return d_chunks;
187 }
188
191 virtual size_t get_chunks_size() const { return d_chunks.size(); }
192
195 virtual const std::vector<unsigned long long> &get_chunk_dimension_sizes() const {
196 return d_chunk_dimension_sizes;
197 }
198
201 virtual unsigned long long get_chunk_size_in_elements() const {
202 unsigned long long elements = 1;
203 for (auto d_chunk_dimension_size : d_chunk_dimension_sizes) {
204 elements *= d_chunk_dimension_size;
205 }
206
207 return elements;
208 }
209
211 virtual void set_uses_fill_value(bool ufv) { d_uses_fill_value = ufv; }
212
214 virtual void set_fill_value_string(const std::string &fv) { d_fill_value_str = fv; }
215
217 virtual void set_fill_value_type(libdap::Type t) { d_fill_value_type = t; }
218
220 virtual bool get_uses_fill_value() const { return d_uses_fill_value; }
221
223 virtual std::string get_fill_value() const { return d_fill_value_str; }
224
226 virtual libdap::Type get_fill_value_type() const { return d_fill_value_type; }
227
228 void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space = "");
229
230 void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space = "", const std::string &encoded = "");
231
232 void print_dmrpp(libdap::XMLWriter &writer, bool constrained = false);
233
235 void set_chunk_dimension_sizes(const std::vector<unsigned long long> &chunk_dims) {
236 for (auto chunk_dim : chunk_dims) {
237 d_chunk_dimension_sizes.push_back(chunk_dim);
238 }
239 }
240
241 // These two functions duplicate code in DMZ but provides access to the DMZ::load_chunks()
242 // method without having to cast a BaseType to a DmrppCommon in order to use it. jhrg 11/12/21
243 virtual void load_chunks(libdap::BaseType *btp);
244 virtual void load_attributes(libdap::BaseType *btp);
245
246 virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string);
247
248 virtual void ingest_compression_type(const std::string &compression_type_string);
249
250 virtual void ingest_byte_order(const std::string &byte_order_string);
251 virtual std::string get_byte_order() const { return d_byte_order; }
252
253 // There are two main versions of add_chunk: One that takes a size and offset
254 // and one that takes a fill value. However, for each of those, there are versions
255 // that take a data URL (or not) and versions that take the 'chunk position in
256 // array' information as a string or as a vector< uint64_t >. Thus, there are
257 // a total of eight of these 'add_chunk()' functions. jhrg 4/22/22
258 virtual unsigned long add_chunk(
259 std::shared_ptr<http::url> d_data_url,
260 const std::string &byte_order,
261 unsigned long long size,
262 unsigned long long offset,
263 const std::string &position_in_array);
264
265 virtual unsigned long add_chunk(
266 std::shared_ptr<http::url> d_data_url,
267 const std::string &byte_order,
268 unsigned long long size,
269 unsigned long long offset,
270 const std::vector<unsigned long long> &position_in_array);
271
272 virtual unsigned long add_chunk(
273 const std::string &byte_order,
274 unsigned long long size,
275 unsigned long long offset,
276 const std::string &position_in_array);
277
278 virtual unsigned long add_chunk(
279 const std::string &byte_order,
280 unsigned long long size,
281 unsigned long long offset,
282 const std::vector<unsigned long long> &position_in_array);
283
284 virtual unsigned long add_chunk(
285 const std::string &byte_order,
286 const std::string &fill_value,
287 libdap::Type fv_type,
288 unsigned long long chunk_size,
289 const std::vector<unsigned long long> &position_in_array);
290
291 virtual void dump(std::ostream & strm) const;
292};
293
294} // namespace dmrpp
295
296#endif // _dmrpp_common_h
297
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
Size and offset information of data included in DMR++ files.
Definition: DmrppCommon.h:94
static std::string d_ns_prefix
The XML namespace prefix to use.
Definition: DmrppCommon.h:136
virtual bool twiddle_bytes() const
Returns true if this object utilizes shuffle compression.
Definition: DmrppCommon.h:168
virtual void set_fill_value_string(const std::string &fv)
Set the fill value (using a string)
Definition: DmrppCommon.h:214
virtual libdap::Type get_fill_value_type() const
Definition: DmrppCommon.h:226
static bool d_print_chunks
if true, print_dap4() prints chunk elements
Definition: DmrppCommon.h:134
virtual bool is_compact_layout() const
Returns true if this object utilizes COMPACT layout.
Definition: DmrppCommon.h:158
virtual std::string get_fill_value() const
Definition: DmrppCommon.h:223
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
Definition: DmrppCommon.cc:166
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
Definition: DmrppCommon.cc:519
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
Definition: DmrppCommon.cc:432
virtual bool get_chunks_loaded() const
Have the chunks been loaded?
Definition: DmrppCommon.h:174
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
Definition: DmrppCommon.h:191
static std::string d_dmrpp_ns
The DMR++ XML namespace.
Definition: DmrppCommon.h:135
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
Definition: DmrppCommon.cc:354
virtual void set_uses_fill_value(bool ufv)
Set the uses_fill_value property.
Definition: DmrppCommon.h:211
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
Definition: DmrppCommon.cc:128
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
Definition: DmrppCommon.h:185
void set_chunk_dimension_sizes(const std::vector< unsigned long long > &chunk_dims)
Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.
Definition: DmrppCommon.h:235
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal ve...
Definition: DmrppCommon.cc:208
void set_filter(const std::string &value)
Set the value of the filters property.
Definition: DmrppCommon.cc:102
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
Definition: DmrppCommon.cc:177
virtual bool get_uses_fill_value() const
Definition: DmrppCommon.h:220
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
Definition: DmrppCommon.h:195
virtual void set_fill_value_type(libdap::Type t)
Set the libdap data type to use with the fill value.
Definition: DmrppCommon.h:217
void set_compact(bool value)
Set the value of the compact property.
Definition: DmrppCommon.h:163
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
Definition: DmrppCommon.cc:455
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
Definition: DmrppCommon.cc:510
virtual unsigned long long get_chunk_size_in_elements() const
Get the number of elements in this chunk.
Definition: DmrppCommon.h:201
virtual std::string get_filters() const
Return the names of all the filters in the order they were applied.
Definition: DmrppCommon.h:147
virtual bool get_attributes_loaded() const
Have the attributes been loaded?
Definition: DmrppCommon.h:178
virtual char * read_atomic(const std::string &name)
read method for the atomic types
Definition: DmrppCommon.cc:328
utility class for the HTTP catalog module
Definition: AllowedHosts.cc:55
hold the value used to fill empty chunks
Definition: DmrppCommon.h:67