bes Updated for version 3.20.13
Chunk.h
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2016 OPeNDAP, Inc.
6// Author: Nathan Potter <ndp@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#ifndef _Chunk_h
25#define _Chunk_h 1
26
27#include <string>
28#include <utility>
29#include <vector>
30#include <memory>
31
32// BES
33#include "url_impl.h"
34
35// libdap4
36#include <libdap/util.h>
37
38
39// This is used to track access to 'cloudydap' accesses in the S3 logs
40// by adding a query string that will show up in those logs. This is
41// activated by using a special BES context with the name 'cloudydap.'
42//
43// We disabled this (via ENABLE_TRACKING_QUERY_PARAMETER) because it
44// used regexes and was a performance killer. jhrg 4/22/22
45#define S3_TRACKING_CONTEXT "cloudydap"
46#define ENABLE_TRACKING_QUERY_PARAMETER 0
47
48namespace dmrpp {
49
51 int8_t int8;
52 int16_t int16;
53 int32_t int32;
54 int64_t int64;
55
56 uint8_t uint8;
57 uint16_t uint16;
58 uint32_t uint32;
59 uint64_t uint64;
60
61 float f;
62 double d;
63};
64
65// Callback functions used by chunk readers
66size_t chunk_header_callback(char *buffer, size_t size, size_t nitems, void *data);
67size_t chunk_write_data(void *buffer, size_t size, size_t nmemb, void *data);
68
69void process_s3_error_response(const std::shared_ptr<http::url> &data_url, const std::string &xml_message);
70
77class Chunk {
78private:
79 std::shared_ptr<http::url> d_data_url;
80 std::string d_query_marker;
81 std::string d_byte_order;
82 std::string d_fill_value;
83 unsigned long long d_size{0};
84 unsigned long long d_offset{0};
85 bool d_uses_fill_value{false};
86 libdap::Type d_fill_value_type{libdap::dods_null_c};
87
88 std::vector<unsigned long long> d_chunk_position_in_array;
89
90 // These are used only during the libcurl callback; they are not duplicated by the
91 // copy ctor or assignment operator.
92
109 bool d_read_buffer_is_mine {true};
110 unsigned long long d_bytes_read {0};
111 char *d_read_buffer {nullptr};
112 unsigned long long d_read_buffer_size {0};
113 bool d_is_read {false};
114 bool d_is_inflated {false};
115 std::string d_response_content_type;
116
117 friend class ChunkTest;
118 friend class DmrppCommonTest;
119 friend class MockChunk;
120
121protected:
122
123 void _duplicate(const Chunk &bs)
124 {
125 d_size = bs.d_size;
126 d_offset = bs.d_offset;
127 d_data_url = bs.d_data_url;
128 d_byte_order = bs.d_byte_order;
129 d_fill_value = bs.d_fill_value;
130 d_uses_fill_value = bs.d_uses_fill_value;
131 d_query_marker = bs.d_query_marker;
132 d_chunk_position_in_array = bs.d_chunk_position_in_array;
133 }
134
135public:
136
146 Chunk() = default;
147
158 Chunk(std::shared_ptr<http::url> data_url, std::string order, unsigned long long size,
159 unsigned long long offset, const std::string &pia_str = "") :
160 d_data_url(std::move(data_url)), d_byte_order(std::move(order)),
161 d_size(size), d_offset(offset)
162 {
163#if ENABLE_TRACKING_QUERY_PARAMETER
165#endif
166 set_position_in_array(pia_str);
167 }
168
179 Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str = "") :
180 d_byte_order(std::move(order)), d_size(size), d_offset(offset) {
181#if ENABLE_TRACKING_QUERY_PARAMETER
183#endif
184 set_position_in_array(pia_str);
185 }
186
197 Chunk(std::shared_ptr<http::url> data_url, std::string order, unsigned long long size, unsigned long long offset,
198 const std::vector<unsigned long long> &pia_vec) :
199 d_data_url(std::move(data_url)), d_byte_order(std::move(order)),
200 d_size(size), d_offset(offset) {
201#if ENABLE_TRACKING_QUERY_PARAMETER
203#endif
204 set_position_in_array(pia_vec);
205 }
206
207
218 Chunk(std::string order, unsigned long long size, unsigned long long offset,
219 const std::vector<unsigned long long> &pia_vec) :
220 d_byte_order(std::move(order)), d_size(size), d_offset(offset) {
221#if ENABLE_TRACKING_QUERY_PARAMETER
223#endif
224 set_position_in_array(pia_vec);
225 }
226
227 Chunk(std::string order, std::string fill_value, libdap::Type fv_type, unsigned long long chunk_size, std::vector<unsigned long long> pia) :
228 d_byte_order(std::move(order)), d_fill_value(std::move(fill_value)), d_size(chunk_size),
229 d_uses_fill_value(true), d_fill_value_type(fv_type), d_chunk_position_in_array(std::move(pia)) {
230 }
231
232 Chunk(const Chunk &h4bs)
233 {
234 _duplicate(h4bs);
235 }
236
237 virtual ~Chunk()
238 {
239 if(d_read_buffer_is_mine)
240 delete[] d_read_buffer;
241 d_read_buffer = nullptr;
242 }
243
248 Chunk &operator=(const Chunk &rhs)
249 {
250 if (this == &rhs) return *this;
251
252 _duplicate(rhs);
253
254 return *this;
255 }
256
258 virtual std::string get_response_content_type() { return d_response_content_type; }
259
261 void set_response_content_type(const std::string &ct) { d_response_content_type = ct; }
262
264 virtual std::string get_byte_order() { return d_byte_order; }
265
267 virtual unsigned long long get_size() const
268 {
269 return d_size;
270 }
271
273 virtual unsigned long long get_offset() const
274 {
275 return d_offset;
276 }
277
279 virtual bool get_uses_fill_value() const { return d_uses_fill_value; }
280
282 virtual std::string get_fill_value() const { return d_fill_value; }
283
285 virtual std::shared_ptr<http::url> get_data_url() const;
286
288 virtual void set_data_url(std::shared_ptr<http::url> data_url)
289 {
290 d_data_url = std::move(data_url);
291 }
292
294 virtual unsigned long long get_bytes_read() const
295 {
296 return d_bytes_read;
297 }
298
303 virtual void set_bytes_read(unsigned long long bytes_read)
304 {
305 d_bytes_read = bytes_read;
306 }
307
322 virtual void set_rbuf_to_size()
323 {
324 if(d_read_buffer_is_mine)
325 delete[] d_read_buffer;
326
327 d_read_buffer = new char[d_size];
328 d_read_buffer_size = d_size;
329 d_read_buffer_is_mine = true;
331 }
332
335 virtual char *get_rbuf()
336 {
337 return d_read_buffer;
338 }
339
351 void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read = 0,
352 bool assume_ownership = true ) {
353 if(d_read_buffer_is_mine)
354 delete[] d_read_buffer;
355
356 d_read_buffer_is_mine = assume_ownership;
357 d_read_buffer = buf;
358 d_read_buffer_size = buf_size;
359
360 set_bytes_read(bytes_read);
361 }
362
364 virtual unsigned long long get_rbuf_size() const
365 {
366 return d_read_buffer_size;
367 }
368
370 virtual const std::vector<unsigned long long> &get_position_in_array() const
371 {
372 return d_chunk_position_in_array;
373 }
374
376
377 void set_position_in_array(const std::string &pia);
378 void set_position_in_array(const std::vector<unsigned long long> &pia);
379
380 virtual void read_chunk();
381 virtual void load_fill_values();
382
383 virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width);
384
385 virtual bool get_is_read() { return d_is_read; }
386 virtual void set_is_read(bool state) { d_is_read = state; }
387
388 virtual std::string get_curl_range_arg_string();
389
390 static void parse_chunk_position_in_array_string(const std::string &pia, std::vector<unsigned long long> &pia_vect);
391
392 virtual void dump(std::ostream & strm) const;
393
394 virtual std::string to_string() const;
395};
396
397} // namespace dmrpp
398
399#endif // _Chunk_h
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
Definition: Chunk.h:303
virtual void dump(std::ostream &strm) const
Definition: Chunk.cc:846
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values, the data URL will not be set.
Definition: Chunk.h:179
virtual char * get_rbuf()
Definition: Chunk.h:335
virtual void read_chunk()
Definition: Chunk.cc:794
void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
Definition: Chunk.cc:486
virtual std::string get_response_content_type()
Get the response type of the last response.
Definition: Chunk.h:258
virtual std::string get_byte_order()
Definition: Chunk.h:264
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
Definition: Chunk.cc:465
virtual std::shared_ptr< http::url > get_data_url() const
Definition: Chunk.cc:868
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values.
Definition: Chunk.h:158
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values, the data URl will not be set.
Definition: Chunk.h:218
virtual void set_data_url(std::shared_ptr< http::url > data_url)
Set the data url for this Chunk's data block.
Definition: Chunk.h:288
virtual std::string get_fill_value() const
Definition: Chunk.h:282
Chunk & operator=(const Chunk &rhs)
Definition: Chunk.h:248
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
Definition: Chunk.h:322
virtual unsigned long long get_offset() const
Definition: Chunk.h:273
virtual unsigned long long get_bytes_read() const
Definition: Chunk.h:294
void set_position_in_array(const std::string &pia)
parse the chunk position string
Definition: Chunk.cc:438
virtual unsigned long long get_rbuf_size() const
Definition: Chunk.h:364
virtual unsigned long long get_size() const
Definition: Chunk.h:267
Chunk()=default
Get an empty chunk.
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values.
Definition: Chunk.h:197
virtual const std::vector< unsigned long long > & get_position_in_array() const
Definition: Chunk.h:370
virtual bool get_uses_fill_value() const
Definition: Chunk.h:279
void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read=0, bool assume_ownership=true)
Set the target read buffer for this chunk.
Definition: Chunk.h:351
virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width)
filter data in the chunk
Definition: Chunk.cc:606
virtual void load_fill_values()
Load the chunk with fill values - temporary implementation.
Definition: Chunk.cc:770
void set_response_content_type(const std::string &ct)
Set the response type of the last response.
Definition: Chunk.h:261