bes Updated for version 3.20.13
SuperChunk.h
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2020 OPeNDAP, Inc.
6// Author: Nathan Potter<ndp@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#ifndef HYRAX_GIT_SUPERCHUNK_H
25#define HYRAX_GIT_SUPERCHUNK_H
26
27#include <vector>
28#include <memory>
29#include <thread>
30#include <queue>
31#include <sstream>
32
33#include "Chunk.h"
34
35namespace dmrpp {
36
37// Forward Declaration
38class DmrppArray;
39
45private:
46 std::string d_id;
47 DmrppArray *d_parent_array;
48 std::shared_ptr<http::url> d_data_url;
49 std::vector<std::shared_ptr<Chunk>> d_chunks;
50 unsigned long long d_offset;
51 unsigned long long d_size;
52 bool d_is_read;
53 char *d_read_buffer;
54
55 bool d_uses_fill_value{false};
56
57 bool is_contiguous(std::shared_ptr<Chunk> candidate_chunk);
58 void map_chunks_to_buffer();
59 void read_aggregate_bytes();
60 void read_fill_value_chunk();
61
62public:
63 // TODO Make the sc_id an uint64 and not a string - the code uses sstream to make the value. jhrg 5/7/22
64 explicit SuperChunk(const std::string sc_id, DmrppArray *parent=nullptr):
65 d_id(sc_id), d_parent_array(parent), d_data_url(nullptr), d_offset(0), d_size(0), d_is_read(false), d_read_buffer(nullptr){}
66
67 virtual ~SuperChunk(){
68 delete[] d_read_buffer;
69 }
70
71 virtual std::string id() const { return d_id; }
72
73 virtual bool add_chunk(std::shared_ptr<Chunk> candidate_chunk);
74
75 std::shared_ptr<http::url> get_data_url() { return d_data_url; }
76 virtual unsigned long long get_size() const { return d_size; }
77 virtual unsigned long long get_offset() const { return d_offset; }
78
79 virtual void read() {
80 retrieve_data(); // TODO process_child_chunks() also calls retrieve_data(). jhrg 5/9/22
82 }
83
84 virtual void read_unconstrained() {
85 retrieve_data(); // TODO process_child_chunks_unconstrained() also calls retrieve_data(). jhrg 5/9/22
87 }
88
89 virtual void retrieve_data();
90 virtual void process_child_chunks();
92
93 virtual bool empty(){ return d_chunks.empty(); }
94
95 std::vector<std::shared_ptr<Chunk>> get_chunks() { return d_chunks; }
96
97 std::string to_string(bool verbose) const;
98 virtual void dump(std::ostream & strm) const;
99};
100
106 std::thread::id parent_thread_id;
107 std::string parent_super_chunk_id;
108 std::shared_ptr<Chunk> chunk;
109 DmrppArray *array;
110 const vector<unsigned long long> &array_shape;
111
112 one_chunk_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s)
113 : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)), array(a), array_shape(a_s) {}
114};
115
122 std::thread::id parent_thread_id;
123 std::string parent_super_chunk_id;
124 std::shared_ptr<Chunk> chunk;
125 DmrppArray *array;
126 const vector<unsigned long long> &array_shape;
127 const vector<unsigned long long> &chunk_shape;
128
129 one_chunk_unconstrained_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s,
130 const vector<unsigned long long> &c_s)
131 : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)),
132 array(a), array_shape(a_s), chunk_shape(c_s) {}
133};
134
135void process_chunks_concurrent(
136 const string &super_chunk_id,
137 std::queue<shared_ptr<Chunk>> &chunks,
138 DmrppArray *array,
139 const std::vector<unsigned long long> &shape );
140
141void process_chunks_unconstrained_concurrent(
142 const string &super_chunk_id,
143 std::queue<std::shared_ptr<Chunk>> &chunks,
144 const std::vector<unsigned long long> &chunk_shape,
145 DmrppArray *array,
146 const std::vector<unsigned long long> &array_shape);
147
148} // namespace dmrpp
149
150#endif // HYRAX_GIT_SUPERCHUNK_H
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
A SuperChunk is a collection of contiguous Chunk objects along with optimized methods for data retrie...
Definition: SuperChunk.h:44
virtual void retrieve_data()
Cause the SuperChunk and all of it's subordinate Chunks to be read.
Definition: SuperChunk.cc:549
virtual bool add_chunk(std::shared_ptr< Chunk > candidate_chunk)
Attempts to add a new Chunk to this SuperChunk.
Definition: SuperChunk.cc:425
std::string to_string(bool verbose) const
Makes a string representation of the SuperChunk.
Definition: SuperChunk.cc:675
virtual void dump(std::ostream &strm) const
Writes the to_string() output to the stream strm.
Definition: SuperChunk.cc:696
virtual void process_child_chunks()
Reads the SuperChunk, inflates/de-shuffles the subordinate chunks as required and copies the values i...
Definition: SuperChunk.cc:596
virtual void process_child_chunks_unconstrained()
Reads the SuperChunk, inflates/deshuffles the subordinate chunks as required and copies the values in...
Definition: SuperChunk.cc:634
Single argument structure for a thread that will process a single Chunk for a constrained array....
Definition: SuperChunk.h:105
Single argument structure for a thread that will process a single Chunk for an unconstrained array....
Definition: SuperChunk.h:121