bes Updated for version 3.20.13
DmrppArray.h
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2016 OPeNDAP, Inc.
6// Author: James Gallagher <jgallagher@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#ifndef _dmrpp_array_h
25#define _dmrpp_array_h 1
26
27#include <string>
28#include <utility>
29#include <vector>
30#include <thread>
31#include <memory>
32#include <queue>
33#include <future>
34#include <list>
35
36#include <libdap/Array.h>
37
38#include "DmrppCommon.h"
39#include "SuperChunk.h"
40
41// The 'read_serial()' method is more closely related to the original code
42// used to read data when the DMR++ handler was initially developed for NASA.
43// I modified that code for a while when we built the prototype version of
44// the handler, but then morphed that into a version that would support parallel
45// access. Defining this symbol will include the old code in the handler,
46// although the DmrppArray::read() method will still have to be hacked to
47// use it. jhrg 5/10/18
48#undef USE_READ_SERIAL
49
50namespace libdap {
51class XMLWriter;
52}
53
54namespace dmrpp {
55
56 class SuperChunk;
68class DmrppArray : public libdap::Array, public dmrpp::DmrppCommon {
69
70private:
71 //void _duplicate(const DmrppArray &ts);
72
73 bool is_projected();
74
75 DmrppArray::dimension get_dimension(unsigned int dim_num);
76
77 void insert_constrained_contiguous(Dim_iter dim_iter, unsigned long *target_index,
78 std::vector<unsigned long long> &subset_addr,
79 const std::vector<unsigned long long> &array_shape, char *data);
80
81 void read_contiguous();
82 void read_contiguous_string();
83
84#ifdef USE_READ_SERIAL
85 virtual void insert_chunk_serial(unsigned int dim, std::vector<unsigned int> *target_element_address,
86 std::vector<unsigned int> *chunk_source_address, Chunk *chunk);
87 virtual void read_chunks_serial();
88#endif
89
90 friend class DmrppArrayTest;
91 // Called from read_chunks_unconstrained() and also using pthreads
92 friend void
93 process_one_chunk_unconstrained(std::shared_ptr<Chunk> chunk, const vector<unsigned long long> &chunk_shape,
94 DmrppArray *array, const vector<unsigned long long> &array_shape);
95
96 // Called from read_chunks()
97 friend void
98 process_one_chunk(std::shared_ptr<Chunk> chunk, DmrppArray *array, const vector<unsigned long long> &constrained_array_shape);
99
100
101
102 virtual void insert_chunk_unconstrained(std::shared_ptr<Chunk> chunk, unsigned int dim,
103 unsigned long long array_offset, const std::vector<unsigned long long> &array_shape,
104 unsigned long long chunk_offset, const std::vector<unsigned long long> &chunk_shape,
105 const std::vector<unsigned long long> &chunk_origin);
106
107 void read_chunks();
108 void read_chunks_unconstrained();
109
110 unsigned long long get_chunk_start(const dimension &thisDim, unsigned int chunk_origin_for_dim);
111
112 std::shared_ptr<Chunk> find_needed_chunks(unsigned int dim, std::vector<unsigned long long> *target_element_address, std::shared_ptr<Chunk> chunk);
113
114 virtual void insert_chunk(
115 unsigned int dim,
116 std::vector<unsigned long long> *target_element_address,
117 std::vector<unsigned long long> *chunk_element_address,
118 std::shared_ptr<Chunk> chunk,
119 const vector<unsigned long long> &constrained_array_shape);
120
121public:
122 DmrppArray(const std::string &n, libdap::BaseType *v) : libdap::Array(n, v, true /*is dap4*/), DmrppCommon() { }
123
124 DmrppArray(const std::string &n, const std::string &d, libdap::BaseType *v) :
125 libdap::Array(n, d, v, true), DmrppCommon() { }
126
127 DmrppArray(const string &n, BaseType *v, shared_ptr<DMZ> dmz) :
128 libdap::Array(n, v, true), DmrppCommon(dmz) { }
129
130 DmrppArray(const string &n, const string &d, BaseType *v, shared_ptr<DMZ> dmz) :
131 libdap::Array(n, d, v, true), DmrppCommon(dmz) { }
132
133 DmrppArray(const DmrppArray &) = default;
134
135 virtual ~DmrppArray() = default;
136
137 DmrppArray &operator=(const DmrppArray &rhs);
138
139 virtual libdap::BaseType *ptr_duplicate() { return new DmrppArray(*this); }
140
141 bool read() override;
142 void set_send_p(bool state) override;
143
144 virtual unsigned long long get_size(bool constrained = false);
145
146 virtual std::vector<unsigned long long> get_shape(bool constrained);
147
148 virtual void print_dap4(libdap::XMLWriter &writer, bool constrained = false);
149
150 virtual void dump(ostream &strm) const;
151};
152
157 std::thread::id parent_thread_id;
158 std::shared_ptr<SuperChunk> super_chunk;
159 DmrppArray *array;
160
161 one_super_chunk_args(std::shared_ptr<SuperChunk> sc, DmrppArray *a)
162 : parent_thread_id(std::this_thread::get_id()), super_chunk(std::move(sc)), array(a) {}
163};
164
165
171 int *fds; // pipe back to parent
172 unsigned char tid; // thread id as a byte
173 std::shared_ptr<Chunk> child_chunk; // this chunk reads data; temporary allocation
174 std::shared_ptr<Chunk> master_chunk; // this chunk gets the data; shared memory, managed by DmrppArray
175
176 one_child_chunk_args(int *pipe, unsigned char id, std::shared_ptr<Chunk> c_c, std::shared_ptr<Chunk> m_c)
177 : fds(pipe), tid(id), child_chunk(c_c), master_chunk(m_c) {}
178
180};
181
182
188 std::shared_ptr<Chunk> child_chunk; // this chunk reads data; temporary allocation
189 std::shared_ptr<Chunk> the_one_chunk; // this chunk gets the data; shared memory, managed by DmrppArray
190
191 one_child_chunk_args_new(std::shared_ptr<Chunk> c_c, std::shared_ptr<Chunk> m_c) : child_chunk(c_c), the_one_chunk(m_c) {}
192
194};
195
196
197bool get_next_future(list<std::future<bool>> &futures, atomic_uint &thread_counter, unsigned long timeout, string debug_prefix);
198
199} // namespace dmrpp
200
201#endif // _dmrpp_array_h
202
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
virtual std::vector< unsigned long long > get_shape(bool constrained)
Get the array shape.
Definition: DmrppArray.cc:596
virtual unsigned long long get_size(bool constrained=false)
Return the total number of elements in this Array.
Definition: DmrppArray.cc:580
bool read() override
Read data for the array.
Definition: DmrppArray.cc:1499
virtual void print_dap4(libdap::XMLWriter &writer, bool constrained=false)
Shadow libdap::Array::print_dap4() - optionally prints DMR++ chunk information.
Definition: DmrppArray.cc:1699
Size and offset information of data included in DMR++ files.
Definition: DmrppCommon.h:94