bes Updated for version 3.20.10
FONcTransmitter.cc
1// FONcTransmitter.cc
2
3// This file is part of BES Netcdf File Out Module
4
5// Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact University Corporation for Atmospheric Research at
23// 3080 Center Green Drive, Boulder, CO 80301
24
25// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26// Please read the full copyright statement in the file COPYRIGHT_UCAR.
27//
28// Authors:
29// pwest Patrick West <pwest@ucar.edu>
30// jgarcia Jose Garcia <jgarcia@ucar.edu>
31// kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32
33#include "config.h"
34
35
36#ifdef HAVE_UNISTD_H
37#include <unistd.h>
38#endif
39
40#include <exception>
41#include <sstream> // std::stringstream
42#include <thread>
43#include <future>
44
45#include <libdap/D4Group.h>
46#include <libdap/D4Attributes.h>
47#include <libdap/BaseType.h>
48#include <libdap/escaping.h>
49
50#include <BESContextManager.h>
51#include <BESDataDDSResponse.h>
52#include <BESDapNames.h>
53#include <BESDataNames.h>
54#include <BESDebug.h>
55#include <BESUtil.h>
56#include <TempFile.h>
57
58
59#include <BESLog.h>
60#include <BESError.h>
61#include <BESDapError.h>
62#include <stringbuffer.h>
63
64#include "FONcBaseType.h"
65#include "FONcRequestHandler.h"
66#include "FONcTransmitter.h"
67#include "FONcTransform.h"
68
69using namespace libdap;
70using namespace std;
71using namespace rapidjson;
72
73#define MODULE "fonc"
74#define prolog string("FONcTransmitter::").append(__func__).append("() - ")
75
76
77#if 0 // Moved to BESUtil.cc
78// size of the buffer used to read from the temporary file built on disk and
79// send data to the client over the network connection (socket/stream)
80// #define OUTPUT_FILE_BLOCK_SIZE 4096
81#endif
82
96{
97 add_method(DATA_SERVICE, FONcTransmitter::send_dap2_data);
98 add_method(DAP4DATA_SERVICE, FONcTransmitter::send_dap4_data);
99}
100
101
119{
120 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
121
122 try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
123
124 // This object closes the file when it goes out of scope.
125 bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
126
127 BESDEBUG(MODULE, prolog << "Building response file " << temp_file.get_name() << endl);
128
129 ostream &strm = dhi.get_output_stream();
130 if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
131
132 BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file.get_name() << endl);
133
134 // Note that 'RETURN_CMD' is the same as the string that determines the file type:
135 // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
136 FONcTransform ft(obj, &dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
137
138#if 0
139 // This is used to signal the BESUtil::file_to_stream_task() this code is done
140 // writing to the file. WIP jhrg 6/4/21
141 atomic<bool> file_write_done(false);
142
143 // Calling the 'packaged_task' here blocks, but we could have run the task in a thread.
144 // See: https://stackoverflow.com/questions/18143661/what-is-the-difference-between-packaged-task-and-async
145 // jhrg 6/4/21
146
147 std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
148 std::future<uint64_t> result = task.get_future();
149 task(temp_file.get_name(), file_write_done, strm);
150#endif
151
152#define TOGGLE_TASK 0
153 // TOGGLE_TASK 1 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
154 // 151.13s user 8.75s system 98% cpu 2:41.69 total
155 // TOGGLE_TASK 0 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
156 // 154.71s user 8.99s system 99% cpu 2:45.27 total
157 // TOGGLE_TASK 0 as above, but using BESUtil::file_to_stream(temp_file.get_name(), strm);
158 // and not ESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
159 // 148.61s user 7.54s system 99% cpu 2:36.35 total
160#if TOGGLE_TASK
161 // This code works without the sleep(1) hack in BESUtil::file_to_stream_task().
162 // Because it is marked as deferred, the task does not start until the future's
163 // get() method is run, after transform() has written all the data. jhrg 6/4/21
164 future<uint64_t> result = async(launch::deferred, &BESUtil::file_to_stream_task, temp_file.get_name(),
165 std::ref(file_write_done), std::ref(strm));
166#endif
167 ft.transform_dap2(strm);
168
169#if 0
170 file_write_done = true;
171 uint64_t tcount = result.get();
172#endif
173
174 //original call before the 'task' hack was added:
175 //BESUtil::file_to_stream(temp_file.get_name(),strm);
176 //jhrg 6/4/21
177
178#if 0
179 // The task can be called like this right here
180 uint64_t tcount = BESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
181
182 // Or it can be run like this...
183 std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
184 std::future<uint64_t> result = task.get_future();
185 task(temp_file.get_name(), file_write_done, strm);
186 uint64_t tcount = result.get();
187#endif
188 //BESDEBUG(MODULE, prolog << "NetCDF file bytes written " << tcount << endl);
189 }
190 catch (Error &e) {
191 throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
192 }
193 catch (BESError &e) {
194 throw;
195 }
196 catch (std::exception &e) {
197 throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
198 }
199 catch (...) {
200 throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
201 }
202
203 BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
204}
205
224{
225 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
226
227 try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
228
229 // This object closes the file when it goes out of scope.
230 bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
231
232 BESDEBUG(MODULE, prolog << "Building response file " << temp_file.get_name() << endl);
233 // Note that 'RETURN_CMD' is the same as the string that determines the file type:
234 // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
235 // FONcTransform ft(loaded_dmr, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
236 FONcTransform ft(obj, &dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
237
238 // Call the transform function for DAP4.
239 ft.transform_dap4();
240
241 ostream &strm = dhi.get_output_stream();
242
243#if !NDEBUG
244 stringstream msg;
245 msg << prolog << "Using ostream: " << (void *) &strm << endl;
246 BESDEBUG(MODULE, msg.str());
247 INFO_LOG( msg.str());
248#endif
249
250 if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
251
252 BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file.get_name() << endl);
253
254 // FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
255 BESUtil::file_to_stream(temp_file.get_name(),strm);
256 }
257 catch (Error &e) {
258 throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
259 }
260 catch (BESError &e) {
261 throw;
262 }
263 catch (std::exception &e) {
264 throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
265 }
266 catch (...) {
267 throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
268 }
269
270 BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
271}
272
273
274
275
276
277
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
Abstract base class representing a specific set of information in response to a request to the BES.
static uint64_t file_to_stream_task(const std::string &file_name, std::atomic< bool > &file_write_done, std::ostream &o_strm)
Definition: BESUtil.cc:1399
static void file_to_stream(const std::string &file_name, std::ostream &o_strm)
Copies the contents of the file identified by file_name to the stream o_strm.
Definition: BESUtil.cc:1226
Transformation object that converts an OPeNDAP DataDDS to a netcdf file.
Definition: FONcTransform.h:61
virtual void transform_dap2(ostream &strm)
Transforms each of the variables of the DataDDS to the NetCDF file.
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
static void send_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
static void send_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
FONcTransmitter()
Construct the FONcTransmitter, adding it with name netcdf to be able to transmit a data response.
Get a new temporary file.
Definition: TempFile.h:46
std::string get_name() const
Definition: TempFile.h:70
main RapidJSON namespace