ndmspc 0.20250304.0
Loading...
Searching...
No Matches
PointRun.cxx
1#include <cstddef>
2#include <fstream>
3#include <iostream>
4#include <sstream>
5#include <string>
6#include <vector>
7#include <chrono>
8#include <thread>
9
10#include "TArrayD.h"
11#include <TFileMerger.h>
12#include <TString.h>
13#include <TMacro.h>
14#include <TROOT.h>
15#include <TH1.h>
16#include <TSystem.h>
17#include <THnSparse.h>
18#include "Core.h"
19#include "RtypesCore.h"
20#include "Utils.h"
21#include "PointRun.h"
22
24ClassImp(Ndmspc::PointRun);
26
27namespace Ndmspc {
28std::string PointRun::fgEnvironment = "";
29PointRun::PointRun(std::string macro) : TObject()
30{
34
35 TH1::AddDirectory(kFALSE);
36 fMacro = Utils::OpenMacro(macro.c_str());
37 if (fMacro) fMacro->Load();
38}
39
46
47bool PointRun::LoadConfig(std::string config, std::string userConfig, std::string environment,
48 std::string userConfigRaw, std::string binning, bool show, std::string outfilename)
49{
53
54 if (!Core::LoadConfig(config, userConfig, environment, userConfigRaw, binning)) return 1;
55
56 if (!gCfg["ndmspc"]["verbose"].is_null() && gCfg["ndmspc"]["verbose"].is_number_integer())
57 fVerbose = gCfg["ndmspc"]["verbose"].get<int>();
58
59 if (!gCfg["ndmspc"]["file"]["cache"].is_null() && gCfg["ndmspc"]["file"]["cache"].is_string()) {
60 std::string cacheDir = gCfg["ndmspc"]["file"]["cache"].get<std::string>();
61 if (!cacheDir.empty()) {
62 Printf("Setting cache directory to '%s' ...", gSystem->ExpandPathName(cacheDir.c_str()));
63 TFile::SetCacheFileDir(gSystem->ExpandPathName(cacheDir.c_str()), 1, 1);
64 }
65 }
66
67 if (show) Printf("%s", gCfg.dump(2).c_str());
68
69 if (!outfilename.empty()) {
70 std::ofstream file(outfilename.c_str());
71 file << gCfg;
72 Printf("Config saved to file '%s' ...", outfilename.c_str());
73 return false;
74 }
75 return true;
76}
77
78bool PointRun::Init(std::string extraPath)
79{
83
84 if (fVerbose >= 2) Printf("Ndmspc::PointRun::Init ...");
85 if (!gCfg["ndmspc"]["process"]["type"].get<std::string>().compare("all") &&
86 gCfg["ndmspc"]["process"]["ranges"].is_null() &&
87 !gCfg["ndmspc"]["output"]["delete"].get<std::string>().compare("onInit")) {
88
89 if (gCfg["ndmspc"]["output"]["host"].get<std::string>().empty()) {
90 gCfg["ndmspc"]["output"]["opt"] = "";
91 }
92
93 std::string outFileName;
94 if (!gCfg["ndmspc"]["output"]["dir"].get<std::string>().empty()) {
95 // outFileName = gCfg["ndmspc"]["output"]["host"].get<std::string>() + "/";
96
97 outFileName += gCfg["ndmspc"]["output"]["dir"].get<std::string>();
98 outFileName += "/";
99
100 std::string environment = gCfg["ndmspc"]["environment"].get<std::string>();
101 outFileName += environment + "/";
102
103 outFileName += Utils::GetCutsPath(gCfg["ndmspc"]["cuts"]);
104 // std::string rebinStr = "";
105 // for (auto & cut : gCfg["ndmspc"]["cuts"]) {
106 // Int_t rebin = 1;
107 // Int_t rebin_start = 1;
108 // Int_t rebin_minimum = 1;
109 // if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
110 // if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
111 // if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
112 //
113 // if (rebin_start > 1) {
114 // rebin_minimum = (rebin_start % rebin);
115 // }
116 // outFileName += cut["axis"].get<std::string>() + "_";
117 // rebinStr += std::to_string(rebin) + "-" + std::to_string(rebin_minimum) + "_";
118 // }
119 // outFileName[outFileName.size() - 1] = '/';
120 // rebinStr[rebinStr.size() - 1] = '/';
121 // outFileName += rebinStr;
122 //
123 outFileName += "bins";
124
125 if (!extraPath.empty()) {
126 outFileName += "/" + extraPath;
127 outFileName.pop_back();
128 }
129 }
130
131 if (!outFileName.empty()) {
132 if (gCfg["ndmspc"]["output"]["host"].is_string() &&
133 !gCfg["ndmspc"]["output"]["host"].get<std::string>().empty()) {
134
135 Printf("Deleting output eos directory '%s' ...", outFileName.c_str());
136 std::string rmUrl =
137 TString::Format("%s/proc/user/?mgm.cmd=rm&mgm.path=%s&mgm.option=rf&mgm.format=json&filetype=raw",
138 gCfg["ndmspc"]["output"]["host"].get<std::string>().c_str(), outFileName.c_str())
139 .Data();
140
141 if (fVerbose >= 2) Printf("rmUrl '%s' ...", rmUrl.c_str());
142 TFile * f = Ndmspc::Utils::OpenFile(rmUrl.c_str());
143 if (!f) return 1;
144 Printf("Directory '%s' deleted", outFileName.c_str());
145 f->Close();
146 }
147 else {
148 Printf("Directory '%s' deleted", outFileName.c_str());
149 gSystem->Exec(TString::Format("rm -rf %s", outFileName.c_str()));
150 }
151 }
152 // gCfg["ndmspc"]["output"]["delete"] = "";
153 }
154 if (fVerbose >= 2) Printf("Ndmspc::PointRun::Init done ...");
155
156 return true;
157}
158
160{
164
165 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::OpenInputs");
166
167 if (fInputFile && fInputList) return fInputList;
168
169 if (gCfg["ndmspc"]["data"]["file"].get<std::string>().empty()) {
170 Printf("Error: Input file is empty !!! Aborting ...");
171 return nullptr;
172 }
173 if (fVerbose >= 0) Printf("Opening file '%s' ...", gCfg["ndmspc"]["data"]["file"].get<std::string>().c_str());
174 fInputFile = Ndmspc::Utils::OpenFile(gCfg["ndmspc"]["data"]["file"].get<std::string>().c_str());
175 if (!fInputFile) {
176 Printf("Error: Cannot open file '%s' !", gCfg["ndmspc"]["data"]["file"].get<std::string>().c_str());
177 return nullptr;
178 }
179
180 if (!fInputList) fInputList = new TList();
181
182 THnSparse *s, *stmp;
183 for (auto & obj : gCfg["ndmspc"]["data"]["objects"]) {
184 if (obj.get<std::string>().empty()) continue;
185
186 std::string dirName;
187 if (!gCfg["ndmspc"]["data"]["directory"].is_null() && gCfg["ndmspc"]["data"]["directory"].is_string())
188 dirName = gCfg["ndmspc"]["data"]["directory"].get<std::string>();
189
190 std::stringstream srcfull(obj.get<std::string>().c_str());
191
192 std::string srcName, sparseName;
193
194 getline(srcfull, srcName, ':');
195 getline(srcfull, sparseName, ':');
196 if (fVerbose >= 2) Printf("srcName=%s customName=%s", srcName.c_str(), sparseName.c_str());
197
198 std::stringstream src(srcName.c_str());
199 std::string item;
200
201 s = nullptr;
202 while (getline(src, item, '+')) {
203
204 std::string objName;
205 if (!dirName.empty()) objName = dirName + "/";
206 objName += item;
207 if (fVerbose >= 1) Printf("Opening obj='%s' ...", objName.c_str());
208 if (s == nullptr) {
209
210 s = (THnSparse *)fInputFile->Get(objName.c_str());
211 if (s == nullptr) {
212 if (fVerbose >= 1) Printf("Warning: Cannot open object '%s' !!!", objName.c_str());
213 continue;
214 }
215
216 if (s && !sparseName.empty()) s->SetName(sparseName.c_str());
217 }
218 else {
219 if (fVerbose >= 1) Printf("Adding obj='%s' ...", objName.c_str());
220 stmp = (THnSparse *)fInputFile->Get(objName.c_str());
221 if (stmp == nullptr) {
222 if (fVerbose >= 1) Printf("Warning: Cannot open object '%s' !!!", objName.c_str());
223 continue;
224 }
225 if (s) s->Add(stmp);
226 }
227 }
228 if (s) {
229 fInputList->Add(s);
230 }
231 else {
232 if (fVerbose >= 1)
233 Printf("Warning : Could not open '%s' from file '%s' !!! Skipping ...", obj.get<std::string>().c_str(),
234 gCfg["ndmspc"]["data"]["file"].get<std::string>().c_str());
235 // return nullptr;
236 }
237 }
238
239 TFunction * fun = gROOT->GetGlobalFunction("NdmspcOpenInputsUser", nullptr, kTRUE);
240 if (fun) {
241 gROOT->ProcessLine(TString::Format("NdmspcOpenInputsUser((Ndmspc::PointRun*)%p),", this));
242 }
243
244 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::OpenInputs");
245
246 return fInputList;
247}
248
250{
254
255 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::CreateResult fResultObject=%p", (void *)fResultObject);
256
257 if (fResultObject) return fResultObject;
258
259 THnSparse * s = (THnSparse *)fInputList->At(0);
260
261 int nDimsParams = 0;
262 int nDimsCuts = 0;
263 int nDimsProccess = fCurrentProcessHistogramAxes.size();
264
265 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
266 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
267 nDimsCuts++;
268 }
269
270 nDimsParams += gCfg["ndmspc"]["result"]["axes"].size();
271
272 const Int_t ndims = nDimsCuts + nDimsParams + nDimsProccess + 1;
273 Int_t bins[ndims];
274 Double_t xmin[ndims];
275 Double_t xmax[ndims];
276 std::vector<std::string> names;
277 std::vector<std::string> titles;
278 std::vector<TAxis *> cutAxes;
279 fMapAxesType = new TH1S("mapAxesType", "Type Axes Map", ndims, 0, ndims);
280 // _currentOutputRootDirectory->Add
281
282 Int_t nParameters = gCfg["ndmspc"]["result"]["parameters"]["labels"].size();
283 if (nParameters <= 0) return nullptr;
284
285 Int_t i = 0;
286
287 bins[i] = nParameters;
288 xmin[i] = 0;
289 xmax[i] = nParameters;
290 names.push_back("parameters");
291 titles.push_back("parameters");
292 fMapAxesType->GetXaxis()->SetBinLabel(i + 1, "par");
293
294 i++;
295
296 // cfg["ndmspc"]["output"]["post"]
297 int iTmp = 0;
298 for (auto & a : fCurrentProcessHistogramAxes) {
299 // a->Print();
300 bins[i] = a->GetNbins();
301 xmin[i] = a->GetXmin();
302 xmax[i] = a->GetXmax();
303 // TODO: handle variable binning
304
305 names.push_back(a->GetName());
306 std::string t = a->GetTitle();
307 if (t.empty()) t = a->GetName();
308 titles.push_back(t);
309 if (iTmp < 2)
310 fMapAxesType->GetXaxis()->SetBinLabel(i + 1, "data");
311 else
312 fMapAxesType->GetXaxis()->SetBinLabel(i + 1, "sys-out");
313 iTmp++;
314 i++;
315 }
316
317 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
318 if (fVerbose >= 3) std::cout << "CreateResult() : " << cut.dump() << std::endl;
319
320 Int_t rebin = 1;
321 Int_t rebin_start = 1;
322 Int_t rebin_minimum = 1;
323 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
324 if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
325 if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
326
327 if (rebin_start > 1) {
328 rebin_minimum = (rebin_start % rebin);
329 }
330
331 TAxis * a = (TAxis *)s->GetListOfAxes()->FindObject(cut["axis"].get<std::string>().c_str());
332 if (a == nullptr) return nullptr;
333
334 if (rebin > 1) {
335 a = (TAxis *)a->Clone();
336 Int_t rebins_max = a->GetNbins() / rebin;
337 if ((a->GetNbins() - rebin_minimum + 1) % rebin == 0) rebins_max++;
338 Double_t arr[rebins_max];
339
340 // Printf("Axis '%s' : rebin_max=%d rebin_minimum=%d nbins=%d", a->GetName(), rebins_max, rebin_minimum,
341 // a->GetNbins());
342 Int_t count = 0;
343 Int_t i;
344 for (i = rebin_minimum; i <= a->GetNbins(); i += rebin) {
345 // Printf("%d %f", i, a->GetBinUpEdge(i));
346 arr[count++] = a->GetBinLowEdge(i);
347 }
348 // Printf("%s %d %d", a->GetName(), count, rebins_max);
349 if (count < rebins_max) arr[count++] = a->GetBinLowEdge(i);
350 // for (Int_t i = 0; i < count; i++) {
351 // Printf("%s %d %f", a->GetName(), i, arr[i]);
352 // }
353 a->Set(count - 1, arr);
354 // Printf("Axis '%s' : %d", a->GetName(), a->GetNbins());
355 }
356 bins[i] = a->GetNbins();
357 xmin[i] = a->GetXmin();
358 xmax[i] = a->GetXmax();
359
360 cutAxes.push_back(a);
361 names.push_back(a->GetName());
362 std::string t = a->GetTitle();
363 if (t.empty()) t = a->GetName();
364 titles.push_back(t);
365 fMapAxesType->GetXaxis()->SetBinLabel(i + 1, "proj");
366 i++;
367 }
368 // exit(1);
369
370 for (auto & value : gCfg["ndmspc"]["result"]["axes"]) {
371 if (!value["labels"].is_null()) {
372 bins[i] = value["labels"].size();
373 xmin[i] = 0;
374 xmax[i] = value["labels"].size();
375 }
376 else if (!value["ranges"].is_null()) {
377 bins[i] = value["ranges"].size();
378 xmin[i] = 0;
379 xmax[i] = value["ranges"].size();
380 }
381 else {
382 Printf("Error: 'labels' or 'ranges' is missing !!!");
383 return nullptr;
384 }
385
386 names.push_back(value["name"].get<std::string>().c_str());
387 titles.push_back(value["name"].get<std::string>().c_str());
388 fMapAxesType->GetXaxis()->SetBinLabel(i + 1, "sys-in");
389 i++;
390 }
391
392 fCurrentPointLabels = names;
393
394 THnSparse * fres = new THnSparseD("results", "Final results", i, bins, xmin, xmax);
395 if (!fres) {
396 Printf("Error: Could not create 'results' THnSparse object !!!");
397 return nullptr;
398 }
399 int iAxis = 0;
400 TAxis * a = fres->GetAxis(iAxis);
401 if (!a) {
402 Printf("Error: 'parameters' axis was not found !!!");
403 return nullptr;
404 }
405 int iLablel = 1;
406 for (auto & n : gCfg["ndmspc"]["result"]["parameters"]["labels"]) {
407 a->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
408 a->SetBinLabel(iLablel++, n.get<std::string>().c_str());
409 }
410
411 iAxis++;
412
413 // cfg["ndmspc"]["output"]["post"]
414 // i = nDimsCuts + iPar + 1;
415 int iLabel;
416 int iPIdx = 0;
417 for (auto & axis : fCurrentProcessHistogramAxes) {
418 if (axis->GetXbins()->GetArray()) fres->GetAxis(iAxis)->Set(axis->GetNbins(), axis->GetXbins()->GetArray());
419 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
420
421 for (iLabel = 1; iLabel < fCurrentProccessHistogram->GetAxis(iPIdx)->GetNbins() + 1; iLabel++) {
422 std::string l = fCurrentProccessHistogram->GetAxis(iPIdx)->GetBinLabel(iLabel);
423 if (!l.empty()) fres->GetAxis(iAxis)->SetBinLabel(iLabel, l.c_str());
424 }
426 iPIdx++;
427 iAxis++;
428 }
429
430 // i = 1;
431 for (auto & a : cutAxes) {
432 // Printf("%s", )
433 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
434 if (a->GetXbins()->GetArray()) fres->GetAxis(iAxis)->Set(a->GetNbins(), a->GetXbins()->GetArray());
435 iAxis++;
436 }
437 int iPar = 0;
438 // int iLabel;
439 // for (auto & [key, value] : cfg["ndmspc"]["result"].items()) {
440 for (auto & value : gCfg["ndmspc"]["result"]["axes"]) {
441 iPar++;
442 iLabel = 1;
443 if (!value["labels"].is_null()) {
444 for (auto & n : value["labels"]) {
445 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
446 fres->GetAxis(iAxis)->SetBinLabel(iLabel++, n.get<std::string>().c_str());
447 }
448 }
449 iLabel = 1;
450 if (!value["ranges"].is_null()) {
451 for (auto & n : value["ranges"]) {
452 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
453 fres->GetAxis(iAxis)->SetBinLabel(iLabel++, n["name"].get<std::string>().c_str());
454 }
455 }
456 iAxis++;
457 }
458
459 if (fVerbose >= 2) fres->Print("all");
460 // return nullptr;
461
462 // TODO! port fres to fResultObject
463 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::CreateResult fResultObject=%p", (void *)fres);
464 return fres;
465}
466
468{
472
473 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::ApplyCuts");
474
476 TString titlePostfix = "";
477 THnSparse * s;
478
479 Int_t iCut = 0;
480 Int_t rebin = 1;
481 Int_t rebin_start = 1;
482
483 fCurrentPoint[iCut] = 0;
484 for (Long64_t i = 0; i < fInputList->GetEntries(); i++) {
485 s = (THnSparse *)fInputList->At(i);
486 iCut = 1;
487 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
488
489 if (cut.is_null()) continue;
490
491 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
492
493 if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
494 if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
495
496 if (cut["axis"].is_string() && cut["axis"].get<std::string>().empty()) {
497 std::cerr << "Error: Axis name is empty ('" << cut << "') !!! Exiting ..." << std::endl;
498 return false;
499 }
500 if (cut["bin"]["min"].get<Int_t>() < 0 || cut["bin"]["max"].get<Int_t>() < 0) {
501 std::cerr << "Error: Bin min or max is less then 0 ('" << cut << "') !!! Exiting ..." << std::endl;
502 return false;
503 }
504
505 Int_t id = s->GetListOfAxes()->IndexOf(s->GetListOfAxes()->FindObject(cut["axis"].get<std::string>().c_str()));
506 if (id == s->GetListOfAxes()->GetEntries()) {
507 Printf("Axis '%s' was not found !!! Skipping ...", cut["axis"].get<std::string>().c_str());
508 return false;
509 }
510
511 Int_t binLocal = Utils::GetBinFromBase(cut["bin"]["min"].get<int>(), rebin, rebin_start);
512 fCurrentPoint[iCut + fCurrentProcessHistogramPoint.size()] = binLocal;
513
514 Int_t binMin = cut["bin"]["min"].get<Int_t>();
515 Int_t binMax = cut["bin"]["max"].get<Int_t>();
516 // Ndmspc::Utils::RebinBins(binMin, binMax, rebin);
517 // Int_t binDiff = cut["bin"]["max"].get<Int_t>() - cut["bin"]["min"].get<Int_t>() + 1;
518 // Int_t binMin = binLocal;
519 // Int_t binMax = binLocal + rebin * binDiff - 1;
520 if (fVerbose >= 2)
521 Printf("cut=%s binLocal=%d binMin=%d binMax=%d", cut["axis"].get<std::string>().c_str(), binLocal, binMin,
522 binMax);
523 s->GetAxis(id)->SetRange(binMin, binMax);
524
525 if (i == 0) {
526 if (s->GetAxis(id)->IsAlphanumeric()) {
527 titlePostfix +=
528 TString::Format("%s[%s bin=%d] ", s->GetAxis(id)->GetName(), s->GetAxis(id)->GetBinLabel(binMin), binMin);
529 }
530 else {
531 titlePostfix += TString::Format("%s[%.2f,%.2f] ", s->GetAxis(id)->GetName(),
532 s->GetAxis(id)->GetBinLowEdge(binMin), s->GetAxis(id)->GetBinUpEdge(binMax));
533 }
534 }
535 iCut++;
536 }
537 }
538 if (!titlePostfix.IsNull()) {
539 titlePostfix.Remove(titlePostfix.Length() - 1);
540 Printf("Processing '%s' ...", titlePostfix.Data());
541 gCfg["ndmspc"]["projection"]["title"] = titlePostfix.Data();
542 }
543
544 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ApplyCuts");
545 return true;
546}
548{
552
553 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::ProcessSinglePoint");
554
555 fIsProcessOk = false;
556
557 if (!ApplyCuts()) return false;
558
559 fIsSkipBin = false;
560
561 if (fResultObject != nullptr) {
562 delete fResultObject;
563 fResultObject = nullptr;
564 }
566 // fResultObject->Print();
567
568 json resultAxes = gCfg["ndmspc"]["result"]["axes"];
569
570 std::vector<std::string> names;
571 for (auto & value : resultAxes) {
572 std::string n = value["name"].get<std::string>();
573 names.push_back(n.c_str());
574 }
575
576 // OutputFileOpen(cfg);
577
578 if (fVerbose >= 1) Printf("Starting User Process() ...");
579 fBinCount++;
580 ProcessRecursiveInner(resultAxes.size() - 1, names);
581
582 if (fVerbose >= 1) Printf("User Process() done ...");
583
584 // Store add fResultObject to final file
586
587 if (!fIsProcessOk) {
588 if (fVerbose >= 0) Printf("Skipping ...");
589 }
590
591 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ProcessSinglePoint");
592 return true;
593}
595{
599
600 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::ProcessRecursive[%d]", i);
601
602 if (i < 0) {
603 return ProcessSinglePoint();
604 }
605
606 THnSparse * s = (THnSparse *)fInputList->At(0);
607 TAxis * a = (TAxis *)s->GetListOfAxes()->FindObject(gCfg["ndmspc"]["cuts"][i]["axis"].get<std::string>().c_str());
608 if (a == nullptr) {
609 Printf("Error: Axis canot be found");
610 return false;
611 }
612 Int_t start = 1;
613 Int_t end = a->GetNbins();
614 Int_t rebin = 1;
615 Int_t rebin_start = 1;
616 Int_t rebin_minimum = 1;
617 if (gCfg["ndmspc"]["cuts"][i]["rebin"].is_number_integer()) rebin = gCfg["ndmspc"]["cuts"][i]["rebin"].get<Int_t>();
618 if (gCfg["ndmspc"]["cuts"][i]["rebin_start"].is_number_integer())
619 rebin_start = gCfg["ndmspc"]["cuts"][i]["rebin_start"].get<Int_t>();
620
621 gCfg["ndmspc"]["cuts"][i]["rebin_minimum"] = rebin_minimum;
622 if (rebin > 1) end /= rebin;
623 if (rebin_start > 1) {
624 rebin_minimum = (rebin_start % rebin);
625 gCfg["ndmspc"]["cuts"][i]["rebin_minimum"] = rebin_minimum;
626 start = (rebin_start / rebin) + 1;
627 end = (a->GetNbins() - rebin_minimum + 1) / rebin;
628 // Printf("%s start=%d end=%d rebin=%d nbins=%d rebin_start=%d rebin_minimum=%d", a->GetName(), start, end, rebin,
629 // a->GetNbins(), rebin_start, rebin_minimum);
630 // exit(1);
631 }
632
633 if (gCfg["ndmspc"]["process"]["ranges"].is_array()) {
634 int range_min = gCfg["ndmspc"]["process"]["ranges"][i][0].get<int>();
635 int range_max = gCfg["ndmspc"]["process"]["ranges"][i][1].get<int>();
636 if (range_max > end || range_min < start || range_min > range_max || range_min > end || range_max < start) {
637 Printf("Error: Process range is out of bounds histogram(after rebin)=[%d,%d] request=[%d,%d] or requested min is "
638 "higher then requested max !!!",
639 start, end, range_min, range_max);
640 gSystem->Exit(1);
641 }
642 start = range_min;
643 if (gCfg["ndmspc"]["process"]["ranges"][i][1] < end) end = range_max;
644 // TODO: Handle rebin and rebin_start
645 }
646
647 for (Int_t iBin = start; iBin <= end; iBin++) {
648 Int_t binMin = (iBin - 1) * rebin + rebin_minimum;
649 Int_t binMax = (iBin * rebin) - 1 + rebin_minimum;
650 if (fVerbose >= 2)
651 Printf("axis=%s rebin=%d rebin_minimum=%d binMin=%d binMax=%d [%f,%f]", a->GetName(), rebin, rebin_minimum,
652 binMin, binMax, a->GetBinLowEdge(binMin), a->GetBinUpEdge(binMax));
653 gCfg["ndmspc"]["cuts"][i]["bin"]["min"] = binMin;
654 gCfg["ndmspc"]["cuts"][i]["bin"]["max"] = binMax;
655 ProcessRecursive(i - 1);
656 }
657
658 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ProcessRecursive[%d]", i);
659
660 return true;
661}
662bool PointRun::ProcessRecursiveInner(Int_t i, std::vector<std::string> & n)
663{
667
668 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::ProcessRecursiveInner[%d]", i);
669
670 if (fIsSkipBin) return false;
671
672 if (!fResultObject) {
673 fIsProcessExit = true;
674 return false;
675 }
676
677 if (fIsProcessExit) gSystem->Exit(1);
678
679 if (i < 0) {
680 TList * outputList = new TList();
681 SetOutputList(outputList);
682 if (fBinCount == 1 && fVerbose >= 0 && !fCurrentPointValue.is_null())
683 Printf("\tPoint: %s", fCurrentPointValue.dump().c_str());
684
685 // TODO! Apply TMacro
686 if (fVerbose >= 2) Printf("Running point macro '%s.C' ...", fMacro->GetName());
687 /*fMacro.Exec();*/
688
689 {
690 /*TRedirectOutputGuard g{"/dev/null"};*/
691 /*Longptr_t ok = fMacro->Exec(TString::Format("(Ndmspc::PointRun*)%p", this));*/
692 /*Longptr_t ok = fMacro->Exec(TString::Format("(Ndmspc::PointRun*)%p", this));*/
693 Longptr_t ok = gROOT->ProcessLine(TString::Format("%s((Ndmspc::PointRun*)%p);", fMacro->GetName(), this));
694 /*fMacro.Exec(TString::Format("(TList*)%p,(json*)%p", fInputList, &gCfg));*/
695
696 /*fMacro.Exec(TString::Format("(TList*)%ld,(json&)%p,(THnSparse "*/
697 /* "*)%ld,(int*)%ld,(std::vector<std::string>*)%ld,(json*)%ld,(TList*)%ld,0,0",*/
698 /* (Longptr_t)fInputList, &gCfg, (Longptr_t)fResultObject,*/
699 /* (Longptr_t)fCurrentPoint, (Longptr_t)&fCurrentPointLabels,*/
700 /* (Longptr_t)&fCurrentPointValue, (Longptr_t)outputList)*/
701 /* .Data());*/
702 /*bool ok = NdmspcPointMacro(_currentInputList, cfg, fResultObject, _currentPoint, _currentPointLabels,*/
703 /* _currentPointValue, outputList, _currentSkipBin, _currentProcessExit);*/
704 /*gSystem->Exit(0);*/
705 if (ok && fVerbose >= 5) outputList->Print();
706 if (ok) {
707 fIsProcessOk = true;
708 }
709 else {
710 return false;
711 }
712 }
713 if (fCurrentOutputFile == nullptr) {
715 }
716
717 TDirectory * outputDir = fCurrentOutputRootDirectory;
718
719 // int iPoint = 1;
720 std::string path;
721 for (int iPoint = 1; iPoint < fResultObject->GetNdimensions(); iPoint++) {
722 path += std::to_string(fCurrentPoint[iPoint]) + "/";
723 }
724
725 // if (gCfg["ndmspc"]["output"]["post"].is_string()) {
726 // std::string post = gCfg["ndmspc"]["output"]["post"].get<std::string>();
727 // if (!post.empty()) {
728 // path += post;
729 // }
730 // }
731
732 // Printf("path='%s'", path.c_str());
733
734 fCurrentOutputRootDirectory->mkdir(path.c_str(), "", true);
735 outputDir = fCurrentOutputRootDirectory->GetDirectory(path.c_str());
736
737 outputDir->cd();
738 outputList->Write();
739 return true;
740 }
741
742 std::string axisName = gCfg["ndmspc"]["result"]["axes"][i]["name"].get<std::string>();
743 if (!gCfg["ndmspc"]["result"]["axes"][i]["labels"].is_null()) {
744 for (auto & v : gCfg["ndmspc"]["result"]["axes"][i]["labels"]) {
745 TAxis * a = (TAxis *)fResultObject->GetListOfAxes()->FindObject(axisName.c_str());
746 Int_t id = fResultObject->GetListOfAxes()->IndexOf(a);
747 fCurrentPoint[id] = a->FindBin(v.get<std::string>().c_str());
748 fCurrentPointValue[axisName] = v;
749 fCurrentPointLabels[id] = v.get<std::string>().c_str();
750 ProcessRecursiveInner(i - 1, n);
751 }
752 }
753 else if (!gCfg["ndmspc"]["result"]["axes"][i]["ranges"].is_null()) {
754 for (auto & v : gCfg["ndmspc"]["result"]["axes"][i]["ranges"]) {
755 TAxis * a = (TAxis *)fResultObject->GetListOfAxes()->FindObject(axisName.c_str());
756 Int_t id = fResultObject->GetListOfAxes()->IndexOf(a);
757 fCurrentPoint[id] = a->FindBin(v["name"].get<std::string>().c_str());
758 fCurrentPointValue[axisName] = v;
759 fCurrentPointLabels[id] = v["name"].get<std::string>().c_str();
760 ProcessRecursiveInner(i - 1, n);
761 }
762 }
763 else {
764 Printf("Error: ProcessRecursiveInner : No 'labels' or 'ranges' !!!");
765 return false;
766 }
767 return true;
768
769 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ProcessRecursiveInner[%d]", i);
770 return true;
771}
773{
777
778 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::OutputFileOpen");
779
780 // TString outputFileName;
781
783
784 if (!gCfg["ndmspc"]["output"]["host"].get<std::string>().empty())
785 fCurrentOutputFileName += gCfg["ndmspc"]["output"]["host"].get<std::string>().c_str();
786 if (!gCfg["ndmspc"]["output"]["dir"].get<std::string>().empty())
787 fCurrentOutputFileName += gCfg["ndmspc"]["output"]["dir"].get<std::string>().c_str();
788
789 if (gCfg["ndmspc"]["cuts"].is_array() && !fCurrentOutputFileName.empty()) {
790
791 // std::string axisName;
792 // std::string rebinStr = "";
793 // // cfgOutput["ndmspc"]["cuts"] = gCfg["ndmspc"]["cuts"];
794 // for (auto & cut : gCfg["ndmspc"]["cuts"]) {
795 // if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
796 // Int_t rebin = 1;
797 // Int_t rebin_start = 1;
798 // Int_t rebin_minimum = 1;
799 //
800 // if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
801 // if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
802 //
803 // if (rebin_start > 1) {
804 // rebin_minimum = (rebin_start % rebin);
805 // }
806 //
807 // if (axisName.length() > 0) {
808 // axisName += "_";
809 // rebinStr += "_";
810 // }
811 // axisName += cut["axis"].get<std::string>();
812 // rebinStr += std::to_string(rebin);
813 // rebinStr += "-";
814 // rebinStr += std::to_string(rebin_minimum);
815 // }
816 std::string cutsName = Utils::GetCutsPath(gCfg["ndmspc"]["cuts"]);
817 // Printf("cutsName='%s'", cutsName.c_str());
818 // exit(1);
819
820 if (cutsName.length() > 0) {
822 fCurrentOutputFileName += gCfg["ndmspc"]["environment"].get<std::string>().c_str();
824 fCurrentOutputFileName += cutsName.c_str();
825 // fCurrentOutputFileName += "/";
826 fCurrentOutputFileName += "bins";
828
829 // TODO: check what is it used for. Remove it if not needed
830 if (gCfg["ndmspc"]["output"]["post"].is_string()) {
831 std::string post = gCfg["ndmspc"]["output"]["post"].get<std::string>();
832 if (!post.empty()) {
835 // _currentOutputFileName += "/bins/" + post;
837 }
838 }
839
840 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
841 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
842 Int_t rebin = 1;
843 Int_t rebin_start = 1;
844 Int_t rebin_minimum = 1;
845
846 if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
847 if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
848
849 if (rebin_start > 1) {
850 rebin_minimum = (rebin_start % rebin);
851 if (rebin_minimum == 0) rebin_minimum = 1;
852 }
853
854 Int_t bin_min = cut["bin"]["min"].get<Int_t>();
855 // Printf("bin_min=%d rebin_start=%d rebin=%d %d", bin_min, rebin_start, rebin,
856 // (bin_min - rebin_minimum) / rebin + 1);
857 Int_t bin_min_converted = (bin_min - rebin_minimum) / rebin + 1;
858 fCurrentOutputFileName += std::to_string(bin_min_converted) + "/";
859 }
860 }
861 }
862 else {
863 fCurrentOutputFileName += gCfg["ndmspc"]["environment"].get<std::string>().c_str();
865 }
866
869
870 if (!gCfg["ndmspc"]["output"]["file"].get<std::string>().empty())
871 fCurrentOutputFileName += gCfg["ndmspc"]["output"]["file"].get<std::string>().c_str();
872
873 fCurrentOutputFileName = gSystem->ExpandPathName(fCurrentOutputFileName.c_str());
875 Ndmspc::Utils::OpenFile(TString::Format("%s%s", fCurrentOutputFileName.c_str(),
876 gCfg["ndmspc"]["output"]["opt"].get<std::string>().c_str())
877 .Data(),
878 "RECREATE");
879 // _currentOutputFile->cd();
880
881 fCurrentOutputFile->mkdir("content");
882 fCurrentOutputRootDirectory = fCurrentOutputFile->GetDirectory("content");
884
885 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::OutputFileOpen");
886}
888{
892
893 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::OutputFileClose");
894
895 if (fCurrentOutputFile == nullptr) return;
896
897 if (fVerbose >= 2) Printf("Closing file '%s' ...", fCurrentOutputFileName.c_str());
899
900 fCurrentOutputFile->cd();
901 fResultObject->Write();
902 fMapAxesType->Write();
903 fCurrentOutputFile->Close();
904
905 fCurrentOutputFile = nullptr;
907
908 if (fVerbose >= 0) Printf("Objects stored in '%s'", fCurrentOutputFileName.c_str());
909
910 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::OutputFileClose");
911}
913{
917
918 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::Finish");
919 if (fInputList) {
920 fInputList->Clear();
921 delete fInputList;
922 fInputList = nullptr;
923 }
924
925 if (fInputFile) {
926 fInputFile->Close();
927 delete fInputFile;
928 fInputFile = nullptr;
929 }
930 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::Finish");
931
932 return true;
933}
935{
939
940 std::string type;
941 if (gCfg["ndmspc"]["process"]["type"].is_string()) type = gCfg["ndmspc"]["process"]["type"].get<std::string>();
942
943 if (type.empty()) {
944 Printf("Warning: [ndmspc][process][type] is missing or is empty in configuration !!! Setting it ot 'single' ...");
945 type = "single";
946 gCfg["ndmspc"]["process"]["type"] = type;
947 }
948
949 TList * inputList = OpenInputs();
950 if (inputList == nullptr) return 1;
951
952 if (!type.compare("single")) {
953 if (!ProcessSinglePoint()) return 3;
954 }
955 else if (!type.compare("all")) {
956 json cuts;
957 int iCut = 0;
958 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
959 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
960 cuts[iCut++] = cut;
961 }
962 gCfg["ndmspc"]["cuts"] = cuts;
963
964 ProcessRecursive(gCfg["ndmspc"]["cuts"].size() - 1);
965 }
966 else {
967 Printf("Error: Value [process][type]='%s' is not supported !!! Exiting ...", type.c_str());
968 return 4;
969 }
970 Finish();
971
972 /*/// TODO! Handle failure*/
973 /*Init();*/
974 /*/// TODO! Handle failure*/
975 /*OpenInputs();*/
976
977 /*Printf("Running point macro '%s.C' ...", fMacro.GetName());*/
978 /*fMacro.Exec();*/
979 return 0;
980}
982{
986
987 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ProcessHistogramRun");
988
989 std::string fileNameHistogram = gCfg["ndmspc"]["data"]["histogram"]["file"].get<std::string>();
990 std::string objName = gCfg["ndmspc"]["data"]["histogram"]["obj"].get<std::string>();
991
992 TFile * fProccessHistogram = Ndmspc::Utils::OpenFile(fileNameHistogram.c_str());
993 if (!fProccessHistogram) {
994 Printf("Error: Proccess input histogram file '%s' could not opened !!!", fileNameHistogram.c_str());
995 return 1;
996 }
997 fCurrentProccessHistogram = (THnSparse *)fProccessHistogram->Get(objName.c_str());
999 Printf("Error: Proccess input histogram object '%s' could not opened !!!", objName.c_str());
1000 return 1;
1001 }
1002 TObjArray * axesArray = fCurrentProccessHistogram->GetListOfAxes();
1003 for (int iAxis = 0; iAxis < axesArray->GetEntries(); iAxis++) {
1004
1005 TAxis * aTmp = (TAxis *)axesArray->At(iAxis);
1006 // Printf("axis=%s", aTmp->GetName());
1007 fCurrentProcessHistogramAxes.push_back(aTmp);
1008 }
1009
1010 if (gCfg["ndmspc"]["data"]["histogram"]["bins"].is_array()) {
1011
1012 for (auto & v : gCfg["ndmspc"]["data"]["histogram"]["bins"]) {
1013 Printf("%s", v.dump().c_str());
1014 int i = 0;
1015 Int_t p[v.size()];
1016 for (auto & idx : v) {
1017 p[i] = idx;
1018 // _currentProcessHistogramPoint.push_back(idx);
1019 i++;
1020 }
1021 fCurrentProccessHistogram->SetBinContent(p, 1);
1022 }
1023 }
1024 /*fCurrentProccessHistogram->Print();*/
1025 if (fCurrentProccessHistogram->GetNbins()) {
1026 Int_t proccessPoint[fCurrentProccessHistogram->GetNdimensions()];
1027 for (int iBin = 0; iBin < fCurrentProccessHistogram->GetNbins(); iBin++) {
1029 fCurrentProccessHistogram->GetBinContent(iBin, proccessPoint);
1030 // Printf("iBin=%d %d %d", iBin, proccessPoint[0], proccessPoint[1]);
1031
1032 std::string path;
1033 for (auto & p : proccessPoint) {
1034 // printf("%d ", p);
1035 fCurrentProcessHistogramPoint.push_back(p);
1036 path += std::to_string(std::abs(p)) + "/";
1037 }
1038 // printf("\n");
1039 std::string fullPath = gCfg["ndmspc"]["data"]["histogram"]["base"].get<std::string>();
1040 fullPath += "/";
1041 fullPath += path;
1042 fullPath += gCfg["ndmspc"]["data"]["histogram"]["filename"].get<std::string>();
1043 // Printf("Path: %s %s", path.c_str(), fullPath.c_str());
1044 gCfg["ndmspc"]["data"]["file"] = fullPath;
1045
1046 gCfg["ndmspc"]["output"]["post"] = path;
1047
1048 // return 0;
1049 Init(path);
1050 Int_t rc = ProcessSingleFile();
1051 if (rc) {
1052 return rc;
1053 }
1054 }
1055 }
1056 else {
1057 Printf("Error: No entries in proccess histogram !!! Nothing to process !!!");
1058 return 1;
1059 }
1060
1061 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::ProcessHistogramRun");
1062 return 0;
1063}
1064bool PointRun::Run(std::string filename, std::string userConfig, std::string environment, std::string userConfigRaw,
1065 std::string binning, bool show, std::string outfilename)
1066{
1067
1071
1072 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::Run");
1073
1074 if (!fMacro) return 1;
1075
1076 if (!LoadConfig(filename, userConfig, environment, userConfigRaw, binning, show, outfilename)) return false;
1077 /*fVerbose = 2;*/
1078
1079 if (!gCfg["ndmspc"]["data"]["histogram"].is_null() && !gCfg["ndmspc"]["data"]["histogram"]["enabled"].is_null() &&
1080 gCfg["ndmspc"]["data"]["histogram"]["enabled"].get<bool>() == true) {
1082 }
1083 else {
1085 }
1086 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::Run");
1087 return true;
1088}
1089
1090bool PointRun::GenerateJobs(std::string jobs, std::string filename, std::string userConfig, std::string environment,
1091 std::string userConfigRaw, std::string outfilename, std::string binnings)
1092{
1096
1097 if (fVerbose >= 2) Printf("[<-] Ndmspc::PointRun::GenerateJobs");
1098
1099 if (!fMacro) return 1;
1100
1101 if (!LoadConfig(filename, userConfig, environment, userConfigRaw, "", true)) return false;
1102
1103 if (outfilename[outfilename.size() - 1] == '/') outfilename.pop_back();
1104
1105 Printf("Generating jobs with split '%s' with binnings '%s' to %s ...", jobs.c_str(), binnings.c_str(),
1106 outfilename.c_str());
1107
1108 gCfg["ndmspc"]["process"]["type"] = "all";
1109
1110 std::vector<std::string> jobsArray = Utils::Tokenize(jobs.c_str(), ':');
1111 std::vector<std::vector<int>> cutBins;
1112
1113 std::vector<std::string> binningsArray = Utils::Tokenize(binnings.c_str(), ',');
1114
1115 std::string binningFirst = "";
1116 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
1117 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
1118 if (!binningFirst.empty()) {
1119 binningFirst += "_";
1120 }
1121
1122 Int_t nbins = cut["nbins"].get<Int_t>();
1123
1124 binningFirst += std::to_string(nbins) + "-1";
1125 }
1126 if (binningFirst.empty()) {
1127 Printf("Error: Binning is empty !!! Exiting ...");
1128 return false;
1129 }
1130 binningsArray.insert(binningsArray.begin(), binningFirst);
1131
1132 for (auto & binning : binningsArray) {
1133 std::vector<std::string> binningAxes = Utils::Tokenize(binning.c_str(), '_');
1134 int i = 0;
1135 int index = -1;
1136 // Printf("Binning: %s", binning.c_str());
1137 cutBins.clear();
1138 for (auto & cut : gCfg["ndmspc"]["cuts"]) {
1139 index++;
1140 if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
1141
1142 Int_t rebin = 1;
1143 Int_t rebin_start = 1;
1144 Int_t rebin_minimum = 1;
1145 Int_t nbins = -1;
1146 if (cut["rebin"].is_number_integer()) rebin = cut["rebin"].get<Int_t>();
1147 if (cut["rebin_start"].is_number_integer()) rebin_start = cut["rebin_start"].get<Int_t>();
1148 if (cut["nbins"].is_number_integer()) nbins = cut["nbins"].get<Int_t>();
1149 if (nbins < 0) {
1150 Printf("Error: Number of bins in '%s' is less then 0 !!! Exiting ...", cut["name"].get<std::string>().c_str());
1151 return false;
1152 }
1153
1154 std::vector<std::string> binningAxis = Utils::Tokenize(binningAxes[i], '-');
1155 if (binningAxis.size() == 2) {
1156 rebin = atoi(binningAxis[0].c_str());
1157 rebin_start = atoi(binningAxis[1].c_str());
1158 gCfg["ndmspc"]["cuts"][index]["rebin"] = rebin;
1159 gCfg["ndmspc"]["cuts"][index]["rebin_start"] = rebin_start;
1160 }
1161 // Printf("rebin=%d rebin_start=%d nbins=%d", rebin, rebin_start, nbins);
1162
1163 if (rebin_start > 1) {
1164 rebin_minimum = (rebin_start % rebin);
1165 if (rebin_minimum == 0) rebin_minimum = 1;
1166 }
1167 int start = (rebin_start / rebin) + 1;
1168 if (rebin == 1) start = rebin_start;
1169 int end = (nbins - rebin_minimum + 1) / rebin;
1170 if (end < start) {
1171 Printf("Error: rebin=%d is higher then nbins=%d !!! Exiting ...", rebin, nbins);
1172 return false;
1173 }
1174 // if (nbins == rebin) end = 1;
1175 // Printf("start=%d end=%d jobsArray=%s", start, end, jobsArray[i].c_str());
1176 cutBins.push_back({start, end, atoi(jobsArray[i].c_str())});
1177 i++;
1178 }
1179 // i = 0;
1180 // for (auto & ranges : cutBins) {
1181 // Printf("%d [%d,%d]", i, ranges[0], ranges[1]);
1182 // gCfg["ndmspc"]["process"]["ranges"][i] = {ranges[0], ranges[1]};
1183 // i++;
1184 // }
1185 int count = 1;
1186 std::string outfilenameTmp = outfilename + "/" + binning;
1187 GenerateRecursiveConfig(0, cutBins, gCfg, outfilenameTmp, count);
1188 std::this_thread::sleep_for(std::chrono::milliseconds(500));
1189 }
1190 if (fVerbose >= 2) Printf("[->] Ndmspc::PointRun::GenerateJobs");
1191 return true;
1192}
1193
1194bool PointRun::GenerateRecursiveConfig(Int_t dim, std::vector<std::vector<int>> & ranges, json & cfg,
1195 std::string & outfilename, int & count)
1196{
1200
1201 if (dim < static_cast<Int_t>(ranges.size())) {
1202 // Printf("dim=%d", dim);
1203 for (int i = ranges[dim][0]; i <= ranges[dim][1]; i += ranges[dim][2]) {
1204 // Printf("Running dim=%d [%d,%d]", dim, i, i + ranges[dim][2] - 1);
1205 int min = i;
1206 int max = i + ranges[dim][2] - 1;
1207 if (max > ranges[dim][1]) max = ranges[dim][1];
1208 cfg["ndmspc"]["process"]["ranges"][dim] = {min, max};
1209 if (!GenerateRecursiveConfig(dim + 1, ranges, cfg, outfilename, count)) return false;
1210 }
1211 return true;
1212 }
1213 // outfilename = "/tmp/test/";
1214 std::string outFilenamePath = outfilename + "/" + std::to_string(count++) + ".json";
1215 if (!Core::SaveConfig(gCfg, outFilenamePath)) return false;
1216 Printf("Jobs saved to '%s' '%s'", outFilenamePath.c_str(), gCfg["ndmspc"]["process"]["ranges"].dump().c_str());
1217 return true;
1218 // return GenerateRecursiveConfig(dim + 1, ranges, cfg);
1219}
1220
1221bool PointRun::Generate(std::string name, std::string inFile, std::string inObjectName)
1222{
1223
1227
1228 Printf("Genrating point run with name '%s' ...", name.c_str());
1229 json cfg = R"({
1230
1231 "user": {
1232 "proj": 0,
1233 "minEntries": 1,
1234 "verbose": 0
1235 },
1236 "ndmspc": {
1237 "environments": {
1238 "local": {
1239 "output": { "dir": "$HOME/.ndmspc/analyses/generated", "host": "" }
1240 },
1241 },
1242 "environment": "local",
1243 "data": {
1244 "file": "input.root",
1245 "objects": ["hNSparse"]
1246 },
1247 "cuts": [],
1248 "result": {
1249 "parameters": { "labels": ["Integral"], "default": "Integral" }
1250 },
1251 "output": {
1252 "host": "",
1253 "dir": "",
1254 "file": "content.root",
1255 "opt": "?remote=1"
1256 },
1257 "process": {
1258 "type": "single"
1259 },
1260 "log": {
1261 "type": "error-only",
1262 "dir": "root://eos.ndmspc.io//eos/ndmspc/scratch/ndmspc/logs"
1263 },
1264 "job":{
1265 "inputs": []
1266 },
1267 "verbose": 0
1268 }
1269})"_json;
1270
1271 std::string macroTemplateHeader = R""""(
1272#include <TROOT.h>
1273#include <TList.h>
1274#include <THnSparse.h>
1275#include <TH1D.h>
1276#include <ndmspc/PointRun.h>
1277#include <ndmspc/Utils.h>
1278)"""";
1279
1280 std::string macroTemplate = R""""(
1281{
1282 json cfg = pr->Cfg();
1283 TList * inputList = pr->GetInputList();
1284 THnSparse * resultObject = pr->GetResultObject();
1285 Int_t * point = pr->GetCurrentPoint();
1286 std::vector<std::string> pointLabels = pr->GetCurrentPointLabels();
1287 json pointValue = pr->GetCurrentPointValue();
1288 TList * outputList = pr->GetOutputList();
1289
1290 int verbose = 0;
1291 if (!cfg["user"]["verbose"].is_null() && cfg["user"]["verbose"].is_number_integer()) {
1292 verbose = cfg["user"]["verbose"].get<int>();
1293 }
1294
1295 THnSparse * hs = (THnSparse *)inputList->At(0);
1296
1297 int projId = cfg["user"]["proj"].get<int>();
1298 TH1D * h = hs->Projection(projId, "O");
1299
1300 TString titlePostfix = "(no cuts)";
1301 if (cfg["ndmspc"]["projection"]["title"].is_string())
1302 titlePostfix = cfg["ndmspc"]["projection"]["title"].get<std::string>();
1303 h->SetNameTitle("h", TString::Format("h - %s", titlePostfix.Data()).Data());
1304 outputList->Add(h);
1305
1306 // Skip bin (do not save any output)
1307 if (h->GetEntries() < cfg["user"]["minEntries"].get<int>())
1308 return false;
1309
1310 Double_t integral = h->Integral();
1311 if (verbose >= 0)
1312 Printf("Integral = %f ", integral);
1313
1314
1315 if (resultObject) {
1316 Ndmspc::Utils::SetResultValueError(cfg, resultObject, "Integral", point, integral, TMath::Sqrt(integral), false, true);
1317 }
1318
1319 if (!gROOT->IsBatch() && !cfg["ndmspc"]["process"]["type"].get<std::string>().compare("single")) {
1320 h->DrawCopy();
1321 }
1322
1323 return true;
1324}
1325
1326)"""";
1327
1328 cfg["ndmspc"]["data"]["file"] = inFile.c_str();
1329 cfg["ndmspc"]["data"]["objects"] = {inObjectName.c_str()};
1330
1331 if (cfg["ndmspc"]["cuts"].size() == 0) {
1332 // Generate all axis
1333 // {"enabled": false, "axis": "hNSparseAxisName", "bin" : {"min":3, "max": 3}, "rebin":1}
1334 TFile * tmpFile = Ndmspc::Utils::OpenFile(inFile.c_str());
1335 if (!tmpFile) {
1336 Printf("Error: Problem opening file '%s' !!! Exiting ...", inFile.c_str());
1337 return 1;
1338 }
1339
1340 THnSparse * tmpSparse = (THnSparse *)tmpFile->Get(inObjectName.c_str());
1341 if (!tmpSparse) {
1342 Printf("Error: Problem opening object '%s' !!! Exiting ...", inObjectName.c_str());
1343 return 1;
1344 }
1345 tmpSparse->ls();
1346 TObjArray * axes = tmpSparse->GetListOfAxes();
1347 TAxis * a;
1348 for (int i = 0; i < axes->GetEntries(); i++) {
1349 a = (TAxis *)axes->At(i);
1350 if (!a) continue;
1351 // a->Print();
1352 Printf("Init axis '%s' with enabled=false", a->GetName());
1353 cfg["ndmspc"]["cuts"][i]["enabled"] = false;
1354 cfg["ndmspc"]["cuts"][i]["axis"] = a->GetName();
1355 cfg["ndmspc"]["cuts"][i]["bin"]["min"] = 1;
1356 cfg["ndmspc"]["cuts"][i]["bin"]["max"] = 1;
1357 cfg["ndmspc"]["cuts"][i]["rebin"] = 1;
1358 }
1359
1360 tmpFile->Close();
1361 }
1362
1363 std::string outputConfig = name + ".json";
1364 std::ofstream fileConfig(outputConfig);
1365 fileConfig << std::setw(2) << cfg << std::endl;
1366 // fileConfig << cfg.dump() << std::endl;
1367
1368 std::string outputMacro = name + ".C";
1369 std::ofstream fileMacro(outputMacro);
1370 fileMacro << macroTemplateHeader.c_str();
1371 fileMacro << "bool " << name.c_str() << "(Ndmspc::PointRun *pr)";
1372 fileMacro << macroTemplate.c_str();
1373
1374 Printf("File '%s.C' and '%s.json' were generated ...", name.c_str(), name.c_str());
1375 return true;
1376}
1377
1378bool PointRun::Merge(int from, int to, std::string config, std::string userConfig, std::string environment,
1379 std::string userConfigRaw, std::string binning, std::string cacheDir, std::string fileOpt)
1380{
1384
1385 if (from >= to && to >= 0) {
1386 Printf("Error: 'from=%d' must be smaller then 'to=%d' !!! Exiting ...", from, to);
1387 return false;
1388 }
1389
1390 std::string fromFile = "content.root";
1391 if (from > 0) {
1392 fromFile = "merged_" + std::to_string(from) + ".root";
1393 }
1394 std::string toFile = "merged_" + std::to_string(to) + ".root";
1395
1396 if (!Core::LoadConfig(config, userConfig, environment, userConfigRaw, binning)) return false;
1397
1398 if (!cacheDir.empty()) {
1399 cacheDir += "_" + std::to_string(gSystem->GetPid());
1400 Printf("Setting cache directory to '%s' ...", gSystem->ExpandPathName(cacheDir.c_str()));
1401 TFile::SetCacheFileDir(gSystem->ExpandPathName(cacheDir.c_str()), 1, 1);
1402 }
1403 if (gCfg["ndmspc"]["output"]["host"].get<std::string>().empty()) {
1404 gCfg["ndmspc"]["output"]["opt"] = "";
1405 }
1406
1407 std::string hostUrl = gCfg["ndmspc"]["output"]["host"].get<std::string>();
1408 // if (hostUrl.empty()) {
1409 // Printf("Error: cfg[ndmspc][output][host] is empty!!!");
1410 // return 2;
1411 // }
1412 std::string path;
1413 if (!hostUrl.empty()) path = hostUrl + "/";
1414 path += gCfg["ndmspc"]["output"]["dir"].get<std::string>() + "/";
1415
1416 path += environment + "/";
1417
1418 // int nDimsCuts = 0;
1419 // std::string rebinStr = "";
1420 // for (auto & cut : gCfg["ndmspc"]["cuts"]) {
1421 // if (cut["enabled"].is_boolean() && cut["enabled"].get<bool>() == false) continue;
1422 // path += cut["axis"].get<std::string>() + "_";
1423 // rebinStr += std::to_string(cut["rebin"].get<Int_t>()) + "_";
1424 // nDimsCuts++;
1425 // }
1426 // path[path.size() - 1] = '/';
1427 // path += rebinStr;
1428 // path[path.size() - 1] = '/';
1429 path += Utils::GetCutsPath(gCfg["ndmspc"]["cuts"]);
1430
1431 path = gSystem->ExpandPathName(path.c_str());
1432
1433 // std::vector<std::string> binsArrayFrom;
1434 std::vector<std::string> binsArrayTo;
1435 int binsize = 0;
1436 if (gCfg["ndmspc"]["data"]["histogram"]["enabled"].get<bool>()) {
1437 std::string binToStr;
1438 for (auto & bin : gCfg["ndmspc"]["data"]["histogram"]["bins"]) {
1439 // binFromStr = "";
1440 binToStr = "";
1441 binsize = bin.size();
1442 if (to < 0) to = binsize + 1;
1443 // if (to > binsize) {
1444 // path.erase(path.size() - 5, path.size());
1445 // }
1446 int iLevelFrom = binsize - from + 1;
1447 int iLevelTo = binsize - to + 1;
1448 for (auto & binElement : bin) {
1449
1450 if (iLevelTo > 0) {
1451 binToStr += std::to_string(binElement.get<Int_t>()) + "/";
1452 }
1453 if (iLevelFrom <= 0) break;
1454 iLevelFrom--;
1455 iLevelTo--;
1456 }
1457 Printf("To: %s", binToStr.c_str());
1458 binsArrayTo.push_back(binToStr);
1459 if (binToStr.empty()) {
1460 toFile = "results.root";
1461 break;
1462 }
1463 }
1464 }
1465 else {
1466 binsArrayTo.push_back("");
1467 }
1468
1469 if (from > binsize) {
1470 Printf("Error: 'from=%d' must be smaller then 'binsize=%d' !!! Exiting ...", from, binsize);
1471 return false;
1472 }
1473 // exit(0);
1474 std::string pathBase = path;
1475 for (auto & bin : binsArrayTo) {
1476
1477 std::string pathFrom = pathBase + "bins/" + bin;
1478 std::string outFile = pathBase;
1479 if (to <= binsize) {
1480 outFile += "bins/" + bin;
1481 }
1482 outFile += toFile;
1483
1484 TUrl url(pathFrom.c_str());
1485 std::string outHost = url.GetHost();
1486 std::string inputDirectory = url.GetFile();
1487 std::string linesMerge = "";
1488
1489 if (outHost.empty()) {
1490 if (gSystem->AccessPathName(pathFrom.c_str())) {
1491 Printf("Error: Nothing to merge, because path '%s' does not exist !!!", pathFrom.c_str());
1492 return false;
1493 }
1494 Printf("Doing local find %s -name %s", pathFrom.c_str(),
1495 gCfg["ndmspc"]["output"]["file"].get<std::string>().c_str());
1496 linesMerge = gSystem->GetFromPipe(TString::Format("find %s -name %s", pathFrom.c_str(),
1497 gCfg["ndmspc"]["output"]["file"].get<std::string>().c_str())
1498 .Data());
1499 // Printf("%s", linesMerge.c_str());
1500 // gSystem->Exit(1);
1501 }
1502 else {
1503
1504 Printf("Doing eos find -f %s", pathFrom.c_str());
1505
1506 // std::string contentFile = gCfg["ndmspc"]["output"]["file"].get<std::string>();
1507 std::string contentFile = fromFile;
1508 TString findUrl;
1509
1510 // Vector of string to save tokens
1511 std::vector<std::string> tokens;
1512
1513 if (!inputDirectory.empty()) {
1514 findUrl =
1515 TString::Format("root://%s//proc/user/"
1516 "?mgm.cmd=find&mgm.find.match=%s&mgm.path=%s&mgm.format=json&mgm.option=f&filetype=raw",
1517 outHost.c_str(), contentFile.c_str(), inputDirectory.c_str());
1518 // Printf("Doing '%s' ...", findUrl.Data());
1519
1520 TFile * f = Ndmspc::Utils::OpenFile(findUrl.Data());
1521 if (!f) return 1;
1522
1523 // Printf("%lld", f->GetSize());
1524
1525 int buffsize = 4096;
1526 char buff[buffsize + 1];
1527
1528 Long64_t buffread = 0;
1529 std::string content;
1530 while (buffread < f->GetSize()) {
1531
1532 if (buffread + buffsize > f->GetSize()) buffsize = f->GetSize() - buffread;
1533
1534 // Printf("Buff %lld %d", buffread, buffsize);
1535 f->ReadBuffer(buff, buffread, buffsize);
1536 buff[buffsize] = '\0';
1537 content += buff;
1538 buffread += buffsize;
1539 }
1540
1541 f->Close();
1542
1543 std::string ss = "mgm.proc.stdout=";
1544 size_t pos = ss.size() + 1;
1545 content = content.substr(pos);
1546
1547 // stringstream class check1
1548 std::stringstream check1(content);
1549
1550 std::string intermediate;
1551
1552 // Tokenizing w.r.t. space '&'
1553 while (getline(check1, intermediate, '&')) {
1554 tokens.push_back(intermediate);
1555 }
1556 }
1557 else {
1558 tokens.push_back(contentFile.c_str());
1559 }
1560 linesMerge = tokens[0];
1561 }
1562
1563 if (linesMerge.empty()) {
1564 Printf("Error: Nothing to merge, because path '%s' does not contain file '%s' !!!", pathFrom.c_str(),
1565 fromFile.c_str());
1566 return false;
1567 }
1568
1569 std::stringstream check2(linesMerge);
1570 std::string line;
1571 std::string tempMergeDir = "/tmp";
1572 if (gCfg["ndmspc"]["output"]["merge"]["tmpdir"].is_string())
1573 tempMergeDir = gCfg["ndmspc"]["output"]["merge"]["tmpdir"].get<std::string>();
1574
1575 std::string outFileLocal = tempMergeDir + "/ndmspc-merged-" + std::to_string(gSystem->GetPid()) + ".root";
1576 bool copy = true;
1577 if (hostUrl.empty()) {
1578 outFileLocal = outFile;
1579 copy = false;
1580 }
1581 outFileLocal = gSystem->ExpandPathName(outFileLocal.c_str());
1582 Printf("Output file: '%s'", outFile.c_str());
1583
1584 TFileMerger m(kFALSE);
1585 m.OutputFile(TString::Format("%s%s", outFileLocal.c_str(), fileOpt.c_str()));
1586 // m.AddObjectNames("results");
1587 // m.AddObjectNames("content");
1588 // Int_t default_mode = TFileMerger::kAll | TFileMerger::kIncremental;
1589 // Int_t mode = default_mode | TFileMerger::kOnlyListed;
1590 while (std::getline(check2, line)) {
1591
1592 if (!outHost.empty()) {
1593 line = TString::Format("root://%s/%s", outHost.c_str(), line.c_str()).Data();
1594 }
1595 Printf("Adding file '%s' ...", line.data());
1596 m.AddFile(line.c_str());
1597 }
1598
1599 Printf("Merging ...");
1600 m.Merge();
1601 // m.PartialMerge(mode);
1602 if (copy && toFile > 0) {
1603 Printf("Copy '%s' to '%s' ...", outFileLocal.c_str(), outFile.c_str());
1604 TFile::Cp(outFileLocal.c_str(), outFile.c_str());
1605 std::string rm = "rm -f " + outFileLocal;
1606 Printf("Doing '%s' ...", rm.c_str());
1607 gSystem->Exec(rm.c_str());
1608 }
1609 Printf("Output: '%s'", outFile.c_str());
1610 std::string rmCache = "rm -rf ";
1611 rmCache += gSystem->ExpandPathName(cacheDir.c_str());
1612 Printf("Removing cache dir: '%s' ...", rmCache.c_str());
1613 gSystem->Exec(rmCache.c_str());
1614 Printf("Done ...");
1615 }
1616 return true;
1617}
1618
1619} // namespace Ndmspc
static bool SaveConfig(json cfg, std::string filename)
Definition Core.cxx:111
static bool LoadConfig(std::string config, std::string userConfig, std::string &environment, std::string userConfigRaw="", std::string binning="")
Definition Core.cxx:16
PointRun object.
Definition PointRun.h:24
bool ProcessRecursiveInner(Int_t i, std::vector< std::string > &n)
Definition PointRun.cxx:662
std::string fCurrentOutputFileName
Current output filename.
Definition PointRun.h:76
void OutputFileOpen()
Definition PointRun.cxx:772
bool GenerateRecursiveConfig(Int_t dim, std::vector< std::vector< int > > &ranges, json &cfg, std::string &outfilename, int &count)
TFile * fInputFile
Input file.
Definition PointRun.h:72
static std::string fgEnvironment
Environment.
Definition PointRun.h:89
int ProcessSingleFile()
Definition PointRun.cxx:934
THnSparse * fCurrentProccessHistogram
Current processed histogram.
Definition PointRun.h:81
bool Init(std::string extraPath="")
Definition PointRun.cxx:78
int fBinCount
Bin Count (TODO! rename to axis level maybe)
Definition PointRun.h:71
static bool Merge(int from=0, int to=1, std::string name="myAnalysis.json", std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binning="", std::string cacheDir="${PWD}/.ndmspc_merge_cache", std::string fileOpt="?remote=1")
Merge.
int ProcessHistogramRun()
Definition PointRun.cxx:981
bool LoadConfig(std::string config, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binning="", bool show=false, std::string outfilename="")
Definition PointRun.cxx:47
THnSparse * CreateResult()
Definition PointRun.cxx:249
TList * OpenInputs()
Definition PointRun.cxx:159
PointRun(std::string macro="NdmspcPointRun.C")
Definition PointRun.cxx:29
TH1S * fMapAxesType
Map axis type histogram.
Definition PointRun.h:82
virtual ~PointRun()
Definition PointRun.cxx:40
bool Run(std::string filename, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binnings="", bool show=false, std::string outfilename="")
TList * fInputList
Input list.
Definition PointRun.h:73
std::vector< std::string > fCurrentPointLabels
Current labels.
Definition PointRun.h:79
THnSparse * fResultObject
Result object.
Definition PointRun.h:74
int fVerbose
Verbose level.
Definition PointRun.h:70
json fCurrentPointValue
Current point value.
Definition PointRun.h:80
void SetOutputList(TList *outList)
Sets output list.
Definition PointRun.h:53
bool fIsProcessOk
Flag that process is ok.
Definition PointRun.h:86
bool fIsSkipBin
Flag to skip bin.
Definition PointRun.h:85
std::vector< TAxis * > fCurrentProcessHistogramAxes
Current process histogram axes.
Definition PointRun.h:83
bool fIsProcessExit
Flag to exit process.
Definition PointRun.h:87
TMacro * fMacro
Macro.
Definition PointRun.h:69
TFile * fCurrentOutputFile
Current output file.
Definition PointRun.h:75
static bool Generate(std::string name="myAnalysis", std::string inFile="myFile.root", std::string inObjectName="myNDHistogram")
TDirectory * fCurrentOutputRootDirectory
Current output root directory.
Definition PointRun.h:77
Int_t fCurrentPoint[32]
Current point.
Definition PointRun.h:78
bool ProcessSinglePoint()
Definition PointRun.cxx:547
bool GenerateJobs(std::string jobs, std::string filename, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string jobDir="/tmp/ndmspc-jobs", std::string binnings="")
std::vector< int > fCurrentProcessHistogramPoint
Current process histoghram point.
Definition PointRun.h:84
void OutputFileClose()
Definition PointRun.cxx:887
bool ProcessRecursive(int i)
Definition PointRun.cxx:594
static std::string GetCutsPath(json cuts)
Definition Utils.cxx:143
static TFile * OpenFile(std::string filename, std::string mode="READ", bool createLocalDir=true)
Definition Utils.cxx:18
static Int_t GetBinFromBase(Int_t bin, Int_t rebin, Int_t rebin_start)
Definition Utils.cxx:175
static TMacro * OpenMacro(std::string filename)
Definition Utils.cxx:100
static std::vector< std::string > Tokenize(std::string_view input, const char delim)
Definition Utils.cxx:256