openPMD-api
 
Loading...
Searching...
No Matches
ADIOS2IOHandler.hpp
1/* Copyright 2017-2025 Fabian Koller and Franz Poeschel, Axel Huebl, Junmin Gu,
2 * Luca Fedeli
3 *
4 * This file is part of openPMD-api.
5 *
6 * openPMD-api is free software: you can redistribute it and/or modify
7 * it under the terms of of either the GNU General Public License or
8 * the GNU Lesser General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * openPMD-api is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License and the GNU Lesser General Public License
16 * for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * and the GNU Lesser General Public License along with openPMD-api.
20 * If not, see <http://www.gnu.org/licenses/>.
21 */
22#pragma once
23
24#include "openPMD/Error.hpp"
25#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
26#include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
27#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
28#include "openPMD/IO/ADIOS/ADIOS2PreloadVariables.hpp"
29#include "openPMD/IO/ADIOS/macros.hpp"
30#include "openPMD/IO/AbstractIOHandler.hpp"
31#include "openPMD/IO/AbstractIOHandlerImpl.hpp"
32#include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
33#include "openPMD/IO/FlushParametersInternal.hpp"
34#include "openPMD/IO/IOTask.hpp"
35#include "openPMD/IO/InvalidatableFile.hpp"
36#include "openPMD/IterationEncoding.hpp"
37#include "openPMD/ThrowError.hpp"
38#include "openPMD/auxiliary/JSON_internal.hpp"
39#include "openPMD/auxiliary/StringManip.hpp"
40#include "openPMD/backend/Variant_internal.hpp"
41#include "openPMD/backend/Writable.hpp"
42#include "openPMD/config.hpp"
43#include <stdexcept>
44
45#if openPMD_HAVE_ADIOS2
46#include <adios2.h>
47#endif
48#if openPMD_HAVE_MPI
49#include <mpi.h>
50#endif
51#include <nlohmann/json.hpp>
52
53#include <array>
54#include <exception>
55#include <future>
56#include <iostream>
57#include <memory> // shared_ptr
58#include <optional>
59#include <set>
60#include <string>
61#include <unordered_map>
62#include <utility> // pair
63#include <vector>
64
65namespace openPMD
66{
67#if openPMD_HAVE_ADIOS2
68
69std::optional<size_t> joinedDimension(adios2::Dims const &dims);
70
71class ADIOS2IOHandler;
72
73namespace detail
74{
75 template <typename, typename>
77 struct GetSpan;
78 struct DatasetReader;
79 struct AttributeReader;
80 struct AttributeWriter;
81 struct AttributeReader;
82 struct AttributeWriter;
83 template <typename>
84 struct AttributeTypes;
85 struct DatasetOpener;
86 struct VariableDefiner;
87 template <typename>
89 struct WriteDataset;
90 class ADIOS2File;
91 struct BufferedPut;
92 struct BufferedGet;
93 struct BufferedAttributeRead;
94 struct BufferedAttributeWrite;
95 struct RunUniquePtrPut;
96} // namespace detail
97
98class ADIOS2IOHandlerImpl
99 : public AbstractIOHandlerImplCommon<ADIOS2FilePosition>
100{
101 template <typename, typename>
102 friend struct detail::DatasetHelper;
103 friend struct detail::GetSpan;
104 friend struct detail::DatasetReader;
105 friend struct detail::AttributeReader;
106 friend struct detail::AttributeWriter;
107 friend struct detail::AttributeReader;
108 friend struct detail::AttributeWriter;
109 template <typename>
110 friend struct detail::AttributeTypes;
111 friend struct detail::DatasetOpener;
112 friend struct detail::VariableDefiner;
113 template <typename>
114 friend struct detail::DatasetTypes;
115 friend struct detail::WriteDataset;
116 friend class detail::ADIOS2File;
117 friend struct detail::BufferedAttributeRead;
118 friend struct detail::RunUniquePtrPut;
119
120 using UseGroupTable = adios_defs::UseGroupTable;
121 using FlushTarget = adios_defs::FlushTarget;
122
123public:
124#if openPMD_HAVE_MPI
125
126 ADIOS2IOHandlerImpl(
128 MPI_Comm,
129 std::string engineType,
130 std::string specifiedExtension);
131
132#endif // openPMD_HAVE_MPI
133
134 explicit ADIOS2IOHandlerImpl(
136 std::string engineType,
137 std::string specifiedExtension);
138
139 ~ADIOS2IOHandlerImpl() override;
140
141 std::future<void> flush(internal::ParsedFlushParams &);
142
143 void
145
147
148 // MPI Collective
149 bool checkFile(std::string fullFilePath) const;
150
151 void
153
154 void createDataset(
156
157 void extendDataset(
159
161
162 void
164
165 void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
166
167 void
169
171
172 void
174
175 void
177
178 void deleteDataset(
180
181 void deleteAttribute(
182 Writable *, Parameter<Operation::DELETE_ATT> const &) override;
183
184 void
186
187 void writeAttribute(
188 Writable *, Parameter<Operation::WRITE_ATT> const &) override;
189
191
192 void
194
196
199
201
202 void
204
205 void listAttributes(
206 Writable *, Parameter<Operation::LIST_ATTS> &parameters) override;
207
209
210 void availableChunks(
212
213 void
215
216 void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
217
222 adios2::Mode
223 adios2AccessMode(std::string const &fullPath, adios_defs::OpenFileAs);
224
225 FlushTarget m_flushTarget = FlushTarget::Disk;
226
227private:
228 adios2::ADIOS m_ADIOS;
229#if openPMD_HAVE_MPI
230 std::optional<MPI_Comm> m_communicator;
231#endif
235 std::string m_engineType;
236 std::optional<std::string> m_realEngineType;
237
238 inline std::string const &realEngineType() const
239 {
240 if (m_realEngineType.has_value())
241 {
242 return *m_realEngineType;
243 }
244 else
245 {
246 return m_engineType;
247 }
248 }
249 inline std::string &realEngineType()
250 {
251 return const_cast<std::string &>(
252 static_cast<ADIOS2IOHandlerImpl const *>(this)->realEngineType());
253 }
254 inline void pretendEngine(std::string facade_engine)
255 {
256 if (!m_realEngineType.has_value())
257 {
258 m_realEngineType = std::move(m_engineType);
259 }
260 m_engineType = std::move(facade_engine);
261 }
262 /*
263 * The filename extension specified by the user.
264 */
265 std::string m_userSpecifiedExtension;
266
267 /*
268 * Empty option: No choice about the group table has been explicitly made,
269 * use default.
270 */
271 std::optional<UseGroupTable> m_useGroupTable;
272
273 enum class UseSpan : char
274 {
275 Yes,
276 No,
277 Auto
278 };
279
280 UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto;
281
282 enum class ModifiableAttributes : char
283 {
284 Yes,
285 No,
286 Unspecified
287 };
288
289 ModifiableAttributes m_modifiableAttributes =
290 ModifiableAttributes::Unspecified;
291
292 inline UseGroupTable useGroupTable() const
293 {
294 if (!m_useGroupTable.has_value())
295 {
296 return UseGroupTable::No;
297 }
298 return m_useGroupTable.value();
299 }
300
301 bool m_writeAttributesFromThisRank = true;
302
303 struct ParameterizedOperator
304 {
305 adios2::Operator op;
306 adios2::Params params;
307 };
308
309 std::vector<ParameterizedOperator> defaultOperators;
310
311 json::TracingJSON m_config;
312 static json::TracingJSON nullvalue;
313
314 template <typename Callback>
315 void
316 init(json::TracingJSON config, Callback &&callbackWriteAttributesFromRank);
317
318 template <typename Key>
319 json::TracingJSON config(Key &&key, json::TracingJSON &cfg)
320 {
321 if (cfg.json().is_object() && cfg.json().contains(key))
322 {
323 return cfg[key];
324 }
325 else
326 {
327 return nullvalue;
328 }
329 }
330
331 template <typename Key>
332 json::TracingJSON config(Key &&key)
333 {
334 return config<Key>(std::forward<Key>(key), m_config);
335 }
336
344 std::optional<std::vector<ParameterizedOperator>>
345 getOperators(json::TracingJSON config);
346
347 // use m_config
348 std::optional<std::vector<ParameterizedOperator>> getOperators();
349
350 template <typename Parameter>
351 std::vector<ParameterizedOperator> getDatasetOperators(
352 Parameter const &, Writable *, std::string const &varName);
353
354 std::string fileSuffix(bool verbose = true) const;
355
356 /*
357 * We need to give names to IO objects. These names are irrelevant
358 * within this application, since:
359 * 1) The name of the file written to is decided by the opened Engine's
360 * name.
361 * 2) The IOs are managed by the unordered_map m_fileData, so we do not
362 * need the ADIOS2 internal management.
363 * Since within one m_ADIOS object, the same IO name cannot be used more
364 * than once, we ensure different names by using the name counter.
365 * This allows to overwrite a file later without error.
366 */
367 int nameCounter{0};
368
369 /*
370 * IO-heavy actions are deferred to a later point. This map stores for
371 * each open file (identified by an InvalidatableFile object) an object
372 * that manages IO-heavy actions, as well as its ADIOS2 objects, i.e.
373 * IO and Engine object.
374 * Not to be accessed directly, use getFileData().
375 */
376 std::unordered_map<InvalidatableFile, std::unique_ptr<detail::ADIOS2File>>
377 m_fileData;
378
379 std::map<std::string, adios2::Operator> m_operators;
380
381 // Overrides from AbstractIOHandlerImplCommon.
382
383 std::string
384 filePositionToString(std::shared_ptr<ADIOS2FilePosition>) override;
385
386 std::shared_ptr<ADIOS2FilePosition> extendFilePosition(
387 std::shared_ptr<ADIOS2FilePosition> const &pos,
388 std::string extend) override;
389
390 // Helper methods.
391
392 std::optional<adios2::Operator>
393 getCompressionOperator(std::string const &compression);
394
395 /*
396 * The name of the ADIOS2 variable associated with this Writable.
397 * To be used for Writables that represent a dataset.
398 */
399 std::string nameOfVariable(Writable *writable);
400
410 std::string nameOfAttribute(Writable *writable, std::string attribute);
411
412 /*
413 * Figure out whether the Writable corresponds with a
414 * group or a dataset.
415 */
416 GroupOrDataset groupOrDataset(Writable *);
417
418 enum class IfFileNotOpen : char
419 {
420 OpenImplicitly,
421 CreateImplicitly,
422 ThrowError,
423 ReopenFileThatWeCreated,
424 ReopenFileFoundOnDisk = OpenImplicitly,
425 };
426
428 getFileData(InvalidatableFile const &file, IfFileNotOpen);
429
430 void dropFileData(InvalidatableFile const &file);
431
432 template <typename T>
433 static void setStepSelectionForVariable(
434 adios2::Variable<T> var,
435 std::string const &varName,
436 size_t step_selection,
437 size_t file_steps,
438 detail::AdiosVariables const &av)
439 {
440 auto var_steps = var.Steps();
441 if (var_steps == 1 && step_selection == 0)
442 {
443 // variable has no steps
444 return;
445 }
446 if (file_steps != var_steps)
447 {
448 if (!av.m_preparsed.has_value())
449 {
450 throw error::ReadError(
451 error::AffectedObject::Dataset,
452 error::Reason::UnexpectedContent,
453 "ADIOS2",
454 "The opened file contains different data per step, but "
455 "variable data was not preparsed. ERROR: Variable " +
456 varName + "' has " + std::to_string(var_steps) +
457 " step(s), but the file has " +
458 std::to_string(file_steps) + " step(s).");
459 }
460 auto preparsed = av.m_preparsed->m_partialVariables.find(varName);
461 if (preparsed == av.m_preparsed->m_partialVariables.end())
462 {
463 throw error::ReadError(
464 error::AffectedObject::Dataset,
465 error::Reason::UnexpectedContent,
466 "ADIOS2",
467 "The opened file contains different data per step, but "
468 "variable data contains no preparsing info on '" +
469 varName + "'. Has " + std::to_string(var_steps) +
470 " step(s), but the file has " +
471 std::to_string(file_steps) + " step(s).");
472 }
473 auto step_index = std::find(
474 preparsed->second.begin(),
475 preparsed->second.end(),
476 step_selection);
477 if (step_index == preparsed->second.end())
478 {
479 throw error::ReadError(
480 error::AffectedObject::Dataset,
481 error::Reason::UnexpectedContent,
482 "ADIOS2",
483 "Tried selecting global step " +
484 std::to_string(step_selection) + " for variable '" +
485 varName +
486 "', but variable is not defined for that step (only "
487 "for steps " +
488 auxiliary::vec_as_string(preparsed->second) +
489 "). Has " + std::to_string(var_steps) +
490 " step(s), but the file has " +
491 std::to_string(file_steps) + " step(s).");
492 }
493 // We need to replace the (global) step selection with the
494 // (local) step index
495 step_selection = step_index - preparsed->second.begin();
496 }
497 var.SetStepSelection({step_selection, 1});
498 }
499
500 /*
501 * Prepare a variable that already exists for an IO
502 * operation, including:
503 * (1) checking that its datatype matches T.
504 * (2) the offset and extent match the variable's shape
505 * (3) setting the offset and extent (ADIOS lingo: start
506 * and count)
507 */
508 template <typename T>
509 adios2::Variable<T> verifyDataset(
510 Offset const &offset,
511 Extent const &extent,
512 adios2::IO &IO,
513 adios2::Engine &engine,
514 std::string const &varName,
515 std::optional<size_t> stepSelection,
516 detail::AdiosVariables const &av)
517 {
518 {
519 auto requiredType = adios2::GetType<T>();
520 auto actualType = IO.VariableType(varName);
521
522 if (requiredType != actualType)
523 {
524 std::stringstream errorMessage;
525 errorMessage << "Trying to access a dataset with wrong type "
526 "(trying to access dataset with type '"
527 << requiredType << "', but has type '"
528 << actualType << "')";
529 throw error::ReadError(
530 error::AffectedObject::Dataset,
531 error::Reason::UnexpectedContent,
532 "ADIOS2",
533 errorMessage.str());
534 };
535 }
536 adios2::Variable<T> var = IO.InquireVariable<T>(varName);
537 if (!var.operator bool())
538 {
539
540 throw std::runtime_error(
541 "[ADIOS2] Internal error: Failed opening ADIOS2 variable.");
542 }
543 if (stepSelection.has_value())
544 {
545 auto file_steps = engine.Steps();
546 setStepSelectionForVariable(
547 var, varName, *stepSelection, file_steps, av);
548 }
549 // TODO leave this check to ADIOS?
550 adios2::Dims shape = var.Shape();
551 auto actualDim = shape.size();
552 {
553 auto requiredDim = extent.size();
554 if (requiredDim != actualDim)
555 {
556 throw error::ReadError(
557 error::AffectedObject::Dataset,
558 error::Reason::UnexpectedContent,
559 "ADIOS2",
560 "Trying to access a dataset with wrong dimensionality "
561 "(trying to access dataset with dimensionality " +
562 std::to_string(requiredDim) +
563 ", but has dimensionality " +
564 std::to_string(actualDim) + ")");
565 }
566 }
567 auto joinedDim = joinedDimension(shape);
568 auto make_runtime_error = [&](char const *message) {
569 std::stringstream s;
570 s << "[ADIOS2IOHandlerImpl::verifyDataset()] " << message;
571 s << "\nNote: Variable '" << varName << "' has shape ";
572 auxiliary::write_vec_to_stream(s, shape)
573 << ", is accessed from offset ";
574 auxiliary::write_vec_to_stream(s, offset) << " with extent ";
575 auxiliary::write_vec_to_stream(s, extent);
576 if (joinedDim.has_value())
577 {
578 s << " (joined dimension on index " << *joinedDim << ").";
579 }
580 else
581 {
582 s << " (no joined dimension).";
583 }
584 return std::runtime_error(s.str());
585 };
586 if (joinedDim.has_value() ||
587 var.ShapeID() == adios2::ShapeID::JoinedArray)
588 {
589 if (!offset.empty())
590 {
591 throw make_runtime_error(
592 "Offset must be an empty vector in case of joined array.");
593 }
594 if (!joinedDim.has_value())
595 {
596 throw make_runtime_error(
597 "Trying to access a dataset as a non-joined array, but it "
598 "has previously been array.");
599 }
600 for (unsigned int i = 0; i < actualDim; i++)
601 {
602 if (*joinedDim != i && extent[i] != shape[i])
603 {
604 throw make_runtime_error(
605 "store_chunk extent of non-joined dimensions "
606 "must be equivalent to the total extent.");
607 }
608 }
609 }
610 else
611 {
612 for (unsigned int i = 0; i < actualDim; i++)
613 {
614 if (!(joinedDim.has_value() && *joinedDim == i) &&
615 offset[i] + extent[i] > shape[i])
616 {
617 throw make_runtime_error("Dataset access out of bounds.");
618 }
619 }
620 }
621
622 var.SetSelection(
623 {adios2::Dims(offset.begin(), offset.end()),
624 adios2::Dims(extent.begin(), extent.end())});
625 return var;
626 }
627
628 struct
629 {
630 bool noGroupBased = false;
631 bool blosc2bp5 = false;
632 } printedWarningsAlready;
633}; // ADIOS2IOHandlerImpl
634
635namespace detail
636{
637 // Helper structs for calls to the switchType function
638
639 template <typename T>
640 inline constexpr bool IsUnsupportedComplex_v =
641 std::is_same_v<T, std::complex<long double>> ||
642 std::is_same_v<T, std::vector<std::complex<long double>>>;
643
645 {
647 {
648 size_t step;
649 adios2::IO &IO;
650 detail::AdiosAttributes const &attributes;
651 template <typename AdiosType>
652 auto call(std::string const &name) const
654 {
655 return attributes.getAttribute<AdiosType>(step, IO, name);
656 }
657 };
658
659 template <typename T>
660 static Datatype call(
661 size_t step,
662 adios2::IO &IO,
663 std::string name,
666
667 template <int n, typename... Params>
668 static Datatype call(Params &&...);
669 };
670
672 {
673 template <typename T>
674 static void call(
676 Writable *writable,
677 const Parameter<Operation::WRITE_ATT> &parameters);
678
679 template <int n, typename... Params>
680 static void call(Params &&...);
681 };
682
684 {
685 template <typename T>
686 static void call(
688 InvalidatableFile const &,
689 std::string const &varName,
691 std::optional<size_t> stepSelection,
692 std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator> const
693 &operators,
694 detail::AdiosVariables const &);
695
696 static constexpr char const *errorMsg = "ADIOS2: openDataset()";
697 };
698
700 {
716 template <typename T>
717 static void call(
718 adios2::IO &IO,
719 std::string const &name,
720 std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator> const
721 &compressions,
722 adios2::Dims const &shape = adios2::Dims(),
723 adios2::Dims const &start = adios2::Dims(),
724 adios2::Dims const &count = adios2::Dims(),
725 bool const constantDims = false);
726
727 static constexpr char const *errorMsg = "ADIOS2: defineVariable()";
728 };
729
731 {
732 template <typename T>
733 static void call(
735 adios2::IO &IO,
736 adios2::Engine &engine,
737 std::string const &varName,
738 bool allSteps);
739
740 template <int n, typename... Params>
741 static void call(Params &&...);
742 };
743
744 // Helper structs to help distinguish valid attribute/variable
745 // datatypes from invalid ones
746
747 /*
748 * This struct's purpose is to have specialisations
749 * for vector and array types, as well as the boolean
750 * type (which is not natively supported by ADIOS).
751 */
752 template <typename T>
754 {
759 static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
760 {
761 auto attr = IO.InquireAttribute<T>(name);
762 if (!attr)
763 {
764 return false;
765 }
766 std::vector<T> data = attr.Data();
767 if (data.size() != 1)
768 {
769 return false;
770 }
771 return data[0] == val;
772 }
773 };
774
775 template <>
776 struct AttributeTypes<std::complex<long double>>
777 {
778 static bool
779 attributeUnchanged(adios2::IO &, std::string, std::complex<long double>)
780 {
781 throw std::runtime_error(
782 "[ADIOS2] Internal error: no support for long double complex "
783 "attribute types");
784 }
785 };
786
787 template <>
788 struct AttributeTypes<std::vector<std::complex<long double>>>
789 {
790 static bool attributeUnchanged(
791 adios2::IO &, std::string, std::vector<std::complex<long double>>)
792 {
793 throw std::runtime_error(
794 "[ADIOS2] Internal error: no support for long double complex "
795 "vector attribute types");
796 }
797 };
798
799 template <typename T>
800 struct AttributeTypes<std::vector<T>>
801 {
802 static bool
803 attributeUnchanged(adios2::IO &IO, std::string name, std::vector<T> val)
804 {
805 auto attr = IO.InquireAttribute<T>(name);
806 if (!attr)
807 {
808 return false;
809 }
810 std::vector<T> data = attr.Data();
811 if (data.size() != val.size())
812 {
813 return false;
814 }
815 for (size_t i = 0; i < val.size(); ++i)
816 {
817 if (data[i] != val[i])
818 {
819 return false;
820 }
821 }
822 return true;
823 }
824 };
825
826 template <>
827 struct AttributeTypes<std::vector<std::string>>
828 {
829 static bool attributeUnchanged(
830 adios2::IO &IO, std::string name, std::vector<std::string> val)
831 {
832 auto attr = IO.InquireAttribute<std::string>(name);
833 if (!attr)
834 {
835 return false;
836 }
837 std::vector<std::string> data = attr.Data();
838 if (data.size() != val.size())
839 {
840 return false;
841 }
842 for (size_t i = 0; i < val.size(); ++i)
843 {
844 if (data[i] != val[i])
845 {
846 return false;
847 }
848 }
849 return true;
850 }
851 };
852
853 template <typename T, size_t n>
854 struct AttributeTypes<std::array<T, n>>
855 {
856 static bool attributeUnchanged(
857 adios2::IO &IO, std::string name, std::array<T, n> val)
858 {
859 auto attr = IO.InquireAttribute<T>(name);
860 if (!attr)
861 {
862 return false;
863 }
864 std::vector<T> data = attr.Data();
865 if (data.size() != n)
866 {
867 return false;
868 }
869 for (size_t i = 0; i < n; ++i)
870 {
871 if (data[i] != val[i])
872 {
873 return false;
874 }
875 }
876 return true;
877 }
878 };
879
880 namespace bool_repr
881 {
882 using rep = detail::bool_representation;
883
884 static constexpr rep toRep(bool b)
885 {
886 return b ? 1U : 0U;
887 }
888
889 static constexpr bool fromRep(rep r)
890 {
891 return r != 0;
892 }
893 } // namespace bool_repr
894
895 template <>
896 struct AttributeTypes<bool>
897 {
898 using rep = detail::bool_representation;
899
900 static constexpr rep toRep(bool b)
901 {
902 return b ? 1U : 0U;
903 }
904
905 static constexpr bool fromRep(rep r)
906 {
907 return r != 0;
908 }
909
910 static bool
911 attributeUnchanged(adios2::IO &IO, std::string name, bool val)
912 {
913 auto attr = IO.InquireAttribute<rep>(name);
914 if (!attr)
915 {
916 return false;
917 }
918 std::vector<rep> data = attr.Data();
919 if (data.size() != 1)
920 {
921 return false;
922 }
923 return data[0] == toRep(val);
924 }
925 };
926} // namespace detail
927#endif // openPMD_HAVE_ADIOS2
928
929class ADIOS2IOHandler : public AbstractIOHandler
930{
931#if openPMD_HAVE_ADIOS2
932
933 friend class ADIOS2IOHandlerImpl;
934
935private:
936 ADIOS2IOHandlerImpl m_impl;
937
938public:
939 ~ADIOS2IOHandler() override
940 {
941 // we must not throw in a destructor
942 try
943 {
944 auto params = internal::defaultParsedFlushParams;
945 this->flush(params);
946 }
947 catch (std::exception const &ex)
948 {
949 std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what()
950 << std::endl;
951 }
952 catch (...)
953 {
954 std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
955 }
956 }
957
958#else
959public:
960#endif
961
962#if openPMD_HAVE_MPI
963
964 ADIOS2IOHandler(
965 std::optional<std::unique_ptr<AbstractIOHandler>> initialize_from,
966 std::string path,
967 Access,
968 MPI_Comm,
969 json::TracingJSON options,
970 std::string engineType,
971 std::string specifiedExtension);
972
973#endif
974
975 ADIOS2IOHandler(
976 std::optional<std::unique_ptr<AbstractIOHandler>> initialize_from,
977 std::string path,
978 Access,
979 json::TracingJSON options,
980 std::string engineType,
981 std::string specifiedExtension);
982
983 std::string backendName() const override
984 {
985 return "ADIOS2";
986 }
987
988 bool fullSupportForVariableBasedEncoding() const override
989 {
990 return true;
991 }
992
993 std::future<void> flush(internal::ParsedFlushParams &) override;
994}; // ADIOS2IOHandler
995} // namespace openPMD
Definition ADIOS2IOHandler.hpp:930
std::future< void > flush(internal::ParsedFlushParams &) override
Process operations in queue according to FIFO.
Definition ADIOS2IOHandler.cpp:2615
std::string backendName() const override
The currently used backend.
Definition ADIOS2IOHandler.hpp:983
Definition ADIOS2IOHandler.hpp:100
adios2::Mode adios2AccessMode(std::string const &fullPath, adios_defs::OpenFileAs)
The ADIOS2 access type to chose for Engines opened within this instance.
Definition ADIOS2IOHandler.cpp:2014
void readDataset(Writable *, Parameter< Operation::READ_DATASET > &) override
Read a chunk of data from an existing dataset.
Definition ADIOS2IOHandler.cpp:1202
void openFile(Writable *, Parameter< Operation::OPEN_FILE > &) override
Open an existing file assuming it conforms to openPMD.
Definition ADIOS2IOHandler.cpp:1018
void touch(Writable *, Parameter< Operation::TOUCH > const &) override
Treat this writable's file as open/active/dirty.
Definition ADIOS2IOHandler.cpp:1999
void writeAttribute(Writable *, Parameter< Operation::WRITE_ATT > const &) override
Create a single attribute and fill the value, possibly overwriting an existing attribute.
Definition ADIOS2IOHandler.cpp:1191
void createFile(Writable *, Parameter< Operation::CREATE_FILE > const &) override
Create a new file in physical storage, possibly overriding an existing file.
Definition ADIOS2IOHandler.cpp:676
void deletePath(Writable *, Parameter< Operation::DELETE_PATH > const &) override
Delete all objects within an existing path.
Definition ADIOS2IOHandler.cpp:1154
void deleteDataset(Writable *, Parameter< Operation::DELETE_DATASET > const &) override
Delete an existing dataset.
Definition ADIOS2IOHandler.cpp:1160
void listDatasets(Writable *, Parameter< Operation::LIST_DATASETS > &) override
List all datasets inside a group, non-recursively.
Definition ADIOS2IOHandler.cpp:1855
void closePath(Writable *, Parameter< Operation::CLOSE_PATH > const &) override
Close an openPMD group.
Definition ADIOS2IOHandler.cpp:1941
void advance(Writable *, Parameter< Operation::ADVANCE > &) override
Advance the file/stream that this writable belongs to.
Definition ADIOS2IOHandler.cpp:1926
void deleteFile(Writable *, Parameter< Operation::DELETE_FILE > const &) override
Delete an existing file from physical storage.
Definition ADIOS2IOHandler.cpp:1148
void listAttributes(Writable *, Parameter< Operation::LIST_ATTS > &parameters) override
List all attributes associated with an object.
Definition ADIOS2IOHandler.cpp:1897
void openDataset(Writable *, Parameter< Operation::OPEN_DATASET > &) override
Open an existing dataset and determine its datatype and extent.
Definition ADIOS2IOHandler.cpp:1114
void extendDataset(Writable *, Parameter< Operation::EXTEND_DATASET > const &) override
Increase the extent of an existing dataset.
Definition ADIOS2IOHandler.cpp:997
void availableChunks(Writable *, Parameter< Operation::AVAILABLE_CHUNKS > &) override
Report chunks that are available for loading from the dataset represented by this writable.
Definition ADIOS2IOHandler.cpp:1973
void closeFile(Writable *, Parameter< Operation::CLOSE_FILE > const &) override
Close the file corresponding with the writable and release file handles.
Definition ADIOS2IOHandler.cpp:1059
void readAttributeAllsteps(Writable *, Parameter< Operation::READ_ATT_ALLSTEPS > &) override
Collective task to read modifiable attributes over steps.
Definition ADIOS2IOHandler.cpp:1607
void createDataset(Writable *, Parameter< Operation::CREATE_DATASET > const &) override
Create a new dataset of given type, extent and storage properties.
Definition ADIOS2IOHandler.cpp:827
void checkFile(Writable *, Parameter< Operation::CHECK_FILE > &) override
Check if the file specified by the parameter is already present on disk.
Definition ADIOS2IOHandler.cpp:739
void writeDataset(Writable *, Parameter< Operation::WRITE_DATASET > &) override
Write a chunk of data into an existing dataset.
Definition ADIOS2IOHandler.cpp:1174
void listPaths(Writable *, Parameter< Operation::LIST_PATHS > &) override
List all paths/sub-groups inside a group, non-recursively.
Definition ADIOS2IOHandler.cpp:1715
void createPath(Writable *, Parameter< Operation::CREATE_PATH > const &) override
Create all necessary groups for a path, possibly recursively.
Definition ADIOS2IOHandler.cpp:793
void getBufferView(Writable *, Parameter< Operation::GET_BUFFER_VIEW > &) override
Get a view into a dataset buffer that can be filled by a user.
Definition ADIOS2IOHandler.cpp:1290
void readAttribute(Writable *, Parameter< Operation::READ_ATT > &) override
Read the value of an existing attribute.
Definition ADIOS2IOHandler.cpp:1382
void openPath(Writable *, Parameter< Operation::OPEN_PATH > const &) override
Open all contained groups in a path, possibly recursively.
Definition ADIOS2IOHandler.cpp:1086
void deregister(Writable *, Parameter< Operation::DEREGISTER > const &) override
Notify the backend that the Writable has been / will be deallocated.
Definition ADIOS2IOHandler.cpp:1993
void deleteAttribute(Writable *, Parameter< Operation::DELETE_ATT > const &) override
Delete an existing attribute.
Definition ADIOS2IOHandler.cpp:1167
Interface for communicating between logical and physically persistent data.
Definition AbstractIOHandler.hpp:206
std::string fullPath(InvalidatableFile)
Definition AbstractIOHandlerImplCommon.hpp:179
Layer to mirror structure of logical data and persistent data in file.
Definition Writable.hpp:76
Definition ADIOS2File.hpp:143
Definition Error.hpp:112
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition JSON_internal.hpp:69
nlohmann::json & json()
Access the underlying JSON value.
Definition JSON.cpp:65
Public definitions of openPMD-api.
Definition Date.cpp:29
Access
File access mode to use during IO.
Definition Access.hpp:58
@ T
time
Definition UnitDimension.hpp:41
Datatype
Concrete datatype of an object available at runtime.
Definition Datatype.hpp:51
STL namespace.
Wrapper around a shared pointer to:
Definition InvalidatableFile.hpp:44
Typesafe description of all required arguments for a specified Operation.
Definition IOTask.hpp:148
Definition ADIOS2PreloadAttributes.hpp:152
Definition ADIOS2PreloadVariables.hpp:36
Definition ADIOS2IOHandler.hpp:647
Definition ADIOS2IOHandler.hpp:645
Definition ADIOS2IOHandler.hpp:754
static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition ADIOS2IOHandler.hpp:759
Definition ADIOS2PreloadAttributes.hpp:132
Definition ADIOS2IOHandler.hpp:672
Definition ADIOS2File.hpp:70
Definition ADIOS2File.hpp:94
Definition ADIOS2IOHandler.hpp:76
Definition ADIOS2IOHandler.hpp:684
Definition ADIOS2File.hpp:79
Definition ADIOS2IOHandler.hpp:88
Definition ADIOS2IOHandler.cpp:1221
Definition ADIOS2IOHandler.hpp:731
Definition ADIOS2File.cpp:163
Definition ADIOS2IOHandler.hpp:700
static void call(adios2::IO &IO, std::string const &name, std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const &compressions, adios2::Dims const &shape=adios2::Dims(), adios2::Dims const &start=adios2::Dims(), adios2::Dims const &count=adios2::Dims(), bool const constantDims=false)
Define a Variable of type T within the passed IO.
Definition ADIOS2IOHandler.cpp:2477
Definition ADIOS2File.hpp:102
Definition FlushParametersInternal.hpp:32