openPMD-api
Series.hpp
1 /* Copyright 2017-2021 Fabian Koller, Axel Huebl
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/config.hpp"
24 #include "openPMD/backend/Attributable.hpp"
25 #include "openPMD/backend/Container.hpp"
26 #include "openPMD/IO/AbstractIOHandler.hpp"
27 #include "openPMD/IO/Access.hpp"
28 #include "openPMD/IO/Format.hpp"
29 #include "openPMD/Iteration.hpp"
30 #include "openPMD/IterationEncoding.hpp"
31 #include "openPMD/Streaming.hpp"
32 #include "openPMD/WriteIterations.hpp"
33 #include "openPMD/auxiliary/Option.hpp"
34 #include "openPMD/auxiliary/Variant.hpp"
35 #include "openPMD/backend/Attributable.hpp"
36 #include "openPMD/backend/Container.hpp"
37 #include "openPMD/config.hpp"
38 #include "openPMD/version.hpp"
39 
40 #if openPMD_HAVE_MPI
41 # include <mpi.h>
42 #endif
43 
44 #include <map>
45 #include <string>
46 
47 // expose private and protected members for invasive testing
48 #ifndef OPENPMD_private
49 # define OPENPMD_private private
50 #endif
51 
52 
53 namespace openPMD
54 {
55 class ReadIterations;
56 class Series;
57 class SeriesInterface;
58 
59 namespace internal
60 {
68 {
69 public:
70  explicit SeriesData() = default;
71 
72  SeriesData( SeriesData const & ) = delete;
73  SeriesData( SeriesData && ) = delete;
74 
75  SeriesData & operator=( SeriesData const & ) = delete;
76  SeriesData & operator=( SeriesData && ) = delete;
77 
78  virtual ~SeriesData() = default;
79 
81 
82  auxiliary::Option< WriteIterations > m_writeIterations;
83  auxiliary::Option< std::string > m_overrideFilebasedFilename;
84  std::string m_name;
85  std::string m_filenamePrefix;
86  std::string m_filenamePostfix;
87  int m_filenamePadding = -1;
88  IterationEncoding m_iterationEncoding{};
89  Format m_format;
97  StepStatus m_stepStatus = StepStatus::NoStep;
98  bool m_parseLazily = false;
99  bool m_lastFlushSuccessful = true;
100 }; // SeriesData
101 
102 class SeriesInternal;
103 } // namespace internal
104 
113 {
114  friend class AttributableInterface;
115  friend class Iteration;
116  friend class Writable;
117  friend class SeriesIterator;
118  friend class internal::SeriesInternal;
119  friend class Series;
120  friend class WriteIterations;
121 
122 protected:
123  // Should not be called publicly, only by implementing classes
125 
126 public:
130  std::string openPMD() const;
136  SeriesInterface& setOpenPMD(std::string const& openPMD);
137 
141  uint32_t openPMDextension() const;
148 
152  std::string basePath() const;
158  SeriesInterface& setBasePath(std::string const& basePath);
159 
164  std::string meshesPath() const;
170  SeriesInterface& setMeshesPath(std::string const& meshesPath);
171 
176  std::string particlesPath() const;
182  SeriesInterface& setParticlesPath(std::string const& particlesPath);
183 
188  std::string author() const;
194  SeriesInterface& setAuthor(std::string const& author);
195 
200  std::string software() const;
207  SeriesInterface& setSoftware(std::string const& newName, std::string const& newVersion = std::string("unspecified"));
208 
213  std::string softwareVersion() const;
221  [[deprecated("Set the version with the second argument of setSoftware()")]]
223 
228  std::string date() const;
234  SeriesInterface& setDate(std::string const& date);
235 
240  std::string softwareDependencies() const;
246  SeriesInterface& setSoftwareDependencies(std::string const& newSoftwareDependencies);
247 
252  std::string machine() const;
258  SeriesInterface& setMachine(std::string const& newMachine);
259 
273 
277  std::string iterationFormat() const;
289 
293  std::string name() const;
294 
300  SeriesInterface& setName(std::string const& name);
301 
308  std::string backend() const;
309 
312  void flush();
313 
314 OPENPMD_private:
315  static constexpr char const * const BASEPATH = "/data/%T/";
316 
317  struct ParsedInput;
318  using iterations_t = decltype(internal::SeriesData::iterations);
319  using iterations_iterator = iterations_t::iterator;
320 
321  internal::SeriesData * m_series = nullptr;
322 
323  inline internal::SeriesData & get()
324  {
325  if( m_series )
326  {
327  return *m_series;
328  }
329  else
330  {
331  throw std::runtime_error(
332  "[Series] Cannot use default-constructed Series." );
333  }
334  }
335 
336  inline internal::SeriesData const & get() const
337  {
338  if( m_series )
339  {
340  return *m_series;
341  }
342  else
343  {
344  throw std::runtime_error(
345  "[Series] Cannot use default-constructed Series." );
346  } }
347 
348  std::unique_ptr< ParsedInput > parseInput(std::string);
349  bool hasExpansionPattern( std::string filenameWithExtension );
350  bool reparseExpansionPattern( std::string filenameWithExtension );
351  void init(std::shared_ptr< AbstractIOHandler >, std::unique_ptr< ParsedInput >);
352  void initDefaults( IterationEncoding );
364  std::future< void > flush_impl(
365  iterations_iterator begin,
366  iterations_iterator end,
367  FlushLevel level,
368  bool flushIOHandler = true );
369  void flushFileBased( iterations_iterator begin, iterations_iterator end );
370  /*
371  * Group-based and variable-based iteration layouts share a lot of logic
372  * (realistically, the variable-based iteration layout only throws out
373  * one layer in the hierarchy).
374  * As a convention, methods that deal with both layouts are called
375  * .*GorVBased, short for .*GroupOrVariableBased
376  */
377  void flushGorVBased( iterations_iterator begin, iterations_iterator end );
378  void flushMeshesPath();
379  void flushParticlesPath();
380  void readFileBased( );
381  void readOneIterationFileBased( std::string const & filePath );
387  void readGorVBased( bool init = true );
388  void readBase();
389  std::string iterationFilename( uint64_t i );
390 
391  enum class IterationOpened : bool
392  {
393  HasBeenOpened,
394  RemainsClosed
395  };
396  /*
397  * For use by flushFileBased, flushGorVBased
398  * Open an iteration, but only if necessary.
399  * Only open if the iteration is dirty and if it is not in deferred
400  * parse state.
401  */
402  IterationOpened openIterationIfDirty( uint64_t index, Iteration iteration );
403  /*
404  * Open an iteration. Ensures that the iteration's m_closed status
405  * is set properly and that any files pertaining to the iteration
406  * is opened.
407  * Does not create files when called in CREATE mode.
408  */
409  void openIteration( uint64_t index, Iteration iteration );
410 
415  iterations_iterator
416  indexOf( Iteration const & );
417 
434  advance(
435  AdvanceMode mode,
437  iterations_iterator it,
438  Iteration & iteration );
439 }; // SeriesInterface
440 
441 namespace internal
442 {
444 {
445  friend struct SeriesShared;
446  friend class openPMD::Iteration;
447  friend class openPMD::Series;
448  friend class openPMD::Writable;
449 
450 public:
451 #if openPMD_HAVE_MPI
453  std::string const & filepath,
454  Access at,
455  MPI_Comm comm,
456  std::string const & options = "{}" );
457 #endif
458 
460  std::string const & filepath,
461  Access at,
462  std::string const & options = "{}" );
463  // @todo make AttributableInterface<>::linkHierarchy non-virtual
464  virtual ~SeriesInternal();
465 };
466 } // namespace internal
467 
478 class Series : public SeriesInterface
479 {
480 private:
481  std::shared_ptr< internal::SeriesInternal > m_series;
482 
483 public:
484  explicit Series();
485 
486 #if openPMD_HAVE_MPI
487  Series(
488  std::string const & filepath,
489  Access at,
490  MPI_Comm comm,
491  std::string const & options = "{}" );
492 #endif
493 
503  Series(
504  std::string const & filepath,
505  Access at,
506  std::string const & options = "{}" );
507 
508  virtual ~Series() = default;
509 
511 
518  operator bool() const;
519 
531 
545 };
546 } // namespace openPMD
547 
548 // Make sure that this one is always included if Series.hpp is included,
549 // otherwise SeriesInterface::readIterations() cannot be used
550 #include "openPMD/ReadIterations.hpp"
openPMD::Writable
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:64
openPMD::internal::AttributableData
Definition: Attributable.hpp:68
openPMD::SeriesInterface::setMeshesPath
SeriesInterface & setMeshesPath(std::string const &meshesPath)
Set the path to mesh records, relative(!) to basePath.
Definition: Series.cpp:151
openPMD::SeriesInterface::machine
std::string machine() const
Definition: Series.cpp:255
openPMD::SeriesInterface::setBasePath
SeriesInterface & setBasePath(std::string const &basePath)
Set the common prefix for all data sets and sub-groups of a specific iteration.
Definition: Series.cpp:134
openPMD::IterationEncoding
IterationEncoding
Encoding scheme of an Iterations Series'.
Definition: IterationEncoding.hpp:32
openPMD::Format
Format
File format to use during IO.
Definition: Format.hpp:30
openPMD::SeriesInterface::setMachine
SeriesInterface & setMachine(std::string const &newMachine)
Indicate the machine or relevant hardware that created the file.
Definition: Series.cpp:261
openPMD::SeriesInterface::setParticlesPath
SeriesInterface & setParticlesPath(std::string const &particlesPath)
Set the path to groups for each particle species, relative(!) to basePath.
Definition: Series.cpp:173
openPMD::SeriesInterface::setSoftwareDependencies
SeriesInterface & setSoftwareDependencies(std::string const &newSoftwareDependencies)
Indicate dependencies of software that were used to create the file.
Definition: Series.cpp:248
openPMD::SeriesInterface::software
std::string software() const
Definition: Series.cpp:202
openPMD::SeriesInterface::author
std::string author() const
Definition: Series.cpp:189
openPMD::SeriesInterface::setOpenPMD
SeriesInterface & setOpenPMD(std::string const &openPMD)
Set the version of the enforced openPMD standard.
Definition: Series.cpp:108
openPMD::SeriesInterface::meshesPath
std::string meshesPath() const
Definition: Series.cpp:145
openPMD::internal::SeriesData::m_stepStatus
StepStatus m_stepStatus
Whether a step is currently active for this iteration.
Definition: Series.hpp:97
openPMD::WriteIterations
Writing side of the streaming API.
Definition: WriteIterations.hpp:47
openPMD::internal::SeriesInternal
Definition: Series.hpp:443
openPMD::AdvanceStatus
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
openPMD::Series
Root level of the openPMD hierarchy.
Definition: Series.hpp:478
openPMD::SeriesInterface::name
std::string name() const
Definition: Series.cpp:334
openPMD::SeriesInterface::date
std::string date() const
Definition: Series.cpp:229
openPMD::auxiliary::Option
Simple Option type based on variantSrc::variant.
Definition: Option.hpp:47
openPMD::SeriesInterface
Implementation for the root level of the openPMD hierarchy.
Definition: Series.hpp:112
openPMD::Iteration
Logical compilation of data from one snapshot (e.g.
Definition: Iteration.hpp:39
openPMD
Public definitions of openPMD-api.
Definition: Date.cpp:29
openPMD::SeriesInterface::setDate
SeriesInterface & setDate(std::string const &date)
Indicate the date of creation.
Definition: Series.cpp:235
openPMD::StepStatus
StepStatus
Used in step-based mode (i.e.
Definition: Streaming.hpp:44
openPMD::SeriesInterface::softwareDependencies
std::string softwareDependencies() const
Definition: Series.cpp:242
openPMD::SeriesInterface::openPMD
std::string openPMD() const
Definition: Series.cpp:102
openPMD::Series::readIterations
ReadIterations readIterations()
Entry point to the reading end of the streaming API.
Definition: Series.cpp:1571
openPMD::Container< Iteration, uint64_t >
openPMD::AdvanceMode
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:33
openPMD::SeriesIterator
Definition: ReadIterations.hpp:50
openPMD::AttributableInterface
Layer to manage storage of attributes associated with file objects.
Definition: Attributable.hpp:115
openPMD::SeriesInterface::setIterationEncoding
SeriesInterface & setIterationEncoding(IterationEncoding iterationEncoding)
Set the encoding style for multiple iterations in this series.
Definition: Series.cpp:274
openPMD::SeriesInterface::backend
std::string backend() const
The currently used backend.
Definition: Series.cpp:374
openPMD::SeriesInterface::setName
SeriesInterface & setName(std::string const &name)
Set the pattern for file names.
Definition: Series.cpp:340
openPMD::SeriesInterface::iterationFormat
std::string iterationFormat() const
Definition: Series.cpp:313
openPMD::SeriesInterface::setIterationFormat
SeriesInterface & setIterationFormat(std::string const &iterationFormat)
Set a pattern describing how to access single iterations in the raw file.
Definition: Series.cpp:319
openPMD::Access
Access
File access mode to use during IO.
Definition: Access.hpp:28
openPMD::Series::writeIterations
WriteIterations writeIterations()
Entry point to the writing end of the streaming API.
Definition: Series.cpp:1577
openPMD::SeriesInterface::softwareVersion
std::string softwareVersion() const
Definition: Series.cpp:216
openPMD::SeriesInterface::basePath
std::string basePath() const
Definition: Series.cpp:128
openPMD::FlushLevel
FlushLevel
Determine what items should be flushed upon Series::flush()
Definition: AbstractIOHandler.hpp:63
openPMD::SeriesInterface::flush
void flush()
Execute all required remaining IO operations to write or read data.
Definition: Series.cpp:380
openPMD::SeriesInterface::setSoftware
SeriesInterface & setSoftware(std::string const &newName, std::string const &newVersion=std::string("unspecified"))
Indicate the software/code/simulation that created the file.
Definition: Series.cpp:208
openPMD::SeriesInterface::setAuthor
SeriesInterface & setAuthor(std::string const &author)
Indicate the author and contact for the information in the file.
Definition: Series.cpp:195
openPMD::SeriesInterface::ParsedInput
Definition: Series.cpp:83
openPMD::ReadIterations
Reading side of the streaming API.
Definition: ReadIterations.hpp:92
openPMD::SeriesInterface::iterationEncoding
IterationEncoding iterationEncoding() const
Definition: Series.cpp:268
openPMD::internal::SeriesData
Data members for Series.
Definition: Series.hpp:67
openPMD::SeriesInterface::setSoftwareVersion
SeriesInterface & setSoftwareVersion(std::string const &softwareVersion)
Indicate the version of the software/code/simulation that created the file.
Definition: Series.cpp:222
openPMD::SeriesInterface::setOpenPMDextension
SeriesInterface & setOpenPMDextension(uint32_t openPMDextension)
Set a 32-bit mask of applied extensions to the openPMD standard.
Definition: Series.cpp:121
openPMD::SeriesInterface::particlesPath
std::string particlesPath() const
Definition: Series.cpp:167
openPMD::SeriesInterface::openPMDextension
uint32_t openPMDextension() const
Definition: Series.cpp:115