ADIOS2IOHandler.hpp

/* Copyright 2017-2021 Fabian Koller and Franz Poeschel
 *
 * This file is part of openPMD-api.
 *
 * openPMD-api is free software: you can redistribute it and/or modify
 * it under the terms of of either the GNU General Public License or
 * the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * openPMD-api is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License and the GNU Lesser General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License
 * and the GNU Lesser General Public License along with openPMD-api.
 * If not, see <http://www.gnu.org/licenses/>.
 */
#pragma once
 
#include "openPMD/IO/AbstractIOHandler.hpp"
#include "openPMD/IO/AbstractIOHandlerImpl.hpp"
#include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
#include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
#include "openPMD/IO/IOTask.hpp"
#include "openPMD/IO/InvalidatableFile.hpp"
#include "openPMD/auxiliary/JSON.hpp"
#include "openPMD/auxiliary/Option.hpp"
#include "openPMD/backend/Writable.hpp"
#include "openPMD/config.hpp"
#include "openPMD/IterationEncoding.hpp"
 
#if openPMD_HAVE_ADIOS2
#    include <adios2.h>
#endif
#if openPMD_HAVE_MPI
#   include <mpi.h>
#endif
#include <nlohmann/json.hpp>
 
#include <array>
#include <exception>
#include <future>
#include <iostream>
#include <memory> // shared_ptr
#include <set>
#include <string>
#include <unordered_map>
#include <utility> // pair
#include <vector>
 
 
namespace openPMD
{
#if openPMD_HAVE_ADIOS2
 
class ADIOS2IOHandler;
 
namespace detail
{
    template < typename, typename > struct DatasetHelper;
    struct GetSpan;
    struct DatasetReader;
    struct AttributeReader;
    struct AttributeWriter;
    struct OldAttributeReader;
    struct OldAttributeWriter;
    template < typename > struct AttributeTypes;
    struct DatasetOpener;
    struct VariableDefiner;
    template < typename > struct DatasetTypes;
    struct WriteDataset;
    struct BufferedActions;
    struct BufferedPut;
    struct BufferedGet;
    struct BufferedAttributeRead;
    struct BufferedAttributeWrite;
} // namespace detail
 
 
namespace ADIOS2Schema
{
    using schema_t = uint64_t;
    /*
     * Original ADIOS schema.
     */
    constexpr schema_t schema_0000_00_00 = 00000000;
    /*
     * This introduces attribute layout via scalar ADIOS variables.
     */
    constexpr schema_t schema_2021_02_09 = 20210209;
 
    enum class SupportedSchema : char
    {
        s_0000_00_00,
        s_2021_02_09
    };
}
using SupportedSchema = ADIOS2Schema::SupportedSchema;
 
class ADIOS2IOHandlerImpl
: public AbstractIOHandlerImplCommon< ADIOS2FilePosition >
{
    template < typename, typename > friend struct detail::DatasetHelper;
    friend struct detail::GetSpan;
    friend struct detail::DatasetReader;
    friend struct detail::AttributeReader;
    friend struct detail::AttributeWriter;
    friend struct detail::OldAttributeReader;
    friend struct detail::OldAttributeWriter;
    template < typename > friend struct detail::AttributeTypes;
    friend struct detail::DatasetOpener;
    friend struct detail::VariableDefiner;
    template < typename > friend struct detail::DatasetTypes;
    friend struct detail::WriteDataset;
    friend struct detail::BufferedActions;
    friend struct detail::BufferedAttributeRead;
 
    static constexpr bool ADIOS2_DEBUG_MODE = false;
 
 
public:
 
#if openPMD_HAVE_MPI
 
    ADIOS2IOHandlerImpl(
        AbstractIOHandler *,
        MPI_Comm,
        nlohmann::json config,
        std::string engineType );
 
#endif // openPMD_HAVE_MPI
 
    explicit ADIOS2IOHandlerImpl(
        AbstractIOHandler *,
        nlohmann::json config,
        std::string engineType );
 
 
    ~ADIOS2IOHandlerImpl() override;
 
    std::future< void > flush( ) override;
 
    void createFile( Writable *,
                     Parameter< Operation::CREATE_FILE > const & ) override;
 
    void createPath( Writable *,
                     Parameter< Operation::CREATE_PATH > const & ) override;
 
    void
    createDataset( Writable *,
                   Parameter< Operation::CREATE_DATASET > const & ) override;
 
    void
    extendDataset( Writable *,
                   Parameter< Operation::EXTEND_DATASET > const & ) override;
 
    void openFile( Writable *,
                   Parameter< Operation::OPEN_FILE > const & ) override;
 
    void closeFile( Writable *,
                    Parameter< Operation::CLOSE_FILE > const & ) override;
 
    void openPath( Writable *,
                   Parameter< Operation::OPEN_PATH > const & ) override;
 
    void closePath( Writable *,
                    Parameter< Operation::CLOSE_PATH > const & ) override;
 
    void openDataset( Writable *,
                      Parameter< Operation::OPEN_DATASET > & ) override;
 
    void deleteFile( Writable *,
                     Parameter< Operation::DELETE_FILE > const & ) override;
 
    void deletePath( Writable *,
                     Parameter< Operation::DELETE_PATH > const & ) override;
 
    void
    deleteDataset( Writable *,
                   Parameter< Operation::DELETE_DATASET > const & ) override;
 
    void deleteAttribute( Writable *,
                          Parameter< Operation::DELETE_ATT > const & ) override;
 
    void writeDataset( Writable *,
                       Parameter< Operation::WRITE_DATASET > const & ) override;
 
    void writeAttribute( Writable *,
                         Parameter< Operation::WRITE_ATT > const & ) override;
 
    void readDataset( Writable *,
                      Parameter< Operation::READ_DATASET > & ) override;
 
    void getBufferView( Writable *,
                      Parameter< Operation::GET_BUFFER_VIEW > & ) override;
 
    void readAttribute( Writable *,
                        Parameter< Operation::READ_ATT > & ) override;
 
    void listPaths( Writable *, Parameter< Operation::LIST_PATHS > & ) override;
 
    void listDatasets( Writable *,
                       Parameter< Operation::LIST_DATASETS > & ) override;
 
    void
    listAttributes( Writable *,
                    Parameter< Operation::LIST_ATTS > & parameters ) override;
 
    void
    advance( Writable*, Parameter< Operation::ADVANCE > & ) override;
 
    void
    availableChunks( Writable*,
                     Parameter< Operation::AVAILABLE_CHUNKS > &) override;
    adios2::Mode adios2AccessMode( std::string const & fullPath );
 
private:
    adios2::ADIOS m_ADIOS;
    /*
     * If the iteration encoding is variableBased, we default to using the
     * 2021_02_09 schema since it allows mutable attributes.
     */
    IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
    std::string m_engineType;
    ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00;
 
    enum class AttributeLayout : char
    {
        ByAdiosAttributes,
        ByAdiosVariables
    };
 
    inline SupportedSchema schema() const
    {
        switch( m_schema )
        {
        case ADIOS2Schema::schema_0000_00_00:
            return SupportedSchema::s_0000_00_00;
        case ADIOS2Schema::schema_2021_02_09:
            return SupportedSchema::s_2021_02_09;
        default:
            throw std::runtime_error(
                "[ADIOS2] Encountered unsupported schema version: " +
                std::to_string( m_schema ) );
        }
    }
 
    inline AttributeLayout attributeLayout() const
    {
        switch( schema() )
        {
        case SupportedSchema::s_0000_00_00:
            return AttributeLayout::ByAdiosAttributes;
        case SupportedSchema::s_2021_02_09:
            return AttributeLayout::ByAdiosVariables;
        }
        throw std::runtime_error( "Unreachable!" );
    }
 
    struct ParameterizedOperator
    {
        adios2::Operator op;
        adios2::Params params;
    };
 
    std::vector< ParameterizedOperator > defaultOperators;
 
    auxiliary::TracingJSON m_config;
    static auxiliary::TracingJSON nullvalue;
 
    void
    init( nlohmann::json config );
 
    template< typename Key >
    auxiliary::TracingJSON
    config( Key && key, auxiliary::TracingJSON & cfg )
    {
        if( cfg.json().is_object() && cfg.json().contains( key ) )
        {
            return cfg[ key ];
        }
        else
        {
            return nullvalue;
        }
    }
 
    template< typename Key >
    auxiliary::TracingJSON
    config( Key && key )
    {
        return config< Key >( std::forward< Key >( key ), m_config );
    }
 
    auxiliary::Option< std::vector< ParameterizedOperator > >
    getOperators( auxiliary::TracingJSON config );
 
    // use m_config
    auxiliary::Option< std::vector< ParameterizedOperator > >
    getOperators();
 
    std::string
    fileSuffix() const;
 
    /*
     * We need to give names to IO objects. These names are irrelevant
     * within this application, since:
     * 1) The name of the file written to is decided by the opened Engine's
     *    name.
     * 2) The IOs are managed by the unordered_map m_fileData, so we do not
     *    need the ADIOS2 internal management.
     * Since within one m_ADIOS object, the same IO name cannot be used more
     * than once, we ensure different names by using the name counter.
     * This allows to overwrite a file later without error.
     */
    int nameCounter{0};
 
    /*
     * IO-heavy actions are deferred to a later point. This map stores for
     * each open file (identified by an InvalidatableFile object) an object
     * that manages IO-heavy actions, as well as its ADIOS2 objects, i.e.
     * IO and Engine object.
     * Not to be accessed directly, use getFileData().
     */
    std::unordered_map< InvalidatableFile,
                        std::unique_ptr< detail::BufferedActions >
    > m_fileData;
 
    std::map< std::string, adios2::Operator > m_operators;
 
    // Overrides from AbstractIOHandlerImplCommon.
 
    std::string
        filePositionToString( std::shared_ptr< ADIOS2FilePosition > ) override;
 
    std::shared_ptr< ADIOS2FilePosition >
    extendFilePosition( std::shared_ptr< ADIOS2FilePosition > const & pos,
                        std::string extend ) override;
 
    // Helper methods.
 
    auxiliary::Option< adios2::Operator >
    getCompressionOperator( std::string const & compression );
 
    /*
     * The name of the ADIOS2 variable associated with this Writable.
     * To be used for Writables that represent a dataset.
     */
    std::string nameOfVariable( Writable * writable );
 
    std::string nameOfAttribute( Writable * writable, std::string attribute );
 
    /*
     * Figure out whether the Writable corresponds with a
     * group or a dataset.
     */
    ADIOS2FilePosition::GD groupOrDataset( Writable * );
 
    enum class IfFileNotOpen : bool
    {
        OpenImplicitly,
        ThrowError
    };
 
    detail::BufferedActions &
    getFileData( InvalidatableFile file, IfFileNotOpen );
 
    void dropFileData( InvalidatableFile file );
 
    /*
     * Prepare a variable that already exists for an IO
     * operation, including:
     * (1) checking that its datatype matches T.
     * (2) the offset and extent match the variable's shape
     * (3) setting the offset and extent (ADIOS lingo: start
     *     and count)
     */
    template < typename T >
    adios2::Variable< T > verifyDataset( Offset const & offset,
                                         Extent const & extent, adios2::IO & IO,
                                         std::string const & var );
}; // ADIOS2IOHandlerImpl
 
/*
 * The following strings are used during parsing of the JSON configuration
 * string for the ADIOS2 backend.
 */
namespace ADIOS2Defaults
{
    using const_str = char const * const;
    constexpr const_str str_engine = "engine";
    constexpr const_str str_type = "type";
    constexpr const_str str_params = "parameters";
    constexpr const_str str_usesteps = "usesteps";
    constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps";
    constexpr const_str str_adios2Schema =
        "__openPMD_internal/openPMD2_adios2_schema";
    constexpr const_str str_isBooleanOldLayout = "__is_boolean__";
    constexpr const_str str_isBooleanNewLayout =
        "__openPMD_internal/is_boolean";
} // namespace ADIOS2Defaults
 
namespace detail
{
    // Helper structs for calls to the switchType function
 
    struct DatasetReader
    {
        openPMD::ADIOS2IOHandlerImpl * m_impl;
 
 
        explicit DatasetReader( openPMD::ADIOS2IOHandlerImpl * impl );
 
 
        template < typename T >
        void operator( )( BufferedGet & bp, adios2::IO & IO,
                          adios2::Engine & engine,
                          std::string const & fileName );
 
        std::string errorMsg = "ADIOS2: readDataset()";
    };
 
    struct OldAttributeReader
    {
        template< typename T >
        Datatype
        operator()(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        template< int n, typename... Params >
        Datatype
        operator()( Params &&... );
    };
 
    struct OldAttributeWriter
    {
        template< typename T >
        void
        operator()(
            ADIOS2IOHandlerImpl * impl,
            Writable * writable,
            const Parameter< Operation::WRITE_ATT > & parameters );
 
 
        template< int n, typename... Params >
        void
        operator()( Params &&... );
    };
 
    struct AttributeReader
    {
        template< typename T >
        Datatype
        operator()(
            adios2::IO & IO,
            detail::PreloadAdiosAttributes const & preloadedAttributes,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        template < int n, typename... Params >
        Datatype operator( )( Params &&... );
    };
 
    struct AttributeWriter
    {
        template < typename T >
        void
        operator()(
            detail::BufferedAttributeWrite & params,
            BufferedActions & fileData );
 
 
        template < int n, typename... Params > void operator( )( Params &&... );
    };
 
    struct DatasetOpener
    {
        ADIOS2IOHandlerImpl * m_impl;
 
        explicit DatasetOpener( ADIOS2IOHandlerImpl * impl );
 
 
        template < typename T >
        void operator( )( InvalidatableFile, const std::string & varName,
                          Parameter< Operation::OPEN_DATASET > & parameters );
 
 
        std::string errorMsg = "ADIOS2: openDataset()";
    };
 
    struct WriteDataset
    {
        ADIOS2IOHandlerImpl * m_handlerImpl;
 
 
        WriteDataset( ADIOS2IOHandlerImpl * handlerImpl );
 
 
        template < typename T >
        void operator( )( BufferedPut & bp, adios2::IO & IO,
                          adios2::Engine & engine );
 
        template < int n, typename... Params > void operator( )( Params &&... );
    };
 
    struct VariableDefiner
    {
        template < typename T >
        void operator( )(
            adios2::IO & IO,
            std::string const & name,
            std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const &
                compressions,
            adios2::Dims const & shape = adios2::Dims(),
            adios2::Dims const & start = adios2::Dims(),
            adios2::Dims const & count = adios2::Dims(),
            bool const constantDims = false );
 
        std::string errorMsg = "ADIOS2: defineVariable()";
    };
 
    struct RetrieveBlocksInfo
    {
        template < typename T >
        void operator( )(
            Parameter< Operation::AVAILABLE_CHUNKS > & params,
            adios2::IO & IO,
            adios2::Engine & engine,
            std::string const & varName );
 
        template < int n, typename... Params >
        void operator( )( Params &&... );
    };
 
    // Helper structs to help distinguish valid attribute/variable
    // datatypes from invalid ones
 
    /*
     * This struct's purpose is to have specialisations
     * for vector and array types, as well as the boolean
     * type (which is not natively supported by ADIOS).
     */
    template< typename T >
    struct AttributeTypes
    {
        static void
        oldCreateAttribute(
            adios2::IO & IO,
            std::string name,
            T value );
 
        static void
        oldReadAttribute(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static void
        createAttribute(
            adios2::IO & IO,
            adios2::Engine & engine,
            detail::BufferedAttributeWrite & params,
            T value );
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static bool
        attributeUnchanged( adios2::IO & IO, std::string name, T val )
        {
            auto attr = IO.InquireAttribute< T >( name );
            if( !attr )
            {
                return false;
            }
            std::vector< T > data = attr.Data();
            if( data.size() != 1 )
            {
                return false;
            }
            return data[ 0 ] == val;
        }
    };
 
    template< > struct AttributeTypes< std::complex< long double > >
    {
        static void
        oldCreateAttribute(
            adios2::IO &,
            std::string,
            std::complex< long double > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex attribute types" );
        }
 
        static void
        oldReadAttribute(
            adios2::IO &,
            std::string,
            std::shared_ptr< Attribute::resource > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex attribute types" );
        }
 
        static void
        createAttribute(
            adios2::IO &,
            adios2::Engine &,
            detail::BufferedAttributeWrite &,
            std::complex< long double > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex attribute types" );
        }
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string,
            std::shared_ptr< Attribute::resource > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex attribute types" );
        }
 
        static bool
        attributeUnchanged(
            adios2::IO &, std::string, std::complex< long double > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex attribute types" );
        }
    };
 
    template< > struct AttributeTypes< std::vector< std::complex< long double > > >
    {
        static void
        oldCreateAttribute(
            adios2::IO &,
            std::string,
            const std::vector< std::complex< long double > > & )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
        }
 
        static void
        oldReadAttribute(
            adios2::IO &,
            std::string,
            std::shared_ptr< Attribute::resource > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
        }
 
        static void
        createAttribute(
            adios2::IO &,
            adios2::Engine &,
            detail::BufferedAttributeWrite &,
            const std::vector< std::complex< long double > > & )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
        }
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string,
            std::shared_ptr< Attribute::resource > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
        }
 
        static bool
        attributeUnchanged(
            adios2::IO &,
            std::string,
            std::vector< std::complex< long double > > )
        {
            throw std::runtime_error(
                "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
        }
    };
 
    template < typename T > struct AttributeTypes< std::vector< T > >
    {
        static void
        oldCreateAttribute(
            adios2::IO & IO,
            std::string name,
            const std::vector< T > & value );
 
        static void
        oldReadAttribute(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static void
        createAttribute(
            adios2::IO & IO,
            adios2::Engine & engine,
            detail::BufferedAttributeWrite & params,
            const std::vector< T > & value );
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static bool
        attributeUnchanged(
            adios2::IO & IO,
            std::string name,
            std::vector< T > val )
        {
            auto attr = IO.InquireAttribute< T >( name );
            if( !attr )
            {
                return false;
            }
            std::vector< T > data = attr.Data();
            if( data.size() != val.size() )
            {
                return false;
            }
            for( size_t i = 0; i < val.size(); ++i )
            {
                if( data[ i ] != val[ i ] )
                {
                    return false;
                }
            }
            return true;
        }
    };
 
    template<>
    struct AttributeTypes< std::vector< std::string > >
    {
        static void
        oldCreateAttribute(
            adios2::IO & IO,
            std::string name,
            const std::vector< std::string > & value );
 
        static void
        oldReadAttribute(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static void
        createAttribute(
            adios2::IO & IO,
            adios2::Engine & engine,
            detail::BufferedAttributeWrite & params,
            const std::vector< std::string > & vec );
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static bool
        attributeUnchanged(
            adios2::IO & IO,
            std::string name,
            std::vector< std::string > val )
        {
            auto attr = IO.InquireAttribute< std::string >( name );
            if( !attr )
            {
                return false;
            }
            std::vector< std::string > data = attr.Data();
            if( data.size() != val.size() )
            {
                return false;
            }
            for( size_t i = 0; i < val.size(); ++i )
            {
                if( data[ i ] != val[ i ] )
                {
                    return false;
                }
            }
            return true;
        }
    };
 
    template < typename T, size_t n >
    struct AttributeTypes< std::array< T, n > >
    {
        static void
        oldCreateAttribute(
            adios2::IO & IO,
            std::string name,
            const std::array< T, n > & value );
 
        static void
        oldReadAttribute(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static void
        createAttribute(
            adios2::IO & IO,
            adios2::Engine & engine,
            detail::BufferedAttributeWrite & params,
            const std::array< T, n > & value );
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static bool
        attributeUnchanged(
            adios2::IO & IO,
            std::string name,
            std::array< T, n > val )
        {
            auto attr = IO.InquireAttribute< T >( name );
            if( !attr )
            {
                return false;
            }
            std::vector< T > data = attr.Data();
            if( data.size() != n )
            {
                return false;
            }
            for( size_t i = 0; i < n; ++i )
            {
                if( data[ i ] != val[ i ] )
                {
                    return false;
                }
            }
            return true;
        }
    };
 
    template <> struct AttributeTypes< bool >
    {
        using rep = detail::bool_representation;
 
        static void
        oldCreateAttribute( adios2::IO & IO, std::string name, bool value );
 
        static void
        oldReadAttribute(
            adios2::IO & IO,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
        static void
        createAttribute(
            adios2::IO & IO,
            adios2::Engine & engine,
            detail::BufferedAttributeWrite & params,
            bool value );
 
        static void
        readAttribute(
            detail::PreloadAdiosAttributes const &,
            std::string name,
            std::shared_ptr< Attribute::resource > resource );
 
 
        static constexpr rep toRep( bool b )
        {
            return b ? 1U : 0U;
        }
 
 
        static constexpr bool fromRep( rep r )
        {
            return r != 0;
        }
 
        static bool
        attributeUnchanged( adios2::IO & IO, std::string name, bool val )
        {
            auto attr = IO.InquireAttribute< rep >( name );
            if( !attr )
            {
                return false;
            }
            std::vector< rep > data = attr.Data();
            if( data.size() != 1 )
            {
                return false;
            }
            return data[ 0 ] == toRep( val );
        }
    };
 
    // Other datatypes used in the ADIOS2IOHandler implementation
 
 
    struct BufferedActions;
 
    /*
     * IO-heavy action to be executed upon flushing.
     */
    struct BufferedAction
    {
        explicit BufferedAction( ) = default;
        virtual ~BufferedAction( ) = default;
 
        BufferedAction( BufferedAction const & other ) = delete;
        BufferedAction( BufferedAction && other ) = default;
 
        BufferedAction & operator=( BufferedAction const & other ) = delete;
        BufferedAction & operator=( BufferedAction && other ) = default;
 
        virtual void run( BufferedActions & ) = 0;
    };
 
    struct BufferedGet : BufferedAction
    {
        std::string name;
        Parameter< Operation::READ_DATASET > param;
 
        void run( BufferedActions & ) override;
    };
 
    struct BufferedPut : BufferedAction
    {
        std::string name;
        Parameter< Operation::WRITE_DATASET > param;
 
        void run( BufferedActions & ) override;
    };
 
    struct OldBufferedAttributeRead : BufferedAction
    {
        Parameter< Operation::READ_ATT > param;
        std::string name;
 
        void run( BufferedActions & ) override;
    };
 
    struct BufferedAttributeRead
    {
        Parameter< Operation::READ_ATT > param;
        std::string name;
 
        void
        run( BufferedActions & );
    };
 
    struct BufferedAttributeWrite : BufferedAction
    {
        std::string name;
        Datatype dtype;
        Attribute::resource resource;
        std::vector< char > bufferForVecString;
 
        void run( BufferedActions & ) override;
    };
 
    struct I_UpdateSpan
    {
        virtual void *update() = 0;
        virtual ~I_UpdateSpan() = default;
    };
 
    template< typename T >
    struct UpdateSpan : I_UpdateSpan
    {
        adios2::detail::Span< T > span;
 
        UpdateSpan( adios2::detail::Span< T > );
 
        void *update() override;
    };
 
    /*
     * Manages per-file information about
     * (1) the file's IO and Engine objects
     * (2) the file's deferred IO-heavy actions
     */
    struct BufferedActions
    {
        BufferedActions( BufferedActions const & ) = delete;
 
        std::string m_file;
        std::string const m_IOName;
        adios2::ADIOS & m_ADIOS;
        adios2::IO m_IO;
        std::vector< std::unique_ptr< BufferedAction > > m_buffer;
        std::map< std::string, BufferedAttributeWrite > m_attributeWrites;
        std::vector< BufferedAttributeRead > m_attributeReads;
        std::vector< std::unique_ptr< BufferedAction > > m_alreadyEnqueued;
        adios2::Mode m_mode;
        std::map< unsigned, std::unique_ptr< I_UpdateSpan > > m_updateSpans;
        detail::WriteDataset const m_writeDataset;
        detail::DatasetReader const m_readDataset;
        detail::AttributeReader const m_attributeReader;
        PreloadAdiosAttributes preloadAttributes;
 
        /*
         * We call an attribute committed if the step during which it was
         * written has been closed.
         * A committed attribute cannot be modified.
         */
        std::set< std::string > uncommittedAttributes;
 
        /*
         * The openPMD API will generally create new attributes for each
         * iteration. This results in a growing number of attributes over time.
         * In streaming-based modes, these will be completely sent anew in each
         * iteration. If the following boolean is true, old attributes will be
         * removed upon CLOSE_GROUP.
         * Should not be set to true in persistent backends.
         * Will be automatically set by BufferedActions::configure_IO depending
         * on chosen ADIOS2 engine and can not be explicitly overridden by user.
         */
        bool optimizeAttributesStreaming = false;
 
        using AttributeMap_t = std::map< std::string, adios2::Params >;
 
        BufferedActions( ADIOS2IOHandlerImpl & impl, InvalidatableFile file );
 
        ~BufferedActions( );
 
        void
        finalize();
 
        adios2::Engine & getEngine( );
        adios2::Engine & requireActiveStep( );
 
        template < typename BA > void enqueue( BA && ba );
 
        template < typename BA > void enqueue( BA && ba, decltype( m_buffer ) & );
 
        template< typename F >
        void
        flush(
            FlushLevel level,
            F && performPutsGets,
            bool writeAttributes,
            bool flushUnconditionally );
 
        void
        flush( FlushLevel, bool writeAttributes = false );
 
        AdvanceStatus
        advance( AdvanceMode mode );
 
        /*
         * Delete all buffered actions without running them.
         */
        void drop( );
 
        AttributeMap_t const &
        availableAttributes();
 
        std::vector< std::string >
        availableAttributesPrefixed( std::string const & prefix );
 
        /*
         * See description below.
         */
        void
        invalidateAttributesMap();
 
        AttributeMap_t const &
        availableVariables();
 
        std::vector< std::string >
        availableVariablesPrefixed( std::string const & prefix );
 
        /*
         * See description below.
         */
        void
        invalidateVariablesMap();
 
    private:
        ADIOS2IOHandlerImpl * m_impl;
        auxiliary::Option< adios2::Engine > m_engine; 
 
        std::string m_engineType;
        /*
         * streamStatus is NoStream for file-based ADIOS engines.
         * This is relevant for the method BufferedActions::requireActiveStep,
         * where a step is only opened if the status is OutsideOfStep, but not
         * if NoStream. The rationale behind this is that parsing a Series
         * works differently for file-based and for stream-based engines:
         * * stream-based: Iterations are parsed as they arrive. For parsing an
         *   iteration, the iteration must be awaited.
         *   BufferedActions::requireActiveStep takes care of this.
         * * file-based: The Series is parsed up front. If no step has been
         *   opened yet, ADIOS2 gives access to all variables and attributes
         *   from all steps. Upon opening a step, only the variables from that
         *   step are shown which hinders parsing. So, until a step is
         *   explicitly opened via ADIOS2IOHandlerImpl::advance, do not open
         *   one.
         *   This is to enable use of ADIOS files without the Streaming API
         *   (i.e. all iterations should be visible to the user upon opening
         *   the Series.)
         *   @todo Add a workflow without up-front parsing of all iterations
         *         for file-based engines.
         *         (This would merely be an optimization since the streaming
         *         API still works with files as intended.)
         *
         */
        enum class StreamStatus
        {
            DuringStep,
            OutsideOfStep,
            StreamOver,
            NoStream,
            Parsing,
            Undecided
        };
        StreamStatus streamStatus = StreamStatus::OutsideOfStep;
        adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
 
        bool delayOpeningTheFirstStep = false;
 
        /*
         * ADIOS2 does not give direct access to its internal attribute and
         * variable maps, but will instead give access to copies of them.
         * In order to avoid unnecessary copies, we buffer the returned map.
         * The downside of this is that we need to pay attention to invalidate
         * the map whenever an attribute/variable is altered. In that case, we
         * fetch the map anew.
         * If empty, the buffered map has been invalidated and needs to be
         * queried from ADIOS2 again. If full, the buffered map is equivalent to
         * the map that would be returned by a call to
         * IO::Available(Attributes|Variables).
         */
        auxiliary::Option< AttributeMap_t > m_availableAttributes;
        auxiliary::Option< AttributeMap_t > m_availableVariables;
 
        /*
         * finalize() will set this true to avoid running twice.
         */
        bool finalized = false;
 
        inline SupportedSchema schema() const
        {
            return m_impl->schema();
        }
 
        void
        configure_IO( ADIOS2IOHandlerImpl & impl );
 
        using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout;
        inline AttributeLayout attributeLayout() const
        {
            return m_impl->attributeLayout();
        }
    };
 
} // namespace detail
#endif // openPMD_HAVE_ADIOS2
 
 
class ADIOS2IOHandler : public AbstractIOHandler
{
#if openPMD_HAVE_ADIOS2
 
friend class ADIOS2IOHandlerImpl;
 
private:
    ADIOS2IOHandlerImpl m_impl;
 
public:
    ~ADIOS2IOHandler( ) override
    {
        // we must not throw in a destructor
        try
        {
            this->flush( );
        }
        catch( std::exception const & ex )
        {
            std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
        }
        catch( ... )
        {
             std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
        }
    }
 
#else
public:
#endif
 
#if openPMD_HAVE_MPI
 
    ADIOS2IOHandler(
        std::string path,
        Access,
        MPI_Comm,
        nlohmann::json options,
        std::string engineType );
 
#endif
 
    ADIOS2IOHandler(
        std::string path,
        Access,
        nlohmann::json options,
        std::string engineType );
 
    std::string backendName() const override { return "ADIOS2"; }
 
    std::future< void > flush( ) override;
}; // ADIOS2IOHandler
} // namespace openPMD