ChunkInfo.hpp

/* Copyright 2020-2025 Franz Poeschel, Axel Huebl
 *
 * This file is part of openPMD-api.
 *
 * openPMD-api is free software: you can redistribute it and/or modify
 * it under the terms of of either the GNU General Public License or
 * the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * openPMD-api is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License and the GNU Lesser General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License
 * and the GNU Lesser General Public License along with openPMD-api.
 * If not, see <http://www.gnu.org/licenses/>.
 */
#pragma once
 
#include "openPMD/config.hpp"
 
#include "openPMD/Dataset.hpp" // Offset, Extent
#include "openPMD/auxiliary/BlockSlicer.hpp"
 
#if openPMD_HAVE_MPI
#include <mpi.h>
#endif
 
#include <map>
#include <memory>
#include <string>
#include <vector>
 
namespace openPMD
{
struct ChunkInfo
{
    Offset offset; 
    Extent extent; 
 
    /*
     * If rank is smaller than zero, will be converted to zero.
     */
    explicit ChunkInfo() = default;
    ChunkInfo(Offset, Extent);
 
    bool operator==(ChunkInfo const &other) const;
};

struct WrittenChunkInfo : ChunkInfo
{
    unsigned int sourceID = 0; 
 
    explicit WrittenChunkInfo() = default;
    /*
     * If rank is smaller than zero, will be converted to zero.
     */
    WrittenChunkInfo(Offset, Extent, int sourceID);
    WrittenChunkInfo(Offset, Extent);
 
    bool operator==(WrittenChunkInfo const &other) const;
};
 
using ChunkTable = std::vector<WrittenChunkInfo>;
 
namespace chunk_assignment
{
    using RankMeta = std::map<unsigned int, std::string>;

    using Assignment = std::map<unsigned int, std::vector<WrittenChunkInfo>>;

    template <typename Chunk_t>
    void mergeChunks(std::vector<Chunk_t> &chunks);

    auto
    mergeChunksFromSameSourceID(std::vector<WrittenChunkInfo> const &chunks)
        -> std::map<unsigned int, std::vector<ChunkInfo>>;

    struct PartialAssignment
    {
        ChunkTable notAssigned;
        Assignment assigned;
 
        explicit PartialAssignment() = default;
        PartialAssignment(ChunkTable notAssigned);
        PartialAssignment(ChunkTable notAssigned, Assignment assigned);
    };

    struct Strategy
    {
        Assignment assign(
            ChunkTable chunkTable,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks);
        virtual Assignment assign(
            PartialAssignment partialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) = 0;
 
        virtual std::unique_ptr<Strategy> clone() const = 0;
 
        virtual ~Strategy() = default;
    };

    struct PartialStrategy
    {
        PartialAssignment assign(
            ChunkTable table,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks);
        virtual PartialAssignment assign(
            PartialAssignment partialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) = 0;
 
        virtual std::unique_ptr<PartialStrategy> clone() const = 0;
 
        virtual ~PartialStrategy() = default;
    };

    struct FromPartialStrategy : Strategy
    {
        FromPartialStrategy(
            std::unique_ptr<PartialStrategy> firstPass,
            std::unique_ptr<Strategy> secondPass);
 
        virtual Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
 
    private:
        std::unique_ptr<PartialStrategy> m_firstPass;
        std::unique_ptr<Strategy> m_secondPass;
    };

    struct RoundRobin : Strategy
    {
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
    };

    struct RoundRobinOfSourceRanks : Strategy
    {
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
    };

    struct Blocks : Strategy
    {
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        [[nodiscard]] std::unique_ptr<Strategy> clone() const override;
    };

    struct BlocksOfSourceRanks : Strategy
    {
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        [[nodiscard]] std::unique_ptr<Strategy> clone() const override;
    };

    struct ByHostname : PartialStrategy
    {
        ByHostname(std::unique_ptr<Strategy> withinNode);
 
        PartialAssignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<PartialStrategy> clone() const override;
 
    private:
        std::unique_ptr<Strategy> m_withinNode;
    };

    struct ByCuboidSlice : Strategy
    {
        ByCuboidSlice(
            std::unique_ptr<auxiliary::BlockSlicer> blockSlicer,
            Extent totalExtent);
 
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
 
    private:
        std::unique_ptr<auxiliary::BlockSlicer> blockSlicer;
        Extent totalExtent;
    };

    struct BinPacking : Strategy
    {
        size_t splitAlongDimension = 0;

        BinPacking(size_t splitAlongDimension = 0);
 
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
    };

    struct FailingStrategy : Strategy
    {
        explicit FailingStrategy();
 
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
    };

    struct DiscardingStrategy : Strategy
    {
        explicit DiscardingStrategy();
 
        Assignment assign(
            PartialAssignment,
            RankMeta const &in,
            RankMeta const &out,
            size_t my_rank,
            size_t num_ranks) override;
 
        virtual std::unique_ptr<Strategy> clone() const override;
    };
} // namespace chunk_assignment
 
namespace host_info
{
    enum class Method
    {
        POSIX_HOSTNAME,
        MPI_PROCESSOR_NAME
    };

    bool methodAvailable(Method);

    std::string byMethod(Method);
 
#if openPMD_HAVE_MPI
    chunk_assignment::RankMeta byMethodCollective(MPI_Comm, Method);
#endif
} // namespace host_info
} // namespace openPMD
 
#undef openPMD_POSIX_AVAILABLE