#include "NVMeSubsys.hpp"

#include "AsioHelper.hpp"
#include "NVMeCache.hpp"
#include "NVMeDrive.hpp"
#include "NVMeError.hpp"
#include "NVMePlugin.hpp"
#include "NVMeUtil.hpp"
#include "Thresholds.hpp"
#include "xyz/openbmc_project/Common/error.hpp"

#include <dlfcn.h>

#include <boost/asio/spawn.hpp>

#include <algorithm>
#include <charconv>
#include <filesystem>
#include <iostream>
#include <stdexcept>
#include <unordered_map>

using SchedulerClockType = std::chrono::steady_clock;
std::unordered_map<std::string, std::shared_ptr<Scheduler<SchedulerClockType>>>
    schedulers;

template <>
std::shared_ptr<Scheduler<SchedulerClockType>>
    Scheduler<SchedulerClockType>::getScheduler(const std::string& path)
{
    auto itr = std::find_if(
        schedulers.begin(), schedulers.end(),
        [&path](const std::pair<std::string,
                                std::shared_ptr<Scheduler<SchedulerClockType>>>&
                    pair) {
        // check if the given path is subdir of the subsystem path
        const std::string& parent = pair.first;
        if (parent.empty() || path.empty())
        {
            return false;
        }
        if (parent.size() > path.size())
        {
            return false;
        }
        if (!path.starts_with(parent))
        {
            return false;
        }
        if (parent.back() != '/' && path.size() > parent.size() &&
            path.at(parent.size()) != '/')
        {
            return false;
        }

        return true;
    });
    if (itr == schedulers.end())
    {
        return {};
    }
    return itr->second;
}

void NVMeSubsystem::createAssociation()
{
    assocIntf = objServer.add_interface(path, association::interface);
    assocIntf->register_property("Associations", makeAssociation());
    assocIntf->initialize();
}

void NVMeSubsystem::updateAssociation()
{
    assocIntf->set_property("Associations", makeAssociation());
}

std::vector<Association> NVMeSubsystem::makeAssociation() const
{
    std::vector<Association> associations;
    std::filesystem::path p(path);

    associations.emplace_back("chassis", "storage", p.parent_path().string());
    associations.emplace_back("chassis", "drive", p.parent_path().string());
    associations.emplace_back("drive", "storage", path);

    for (const auto& [_, prog] : createProgress)
    {
        associations.emplace_back("awaiting", "awaited", prog->path);
    }

    for (const auto& [_, vol] : volumes)
    {
        associations.emplace_back("containing", "contained", vol->path);
    }

    return associations;
}

// get temporature from a NVMe Basic reading.
static double getTemperatureReading(int8_t reading)
{
    if (reading == static_cast<int8_t>(0x80) ||
        reading == static_cast<int8_t>(0x81))
    {
        // 0x80 = No temperature data or temperature data is more the 5 s
        // old 0x81 = Temperature sensor failure
        return std::numeric_limits<double>::quiet_NaN();
    }

    return reading;
}

std::shared_ptr<NVMeSubsystem> NVMeSubsystem::create(
    boost::asio::io_context& io, sdbusplus::asio::object_server& objServer,
    const std::shared_ptr<sdbusplus::asio::connection>& conn,
    const std::string& path, const std::string& name,
    const SensorData& configData, NVMeIntf intf, bool enableFeatureLockdown)
{
    auto self = std::make_shared<NVMeSubsystem>(io, objServer, conn, path, name,
                                                configData, std::move(intf),
                                                enableFeatureLockdown);
    self->init();
    return self;
}

NVMeSubsystem::NVMeSubsystem(
    boost::asio::io_context& io, sdbusplus::asio::object_server& objServer,
    const std::shared_ptr<sdbusplus::asio::connection>& conn,
    const std::string& path, const std::string& name,
    const SensorData& configData, NVMeIntf intf, bool enableFeatureLockdown) :
    NVMeStorage(objServer, *dynamic_cast<sdbusplus::bus_t*>(conn.get()),
                path.c_str()),
    path(path), io(io), objServer(objServer), conn(conn), name(name),
    config(configData), nvmeIntf(std::move(intf)), status(Status::Stop),
    featureLockdownEnabled(enableFeatureLockdown)
{}

// Performs initialisation after shared_from_this() has been set up.
void NVMeSubsystem::init()
{
    NVMeIntf::Protocol protocol{NVMeIntf::Protocol::NVMeBasic};
    try
    {
        protocol = nvmeIntf.getProtocol();
    }
    catch (const std::runtime_error&)
    {
        throw std::runtime_error("NVMe interface is null");
    }

    // initiate the common interfaces (thermal sensor, Drive and Storage)
    if (protocol != NVMeIntf::Protocol::NVMeBasic &&
        protocol != NVMeIntf::Protocol::NVMeMI)
    {
        throw std::runtime_error("Unsupported NVMe interface");
    }

    /* xyz.openbmc_project.Inventory.Item.Storage */
    NVMeStorage::init(
        std::static_pointer_cast<NVMeStorage>(shared_from_this()));

    /* xyz.openbmc_project.Inventory.Item.Drive */
    drive = std::make_shared<NVMeDrive>(io, conn, path, weak_from_this());
    drive->protocol(NVMeDrive::DriveProtocol::NVMe);
    drive->type(NVMeDrive::DriveType::SSD);
    createAssociation();
}

void NVMeSubsystem::initializeFeatureStore()
{
    if (featureStore)
    {
        return;
    }

    std::shared_ptr<NVMeControllerEnabled> ctrl;
    try
    {
        ctrl = getPrimaryController();
    }
    catch (const std::exception& e)
    {
        // Suppress error if no controller is available yet (e.g. during initial
        // scan)
        return;
    }

    if (!ctrl)
    {
        return;
    }

    this->featureStore = std::make_shared<FeatureStore>(
        io, objServer, conn, path, ctrl->getNvmeIntf(), ctrl->getMiCtrl(), 0,
        FeatureScope::Subsystem);

    this->featureStore->init([selfWeak{weak_from_this()},
                              path = this->path](const std::error_code& ec) {
        if (ec)
        {
            if (ec == std::errc::no_such_device ||
                ec == std::errc::not_supported)
            {
                lg2::info(
                    "[{PATH}]Feature store not supported for subsystem: {ERROR}",
                    "PATH", path, "ERROR", ec.message());
            }
            else
            {
                lg2::error(
                    "[{PATH}]Failed to initialize feature store for subsystem: {ERROR}",
                    "PATH", path, "ERROR", ec.message());
            }
            auto self = selfWeak.lock();
            if (self)
            {
                self->featureStore.reset();
            }
        }
    });
}

NVMeSubsystem::~NVMeSubsystem()
{
    objServer.remove_interface(assocIntf);
}

void NVMeSubsystem::processSecondaryControllerList(
    nvme_secondary_ctrl_list* secCntlrList)
{
    auto findPrimary = controllers.begin();
    int secCntlrCount = 0;
    if (secCntlrList != nullptr)
    {
        // all sc_entry pointing to a single pcid, so we only check
        // the first entry.
        findPrimary = controllers.find(secCntlrList->sc_entry[0].pcid);
        if (findPrimary == controllers.end())
        {
            lg2::error("fail to match primary controller from "
                       "identify secondary cntrl list");
            status = Status::Aborting;
            markFunctional(false);
            markAvailable(false);
            return;
        }
        secCntlrCount = secCntlrList->num;
    }

    // Enable primary controller since they are required to work
    auto& pc = findPrimary->second.first;
    primaryController = NVMeControllerEnabled::create(std::move(*pc));
    // replace with the new controller object
    pc = primaryController;

    std::vector<std::shared_ptr<NVMeController>> secCntrls;
    for (int i = 0; i < secCntlrCount; i++)
    {
        auto findSecondary = controllers.find(secCntlrList->sc_entry[i].scid);
        if (findSecondary == controllers.end())
        {
            lg2::error("[{PATH}] fail to match secondary controller from "
                       "identify secondary cntrl list",
                       "PATH", path);
            break;
        }

        auto& secondaryController = findSecondary->second.first;

        // Check Secondary Controller State
        if (secCntlrList->sc_entry[i].scs != 0)
        {
            secondaryController =
                NVMeControllerEnabled::create(std::move(*secondaryController));
        }
        secondaryController->setSecondary();
        secCntrls.push_back(secondaryController);
    }
    primaryController->setPrimary(secCntrls);
    initializeFeatureStore();

    (void)boost::asio::spawn(
        io,
        [self{shared_from_this()}](const boost::asio::yield_context& yield) {
        try
        {
            self->fillDrive(yield);
            self->updateVolumes(yield);
            self->querySupportedFormats(yield);
            lg2::info("[{PATH}]finished NS enum", "PATH", self->path);
        }
        catch (const std::exception& e)
        {
            lg2::error("[{PATH}]failed starting the subsystem: {ERROR}", "PATH",
                       self->path, "ERROR", e.what());
            self->status = Status::Aborting;
            self->markFunctional(false);
            self->markAvailable(false);
            return;
        }
        // start controller
        for (auto& [_, pair] : self->controllers)
        {
            // create controller plugin
            if (self->plugin)
            {
                pair.second = self->plugin->createControllerPlugin(
                    *pair.first, self->config);
            }
            pair.first->start(pair.second);
        }
        // start plugin
        if (self->plugin)
        {
            self->plugin->start();
        }

        // start Metric Scheduler
        auto scheduler =
            Scheduler<SchedulerClockType>::getScheduler(self->path);
        assert(
            scheduler &&
            "scheduler should be created at the beginning of Initializing Status");
        scheduler->start();
        scheduler->dequeue();

        self->status = Status::Start;
    }, boost::asio::detached);
}

void NVMeSubsystem::markFunctional(bool toggle)
{
    if (ctemp)
    {
        ctemp->markFunctional(toggle);
    }

    if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeBasic)
    {
        return;
    }

    // disable the subsystem
    if (!toggle)
    {
        if (status == Status::Intiatilzing)
        {
            throw std::runtime_error(
                "cannot stop: the subsystem is intiatilzing");
        }

        if (status == Status::Terminating || status == Status::Stop)
        {
            return;
        }

        assert(status == Status::Start || status == Status::Aborting);

        status = Status::Terminating;

        // erase the scheduler to stop furthur Metric scheduling
        schedulers.erase(path);

        if (plugin)
        {
            plugin->stop();
        }
        // TODO: the controller should be stopped after controller level polling
        // is enabled

        // Tell any progress objects we're aborting the operation
        for (auto& [_, v] : createProgress)
        {
            v->abort();
        }

        // Tell the controllers they mustn't post further jobs
        if (primaryController)
        {
            primaryController->stop();
        }

        for (auto& [_, v] : controllers)
        {
            v.first->stop();
        }

        // Avoid triggering C*V D-Bus updates by clearing internal state
        // directly. The controller and volume objects and interfaces will be
        // removed which will update the mapper.
        attached.clear();
        volumes.clear();
        primaryController.reset();
        controllers.clear();
        // plugin.reset();

        updateAssociation();

        if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeMI)
        {
            auto nvme =
                std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

            bool posted =
                nvme->flushOperations([self{shared_from_this()}]() mutable {
                self->status = Status::Stop;
            });

            if (!posted)
            {
                lg2::error(
                    "[{PATH}]Failed to flush operations, subsystem has stalled!",
                    "PATH", path);
            }
        }

        return;
    }

    if (status == Status::Intiatilzing)
    {
        throw std::runtime_error("cannot start: the subsystem is intiatilzing");
    }

    if (status == Status::Aborting)
    {
        throw std::runtime_error(
            "cannot start: subsystem initialisation has aborted and must transition to stopped");
    }

    if (status == Status::Start || status == Status::Terminating)
    {
        // Prevent consecutive calls to NVMeMiIntf::miScanCtrl()
        //
        // NVMeMiIntf::miScanCtrl() calls nvme_mi_scan_ep(..., true), which
        // forces a rescan and invalidates any nvme_mi_ctrl objects created on a
        // previous scan.
        //
        // We require a transition through Status::Stop (via
        // `markFunctional(false)`) so that the lifetime of the NVMeController
        // instances in this->controllers do not exceed the lifetime of their
        // associated nvme_mi_ctrl object.
        return;
    }

    assert(status == Status::Stop);

    status = Status::Intiatilzing;

    // create scheduler
    assert(schedulers.find(path) == schedulers.end() &&
           "scheduler should exsit only between [Intiatilzing, Terminating)");
    schedulers.emplace(path,
                       std::make_shared<Scheduler<SchedulerClockType>>(io));

    markAvailable(toggle);

    // add controllers for the subsystem
    if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeMI)
    {
        auto nvme =
            std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());
        nvme->miScanCtrl(
            [self{shared_from_this()},
             nvme](const std::error_code& ec,
                   const std::vector<nvme_mi_ctrl_t>& ctrlList) mutable {
            if (ec || ctrlList.empty())
            {
                // TODO: mark the subsystem invalid and reschedule refresh
                lg2::error(
                    "[{PATH}]fail to scan controllers for the nvme subsystem: {ERROR}",
                    "PATH", self->path, "ERROR", ec.message());
                self->status = Status::Aborting;
                self->markFunctional(false);
                self->markAvailable(false);
                return;
            }

            for (auto* c : ctrlList)
            {
                uint16_t index = getCtrlId(c);
                std::filesystem::path path = std::filesystem::path(self->path) /
                                             "controllers" /
                                             std::to_string(index);

                try
                {
                    auto nvmeController = std::make_shared<NVMeController>(
                        self->io, self->objServer, self->conn, path.string(),
                        self->config, nvme, c, self->weak_from_this(),
                        self->featureLockdownEnabled);

                    self->controllers.insert({index, {nvmeController, {}}});
                }
                catch (const std::exception& e)
                {
                    lg2::error(
                        "[{PATH}]failed to create controller: {CONTROLLER}, reason: {REASON}",
                        "PATH", path, "CONTROLLER", index, "REASON", e.what());
                }
            }
            // self->createStorageAssociation();

            /* Begin of patch context
             *
             *
             *
             */
            /*
             *
             *
             * End of patch context
             */

            /*
            find primary controller and make association
            The controller is SR-IOV, meaning all controllers (within a
            subsystem) are pointing to a single primary controller. So we
            only need to do identify on an arbatary controller.
            If the controller list contains a single controller. Skip
            identifying the secondary controller list. It will be the primary
            controller.
            */
            if (ctrlList.size() == 1)
            {
                self->processSecondaryControllerList(nullptr);
                return;
            }
            auto* ctrl = ctrlList.back();
            nvme->adminIdentify(
                ctrl, nvme_identify_cns::NVME_IDENTIFY_CNS_SECONDARY_CTRL_LIST,
                0, 0,
                [self{self->shared_from_this()}](const nvme_ex_ptr& ex,
                                                 std::span<uint8_t> data) {
                if (ex || data.size() < sizeof(nvme_secondary_ctrl_list))
                {
                    lg2::error(
                        "[{PATH}]fail to identify secondary controller list",
                        "PATH", self->path);
                    self->status = Status::Aborting;
                    self->markFunctional(false);
                    self->markAvailable(false);
                    return;
                }
                nvme_secondary_ctrl_list* listHdr =
                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
                    reinterpret_cast<nvme_secondary_ctrl_list*>(data.data());

                if (listHdr->num == 0)
                {
                    lg2::error(
                        "[{PATH}]empty identify secondary controller list",
                        "PATH", self->path);
                    self->status = Status::Aborting;
                    self->markFunctional(false);
                    self->markAvailable(false);
                    return;
                }
                self->processSecondaryControllerList(listHdr);
            });
        });
    }
}

void NVMeSubsystem::markAvailable(bool toggle)
{
    if (ctemp)
    {
        ctemp->markAvailable(toggle);
    }

    if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeBasic)
    {
        return;
    }

    if (toggle)
    {
        // TODO: make the Available interface true
        unavailableCount = 0;
        return;
    }
    // TODO: make the Available interface false
    unavailableCount = unavailableMaxCount;
}

std::shared_ptr<NVMeControllerEnabled>
    NVMeSubsystem::getPrimaryController() const
{
    if (!primaryController)
    {
        lg2::error(
            "[{PATH}]dbus call for inactive NVMe subsystem {SUBSYSTEM}. Returning Unavailable",
            "PATH", path, "SUBSYSTEM", name);
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }
    return primaryController;
}

/* Begin of patch context
 *
 *
 *
 */
/*
 *
 *
 * End of patch context
 */

void NVMeSubsystem::start()
{
    for (const auto& [_, lib] : pluginLibMap)
    {
        createplugin_t pluginFunc =
            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
            reinterpret_cast<createplugin_t>(::dlsym(lib, "createPlugin"));
        auto p = pluginFunc(shared_from_this(), config);
        if (p)
        {
            plugin = p;
            break;
        }
    }
    // add thermal sensor for the subsystem
    std::optional<std::string> sensorName = createSensorNameFromPath(path);
    if (!sensorName)
    {
        // fail to parse sensor name from path, using name instead.
        sensorName.emplace(name);
    }

    std::vector<thresholds::Threshold> sensorThresholds;
    if (!parseThresholdsFromConfig(config, sensorThresholds))
    {
        lg2::error("[{PATH}]error populating thresholds for {SENSOR}", "PATH",
                   path, "SENSOR", *sensorName);
        throw std::runtime_error("error populating thresholds for " +
                                 *sensorName);
    }

    if (ctemp || ctempTimer)
    {
        throw std::logic_error(
            "NVMeSubsystem::start() called from invalid state");
    }

    assert(!ctemp && !ctempTimer);

    PowerState powerState = PowerState::always;
    auto sensorBase = config.find(configInterfaceName(nvme::sensorType));
    if (sensorBase == config.end())
    {
        lg2::warning("[{PATH}]cannot find sensor config {CONFIG} for {NAME}",
                     "PATH", path, "CONFIG",
                     configInterfaceName(nvme::sensorType), "NAME", name);
    }
    else
    {
        const SensorBaseConfigMap& sensorConfig = sensorBase->second;
        powerState = getPowerState(sensorConfig);
    }

    /* Begin of patch context
     *
     *
     *
     */
    /*
     *
     *
     * End of patch context
     */

    ctemp = std::make_shared<NVMeSensor>(objServer, io, conn, *sensorName,
                                         std::move(sensorThresholds), path,
                                         powerState);
    ctempTimer = std::make_shared<boost::asio::steady_timer>(io);

    // start to poll value for CTEMP sensor.
    if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeBasic)
    {
        auto intf =
            std::get<std::shared_ptr<NVMeBasicIntf>>(nvmeIntf.getInferface());
        ctemp_fetch_t<NVMeBasicIntf::DriveStatus*> dataFetcher =
            [intf, self{shared_from_this()},
             timer = std::weak_ptr<boost::asio::steady_timer>(ctempTimer)](
                std::function<void(const std::error_code&,
                                   NVMeBasicIntf::DriveStatus*)>&& cb) {
            /* Potentially defer sampling the sensor sensor if it is in error */
            if (!self->ctemp->sample())
            {
                cb(std::make_error_code(std::errc::operation_canceled),
                   nullptr);
                return;
            }

            intf->getStatus(std::move(cb));
        };
        ctemp_process_t<NVMeBasicIntf::DriveStatus*> dataProcessor =
            [self{shared_from_this()},
             timer = std::weak_ptr<boost::asio::steady_timer>(ctempTimer)](
                const std::error_code& error,
                NVMeBasicIntf::DriveStatus* status) {
            // deferred sampling
            if (error == std::errc::operation_canceled)
            {
                return;
            }
            // The device is physically absent
            if (error == std::errc::no_such_device)
            {
                lg2::error(
                    "[{PATH}]error reading ctemp from subsystem, reason: {ERROR}",
                    "PATH", self->path, "ERROR", error.message());
                self->markFunctional(false);
                self->markAvailable(false);
                return;
            }
            // other communication errors
            if (error)
            {
                lg2::error(
                    "[{PATH}]error reading ctemp from subsystem, reason: {ERROR}",
                    "PATH", self->path, "ERROR", error.message());
                self->ctemp->incrementError();
                return;
            }

            if (status == nullptr)
            {
                lg2::error("[{PATH}]empty data returned by data fetcher",
                           "PATH", self->path);
                self->markFunctional(false);
                return;
            }

            uint8_t flags = status->Status;
            if (((flags & NVMeBasicIntf::StatusFlags::
                              NVME_MI_BASIC_SFLGS_DRIVE_NOT_READY) != 0) ||
                ((flags & NVMeBasicIntf::StatusFlags::
                              NVME_MI_BASIC_SFLGS_DRIVE_FUNCTIONAL) == 0))
            {
                lg2::error(
                    "[{PATH}]health poll returns drive not ready or drive not functional",
                    "PATH", self->path);
                self->markFunctional(false);
                return;
            }
            self->ctemp->updateValue(
                getTemperatureReading(static_cast<int8_t>(status->Temp)));
        };

        pollCtemp(ctempTimer, pollingInterval, dataFetcher, dataProcessor);
    }
    else if (nvmeIntf.getProtocol() == NVMeIntf::Protocol::NVMeMI)
    {
        auto intf =
            std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

        ctemp_fetch_t<nvme_mi_nvm_ss_health_status*> dataFetcher =
            [intf, self{shared_from_this()},
             timer = std::weak_ptr<boost::asio::steady_timer>(ctempTimer)](
                std::function<void(const std::error_code&,
                                   nvme_mi_nvm_ss_health_status*)>&& cb) {
            // do not poll the health status if subsystem is in cooldown
            if (self->unavailableCount > 0)
            {
                cb(std::make_error_code(std::errc::operation_canceled),
                   nullptr);
                return;
            }

            // do not poll the health status if the subsystem is intiatilzing
            if (self->status == Status::Intiatilzing)
            {
                lg2::info(
                    "[{PATH}]subsystem is initializing, cancel the health poll",
                    "PATH", self->path);
                cb(std::make_error_code(std::errc::operation_canceled),
                   nullptr);
                return;
            }
            intf->miSubsystemHealthStatusPoll(std::move(cb));
        };
        ctemp_process_t<nvme_mi_nvm_ss_health_status*> dataProcessor =
            [self{shared_from_this()},
             timer = std::weak_ptr<boost::asio::steady_timer>(ctempTimer)](
                const std::error_code& error,
                nvme_mi_nvm_ss_health_status* status) {
            if (self->unavailableCount > 0)
            {
                self->unavailableCount--;
                return;
            }

            if (error == std::errc::operation_canceled)
            {
                lg2::info("[{PATH}]processing health data has been cancelled",
                          "PATH", self->path);
                return;
            }

            if (self->status == Status::Intiatilzing)
            {
                lg2::info(
                    "[{PATH}]subsystem is initializing, do not process the status",
                    "PATH", self->path);
                return;
            }

            if (error == std::errc::no_such_device)
            {
                lg2::error(
                    "[{PATH}]error reading ctemp from subsystem, reason: {ERROR}",
                    "PATH", self->path, "ERROR", error.message());
                // stop the subsystem
                self->markFunctional(false);
                self->markAvailable(false);
                return;
            }
            if (error)
            {
                lg2::error(
                    "[{PATH}]error reading ctemp from subsystem, reason: {ERROR}",
                    "PATH", self->path, "ERROR", error.message());
                self->ctemp->incrementError();
                if (self->ctemp->inError())
                {
                    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(
                        self->nvmeIntf.getInferface());
                    intf->recover();
                    // stop the subsystem
                    self->markFunctional(false);
                    self->markAvailable(false);
                }
                return;
            }

            // Drive Functional
            bool df = (status->nss & 0x20) != 0;
            if (!df)
            {
                // stop the subsystem
                lg2::error("[{PATH}]health poll returns df status 0", "PATH",
                           self->path);
                self->markFunctional(false);
                return;
            }

            self->markFunctional(true);

            // TODO: update the drive interface

            self->ctemp->updateValue(
                getTemperatureReading(static_cast<int8_t>(status->ctemp)));
            return;
        };

        pollCtemp(ctempTimer, pollingInterval, dataFetcher, dataProcessor);
    }
}

void NVMeSubsystem::stop()
{
    if (ctempTimer)
    {
        ctempTimer->cancel();
        ctempTimer.reset();
        ctemp.reset();
    }

    if (status == Status::Intiatilzing)
    {
        lg2::debug("[{PATH}]status init", "PATH", path);
        auto timer = std::make_shared<boost::asio::steady_timer>(
            io, std::chrono::milliseconds(100));
        timer->async_wait(
            [self{shared_from_this()}, timer](boost::system::error_code ec) {
            if (ec)
            {
                return;
            }
            self->stop();
        });
    }
    else
    {
        lg2::debug("[{PATH}]status else", "PATH", path);
        markFunctional(false);

        // There's been an explicit request to stop the subsystem. If it has
        // entered an unavailable state, reset that too. If the subsystem
        // continues to be unavailable beyond a subsequent invocation of start()
        // this will be detected in the usual fashion. Put another way: Don't
        // unnecessarily impede the progress of a subsequent start().
        unavailableCount = 0;
    }

    if (plugin)
    {
        plugin.reset();
    }

    if (featureStore)
    {
        featureStore.reset();
    }
}

sdbusplus::message::object_path
    NVMeSubsystem::createVolume(boost::asio::yield_context yield, uint64_t size,
                                size_t lbaFormat, bool metadataAtEnd)
{
    if (status != Status::Start)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }

    // #0 (sequence of runtime/callbacks)
    auto progId = getRandomId();

    auto pc = getPrimaryController();
    if (!pc)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    using submit_callback_t = void(std::tuple<nvme_ex_ptr>);
    auto [ex] = boost::asio::async_initiate<boost::asio::yield_context,
                                            submit_callback_t>(
        [weak{weak_from_this()}, progId, intf, ctrl, size, lbaFormat,
         metadataAtEnd, path = this->path](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));

        // #1
        intf->createNamespace(
            ctrl, size, lbaFormat, metadataAtEnd,

            // submitted_cb
            [h](const nvme_ex_ptr& ex) mutable {
            // #2

            // Async completion of the createNamespace call.
            // The actual nvme_mi_admin_ns_mgmt_create() call is still running
            // in a separate thread. Pass the error status back out.
            h(std::make_tuple(ex));
        },

            // finished_cb
            [weak, progId, path](const nvme_ex_ptr& ex,
                                 NVMeNSIdentify newns) mutable {
            // #5. This will only be called once #4 completes.
            // It will not be called if the submit failed.
            auto self = weak.lock();
            if (!self)
            {
                lg2::warning(
                    "[{PATH}]createNamespace completed while nvmesensor was exiting",
                    "PATH", path);
                return;
            }
            // The NS create has completed (either successfully or not)
            self->createVolumeFinished(progId, ex, newns);
        });
    },
        yield);

    // #3

    // Exception must be thrown outside of the async block
    if (ex)
    {
        // TODO: (b/375054188) Exception handling to be refactored
        // NOLINTNEXTLINE(cert-err09-cpp,cert-err60-cpp,cert-err61-cpp,misc-throw-by-value-catch-by-reference)
        throw *ex;
    }

    // Progress endpoint for clients to poll, if the submit was successful.
    std::string progPath = path + "/CreateProgress/" + progId;

    auto prog = std::make_shared<NVMeCreateVolumeProgress>(conn, progPath);
    if (!createProgress.insert({progId, prog}).second)
    {
        throw std::logic_error("duplicate progress id");
    }

    updateAssociation();

    // #4
    return progPath;
}

void NVMeSubsystem::createVolumeFinished(const std::string& progId,
                                         const nvme_ex_ptr& ex,
                                         NVMeNSIdentify ns)
{
    try
    {
        auto p = createProgress.find(progId);
        if (p == createProgress.end())
        {
            throw std::logic_error("Missing progress entry");
        }
        auto prog = p->second;

        if (prog->status() == OperationStatus::Aborted)
        {
            return;
        }
        assert(status == Status::Start);

        if (ex)
        {
            prog->createFailure(ex);
            return;
        }

        std::shared_ptr<NVMeVolume> vol;
        try
        {
            vol = addVolume(ns);
        }
        // TODO: (b/375054188) Exception handling to be refactored
        // NOLINTNEXTLINE(cert-err09-cpp,cert-err60-cpp,cert-err61-cpp,misc-throw-by-value-catch-by-reference)
        catch (nvme_ex_ptr e)
        {
            prog->createFailure(e);
            return;
        }

        prog->createSuccess(vol);
        updateAssociation();
    }
    catch (const std::exception& e)
    {
        lg2::error("[{PATH}]Unhandled error in createVolumeFinished: {ERROR}",
                   "PATH", path, "ERROR", e.what());
    }
}

static uint8_t determineHardwareMset(uint8_t mc)
{
    constexpr uint8_t nvmeNsMcExtended = 1 << 0;
    constexpr uint8_t nvmeNsMcSeparate = 1 << 1;

    bool mtelbas = (mc & nvmeNsMcExtended) != 0;
    bool mtsbs = (mc & nvmeNsMcSeparate) != 0;

    // dbusMset: 1 (Contiguous), 0 (Separate).
    return (mtelbas && !mtsbs) ? 1 : 0;
}

void NVMeSubsystem::format(
    boost::asio::yield_context yield, uint8_t lbaFormat,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::SecureEraseSetting
        secureEraseSetting,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::DataProtectionType
        dataProtectionType)
{
    uint8_t secureEraseSettingVal = static_cast<uint8_t>(secureEraseSetting);
    uint8_t dataProtectionTypeVal = static_cast<uint8_t>(dataProtectionType);

    formatInternal(std::move(yield), lbaFormat, secureEraseSettingVal,
                   dataProtectionTypeVal, std::nullopt);
}

void NVMeSubsystem::formatController(
    boost::asio::yield_context yield, uint8_t lbaFormat,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::SecureEraseSetting
        secureEraseSetting,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::DataProtectionType
        dataProtectionType,
    uint16_t ctrlId)
{
    uint8_t secureEraseSettingVal = static_cast<uint8_t>(secureEraseSetting);
    uint8_t dataProtectionTypeVal = static_cast<uint8_t>(dataProtectionType);

    formatInternal(std::move(yield), lbaFormat, secureEraseSettingVal,
                   dataProtectionTypeVal, ctrlId);
}

void NVMeSubsystem::formatNamespace(
    boost::asio::yield_context yield, uint8_t lbaFormat,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::SecureEraseSetting
        secureEraseSetting,
    sdbusplus::xyz::openbmc_project::NVMe::server::Format::DataProtectionType
        dataProtectionType,
    uint32_t nsid)
{
    uint8_t secureEraseSettingVal = static_cast<uint8_t>(secureEraseSetting);
    uint8_t dataProtectionTypeVal = static_cast<uint8_t>(dataProtectionType);

    formatInternal(std::move(yield), lbaFormat, secureEraseSettingVal,
                   dataProtectionTypeVal, std::nullopt, nsid);
}

void NVMeSubsystem::formatInternal(boost::asio::yield_context yield,
                                   uint8_t lbaFormat,
                                   uint8_t secureEraseSetting,
                                   uint8_t dataProtectionType,
                                   std::optional<uint16_t> ctrlId,
                                   std::optional<uint32_t> targetNsid)
{
    if (status != Status::Start)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }

    std::shared_ptr<NVMeControllerEnabled> targetCtrl;
    if (ctrlId)
    {
        auto it = controllers.find(*ctrlId);
        if (it == controllers.end() || !it->second.first)
        {
            throw sdbusplus::xyz::openbmc_project::Common::Error::
                InvalidArgument();
        }
        targetCtrl =
            std::dynamic_pointer_cast<NVMeControllerEnabled>(it->second.first);
    }
    else
    {
        targetCtrl = getPrimaryController();
    }

    if (!targetCtrl)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }

    nvme_mi_ctrl_t ctrl = targetCtrl->getMiCtrl();
    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());
    // UserDataErase (1) is not supported for secureEraseSetting.
    // Only None (0) and CryptographicErase (2) are allowed.
    constexpr uint8_t sesNone = 0;
    constexpr uint8_t sesCryptoErase = 2;
    if (secureEraseSetting != sesNone && secureEraseSetting != sesCryptoErase)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::InvalidArgument();
    }

    // Check FNA restrictions for individual namespace formatting and
    // cryptographic erase
    if (secureEraseSetting == sesCryptoErase &&
        (fna & nvme::fna::cryptoEraseSupport) == 0)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::InvalidArgument();
    }

    if (targetNsid)
    {
        if ((fna & nvme::fna::fmtAllRestriction) != 0)
        {
            lg2::error(
                "Format NVM does not support formatting individual namespace based on FNA bit 0");
            throw sdbusplus::xyz::openbmc_project::Common::Error::NotAllowed();
        }
        if (((fna & nvme::fna::secureEraseAllRestriction) != 0) &&
            secureEraseSetting != sesNone)
        {
            lg2::error(
                "Format NVM does not support secure erase on individual namespace based on FNA bit 1");
            throw sdbusplus::xyz::openbmc_project::Common::Error::NotAllowed();
        }
    }

    bool broadcastFormatRestricted = (fna & nvme::fna::nsidAllRestriction) != 0;

    // Determine target namespaces
    std::vector<uint32_t> targets;
    if (targetNsid)
    {
        targets.push_back(*targetNsid);
    }
    else if (ctrlId)
    {
        // Controller-level format: only format namespaces attached to this
        // controller
        auto attachedIt = attached.find(*ctrlId);
        if (attachedIt != attached.end())
        {
            for (uint32_t ns : attachedIt->second)
            {
                targets.push_back(ns);
            }
        }
    }
    else if (broadcastFormatRestricted)
    {
        // Subsystem-level format without broadcast support: iterate through all
        // discovered namespaces
        for (const auto& [ns, _] : volumes)
        {
            targets.push_back(ns);
        }
    }
    else
    {
        // Subsystem-level format with broadcast support (NSID = FFFFFFFFh)
        targets.push_back(NVME_NSID_ALL);
    }

    // Execute format on targets
    for (uint32_t targetNsid : targets)
    {
        uint8_t metadataCapabilities = 0;
        if (targetNsid == NVME_NSID_ALL)
        {
            if (!volumes.empty())
            {
                metadataCapabilities = volumes.begin()
                                           ->second->getNSIdentify()
                                           .metadataCapabilities;
            }
        }
        else
        {
            auto it = volumes.find(targetNsid);
            if (it != volumes.end())
            {
                metadataCapabilities =
                    it->second->getNSIdentify().metadataCapabilities;
            }
            else
            {
                lg2::warning(
                    "Target namespace {NSID} not found in volumes list during format",
                    "NSID", targetNsid);
            }
        }

        uint8_t mset = determineHardwareMset(metadataCapabilities);
        // Default Protection Information Location (PIL) to 0 (LastEightBytes of
        // metadata) as per Design Doc Section 5.1
        uint8_t protectionInformationLocation = 0;

        using callback_t = void(std::tuple<std::error_code, int>);
        auto [err, nvmeStatus] =
            boost::asio::async_initiate<boost::asio::yield_context, callback_t>(
                [intf, ctrl, targetNsid, lbaFormat, secureEraseSetting, mset,
                 dataProtectionType,
                 protectionInformationLocation](auto&& handler) {
            auto h = asio_helper::CopyableCallback(
                std::forward<decltype(handler)>(handler));

            intf->adminFormat(
                ctrl, targetNsid, lbaFormat, secureEraseSetting, mset,
                dataProtectionType, protectionInformationLocation,
                [h](const std::error_code& err, int nvmeStatus) mutable {
                h(std::make_tuple(err, nvmeStatus));
            });
        },
                yield);

        checkLibNVMeError(err, nvmeStatus, "Format");
    }

    // Re-identify and update D-Bus properties for all formatted namespaces
    std::vector<uint32_t> refreshTargets;
    if (targetNsid)
    {
        refreshTargets.push_back(*targetNsid);
    }
    else if (ctrlId)
    {
        auto attachedIt = attached.find(*ctrlId);
        if (attachedIt != attached.end())
        {
            refreshTargets.insert(refreshTargets.end(),
                                  attachedIt->second.begin(),
                                  attachedIt->second.end());
        }
    }
    else
    {
        for (const auto& [ns, _] : volumes)
        {
            refreshTargets.push_back(ns);
        }
    }

    for (uint32_t ns : refreshTargets)
    {
        try
        {
            NVMeNSIdentify newNs = identifyNamespace(yield, ns);
            auto it = volumes.find(ns);
            if (it != volumes.end())
            {
                it->second->updateNSIdentify(newNs);
            }
        }
        catch (const std::exception& e)
        {
            lg2::error(
                "Failed to re-identify namespace {NSID} after format: {ERROR}",
                "NSID", ns, "ERROR", e.what());
        }
    }
}

std::string NVMeSubsystem::volumePath(uint32_t nsid) const
{
    return path + "/volumes/" + std::to_string(nsid);
}

NVMeNSIdentify
    NVMeSubsystem::identifyNamespace(boost::asio::yield_context yield,
                                     uint32_t nsid)
{
    assert((status == Status::Start || status == Status::Intiatilzing) &&
           std::format("Subsystem not in Start state, have {}",
                       static_cast<int>(status))
               .c_str());

    auto pc = getPrimaryController();
    if (!pc)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    using admin_identify_t = void(std::tuple<nvme_ex_ptr, std::span<uint8_t>>);

    auto [ex, data] = boost::asio::async_initiate<boost::asio::yield_context,
                                                  admin_identify_t>(
        [intf, ctrl, nsid](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));
        intf->adminIdentify(
            ctrl, nvme_identify_cns::NVME_IDENTIFY_CNS_ALLOCATED_NS, nsid,
            NVME_CNTLID_NONE,
            [h](const nvme_ex_ptr& ex, std::span<uint8_t> data) mutable {
            h(std::make_tuple(ex, data));
        });
    }, yield);

    if (ex)
    {
        // TODO: (b/375054188) Exception handling to be refactored
        // NOLINTNEXTLINE(cert-err09-cpp,cert-err60-cpp,cert-err61-cpp,misc-throw-by-value-catch-by-reference)
        throw *ex;
    }

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    nvme_id_ns& id = *reinterpret_cast<nvme_id_ns*>(data.data());

    // msb 6:5 and lsb 3:0
    size_t lbafIndex = ((id.flbas >> 1) & 0x30) | (id.flbas & 0x0f);
    size_t blockSize = 1UL << id.lbaf[lbafIndex].ds;
    bool metadataAtEnd = (id.flbas & (1 << 4)) != 0;

    return NVMeNSIdentify{
        .namespaceId = nsid,
        .size = ::le64toh(id.nsze * blockSize),
        .capacity = ::le64toh(id.ncap * blockSize),
        .blockSize = blockSize,
        .lbaFormat = lbafIndex,
        .metadataAtEnd = metadataAtEnd,
        .metadataCapabilities = id.mc,
    };
}

void NVMeSubsystem::addIdentifyNamespace(boost::asio::yield_context yield,
                                         uint32_t nsid)
{
    NVMeNSIdentify ns = identifyNamespace(yield, nsid);
    addVolume(ns);

    // determine attached controllers
    auto pc = getPrimaryController();
    if (!pc)
    {
        return;
    }
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();
    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());
    using admin_identify_t = void(std::tuple<nvme_ex_ptr, std::span<uint8_t>>);

    auto [ex, data] = boost::asio::async_initiate<boost::asio::yield_context,
                                                  admin_identify_t>(
        [intf, ctrl, nsid](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));
        intf->adminIdentify(
            ctrl, nvme_identify_cns::NVME_IDENTIFY_CNS_NS_CTRL_LIST, nsid,
            NVME_CNTLID_NONE,
            [h](const nvme_ex_ptr& ex, std::span<uint8_t> data) mutable {
            h(std::make_tuple(ex, data));
        });
    }, yield);

    if (ex)
    {
        // TODO: (b/375054188) Exception handling to be refactored
        // NOLINTNEXTLINE(cert-err09-cpp,cert-err60-cpp,cert-err61-cpp,misc-throw-by-value-catch-by-reference)
        throw *ex;
    }

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    nvme_ctrl_list& list = *reinterpret_cast<nvme_ctrl_list*>(data.data());
    uint16_t num = ::le16toh(list.num);
    if (num == NVME_ID_CTRL_LIST_MAX)
    {
        lg2::warning("[{PATH}]full ctrl list returned", "PATH", path);
    }

    for (auto i = 0; i < num; i++)
    {
        uint16_t c = ::le16toh(list.identifier[i]);
        attachCtrlVolume(c, nsid);
    }
}
void NVMeSubsystem::updateVolumes(boost::asio::yield_context yield)
{
    assert((status == Status::Start || status == Status::Intiatilzing) &&
           std::format("Subsystem not in Start state, have {}",
                       static_cast<int>(status))
               .c_str());

    auto pc = getPrimaryController();
    if (!pc)
    {
        return;
    }
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());
    using admin_list_ns_t =
        void(std::tuple<nvme_ex_ptr, std::vector<uint32_t>>);
    auto [ex, ns] = boost::asio::async_initiate<boost::asio::yield_context,
                                                admin_list_ns_t>(
        [intf, ctrl](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));
        intf->adminListNamespaces(ctrl,
                                  [h](const nvme_ex_ptr& ex,
                                      const std::vector<uint32_t>& ns) mutable {
            h(std::make_tuple(ex, ns));
        });
    }, yield);

    if (ex)
    {
        // TODO: (b/375054188) Exception handling to be refactored
        // NOLINTNEXTLINE(cert-err09-cpp,cert-err60-cpp,cert-err61-cpp,misc-throw-by-value-catch-by-reference)
        throw *ex;
    }

    std::vector<uint32_t> existing;
    existing.reserve(volumes.size());
    for (auto& [n, _] : volumes)
    {
        existing.push_back(n);
    }

    std::vector<uint32_t> additions;
    std::vector<uint32_t> deletions;

    // namespace lists are ordered
    std::set_difference(ns.begin(), ns.end(), existing.begin(), existing.end(),
                        std::back_inserter(additions));

    std::set_difference(existing.begin(), existing.end(), ns.begin(), ns.end(),
                        std::back_inserter(deletions));

    lg2::info(
        "[{PATH}]subsystem enum {NS_SIZE} NS, {ADDITIONS} will be added, {DELETIONS} will be deleted",
        "PATH", path, "NS_SIZE", ns.size(), "ADDITIONS", additions.size(),
        "DELETIONS", deletions.size());

    for (auto n : deletions)
    {
        forgetVolume(volumes.find(n)->second);
    }

    for (auto n : additions)
    {
        addIdentifyNamespace(yield, n);
    }
}

void NVMeSubsystem::fillDrive(boost::asio::yield_context yield)
{
    assert(status == Status::Intiatilzing);
    auto pc = getPrimaryController();
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    using admin_identify_t = void(std::tuple<nvme_ex_ptr, std::span<uint8_t>>);

    auto [ex, data] = boost::asio::async_initiate<boost::asio::yield_context,
                                                  admin_identify_t>(
        [intf, ctrl](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));
        intf->adminIdentify(
            ctrl, nvme_identify_cns::NVME_IDENTIFY_CNS_CTRL, NVME_NSID_NONE,
            NVME_CNTLID_NONE,
            [h](const nvme_ex_ptr& ex, std::span<uint8_t> data) mutable {
            h(std::make_tuple(ex, data));
        });
    }, yield);

    if (ex)
    {
        throw std::runtime_error(
            std::format("{}: Error for controller identify", name));
    }

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    nvme_id_ctrl& id = *reinterpret_cast<nvme_id_ctrl*>(data.data());
    fna = id.fna;
    drive->serialNumber(nvmeString((const char*)id.sn, sizeof(id.sn)));
    drive->model(nvmeString((const char*)id.mn, sizeof(id.mn)));
    drive->capacity(char128ToUint64((const unsigned char*)id.tnvmcap));

    pc->setNvmeVersion(::le32toh(id.ver));

    auto fwVer = nvmeString((const char*)id.fr, sizeof(id.fr));
    if (!fwVer.empty())
    {
        // Formatting per
        // https://gerrit.openbmc.org/c/openbmc/phosphor-dbus-interfaces/+/43458/2/xyz/openbmc_project/Software/Version.interface.yaml#47
        // TODO find/write a better reference
        std::string v("xyz.openbmc_project.NVMe.ControllerFirmwareVersion");
        auto pc = getPrimaryController();
        pc->version(v);
        pc->purpose(SoftwareVersion::VersionPurpose::Other);
        pc->extendedVersion(v + ":" + fwVer);
    }
}

std::shared_ptr<NVMeVolume> NVMeSubsystem::getVolume(
    const sdbusplus::message::object_path& volPath) const
{
    if (volPath.parent_path() != path + "/volumes")
    {
        lg2::error(
            "[{PATH}]getVolume path '{VOLPATH}' doesn't match parent {PARENT_PATH}",
            "PATH", path, "VOLPATH", volPath.str, "PARENT_PATH", path);
        return nullptr;
    }

    std::string id = volPath.filename();
    uint32_t nsid = 0;
    auto e = std::from_chars(id.data(), id.data() + id.size(), nsid);
    if (e.ptr != id.data() + id.size() || e.ec != std::errc())
    {
        lg2::error("[{PATH}]getVolume path '{VOLPATH}' bad nsid", "PATH", path,
                   "VOLPATH", volPath.str);
        return nullptr;
    }

    auto v = volumes.find(nsid);
    if (v == volumes.end())
    {
        lg2::error("[{PATH}]getVolume nsid {NSID} not found", "PATH", path,
                   "NSID", nsid);
        return nullptr;
    }

    return v->second;
}

std::vector<uint32_t> NVMeSubsystem::attachedVolumes(uint16_t ctrlId) const
{
    std::vector<uint32_t> vols;

    if (!controllers.contains(ctrlId))
    {
        lg2::error("[{PATH}]attachedVolumes bad controller {CTRLID}", "PATH",
                   path, "CTRLID", ctrlId);
        return vols;
    }

    try
    {
        std::ranges::copy(attached.at(ctrlId), std::back_inserter(vols));
    }
    catch (std::out_of_range&)
    {
        // no volumes attached
    }
    return vols;
}

void NVMeSubsystem::attachCtrlVolume(uint16_t c, uint32_t ns)
{
    assert((status == Status::Start || status == Status::Intiatilzing) &&
           std::format("Subsystem not in Start state, have {}",
                       static_cast<int>(status))
               .c_str());

    if (!controllers.contains(c))
    {
        throw std::runtime_error(
            std::format("attachCtrlVolume bad controller {}", c));
    }
    if (!volumes.contains(ns))
    {
        throw std::runtime_error(std::format("attachCtrlVolume bad ns {}", ns));
    }
    attached[c].insert(ns);
    lg2::info("[{PATH}] attached insert {C} {NS}", "PATH", path, "C", c, "NS",
              ns);
    controllers[c].first->updateAssociation();
}

void NVMeSubsystem::detachCtrlVolume(uint16_t c, uint32_t ns)
{
    assert(status == Status::Start &&
           std::format("Subsystem is not in Start state: {}",
                       static_cast<int>(status))
               .c_str());

    if (!controllers.contains(c))
    {
        throw std::runtime_error(
            std::format("detachCtrlVolume bad controller {}", c));
    }
    if (!volumes.contains(ns))
    {
        throw std::runtime_error(std::format("detachCtrlVolume bad ns {}", ns));
    }
    attached[c].erase(ns);
    lg2::info("[{PATH}] attached erase {C} {NS}", "PATH", path, "C", c, "NS",
              ns);
    controllers[c].first->updateAssociation();
}

void NVMeSubsystem::detachAllCtrlVolume(uint32_t ns)
{
    assert(status == Status::Start &&
           std::format("Subsystem is not in Start state: {}",
                       static_cast<int>(status))
               .c_str());

    if (!volumes.contains(ns))
    {
        throw std::runtime_error(std::format("detachCtrlVolume bad ns {}", ns));
    }
    // remove from attached controllers list
    for (auto& [c, attach_vols] : attached)
    {
        if (attach_vols.erase(ns) == 1)
        {
            controllers[c].first->updateAssociation();
        }
    }
}

// Will throw a nvme_ex_ptr if the NS already exists */
std::shared_ptr<NVMeVolume> NVMeSubsystem::addVolume(const NVMeNSIdentify& ns)
{
    assert((status == Status::Start || status == Status::Intiatilzing) &&
           std::format("Subsystem not in Start state, have {}",
                       static_cast<int>(status))
               .c_str());

    if (volumes.contains(ns.namespaceId))
    {
        std::string errMsg = std::string("Internal error, NSID exists " +
                                         std::to_string(ns.namespaceId));
        lg2::error("[{PATH}] {ERROR}", "PATH", path, "ERROR", errMsg);
        throw makeLibNVMeError(errMsg);
    }

    auto vol = NVMeVolume::create(objServer, conn, shared_from_this(), ns);
    volumes.insert({ns.namespaceId, vol});

    updateAssociation();
    return vol;
}

void NVMeSubsystem::forgetVolume(const std::shared_ptr<NVMeVolume>& volume)
{
    // remove any progress references
    for (const auto& [progId, prog] : createProgress)
    {
        std::string s = prog->volumePath();
        if (prog->volumePath() == volume->path)
        {
            createProgress.erase(progId);
            break;
        }
    }

    // remove from attached controllers list
    detachAllCtrlVolume(volume->namespaceId());

    if (volumes.erase(volume->namespaceId()) != 1)
    {
        throw std::runtime_error(std::format(
            "volume {} disappeared unexpectedly", volume->namespaceId()));
    }

    updateAssociation();
}

void NVMeSubsystem::querySupportedFormats(boost::asio::yield_context yield)
{
    assert((status == Status::Start || status == Status::Intiatilzing) &&
           std::format("Subsystem not in Start state, have {}",
                       static_cast<int>(status))
               .c_str());

    auto pc = getPrimaryController();
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    using admin_identify_t = void(std::tuple<nvme_ex_ptr, std::span<uint8_t>>);
    auto [ex, data] = boost::asio::async_initiate<boost::asio::yield_context,
                                                  admin_identify_t>(
        [intf, ctrl](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));
        intf->adminIdentify(
            ctrl, nvme_identify_cns::NVME_IDENTIFY_CNS_NS, NVME_NSID_ALL,
            NVME_CNTLID_NONE,
            [h](const nvme_ex_ptr& ex, std::span<uint8_t> data) mutable {
            h(std::make_tuple(ex, data));
        });
    }, yield);

    if (ex)
    {
        throw std::runtime_error(
            std::format("{}: Error getting LBA formats :{}", name, ex->what()));
    }

    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    nvme_id_ns& id = *reinterpret_cast<nvme_id_ns*>(data.data());

    // nlbaf is 0’s based
    size_t nlbaf = id.nlbaf + 1;
    if (nlbaf > 64)
    {
        throw std::runtime_error(std::format("{}: Bad nlbaf {}", name, nlbaf));
    }

    lg2::debug("[{PATH}] Got nlbaf {NLBAF}", "PATH", path, "NLBAF", nlbaf);
    std::vector<LBAFormat> formats;
    for (size_t i = 0; i < nlbaf; i++)
    {
        size_t blockSize = 1UL << id.lbaf[i].ds;
        size_t metadataSize = id.lbaf[i].ms;
        RelPerf rp = relativePerformanceFromRP(id.lbaf[i].rp);
        lg2::debug("[{PATH}] lbaf {LBAF} blocksize {BLOCKSIZE}", "PATH", path,
                   "LBAF", i, "BLOCKSIZE", blockSize);
        formats.push_back({.index = i,
                           .blockSize = blockSize,
                           .metadataSize = metadataSize,
                           .relativePerformance = rp});
    }
    setSupportedFormats(formats);
}

void NVMeSubsystem::deleteVolume(boost::asio::yield_context yield,
                                 const std::shared_ptr<NVMeVolume>& volume)
{
    if (status != Status::Start)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }

    auto pc = getPrimaryController();
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    using callback_t = void(std::tuple<std::error_code, int>);
    auto [err, nvmeStatus] =
        boost::asio::async_initiate<boost::asio::yield_context, callback_t>(
            [intf, ctrl, nsid{volume->namespaceId()}](auto&& handler) {
        auto h = asio_helper::CopyableCallback(
            std::forward<decltype(handler)>(handler));

        intf->adminDeleteNamespace(
            ctrl, nsid,
            [h](const std::error_code& err, int nvmeStatus) mutable {
            h(std::make_tuple(err, nvmeStatus));
        });
    }, yield);

    // exception must be thrown outside of the async block
    checkLibNVMeError(err, nvmeStatus, "Delete");

    forgetVolume(volume);
}

// submitCb is called once the sanitize has been submitted
void NVMeSubsystem::startSanitize(
    const NVMeSanitizeParams& params,
    std::function<void(nvme_ex_ptr ex)>&& submitCb)
{
    if (status != Status::Start)
    {
        throw sdbusplus::xyz::openbmc_project::Common::Error::Unavailable();
    }

    auto pc = getPrimaryController();
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    intf->adminSanitize(
        ctrl, params.nvmeAction(), params.passes, params.pattern,
        params.patternInvert,
        [submitCb](nvme_ex_ptr ex) { submitCb(std::move(ex)); });
}

void NVMeSubsystem::sanitizeStatus(
    std::function<void(nvme_ex_ptr ex, bool inProgress, bool failed,
                       bool completed, uint16_t sstat, uint16_t sprog,
                       uint32_t scdw10)>&& cb)
{
    if (status != Status::Start)
    {
        lg2::error("[{PATH}]Subsystem not in Start state, have {STATUS}",
                   "PATH", path, "STATUS", static_cast<int>(status));
        return;
    }

    auto pc = getPrimaryController();
    nvme_mi_ctrl_t ctrl = pc->getMiCtrl();

    auto intf = std::get<std::shared_ptr<NVMeMiIntf>>(nvmeIntf.getInferface());

    intf->adminGetLogPage(
        ctrl, NVME_LOG_LID_SANITIZE, NVME_NSID_NONE, 0, 0,
        [self{shared_from_this()}, cb](const std::error_code& ec,
                                       std::span<uint8_t> data) {
        if (ec)
        {
            std::string msg = "GetLogPage failed: " + ec.message();
            auto ex = makeLibNVMeError(msg);
            cb(ex, false, false, false, 0, 0, 0);
            return;
        }

        nvme_sanitize_log_page* log =
            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
            reinterpret_cast<nvme_sanitize_log_page*>(data.data());
        uint8_t sanStatus = log->sstat & NVME_SANITIZE_SSTAT_STATUS_MASK;
        cb(nvme_ex_ptr(), sanStatus == NVME_SANITIZE_SSTAT_STATUS_IN_PROGESS,
           sanStatus == NVME_SANITIZE_SSTAT_STATUS_COMPLETED_FAILED,
           sanStatus == NVME_SANITIZE_SSTAT_STATUS_COMPLETE_SUCCESS, log->sstat,
           log->sprog, log->scdw10);
    });
}