bmc/state_monitor_bmc.cc

#include "bmc/state_monitor_bmc.h"

#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "one/node_entities_api.pb.h"
#include "one/resolved_entities.pb.h"
#include "bmc/address_lookup.h"
#include "bmc/daemon_context_bmc.h"
#include "bmc/gnic_monitor.h"
#include "bmc/health_state_ping.h"
#include "bmc/redfish.h"
#include "bmc/remote_state_ping.h"
#include "daemon_context.h"
#include "internal_boot_counter.h"
#include "safepower_agent.pb.h"
#include "safepower_agent_config.pb.h"
#include "system_state.pb.h"
#include "state_updater.h"
#include "static_state.h"
#include "absl/base/no_destructor.h"
#include "absl/functional/any_invocable.h"
#include "absl/functional/bind_front.h"
#include "absl/log/log.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "nlohmann/json.hpp"
#include "nlohmann/json_fwd.hpp"

namespace safepower_agent {

using safepower_agent_config::GpowerdConfig;
using safepower_agent_config::StateGatheringInfo;

static constexpr int kMinCollectionIntervalMs = 10;

static void GetRedfishBootNumCallback(
    StaticStateLoader::GetStateCallback callback,
    absl::StatusOr<nlohmann::json> js) {
  if (!js.ok()) {
    callback(js.status());
    return;
  }
  auto boot_info = js->find("BootInfo");
  if (boot_info == js->end()) {
    callback(absl::NotFoundError(
        absl::StrCat("BootInfo not found in ", js->dump())));
    return;
  }
  if (!boot_info->is_object()) {
    callback(absl::InvalidArgumentError(
        absl::StrCat("BootInfo is not an object in ", js->dump())));
    return;
  }
  auto boot_num = boot_info->find("BootCount");
  if (boot_num == boot_info->end()) {
    callback(absl::NotFoundError(
        absl::StrCat("BootCount not found in ", js->dump())));
    return;
  }
  auto* boot_num_int = boot_num->get_ptr<int64_t*>();
  if (boot_num_int == nullptr) {
    callback(absl::InvalidArgumentError(
        absl::StrCat("BootCount is not an int64_t in ", js->dump())));
    return;
  }
  absl::string_view entity_tag =
      DaemonContext::Get().offline_node_entities().entity_tag();
  safepower_agent_proto::SystemState state;
  (*state.mutable_node_state())[entity_tag]
      .mutable_boot_state()
      ->set_boot_counter(*boot_num_int);
  callback(state);
}

static void GetInternalBootNumCallback(
    StaticStateLoader::GetStateCallback callback) {
  static absl::NoDestructor<InternalBootCounter> internal_boot_counter(
      DaemonContext::Get()
          .config()
          .gpowerd_config()
          .internal_boot_count_config()
          .boot_id_path());
  std::string entity_tag(
      DaemonContext::Get().offline_node_entities().entity_tag());
  if (entity_tag.empty()) {
    callback(absl::NotFoundError("Entity tag is empty, unable to get bootnum"));
    return;
  }

  absl::Status status = internal_boot_counter->CheckAndIncrement();
  if (!status.ok()) {
    callback(absl::NotFoundError(
        absl::StrCat("Failed to check and increment boot counter: ", status)));
    return;
  }
  std::optional<int> internal_bootnum = internal_boot_counter->value();
  if (!internal_bootnum.has_value()) {
    callback(absl::NotFoundError("Local boot counter is not available"));
    return;
  }

  safepower_agent_proto::SystemState state;
  auto& node_state = (*state.mutable_node_state())[entity_tag];
  node_state.mutable_boot_state()->set_boot_counter(*internal_bootnum);
  callback(state);
}

StateMonitorBMC::StateMonitorBMC(
    std::shared_ptr<StateUpdater<safepower_agent_proto::SystemState>> reactor)
    : reactor_(reactor) {}

absl::Status StateMonitorBMC::InitializeBootnumLoader() {
  const auto config = DaemonContext::Get().config().gpowerd_config();
  if (DaemonContext::Get().offline_node_entities().entity_tag().empty()) {
    return absl::FailedPreconditionError(
        "offline_node_entities is not initialized in DaemonContext: No "
        "entity_tag found");
  }
  if (bootnum_loader_) {
    return absl::InternalError("StateMonitorBMC already initialized");
  }

  // Check if the config specifies using gpowerd for the boot count. Otherwise,
  // Redfish will be used by default.
  if (config.has_internal_boot_count_config()) {
    LOG(INFO) << "Bootnum loader initialized to use internal boot count";
    bootnum_loader_ = StaticStateLoader::Create(
        "bootnum_loader", GetInternalBootNumCallback, reactor_,
        StaticStateLoaderConfig());
    LOG(INFO) << "Bootnum loader initialized to use internal boot count";
    return absl::OkStatus();
  }
  // Otherwise bootnum will come from redfish.
  LOG(INFO) << "Bootnum loader initialized to use redfish boot count";

  StaticStateLoaderConfig loader_config = {
      .retry_interval = absl::Milliseconds(
          config.redfish_boot_count_config().retry_interval_ms()),
      .max_retries = -1,
  };
  if (config.redfish_boot_count_config().has_timeout_sec()) {
    loader_config.timeout =
        absl::Seconds(config.redfish_boot_count_config().timeout_sec());
  }
  bootnum_loader_ = StaticStateLoader::Create(
      "bootnum_loader",
      [c = config.redfish_boot_count_config()](
          StaticStateLoader::GetStateCallback callback) {
        LOG(INFO) << "Attempting to get bootnum from " << c.ip() << ":"
                  << c.port() << c.uri();
        Redfish::Get(
            c.uri(),
            absl::bind_front(&GetRedfishBootNumCallback, std::move(callback)),
            c.ip(), c.port());
      },
      reactor_, loader_config);
  return absl::OkStatus();
}

absl::Status StateMonitorBMC::RescheduleUntilSuccess(
    absl::AnyInvocable<absl::Status()> func, absl::string_view name,
    absl::Duration interval) {
  // Run the function once to check if it is already successful.
  absl::Status first_run_status = func();
  if (first_run_status.ok()) {
    return first_run_status;
  }
  // If the timeout is zero or negative, don't schedule the retry, and return
  // the first run status.
  if (interval <= absl::ZeroDuration()) {
    LOG(WARNING) << "Timeout is zero or negative, not scheduling " << name;
    return first_run_status;
  }
  std::string retry_name = absl::StrCat("retry_until_success_", name);
  auto setup_status = DaemonContext::Get().scheduler().PeriodicCall(
      [func = std::move(func), retry_name]() mutable {
        // Run the function again, and cancel the retry if it succeeds.
        absl::Status status = func();
        if (status.ok()) {
          auto status = DaemonContext::Get().scheduler().CancelCall(retry_name);
          if (!status.ok()) {
            LOG(ERROR) << "Failed to cancel " << retry_name << ": " << status;
          }
        }
      },
      interval, retry_name);
  if (!setup_status.ok()) {
    LOG(ERROR) << "Failed to schedule " << retry_name << ": " << setup_status;
  }
  return setup_status;
}

void StateMonitorBMC::BuildFromConfig(const GpowerdConfig& config) {
  std::weak_ptr<StateMonitorBMC> weak_self = shared_from_this();
  if (config.has_ping_config() || config.health_check_size() > 0) {
    absl::Status icmp_ping_status_setup = RescheduleUntilSuccess(
        [weak_self]() {
          auto self = weak_self.lock();
          if (self == nullptr) {
            // state monitor is destroyed, no need to continue, ping state
            // monitor is not created.
            return absl::OkStatus();
          }
          return self->CreateIMCMonitors();
        },
        "ICMP ping state monitor",
        absl::Milliseconds(config.ping_config().retry_timeout_ms()));
    if (!icmp_ping_status_setup.ok()) {
      LOG(ERROR) << "  " << icmp_ping_status_setup << "  continuing";
    }
  }
  absl::Status bootnum_status_setup = RescheduleUntilSuccess(
      [weak_self]() {
        auto self = weak_self.lock();
        if (self == nullptr) {
          // state monitor is destroyed, no need to continue, bootnum loader
          // is not created.
          LOG(INFO) << "StateMonitorBMC is destroyed, not creating bootnum "
                       "loader";
          return absl::OkStatus();
        }
        return self->InitializeBootnumLoader();
      },
      "bootnum_loader",
      absl::Seconds(config.redfish_boot_count_config().timeout_sec()));
  if (!bootnum_status_setup.ok()) {
    LOG(ERROR) << "Failed to initialize bootnum loader setup: "
               << bootnum_status_setup << "  continuing";
  }

  for (auto& resource_to_monitor : config.state_monitor_config()) {
    if (resource_to_monitor.state_gathering_info().info_case() ==
        StateGatheringInfo::kRedfish) {
      auto prototype = resource_to_monitor.state_gathering_info().prototype();
      if (prototype.has_power_state() &&
          (prototype.power_state().has_state())) {
        // create bmcweb state monitor for power state
        if (absl::Status status =
                CreateRedfishUpdatePowerState(resource_to_monitor);
            !status.ok()) {
          LOG(ERROR) << "failed to create bmcweb state monitor for "
                     << resource_to_monitor.state_name() << ": " << status;
        }
      } else {
        LOG(ERROR) << "update function is not supported  TODO b/372971177";
      }
    } else if (resource_to_monitor.state_gathering_info().info_case() ==
               StateGatheringInfo::kDbus) {
      (void)CreateDbusStateMonitor("", "", "", "", "", 1000);
    } else {
      LOG(ERROR) << "state monitor type is not supported; "
                 << resource_to_monitor.state_gathering_info().info_case();
    }
  }
}

void StateMonitorBMC::InsertStateMonitors(
    std::string node_entity_tag, absl::StatusOr<std::string> ip_address) {
  if (!ip_address.ok()) {
    LOG(ERROR) << "Failed to lookup address for node: " << node_entity_tag
               << " with status: " << ip_address.status();
    return;
  }

  const auto& config = DaemonContext::Get().config().gpowerd_config();
  if (config.has_ping_config()) {
    LOG(INFO) << "creating ping state monitor for: " << node_entity_tag;
    state_monitors_.push_back(std::make_unique<PingStateMonitor>(
        node_entity_tag, *ip_address,
        DaemonContext::Get().config().gpowerd_config().ping_config(),
        reactor_));
    absl::Status status = state_monitors_.back()->Start();
    if (!status.ok()) {
      LOG(DFATAL) << "Failed to start ping state monitor for "
                  << state_monitors_.back()->node_entity_tag() << ": "
                  << status;
    }
  }

  for (const auto& health_check : config.health_check()) {
    LOG(INFO) << "creating health state monitor for: "
              << health_check.daemon_name() << " on " << node_entity_tag;
    state_monitors_.push_back(std::make_unique<HealthStateMonitor>(
        node_entity_tag, *ip_address, health_check, reactor_));
    absl::Status status = state_monitors_.back()->Start();
    if (!status.ok()) {
      LOG(DFATAL) << "Failed to start health state monitor for "
                  << health_check.daemon_name() << " on " << node_entity_tag
                  << ": " << status;
    }
  }
}

absl::Status StateMonitorBMC::CreateIMCMonitors() {
  LOG(INFO) << "Creating state monitors";
  if (!DaemonContext::Get().offline_node_entities().has_resolved_config()) {
    return absl::FailedPreconditionError(
        "offline_node_entities is not initialized in DaemonContext: No "
        "resolved_config found");
  }
  // TODO(b/475859943): return the error here and in insert state monitor
  auto address_lookup_result = AddressLookup::Create();
  std::optional<AddressLookup> address_lookup;
  if (address_lookup_result.ok()) {
    address_lookup = std::move(*address_lookup_result);
  } else {
    LOG(ERROR) << "Failed to initialize address lookup: "
               << address_lookup_result.status();
  }

  for (const auto& [node_entity_tag, node_entity] : DaemonContext::Get()
                                                        .offline_node_entities()
                                                        .resolved_config()
                                                        .entities()) {
    if (node_entity.node_type_info().os_type() ==
            net_model_unm::proto::NOST_NODEOS &&
        node_entity.node_type_info().is_control()) {
      auto ip_address = GetIPv6Address(node_entity_tag, address_lookup);
      InsertStateMonitors(node_entity_tag, std::move(ip_address));
    }
  }

  imc_fru_scanner_ = ImcFruScanner::Create(
      DaemonContext::Get().config().gpowerd_config().gnic_monitor_config(),
      reactor_);
  if (absl::Status status = imc_fru_scanner_->Start(); !status.ok()) {
    LOG(ERROR) << "Failed to start IMC FRU scanner: " << status;
  }

  LOG(INFO) << "DONE creating ping state monitors; started IMC FRU scan";
  return absl::OkStatus();
}

absl::Status StateMonitorBMC::CreateRedfishUpdatePowerState(
    const safepower_agent_config::StateMonitorConfig& config) {
  if (!config.state_gathering_info().has_redfish()) {
    LOG(DFATAL) << "StateGatheringInfo does not have RedfishInfo";
    return absl::InvalidArgumentError(
        "StateGatheringInfo does not have RedfishInfo");
  }
  if (config.state_name().empty()) {
    return absl::InvalidArgumentError("state_name is empty");
  }
  if (config.system_component().node_entity_tag().empty()) {
    return absl::InvalidArgumentError(
        "system_component.node_entity_tag is empty");
  }
  if (config.state_gathering_info().redfish().uri().empty()) {
    return absl::InvalidArgumentError("redfish.uri is empty");
  }
  if (config.state_gathering_info().redfish().json_key().empty()) {
    return absl::InvalidArgumentError("redfish.json_key is empty");
  }
  if (config.state_gathering_info().collection_interval_ms() <=
      kMinCollectionIntervalMs) {
    return absl::InvalidArgumentError(absl::StrCat(
        "collection_interval_ms must be > ", kMinCollectionIntervalMs));
  }
  auto capture_config =
      std::make_shared<safepower_agent_config::StateMonitorConfig>(config);
  if (capture_config->system_component().node_entity_tag() == "@self") {
    std::string entity_tag =
        DaemonContext::Get().offline_node_entities().entity_tag();
    if (entity_tag.empty()) {
      LOG(WARNING) << "Entity tag is empty, keeping state monitor as "
                   << "@self";
    } else {
      capture_config->mutable_system_component()->set_node_entity_tag(
          entity_tag);
    }
  }
  LOG(INFO) << "creating bmcweb state monitor for:"
            << capture_config->state_name();

  return DaemonContextBMC::Get().scheduler().PeriodicCall(
      [capture_config, weak_this = weak_from_this()] {
        auto self = weak_this.lock();
        if (!self) {
          return;
        }
        Redfish::Get(
            capture_config->state_gathering_info().redfish().uri(),
            [weak_this, capture_config](absl::StatusOr<nlohmann::json> js) {
              auto self = weak_this.lock();
              if (!self) {
                return;
              }
              auto redfish = capture_config->state_gathering_info().redfish();
              if (!js.ok()) {
                // gpowerd will be unable to get gbmc state before bmcweb is
                // up, and this is expected.
                VLOG(1) << "Failed to get " << redfish.uri()
                        << " from BMC: " << js.status();
                LOG_EVERY_N_SEC(INFO, 10)
                    << "Failed to get " << redfish.uri()
                    << " from BMC: " << js.status() << " COUNTER:" << COUNTER;
                return;
              }

              auto js_value = js->find(redfish.json_key());
              std::string state_str = js_value != js->end() ? *js_value : "";
              safepower_agent_proto::PowerStateSpecifier temp_power_state =
                  self->ConvertStatePowerStringToEnum(state_str);
              safepower_agent_proto::PowerStateSpecifier last_ps =
                  self->reactor_->WithState(
                      [&](const safepower_agent_proto::SystemState& state) {
                        auto node_it = state.node_state().find(
                            capture_config->system_component()
                                .node_entity_tag());
                        if (node_it != state.node_state().end()) {
                          return node_it->second.power_state().state();
                        }
                        return safepower_agent_proto::POWER_STATE_UNSPECIFIED;
                      });
              if (last_ps == temp_power_state) return;
              absl::string_view last_ps_str = PowerStateSpecifier_Name(last_ps);
              absl::string_view temp_ps_str =
                  PowerStateSpecifier_Name(temp_power_state);
              LOG(INFO) << absl::StrFormat(
                  "%s changed from: %d (%s) to: %d (%s)",
                  capture_config->state_name(), last_ps, last_ps_str,
                  temp_power_state, temp_ps_str);

              // add new state power state to a new system proto
              safepower_agent_proto::SystemState new_state;
              safepower_agent_proto::PowerState ps;
              ps.set_state(temp_power_state);

              safepower_agent_proto::NodeState ns;
              *(ns.mutable_power_state()) = ps;

              new_state.mutable_node_state()->insert(
                  {capture_config->system_component().node_entity_tag(), ns});

              self->reactor_->UpdateState(new_state);
            },
            capture_config->state_gathering_info().redfish().ip(),
            capture_config->state_gathering_info().redfish().port());
      },
      absl::Milliseconds(
          capture_config->state_gathering_info().collection_interval_ms()),
      capture_config->state_name());
}

safepower_agent_proto::PowerStateSpecifier
StateMonitorBMC::ConvertStatePowerStringToEnum(absl::string_view state_str) {
  if (state_str == "On") {
    return safepower_agent_proto::POWER_STATE_ON;
  } else if (state_str == "Off") {
    return safepower_agent_proto::POWER_STATE_OFF;
  } else if (state_str == "Paused") {  // not in bmcweb
    return safepower_agent_proto::POWER_STATE_PAUSED;
  } else if (state_str == "PoweringOn") {
    return safepower_agent_proto::POWER_STATE_POWERING_ON;
  } else if (state_str == "PoweringOff") {
    return safepower_agent_proto::POWER_STATE_POWERING_OFF;
  } else {
    LOG(ERROR) << "failed to map power state string to enum: " << state_str;
    return safepower_agent_proto::POWER_STATE_UNSPECIFIED;
  }
}

absl::Status StateMonitorBMC::CreateDbusStateMonitor(
    std::string state_name, std::string dbus_path, std::string dbus_method,
    std::string dbus_interface, std::string dbus_value,
    int collection_interval_ms) {
  LOG(INFO) << "creating dbus state monitor for:" << state_name;
  LOG(INFO) << "dbus_path: " << dbus_path;
  LOG(INFO) << "dbus_method: " << dbus_method;
  LOG(INFO) << "dbus_interface: " << dbus_interface;
  LOG(INFO) << "dbus_value: " << dbus_value;
  LOG(INFO) << "collection_interval_ms: " << collection_interval_ms;
  return absl::UnimplementedError("CreateDbusStateMonitor is not implemented");
}

std::shared_ptr<StateMonitorBMC> StateMonitorBMC::Create(
    std::shared_ptr<StateUpdater<safepower_agent_proto::SystemState>> reactor) {
  return std::shared_ptr<StateMonitorBMC>(
      new StateMonitorBMC(std::move(reactor)));
}

}  // namespace safepower_agent