blob: 12a042699abef8369a521cd78eaf9401bd482879 [file]
#include "tlbmc/hal/power_fault_detector/power_fault_detector.h"
#include <filesystem> // NOLINT
#include <fstream>
#include <memory>
#include <optional>
#include <sstream>
#include <string>
#include <system_error> // NOLINT
#include <utility>
#include <vector>
#include "gbmc-hal/api/app/power/fault_logger.h"
#include "gbmc-hal/api/system/registries.h"
#include "gbmc-hal/api/system/registry_configs.h"
#include "gbmc-hal/api/system/transport/i2c_bus.h"
#include "absl/container/flat_hash_map.h"
#include "absl/container/flat_hash_set.h"
#include "absl/functional/any_invocable.h"
#include "absl/log/log.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "g3/macros.h"
#include "gbmc_sel_defs.h"
#include "gbmc_sel_pub.h"
#include "gpio_config.pb.h"
#include "power_control.pb.h"
#include "power_fault_detector_config.pb.h"
#include "tlbmc/scheduler/scheduler.h"
namespace milotic_tlbmc {
absl::StatusOr<std::unique_ptr<PowerFaultDetector>> PowerFaultDetector::Create(
const PowerFaultDetectorConfig& power_fault_detector_config,
milotic_tlbmc::TaskScheduler* task_scheduler,
std::shared_ptr<platforms::gbmc::hal::I2cBus> i2c_bus,
gbmc_sel_framework::JournalSenderFunc journal_sender) {
absl::flat_hash_map<std::string,
std::unique_ptr<platforms::gbmc::hal::PowerFaultLogger>>
boards;
// Validate config for duplicate GPIOs
auto check_duplicates = [](const StateMonitorConfig& config,
absl::string_view state_name) -> absl::Status {
absl::flat_hash_set<std::string> seen_gpios;
for (const auto& action : config.gpio_actions()) {
if (!seen_gpios.insert(action.gpio_name()).second) {
return absl::InvalidArgumentError(absl::StrCat(
"Duplicate GPIO action found for '", action.gpio_name(), "' in ",
state_name, " config. Only one action per GPIO is allowed."));
}
}
return absl::OkStatus();
};
ECCLESIA_RETURN_IF_ERROR(
check_duplicates(power_fault_detector_config.s0_config(), "S0"));
ECCLESIA_RETURN_IF_ERROR(
check_duplicates(power_fault_detector_config.s5_config(), "S5"));
for (const auto& [name, info] : power_fault_detector_config.known_boards()) {
platforms::gbmc::hal::PowerFaultLoggerConfig config{
.board_config_path = info.board_config_path(),
.component_config_path = info.component_config_path(),
.i2c_bus = i2c_bus,
.mock_bus = (i2c_bus != nullptr),
};
ECCLESIA_ASSIGN_OR_RETURN(
boards[name], platforms::gbmc::hal::PowerFaultLoggerRegistry::Create(
"PowerFaultLogger", config));
}
auto detector = absl::WrapUnique(
new PowerFaultDetector(power_fault_detector_config, std::move(boards),
task_scheduler, std::move(journal_sender)));
return detector;
}
PowerFaultDetector::PowerFaultDetector(
const PowerFaultDetectorConfig& power_fault_detector_config,
absl::flat_hash_map<std::string,
std::unique_ptr<platforms::gbmc::hal::PowerFaultLogger>>
boards,
milotic_tlbmc::TaskScheduler* task_scheduler,
gbmc_sel_framework::JournalSenderFunc journal_sender)
: life_token_(std::make_shared<LifeToken>()),
power_fault_detector_config_(power_fault_detector_config),
boards_(std::move(boards)),
task_scheduler_(task_scheduler),
current_state_(PowerState::POWER_STATE_OFF),
journal_sender_(std::move(journal_sender)) {
CheckPreviousPowerFault();
}
PowerFaultDetector::~PowerFaultDetector() {
if (life_token_ != nullptr) {
// Reset the shared pointer to release the life token. This will prevent
// any pending tasks from executing.
life_token_.reset();
}
}
absl::flat_hash_map<std::string, absl::AnyInvocable<void(GpioEventType)>>
PowerFaultDetector::GetGpioMonitorCallbackBundle() const {
absl::flat_hash_map<std::string, absl::AnyInvocable<void(GpioEventType)>>
bundle;
// Helper to add unique GPIOs to the bundle
auto add_gpios_from_config = [this,
&bundle](const StateMonitorConfig& config) {
std::weak_ptr<LifeToken> weak_life_token = life_token_;
for (const auto& action : config.gpio_actions()) {
bundle.try_emplace(action.gpio_name(),
[this, name = action.gpio_name(),
weak_life_token](GpioEventType event) {
// Check if the detector is still alive
auto locked_life_token = weak_life_token.lock();
if (locked_life_token == nullptr) {
return;
}
this->HandleGpioEvent(name);
});
}
};
add_gpios_from_config(power_fault_detector_config_.s0_config());
add_gpios_from_config(power_fault_detector_config_.s5_config());
return bundle;
}
void PowerFaultDetector::OnPowerStateChange(PowerStateEvent event) {
switch (event) {
case PowerStateEvent::kPowerGood:
current_state_ = PowerState::POWER_STATE_ON;
break;
case PowerStateEvent::kPowerLost:
current_state_ = PowerState::POWER_STATE_OFF;
break;
}
}
void PowerFaultDetector::HandleGpioEvent(absl::string_view gpio_name) const {
const StateMonitorConfig* active_config =
&power_fault_detector_config_.s5_config();
if (current_state_ == PowerState::POWER_STATE_ON) {
active_config = &power_fault_detector_config_.s0_config();
}
for (const auto& action : active_config->gpio_actions()) {
if (action.gpio_name() == gpio_name) {
absl::flat_hash_map<std::string, std::optional<std::string>>
boards_to_record;
for (const auto& board : action.boards_to_record()) {
boards_to_record[board] = std::nullopt;
}
if (task_scheduler_ != nullptr) {
std::weak_ptr<LifeToken> weak_life_token = life_token_;
task_scheduler_->ScheduleOneShotAsync(
[this, boards = std::move(boards_to_record),
trigger = action.gpio_name(),
weak_life_token](absl::AnyInvocable<void()> on_done) {
auto locked_life_token = weak_life_token.lock();
if (locked_life_token == nullptr) {
on_done();
return;
}
this->CollectPowerFault(boards, trigger);
on_done();
},
absl::Milliseconds(1));
}
// We assume one GPIO might trigger multiple board recordings,
// but usually defined in one GpioAction.
// If a GPIO is listed multiple times in one state (bad config), we assume
// first match or merge. Here we take the first match.
break;
}
}
}
void PowerFaultDetector::CheckPreviousPowerFault() const {
absl::Duration delay =
absl::Seconds(power_fault_detector_config_.initial_check_delay_seconds());
if (delay <= absl::ZeroDuration()) {
delay = absl::Milliseconds(1);
}
if (task_scheduler_ != nullptr) {
std::weak_ptr<LifeToken> weak_life_token = life_token_;
task_scheduler_->ScheduleOneShotAsync(
[this, weak_life_token](absl::AnyInvocable<void()> on_done) {
auto locked_life_token = weak_life_token.lock();
if (locked_life_token == nullptr) {
on_done();
return;
}
this->CheckPreviousPowerFaultTask();
on_done();
},
delay);
}
}
void PowerFaultDetector::CollectPowerFault(
const absl::flat_hash_map<std::string, std::optional<std::string>>&
boards_to_collect,
absl::string_view trigger_reason) const {
if (boards_to_collect.empty()) {
return;
}
// Create timestamp
absl::Time now = absl::Now();
std::string timestamp =
absl::FormatTime("%Y%m%d_%H%M%S", now, absl::LocalTimeZone());
// Append milliseconds
absl::StrAppend(&timestamp, "_",
absl::ToInt64Milliseconds(now - absl::UnixEpoch()) % 1000);
std::filesystem::path output_dir(
power_fault_detector_config_.log_output_directory());
std::filesystem::path tmp_folder_path = output_dir / (timestamp + ".tmp");
std::filesystem::path final_folder_path = output_dir / timestamp;
// Ensure output directory exists
std::error_code ec;
std::filesystem::create_directories(output_dir, ec);
if (ec) {
LOG(ERROR) << "Failed to create log directory: " << ec.message();
return;
}
// Create tmp folder
std::filesystem::create_directory(tmp_folder_path, ec);
if (ec) {
LOG(ERROR) << "Failed to create tmp folder: " << tmp_folder_path.string()
<< ": " << ec.message();
return;
}
std::stringstream summary_ss;
summary_ss << absl::Substitute(
"Power Fault Detected. Trigger: $0, State: $1\n", trigger_reason,
(current_state_ == PowerState::POWER_STATE_ON ? "S0" : "S5"));
int files_created = 0;
for (const auto& [board_name, discard_pattern] : boards_to_collect) {
auto it = boards_.find(board_name);
if (it == boards_.end()) {
LOG(ERROR) << "Board not found: " << board_name;
summary_ss << absl::Substitute("--- Board: $0 NOT FOUND ---\n",
board_name);
continue;
}
std::stringstream board_ss;
board_ss << absl::Substitute("--- Board: $0 ---\n", board_name);
it->second->LogFaults(board_ss);
board_ss << "\n";
// Check discard pattern
if (discard_pattern.has_value() &&
absl::StrContains(board_ss.str(), *discard_pattern)) {
LOG(INFO) << "Discarding clean log for board: " << board_name
<< " Content: " << board_ss.str();
continue;
}
std::filesystem::path board_log_path =
tmp_folder_path / (board_name + ".log");
std::ofstream log_file(board_log_path);
if (log_file.is_open()) {
log_file << absl::Substitute(
"Power Fault Detected. Trigger: $0, State: $1\n", trigger_reason,
(current_state_ == PowerState::POWER_STATE_ON ? "S0" : "S5"));
log_file << board_ss.str();
log_file.close();
files_created++;
} else {
LOG(ERROR) << "Failed to open log file: " << board_log_path.string();
}
summary_ss << board_ss.str();
}
if (files_created == 0) {
LOG(INFO)
<< "No logs created (all discarded or empty). Deleting tmp folder.";
std::filesystem::remove_all(tmp_folder_path, ec);
return;
}
// Write summary log
std::filesystem::path summary_log_path = tmp_folder_path / "summary.log";
std::ofstream summary_file(summary_log_path);
if (summary_file.is_open()) {
summary_file << summary_ss.str();
summary_file.close();
} else {
LOG(ERROR) << "Failed to open summary log file: "
<< summary_log_path.string();
}
// Rename folder
std::filesystem::rename(tmp_folder_path, final_folder_path, ec);
if (ec) {
LOG(ERROR) << "Failed to rename tmp folder: " << ec.message();
} else {
LOG(INFO) << "Power Fault Logs saved to: " << final_folder_path.string();
PublishPowerFaultSel(trigger_reason, timestamp);
}
}
void PowerFaultDetector::PublishPowerFaultSel(
absl::string_view trigger_reason, absl::string_view entry_name) const {
bool is_initial_check = (trigger_reason == "Initial Check");
gbmc_sel_framework::EventSourceComponent component =
is_initial_check ? gbmc_sel_framework::EventSourceComponent::
EVENT_SOURCE_COMPONENT_BlackBox
: gbmc_sel_framework::EventSourceComponent::
EVENT_SOURCE_COMPONENT_GPIO;
std::vector<std::string> additional_data = {
"GBMC_SYSTEM_EVENT_TYPE=PowerFaultEvent",
absl::StrCat("GBMC_POWER_FAULT_LOG_ENTRY=", entry_name)};
std::vector<absl::string_view> additional_data_views;
additional_data_views.reserve(additional_data.size());
for (const auto& s : additional_data) {
additional_data_views.push_back(s);
}
absl::Status status;
if (journal_sender_) {
status = gbmc_sel_framework::PublishGBMCSystemEvent(
gbmc_sel_framework::EventSourceType::EVENT_SOURCE_SCM, component,
gbmc_sel_framework::EventSeverity::EVENT_SEVERITY_ERROR,
gbmc_sel_framework::EventAction::EVENT_ACTION_ASSERT,
/*event_message=*/
absl::StrCat("Power Fault Detected. Trigger: ", trigger_reason),
/*additional_data=*/additional_data_views,
/*journal_sender=*/[this](const struct iovec* iov, int iovcnt) {
return journal_sender_(iov, iovcnt);
});
} else {
status = gbmc_sel_framework::PublishGBMCSystemEvent(
gbmc_sel_framework::EventSourceType::EVENT_SOURCE_SCM, component,
gbmc_sel_framework::EventSeverity::EVENT_SEVERITY_ERROR,
gbmc_sel_framework::EventAction::EVENT_ACTION_ASSERT,
/*event_message=*/
absl::StrCat("Power Fault Detected. Trigger: ", trigger_reason),
/*additional_data=*/additional_data_views);
}
if (!status.ok()) {
LOG(ERROR) << "Failed to publish SEL event: " << status.message();
}
}
void PowerFaultDetector::CheckPreviousPowerFaultTask() const {
absl::flat_hash_map<std::string, std::optional<std::string>> boards_to_check;
const auto& initial_checks = power_fault_detector_config_.initial_checks();
if (!initial_checks.empty()) {
for (const auto& check_config : initial_checks) {
if (check_config.has_discard_pattern()) {
boards_to_check[check_config.board_name()] =
check_config.discard_pattern();
} else {
boards_to_check[check_config.board_name()] = std::nullopt;
}
}
} else {
// Fallback: check all boards with no discard pattern
for (const auto& [name, _] : boards_) {
boards_to_check[name] = std::nullopt;
}
}
CollectPowerFault(boards_to_check, "Initial Check");
}
void PowerFaultDetector::DisableFaultDetection() const {
for (const auto& [board_name, logger] : boards_) {
if (logger == nullptr) {
continue;
}
absl::Status status = logger->DisableFaultDetection();
if (!status.ok()) {
LOG(ERROR) << "Failed to disable fault detection for board " << board_name
<< ": " << status.message();
}
}
}
} // namespace milotic_tlbmc