| #include "tlbmc/hal/power_fault_detector/power_fault_detector.h" |
| |
| #include <filesystem> // NOLINT |
| #include <fstream> |
| #include <memory> |
| #include <optional> |
| #include <sstream> |
| #include <string> |
| #include <system_error> // NOLINT |
| #include <utility> |
| #include <vector> |
| |
| #include "gbmc-hal/api/app/power/fault_logger.h" |
| #include "gbmc-hal/api/system/registries.h" |
| #include "gbmc-hal/api/system/registry_configs.h" |
| #include "gbmc-hal/api/system/transport/i2c_bus.h" |
| #include "absl/container/flat_hash_map.h" |
| #include "absl/container/flat_hash_set.h" |
| #include "absl/functional/any_invocable.h" |
| #include "absl/log/log.h" |
| #include "absl/memory/memory.h" |
| #include "absl/status/status.h" |
| #include "absl/status/statusor.h" |
| #include "absl/strings/match.h" |
| #include "absl/strings/str_cat.h" |
| #include "absl/strings/string_view.h" |
| #include "absl/strings/substitute.h" |
| #include "absl/time/clock.h" |
| #include "absl/time/time.h" |
| #include "g3/macros.h" |
| #include "gbmc_sel_defs.h" |
| #include "gbmc_sel_pub.h" |
| #include "gpio_config.pb.h" |
| #include "power_control.pb.h" |
| #include "power_fault_detector_config.pb.h" |
| #include "tlbmc/scheduler/scheduler.h" |
| |
| namespace milotic_tlbmc { |
| |
| absl::StatusOr<std::unique_ptr<PowerFaultDetector>> PowerFaultDetector::Create( |
| const PowerFaultDetectorConfig& power_fault_detector_config, |
| milotic_tlbmc::TaskScheduler* task_scheduler, |
| std::shared_ptr<platforms::gbmc::hal::I2cBus> i2c_bus, |
| gbmc_sel_framework::JournalSenderFunc journal_sender) { |
| absl::flat_hash_map<std::string, |
| std::unique_ptr<platforms::gbmc::hal::PowerFaultLogger>> |
| boards; |
| |
| // Validate config for duplicate GPIOs |
| auto check_duplicates = [](const StateMonitorConfig& config, |
| absl::string_view state_name) -> absl::Status { |
| absl::flat_hash_set<std::string> seen_gpios; |
| for (const auto& action : config.gpio_actions()) { |
| if (!seen_gpios.insert(action.gpio_name()).second) { |
| return absl::InvalidArgumentError(absl::StrCat( |
| "Duplicate GPIO action found for '", action.gpio_name(), "' in ", |
| state_name, " config. Only one action per GPIO is allowed.")); |
| } |
| } |
| return absl::OkStatus(); |
| }; |
| ECCLESIA_RETURN_IF_ERROR( |
| check_duplicates(power_fault_detector_config.s0_config(), "S0")); |
| ECCLESIA_RETURN_IF_ERROR( |
| check_duplicates(power_fault_detector_config.s5_config(), "S5")); |
| |
| for (const auto& [name, info] : power_fault_detector_config.known_boards()) { |
| platforms::gbmc::hal::PowerFaultLoggerConfig config{ |
| .board_config_path = info.board_config_path(), |
| .component_config_path = info.component_config_path(), |
| .i2c_bus = i2c_bus, |
| .mock_bus = (i2c_bus != nullptr), |
| }; |
| ECCLESIA_ASSIGN_OR_RETURN( |
| boards[name], platforms::gbmc::hal::PowerFaultLoggerRegistry::Create( |
| "PowerFaultLogger", config)); |
| } |
| |
| auto detector = absl::WrapUnique( |
| new PowerFaultDetector(power_fault_detector_config, std::move(boards), |
| task_scheduler, std::move(journal_sender))); |
| return detector; |
| } |
| |
| PowerFaultDetector::PowerFaultDetector( |
| const PowerFaultDetectorConfig& power_fault_detector_config, |
| absl::flat_hash_map<std::string, |
| std::unique_ptr<platforms::gbmc::hal::PowerFaultLogger>> |
| boards, |
| milotic_tlbmc::TaskScheduler* task_scheduler, |
| gbmc_sel_framework::JournalSenderFunc journal_sender) |
| : life_token_(std::make_shared<LifeToken>()), |
| power_fault_detector_config_(power_fault_detector_config), |
| boards_(std::move(boards)), |
| task_scheduler_(task_scheduler), |
| current_state_(PowerState::POWER_STATE_OFF), |
| journal_sender_(std::move(journal_sender)) { |
| CheckPreviousPowerFault(); |
| } |
| |
| PowerFaultDetector::~PowerFaultDetector() { |
| if (life_token_ != nullptr) { |
| // Reset the shared pointer to release the life token. This will prevent |
| // any pending tasks from executing. |
| life_token_.reset(); |
| } |
| } |
| |
| absl::flat_hash_map<std::string, absl::AnyInvocable<void(GpioEventType)>> |
| PowerFaultDetector::GetGpioMonitorCallbackBundle() const { |
| absl::flat_hash_map<std::string, absl::AnyInvocable<void(GpioEventType)>> |
| bundle; |
| |
| // Helper to add unique GPIOs to the bundle |
| auto add_gpios_from_config = [this, |
| &bundle](const StateMonitorConfig& config) { |
| std::weak_ptr<LifeToken> weak_life_token = life_token_; |
| for (const auto& action : config.gpio_actions()) { |
| bundle.try_emplace(action.gpio_name(), |
| [this, name = action.gpio_name(), |
| weak_life_token](GpioEventType event) { |
| // Check if the detector is still alive |
| auto locked_life_token = weak_life_token.lock(); |
| if (locked_life_token == nullptr) { |
| return; |
| } |
| this->HandleGpioEvent(name); |
| }); |
| } |
| }; |
| |
| add_gpios_from_config(power_fault_detector_config_.s0_config()); |
| add_gpios_from_config(power_fault_detector_config_.s5_config()); |
| |
| return bundle; |
| } |
| |
| void PowerFaultDetector::OnPowerStateChange(PowerStateEvent event) { |
| switch (event) { |
| case PowerStateEvent::kPowerGood: |
| current_state_ = PowerState::POWER_STATE_ON; |
| break; |
| case PowerStateEvent::kPowerLost: |
| current_state_ = PowerState::POWER_STATE_OFF; |
| break; |
| } |
| } |
| |
| void PowerFaultDetector::HandleGpioEvent(absl::string_view gpio_name) const { |
| const StateMonitorConfig* active_config = |
| &power_fault_detector_config_.s5_config(); |
| if (current_state_ == PowerState::POWER_STATE_ON) { |
| active_config = &power_fault_detector_config_.s0_config(); |
| } |
| |
| for (const auto& action : active_config->gpio_actions()) { |
| if (action.gpio_name() == gpio_name) { |
| absl::flat_hash_map<std::string, std::optional<std::string>> |
| boards_to_record; |
| for (const auto& board : action.boards_to_record()) { |
| boards_to_record[board] = std::nullopt; |
| } |
| |
| if (task_scheduler_ != nullptr) { |
| std::weak_ptr<LifeToken> weak_life_token = life_token_; |
| task_scheduler_->ScheduleOneShotAsync( |
| [this, boards = std::move(boards_to_record), |
| trigger = action.gpio_name(), |
| weak_life_token](absl::AnyInvocable<void()> on_done) { |
| auto locked_life_token = weak_life_token.lock(); |
| if (locked_life_token == nullptr) { |
| on_done(); |
| return; |
| } |
| this->CollectPowerFault(boards, trigger); |
| on_done(); |
| }, |
| absl::Milliseconds(1)); |
| } |
| // We assume one GPIO might trigger multiple board recordings, |
| // but usually defined in one GpioAction. |
| // If a GPIO is listed multiple times in one state (bad config), we assume |
| // first match or merge. Here we take the first match. |
| break; |
| } |
| } |
| } |
| |
| void PowerFaultDetector::CheckPreviousPowerFault() const { |
| absl::Duration delay = |
| absl::Seconds(power_fault_detector_config_.initial_check_delay_seconds()); |
| if (delay <= absl::ZeroDuration()) { |
| delay = absl::Milliseconds(1); |
| } |
| |
| if (task_scheduler_ != nullptr) { |
| std::weak_ptr<LifeToken> weak_life_token = life_token_; |
| task_scheduler_->ScheduleOneShotAsync( |
| [this, weak_life_token](absl::AnyInvocable<void()> on_done) { |
| auto locked_life_token = weak_life_token.lock(); |
| if (locked_life_token == nullptr) { |
| on_done(); |
| return; |
| } |
| this->CheckPreviousPowerFaultTask(); |
| on_done(); |
| }, |
| delay); |
| } |
| } |
| |
| void PowerFaultDetector::CollectPowerFault( |
| const absl::flat_hash_map<std::string, std::optional<std::string>>& |
| boards_to_collect, |
| absl::string_view trigger_reason) const { |
| if (boards_to_collect.empty()) { |
| return; |
| } |
| |
| // Create timestamp |
| absl::Time now = absl::Now(); |
| std::string timestamp = |
| absl::FormatTime("%Y%m%d_%H%M%S", now, absl::LocalTimeZone()); |
| // Append milliseconds |
| absl::StrAppend(×tamp, "_", |
| absl::ToInt64Milliseconds(now - absl::UnixEpoch()) % 1000); |
| |
| std::filesystem::path output_dir( |
| power_fault_detector_config_.log_output_directory()); |
| std::filesystem::path tmp_folder_path = output_dir / (timestamp + ".tmp"); |
| std::filesystem::path final_folder_path = output_dir / timestamp; |
| |
| // Ensure output directory exists |
| std::error_code ec; |
| std::filesystem::create_directories(output_dir, ec); |
| if (ec) { |
| LOG(ERROR) << "Failed to create log directory: " << ec.message(); |
| return; |
| } |
| |
| // Create tmp folder |
| std::filesystem::create_directory(tmp_folder_path, ec); |
| if (ec) { |
| LOG(ERROR) << "Failed to create tmp folder: " << tmp_folder_path.string() |
| << ": " << ec.message(); |
| return; |
| } |
| |
| std::stringstream summary_ss; |
| summary_ss << absl::Substitute( |
| "Power Fault Detected. Trigger: $0, State: $1\n", trigger_reason, |
| (current_state_ == PowerState::POWER_STATE_ON ? "S0" : "S5")); |
| |
| int files_created = 0; |
| |
| for (const auto& [board_name, discard_pattern] : boards_to_collect) { |
| auto it = boards_.find(board_name); |
| if (it == boards_.end()) { |
| LOG(ERROR) << "Board not found: " << board_name; |
| summary_ss << absl::Substitute("--- Board: $0 NOT FOUND ---\n", |
| board_name); |
| continue; |
| } |
| |
| std::stringstream board_ss; |
| board_ss << absl::Substitute("--- Board: $0 ---\n", board_name); |
| it->second->LogFaults(board_ss); |
| board_ss << "\n"; |
| |
| // Check discard pattern |
| if (discard_pattern.has_value() && |
| absl::StrContains(board_ss.str(), *discard_pattern)) { |
| LOG(INFO) << "Discarding clean log for board: " << board_name |
| << " Content: " << board_ss.str(); |
| continue; |
| } |
| |
| std::filesystem::path board_log_path = |
| tmp_folder_path / (board_name + ".log"); |
| std::ofstream log_file(board_log_path); |
| if (log_file.is_open()) { |
| log_file << absl::Substitute( |
| "Power Fault Detected. Trigger: $0, State: $1\n", trigger_reason, |
| (current_state_ == PowerState::POWER_STATE_ON ? "S0" : "S5")); |
| log_file << board_ss.str(); |
| log_file.close(); |
| files_created++; |
| } else { |
| LOG(ERROR) << "Failed to open log file: " << board_log_path.string(); |
| } |
| summary_ss << board_ss.str(); |
| } |
| |
| if (files_created == 0) { |
| LOG(INFO) |
| << "No logs created (all discarded or empty). Deleting tmp folder."; |
| std::filesystem::remove_all(tmp_folder_path, ec); |
| return; |
| } |
| |
| // Write summary log |
| std::filesystem::path summary_log_path = tmp_folder_path / "summary.log"; |
| std::ofstream summary_file(summary_log_path); |
| if (summary_file.is_open()) { |
| summary_file << summary_ss.str(); |
| summary_file.close(); |
| } else { |
| LOG(ERROR) << "Failed to open summary log file: " |
| << summary_log_path.string(); |
| } |
| |
| // Rename folder |
| std::filesystem::rename(tmp_folder_path, final_folder_path, ec); |
| if (ec) { |
| LOG(ERROR) << "Failed to rename tmp folder: " << ec.message(); |
| } else { |
| LOG(INFO) << "Power Fault Logs saved to: " << final_folder_path.string(); |
| |
| PublishPowerFaultSel(trigger_reason, timestamp); |
| } |
| } |
| |
| void PowerFaultDetector::PublishPowerFaultSel( |
| absl::string_view trigger_reason, absl::string_view entry_name) const { |
| bool is_initial_check = (trigger_reason == "Initial Check"); |
| gbmc_sel_framework::EventSourceComponent component = |
| is_initial_check ? gbmc_sel_framework::EventSourceComponent:: |
| EVENT_SOURCE_COMPONENT_BlackBox |
| : gbmc_sel_framework::EventSourceComponent:: |
| EVENT_SOURCE_COMPONENT_GPIO; |
| |
| std::vector<std::string> additional_data = { |
| "GBMC_SYSTEM_EVENT_TYPE=PowerFaultEvent", |
| absl::StrCat("GBMC_POWER_FAULT_LOG_ENTRY=", entry_name)}; |
| std::vector<absl::string_view> additional_data_views; |
| additional_data_views.reserve(additional_data.size()); |
| for (const auto& s : additional_data) { |
| additional_data_views.push_back(s); |
| } |
| |
| absl::Status status; |
| if (journal_sender_) { |
| status = gbmc_sel_framework::PublishGBMCSystemEvent( |
| gbmc_sel_framework::EventSourceType::EVENT_SOURCE_SCM, component, |
| gbmc_sel_framework::EventSeverity::EVENT_SEVERITY_ERROR, |
| gbmc_sel_framework::EventAction::EVENT_ACTION_ASSERT, |
| /*event_message=*/ |
| absl::StrCat("Power Fault Detected. Trigger: ", trigger_reason), |
| /*additional_data=*/additional_data_views, |
| /*journal_sender=*/[this](const struct iovec* iov, int iovcnt) { |
| return journal_sender_(iov, iovcnt); |
| }); |
| } else { |
| status = gbmc_sel_framework::PublishGBMCSystemEvent( |
| gbmc_sel_framework::EventSourceType::EVENT_SOURCE_SCM, component, |
| gbmc_sel_framework::EventSeverity::EVENT_SEVERITY_ERROR, |
| gbmc_sel_framework::EventAction::EVENT_ACTION_ASSERT, |
| /*event_message=*/ |
| absl::StrCat("Power Fault Detected. Trigger: ", trigger_reason), |
| /*additional_data=*/additional_data_views); |
| } |
| if (!status.ok()) { |
| LOG(ERROR) << "Failed to publish SEL event: " << status.message(); |
| } |
| } |
| |
| void PowerFaultDetector::CheckPreviousPowerFaultTask() const { |
| absl::flat_hash_map<std::string, std::optional<std::string>> boards_to_check; |
| const auto& initial_checks = power_fault_detector_config_.initial_checks(); |
| |
| if (!initial_checks.empty()) { |
| for (const auto& check_config : initial_checks) { |
| if (check_config.has_discard_pattern()) { |
| boards_to_check[check_config.board_name()] = |
| check_config.discard_pattern(); |
| } else { |
| boards_to_check[check_config.board_name()] = std::nullopt; |
| } |
| } |
| } else { |
| // Fallback: check all boards with no discard pattern |
| for (const auto& [name, _] : boards_) { |
| boards_to_check[name] = std::nullopt; |
| } |
| } |
| |
| CollectPowerFault(boards_to_check, "Initial Check"); |
| } |
| |
| void PowerFaultDetector::DisableFaultDetection() const { |
| for (const auto& [board_name, logger] : boards_) { |
| if (logger == nullptr) { |
| continue; |
| } |
| absl::Status status = logger->DisableFaultDetection(); |
| if (!status.ok()) { |
| LOG(ERROR) << "Failed to disable fault detection for board " << board_name |
| << ": " << status.message(); |
| } |
| } |
| } |
| |
| } // namespace milotic_tlbmc |