tlbmc/thermal/zone_manager.h

#ifndef THIRD_PARTY_MILOTIC_EXTERNAL_CC_TLBMC_THERMAL_ZONE_MANAGER_H_
#define THIRD_PARTY_MILOTIC_EXTERNAL_CC_TLBMC_THERMAL_ZONE_MANAGER_H_

#include <algorithm>
#include <atomic>
#include <cstdint>
#include <fstream>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "absl/base/attributes.h"
#include "absl/base/nullability.h"
#include "absl/base/thread_annotations.h"
#include "absl/container/flat_hash_map.h"
#include "absl/log/log.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include <nlohmann/json.hpp>
#include "tlbmc/collector/fru_collector.h"
#include "tlbmc/collector/sensor_collector_aggregator.h"
#include "tlbmc/configs/entity_config.h"
#include "thermal_config.pb.h"
#include "tlbmc/sensors/sensor.h"
#include "tlbmc/thermal/controller/controller.h"
#include "tlbmc/thermal/controller_info/controller_info.h"
#include "tlbmc/thermal/debug_mode.h"
#include "tlbmc/thermal/failsafe_logger.h"
#include "tlbmc/thermal/fan_info/fan_info.h"
#include "tlbmc/thermal/sensor_info/sensor_info.h"

namespace milotic_tlbmc {
namespace thermal {

constexpr double kDefaultZoneFailsafePercent = 100;
constexpr double kDefaultFanRpmValue = 0.0;

// `SetpointInfo` stores the thermal control setpoint value, and the name of
// the sensor that provides the setpoint value.
struct SetpointInfo {
  void Clear() {
    setpoint_value = kDefaultSetpointValue;
    sensor_name = "";
  }

  double setpoint_value = kDefaultSetpointValue;
  std::string sensor_name;
  std::string previous_sensor_name;
};

struct MaximumSetpointInfo {
  void Clear() { setpoint_info.Clear(); }

  // `setpoint_info` will be updated during each thermal control iteration.
  SetpointInfo setpoint_info = SetpointInfo();
};

struct RpmCeilings {
  void Clear() { rpm_ceilings.clear(); }

  // `rpm_ceilings` will be updated during each thermal control iteration, if
  // there are ceiling stepwise controllers. It is thread-safe at all times.
  std::vector<double> rpm_ceilings;
};

struct FailsafeDevice {
  std::string device_name;
  double failsafe_value;

  FailsafeDevice() = default;
  FailsafeDevice(absl::string_view device_name, double failsafe_value)
      : device_name(device_name), failsafe_value(failsafe_value) {}
};

/*
 * `FailsafeDeviceInfo` stores the failsafe info of devices currently in the
 * failsafe mode.
 *
 * This class is thread-safe.
 */
class FailsafeDeviceInfo {
 public:
  bool AnyDeviceInFailsafeMode() const {
    return !device_name_to_failsafe_percent_.empty();
  }

  double GetMaxFailsafePercent() {
    double max_failsafe_percent = 0.0;
    for (const auto& [device_name, failsafe_percent] :
         device_name_to_failsafe_percent_) {
      max_failsafe_percent = std::max(max_failsafe_percent, failsafe_percent);
    }
    return max_failsafe_percent;
  }

  void AddFailsafePercent(absl::string_view device_name,
                          double failsafe_percent) {
    device_name_to_failsafe_percent_[device_name] = failsafe_percent;
  }

  void RemoveFailsafeStateForDevice(absl::string_view device_name) {
    device_name_to_failsafe_percent_.erase(device_name);
  }

  absl::flat_hash_map<std::string, double> device_name_to_failsafe_percent_;
};

struct SensorReadingInfo {
  bool trigger_failsafe;
  std::optional<double> sensor_reading = std::nullopt;
  std::string fail_reason;

  SensorReadingInfo(bool trigger_failsafe, absl::string_view fail_reason)
      : trigger_failsafe(trigger_failsafe), fail_reason(fail_reason) {}
  SensorReadingInfo(bool trigger_failsafe, std::optional<double> sensor_reading,
                    absl::string_view fail_reason)
      : trigger_failsafe(trigger_failsafe),
        sensor_reading(sensor_reading),
        fail_reason(fail_reason) {}
};

struct ZoneManagerParameters {
  int id;
  uint64_t ms_per_fan_cycle = 100;
  uint64_t ms_per_thermal_cycle = 1000;
  uint32_t failsafe_log_count_per_sec = 20;
  DebugMode debug_mode;
  SensorCollectorAggregator* absl_nonnull aggregator ABSL_REQUIRE_EXPLICIT_INIT;
  FruCollector* absl_nonnull fru_collector ABSL_REQUIRE_EXPLICIT_INIT;
  EntityConfig* absl_nonnull entity_config ABSL_REQUIRE_EXPLICIT_INIT;
  double zone_failsafe_percent = kDefaultZoneFailsafePercent;
  double setpoint_upper_bound = std::numeric_limits<double>::max();
  double setpoint_lower_bound = std::numeric_limits<double>::lowest();
  double minimum_thermal_setpoint = 0.0;
  std::vector<ZoneManagerSensorInfo> input_sensors;
  std::vector<ZoneManagerFanInfo> output_fans;
  std::string sample_data_file_prefix = "/tmp/tlbmc_thermal_data";
};

/**
 * A `ZoneManager` provides thermal control services for one thermal zone.
 *
 * Multiple sensors, fans, and thermal loops may be utilized by a `ZoneManager`.
 * The `ZoneManager` will aggregate the sensor readings, calculate the thermal
 * loop output through thermal loop controllers, and update the fan speed
 * accordingly.
 *
 * This class is thread-safe.
 *
 * The amount of work done here, where lock will be used, is insignificant.
 * Therefore, shared locks are not used.
 *
 * This class is created based on `zone.*` files at
 * https://source.corp.google.com/piper///depot/google3/third_party/openbmc_phosphor_pid_control/pid/
 */
class ZoneManager {
 public:
  ~ZoneManager() = default;

  static absl::StatusOr<std::unique_ptr<ZoneManager>> Create(
      const ZoneManagerParameters& params);

  // Complex calculation functions
  /*
   * `AddSetpoint` is for thermal controllers to feed their output to the zone
   * manager.
   *
   * If the setpoint is accumulative, the setpoint value will be added up.
   * Otherwise, the maximum setpoint value will be updated if the new setpoint
   * value is larger.
   */
  void AddSetpoint(absl::string_view controller_id, double setpoint);
  /*
   * `AddAdditiveSetpoint` is for additive thermal controllers (e.g., DFF
   * controllers) to feed their output to the zone manager.
   *
   * `setpoint` will be added up to the affected setpoint regardless of the
   * accumulative mode. Note that, ideally, the additive controllers are to
   * quickly raise the setpoint in advance to the temperature raise. Therefore,
   * non-positive setpoints will be ignored and no-op'ed.
   */
  void AddAdditiveSetpoint(
      absl::string_view controller_id, double setpoint,
      const std::vector<std::string>& affected_controller_ids);
  /*
   * `DetermineMaximumSetpoint` is where zone manager determines the maximum
   * setpoint value among all thermal controllers.
   *
   * This is used after all thermal controllers' setpoints are updated, that is,
   * when all thermal loops are finished in this iteration.
   */
  void DetermineMaximumSetpoint();
  /*
   * `InitializeThermalStates` will do:
   *    1. Reset all sensor readings.
   *    2. Mark all sensors as failsafe mode.
   */
  void InitializeThermalStates();

  // Thermal controller related util functions.
  // `ProcessThermalControllers` processes all thermal controllers in the zone
  // manager: The non-supportive controllers are processed first, and then the
  // supportive controllers are processed.
  // After the initialization of the thermal control service, it is expected to
  // be thread-safe without mutex protection unless tuning mode is enabled.
  void ProcessThermalControllers();
  // `ProcessFanControllers` processes all fan controllers in the zone manager.
  // After the initialization of the thermal control service, it is expected to
  // be thread-safe without mutex protection unless tuning mode is enabled.
  void ProcessFanControllers();
  // `AddThermalController` adds a thermal controller to the zone manager. It is
  // thread-safe at all times.
  void AddThermalController(absl::string_view controller_name,
                            ThermalController* absl_nonnull controller,
                            bool is_enabled = true);
  // `GetControllerNames` is thread-safe only if tuning mode is enabled.
  std::vector<std::string> GetControllerNames();
  // `SetThermalControllerEnabled` writes. It is thread-safe at all times.
  void SetThermalControllerEnabled(absl::string_view controller_name,
                                   bool is_enabled);
  // `IsControllerEnabled` is thread-safe only if tuning mode is
  // enabled.
  bool IsControllerEnabled(absl::string_view controller_name);
  // `GetAllThermalControllers` returns all the non-null thermal controller
  // pointers. It is thread-safe only if tuning mode is enabled.
  std::vector<ThermalController*> GetAllThermalControllers();
  // `GetAllFanControllers` returns all the non-null fan controller pointers.
  // It is thread-safe only if tuning mode is enabled.
  std::vector<ThermalController*> GetAllFanControllers();
  // `SetThermalControllerSetpoint` writes. It is thread-safe at all times.
  void SetThermalControllerSetpoint(absl::string_view controller_name,
                                    double setpoint);
  // `GetThermalControllerSetpoint` returns the setpoint of a controller. It is
  // not thread-safe at all times.
  double GetThermalControllerSetpoint(absl::string_view controller_name);

  // Sensor & Fan related util functions.
  // `AddSensor` adds a sensor to the zone manager. It is thread-safe at all
  // times.
  void AddSensor(const ZoneManagerSensorInfo& sensor_info);
  // `GetSensorNames` returns the names of all sensors. It is thread-safe after
  // the initialization of tlBMC thermal control service, as sensor names should
  // not be updated after initialization.
  const std::vector<std::string>& GetSensorNames() const;
  // `SetSensorValues` updates the reading of a sensor. It is thread-safe at all
  // times.
  void SetSensorValues(absl::string_view sensor_name, double sensor_reading);
  // `UpdateAllSensorValues` updates the reading of all sensors owned by this
  // zone. It is thread-safe at all times.
  void UpdateAllSensorValues();
  // `GetSensorReadings` returns both the scaled and unscaled sensor reading
  // value. It is thread-safe only if tuning mode is enabled, as the sensor
  // readings should not change once a thermal loop begins.
  SensorValues GetSensorReadings(absl::string_view sensor_name);
  // `GetSensorMetadata` returns the metadata of a sensor. Its thread-safety
  // is provided by `SensorInfo::GetSensorMetadata`.
  absl::StatusOr<const SensorMetadata&> GetSensorMetadata(
      absl::string_view sensor_name);
  // `IsSensorMissingAcceptable` returns true if the sensor is missing and it
  // is acceptable. Its thread-safety is provided by
  // `SensorInfo::IsSensorMissingAcceptable`.
  bool IsSensorMissingAcceptable(absl::string_view sensor_name);
  // `AddFan` adds a sensor to the zone manager. It is thread-safe at all
  // times.
  void AddFan(const ZoneManagerFanInfo& fan_info);
  // `GetFanNames` returns the names of all fans. It is thread-safe after
  // the initialization of tlBMC thermal control service, as fan names should
  // not be updated after initialization.
  const std::vector<std::string>& GetFanNames() const;
  // `UpdateLocallyRecordedFanPwmAndCheckFailsafe` updates the locally
  // recorded PWM of a fan. A failsafe mode check will be performed, in case the
  // fan PWM is abnormal; and it will add the failsafe percent if the PWM is
  // abnormal. Its thread-safety is the provided by
  // `FanInfo::UpdateLocallyRecordedFanPwmAndCheckFailsafe`.
  void UpdateLocallyRecordedFanPwmAndCheckFailsafe(absl::string_view fan_name,
                                                   double pwm);
  // `GetFanMetadata` returns the metadata of a fan. Its thread-safety is
  // provided by `FanInfo::GetFanMetadata`.
  absl::StatusOr<const FanMetadata&> GetFanMetadata(absl::string_view fan_name);
  // `GetFanPwm` returns both the PWM of a fan. Its thread-safety is provided
  // by `FanInfo::GetFanPwm`.
  double GetFanPwm(absl::string_view fan_name);
  // `WriteFanSpeed` updates the PWM speed of a real fan. It is thread-safe
  // at all times.
  void WriteFanSpeed(absl::string_view fan_name, double pwm);
  // `WriteFanSpeeds` updates all the locally recorded PWM speeds to real
  // fans. It is thread-safe at all times.
  void WriteFanSpeeds();
  // `GetFailsafeMode` returns true if the zone is in failsafe mode (i.e., any
  // device is in failsafe mode). It is thread-safe at all times.
  bool GetFailsafeMode() const;
  // `GetFailsafePercent` returns the maximum failsafe percentage of all
  // devices in failsafe mode; if there is no device in failsafe mode, it
  // returns the default failsafe percentage of this zone. It is thread-safe at
  // all times.
  double GetFailsafePercent();
  // `GetZoneFailsafePercent` returns the default failsafe percentage of this
  // zone. It is thread-safe at all times.
  double GetZoneFailsafePercent() const { return zone_failsafe_percent_; }
  // `AddFailsafePercent` adds the failsafe percentage of a device. It is
  // thread-safe at all times.
  void AddFailsafePercent(absl::string_view device_name,
                          double failsafe_percent);
  // `RemoveFailsafeStateForDevice` removes the failsafe state of a device. It
  // is thread-safe at all times.
  void RemoveFailsafeStateForDevice(absl::string_view device_name);
  // `OutputFailsafeLog` outputs a failsafe log via `failsafe_logger_` with rate
  // control to avoid log spamming.
  void OutputFailsafeLog(absl::string_view location, absl::string_view reason);
  // `WriteCurrentPwmToFans` writes, correspondingly, the current PWM to every
  // fan.
  // A use case: When the manual mode is triggered, all thermal processes are
  // stopped, and the fan will not get any PWM write signals from tlBMC Thermal
  // Control, which may cause the hardware level failsafe mode unwantedly. To
  // avoid this, we consistently write the current PWM to fans to keep them
  // busy.
  void WriteCurrentPwmToFans();
  void SetManualPwm(absl::string_view fan_name, double pwm);

  // Returns a JSON object representing the zone manager.
  nlohmann::json ToJson() const;

  // Setpoint variable util functions.
  // `ClearSetpoints` resets all recorded setpoints. It is thread-safe at all
  // times.
  void ClearSetpoints();
  // `AdjustSetpoint` offsets the setpoint of each thermal controller in this
  // zone by `offset`, if there is a setpoint in the controller. It is
  // thread-safe at all times.
  void AdjustSetpoint(double offset);
  // `IsSetpointAccumulative` is thread-safe at all times, as
  // `is_setpoint_accumulative_` is atomic.
  void SetIsSetpointAccumulative(bool is_setpoint_accumulative) {
    is_setpoint_accumulative_ = is_setpoint_accumulative;
  }
  // `IsSetpointAccumulative` is thread-safe at all times, as
  // `is_setpoint_accumulative_` is atomic.
  bool IsSetpointAccumulative() { return is_setpoint_accumulative_; }
  // `SetMaximumSetpoint` updates the maximum setpoint value and its (previous)
  // name. It is thread-safe at all times.
  void SetMaximumSetpoint(absl::string_view setpoint_sensor_name,
                          double setpoint);
  // `GetMaximumSetpointInfo` is thread-safe at all times.
  SetpointInfo GetMaximumSetpointInfo() const {
    return maximum_setpoint_info_.setpoint_info;
  }
  // `GetMaximumSetpointValue` is thread-safe at all times.
  double GetMaximumSetpointValue() const {
    return maximum_setpoint_info_.setpoint_info.setpoint_value;
  }
  // `GetMinimumThermalSetpoint` is thread-safe at all times, as
  // `minimum_thermal_setpoint_` is atomic.
  double GetMinimumThermalSetpoint() const { return minimum_thermal_setpoint_; }
  // `AddRpmCeiling` is thread-safe at all times.
  void AddRpmCeiling(double rpm) { rpm_ceilings_.rpm_ceilings.push_back(rpm); }
  // `ClearRpmCeilings` is thread-safe at all times.
  void ClearRpmCeilings() { rpm_ceilings_.rpm_ceilings.clear(); }
  // `GetMinimumRpmCeiling` is thread-safe at all times.
  double GetMinimumRpmCeiling() {
    return rpm_ceilings_.rpm_ceilings.empty()
               ? std::numeric_limits<double>::max()
               : *std::min_element(rpm_ceilings_.rpm_ceilings.begin(),
                                   rpm_ceilings_.rpm_ceilings.end());
  }

  // Debug mode functions.
  static void LogThermalMessage(absl::string_view message);
  void TryLogThermalDebugMessage(absl::string_view message) const;
  bool GetDebugEnabled() const { return debug_mode_.debug_enabled; }
  bool GetDebugPidEnabled() const {
    return debug_mode_.debug_pid_enabled || debug_mode_.debug_enabled;
  }
  bool GetDebugDffEnabled() const {
    return debug_mode_.debug_dff_enabled || debug_mode_.debug_enabled;
  }
  bool GetTuningEnabled() const { return debug_mode_.tuning_enabled; }
  void AppendSampledData(absl::string_view data) {
    absl::MutexLock lock(mutex_);
    absl::StrAppend(&sampled_data_, data);
  }
  void DumpSampledData();
  void ClearSampledData() {
    absl::MutexLock lock(mutex_);
    sampled_data_.clear();
  }

  int GetId() const { return id_; }
  void SetManualMode(bool is_manual_mode) { is_manual_mode_ = is_manual_mode; }
  bool GetManualMode() const { return is_manual_mode_; }
  uint64_t GetMsPerFanCycle() const { return ms_per_fan_cycle_; }
  uint64_t GetMsPerThermalCycle() const { return ms_per_thermal_cycle_; }

 protected:
  explicit ZoneManager(const ZoneManagerParameters& zone_manager_params)
      : id_(zone_manager_params.id),
        ms_per_fan_cycle_(zone_manager_params.ms_per_fan_cycle),
        ms_per_thermal_cycle_(zone_manager_params.ms_per_thermal_cycle),
        aggregator_(zone_manager_params.aggregator),
        fru_collector_(zone_manager_params.fru_collector),
        entity_config_(zone_manager_params.entity_config),
        zone_failsafe_percent_(zone_manager_params.zone_failsafe_percent),
        failsafe_logger_(zone_manager_params.failsafe_log_count_per_sec),
        setpoint_upper_bound_(zone_manager_params.setpoint_upper_bound),
        setpoint_lower_bound_(zone_manager_params.setpoint_lower_bound),
        minimum_thermal_setpoint_(zone_manager_params.minimum_thermal_setpoint),
        debug_mode_(zone_manager_params.debug_mode),
        sample_data_file_path_(
            absl::StrCat(zone_manager_params.sample_data_file_prefix, "_zone",
                         id_, ".txt")) {
    for (const ZoneManagerSensorInfo& input_sensor :
         zone_manager_params.input_sensors) {
      AddSensor(input_sensor);
    }
    for (const ZoneManagerFanInfo& output_fan :
         zone_manager_params.output_fans) {
      AddFan(output_fan);
    }
    maximum_setpoint_info_.setpoint_info.setpoint_value = kDefaultSetpointValue;

    if (GetDebugEnabled()) {
      sample_data_file_.open(sample_data_file_path_,
                             std::ofstream::out | std::ofstream::trunc);
      sampled_data_.clear();
    }
  }

  void DumpSampledDataHeader();

  // `ExtractSensorReadingInfo` returns the sensor reading info of the given
  // sensor.
  SensorReadingInfo ExtractSensorReadingInfo(absl::string_view sensor_name,
                                             const Sensor* sensor);
  // `ExtractTachSensorReadingInfoAndCheckFailsafe` returns the sensor reading
  // info of the given tach sensor, and `trigger_failsafe` is true if and only
  // if all its sibling tach sensors are failing.
  SensorReadingInfo ExtractTachSensorReadingInfoAndCheckFailsafe(
      absl::string_view fan_tach_name, const Sensor* sensor);

 private:
  absl::Mutex mutex_;
  // Zone info level variables.
  // `id_` should be immutable after initialization, so no need for atomic
  // protection.
  const int id_;

  // If `is_manual_mode_` is true, no PWM sensor writes requested by thermal
  // controllers will be processed unless the zone is in failsafe mode; and fan
  // speeds is expected to be manually set by the RedFish `PATCH` commands.
  // However, if the zone enters failsafe mode, to avoid overheating, the manual
  // mode will be overridden to false.
  std::atomic<bool> is_manual_mode_ = false;
  absl::flat_hash_map<std::string, double> fan_name_to_manual_pwm_
      ABSL_GUARDED_BY(mutex_);

  // The time interval between two fan cycles.
  const uint64_t ms_per_fan_cycle_;
  // The time interval between two thermal cycles.
  const uint64_t ms_per_thermal_cycle_;

  // Thermal controller level variables.
  // These variables will be read-only after initialization, if tuning mode is
  // disabled.
  ControllerInfo controller_info_;

  // Sensor & Fan level variables.
  SensorCollectorAggregator* aggregator_;
  // `fru_collector_` is used to precisely identify the status of a sensor.
  // For example, it may avoid entering failsafe mode if the board associated
  // with the sensor is not present.
  FruCollector* fru_collector_;
  // `entity_config_` is used to fetch entity info associated with a sensor or a
  // fan. For example, it may provide all the tach sensor names of a fan.
  EntityConfig* entity_config_;
  // `sensor_names_` should not be updated after initialization of tlBMC thermal
  // control service. Therefore, no need for mutex protection at all times.
  std::vector<std::string> sensor_names_;
  // `sensor_info_` contains all data of sensors that is needed for a thermal
  // control loop. It will be updated only during the initialization of a
  // thermal control loop, and it will be read-only after that.
  SensorInfo sensor_info_;
  // `fan_names_` should not be updated after initialization of tlBMC thermal
  // control service. Therefore, no need for mutex protection at all times.
  std::vector<std::string> fan_names_;
  // `fan_info_` contains RPM and PWM of fans needed for fan thermal
  // controllers. It will be updated only during the initialization of a thermal
  // control loop, and it will be read-only after that.
  FanInfo fan_info_;
  // `device_name_to_failsafe_info_` maps the names of devices currently in
  // failsafe mode to the corresponding failsafe information.
  FailsafeDeviceInfo failsafe_device_info_;
  // `zone_failsafe_percent_` is the default failsafe percentage of this zone.
  const double zone_failsafe_percent_;
  // `failsafe_logger_` is used to output failsafe logs with rate control to
  // avoid log spamming.
  FailsafeLoggerCore failsafe_logger_;

  // Setpoint calculation level variables, which are thread-safe at all times,
  // if mutable.
  std::atomic<bool> is_setpoint_accumulative_ = false;
  // The maximum setpoint that can be processed (not output) by
  // `DetermineMaximumSetpoint` of this thermal zone.
  const double setpoint_upper_bound_;
  // The minimum setpoint that can be processed (not output) by
  // `DetermineMaximumSetpoint` of this thermal zone.
  const double setpoint_lower_bound_;
  MaximumSetpointInfo maximum_setpoint_info_;
  // `minimum_thermal_setpoint_` is the minimum thermal setpoint for the zone to
  // operate safely, which should not be changed after initialization.
  const double minimum_thermal_setpoint_;
  // `rpm_ceilings_` is used to limit the maximum fan speed, which is updated by
  // stepwise controllers.
  RpmCeilings rpm_ceilings_;

  // Debug mode related variables.
  // These variables will be read-only after initialization, if tuning mode is
  // disabled.
  DebugMode debug_mode_;
  // The format of a line in the sample data file is:
  // `[timestamp],[sensor_readings],[thermal_sepoints],[fan_controller_pwms]`
  const std::string sample_data_file_path_;
  std::ofstream sample_data_file_;
  std::string sampled_data_;
  std::atomic<bool> is_sampled_data_header_written_ = false;
};

}  // namespace thermal
}  // namespace milotic_tlbmc

#endif  // THIRD_PARTY_MILOTIC_EXTERNAL_CC_TLBMC_THERMAL_ZONE_MANAGER_H_