streaming_telemetry/proto/voyager_telemetry.proto

/*
 * Copyright 2023 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: Arun Satyanarayana arunsa@google.com
 * Contributors: Yongbing Chen yongbingchen@google.com
 */

syntax = "proto3";

package third_party_voyager;

import "google/protobuf/any.proto";

enum FqpType {
  // When the type is not specified, the rest of the Fqp message is sufficient
  // to identify a resource uniquely
  FQP_TYPE_NOT_SET = 0;
  // When Fqp is this type, then Fqp.specifier represents an odata.type, which
  // can be used to select all telemetry sources of this type.
  FQP_TYPE_REDFISH_RESOURCE = 1;
}

enum FiltOp {
  OPERATOR_UNSUPPORTED = 0;
  OPERATOR_EQUAL = 1;
  OPERATOR_LESS_THAN = 2;
  OPERATOR_GREATER_THAN = 3;
  OPERATOR_AND = 4;
  OPERATOR_OR = 5;
  OPERATOR_NOT = 6;
}

/*
 * The number of operands is decided by the operator.
 * Nested LogicalOperation is allowed.
 */
message LogicalOperation {
  FiltOp logical_operator = 1;
  repeated FiltExpr operands = 2;
}

message Filters {
  repeated FiltExpr expressions = 1;
}

/*
 * The field is a Property in Redfish. In other data models, it is typically
 * a scalar value. An example Redfish filter expr is:
 * "SystemType=Physical&PowerState=On".
 */
message Comparison {
  string field = 1;
  FiltOp filter_operator = 2;
  string value = 3;
}

message FiltExpr {
  oneof expr {
    Comparison comparison = 1;
    LogicalOperation logical_operation = 2;
  }
}

/*
 * Fqp (Fully Qualified Path) is used for identifying a telemetry resource
 * (Redfish Resource, Yang Container or other similar metrics group).
 *
 * This Protobuf allows a generic path with placeholders in the Identifier
 * segments. The Identifiers are in the identifiers map.
 * For FQP_TYPE_NOT_SET, the FQP: /redfish/v1/Systems/izumi/Processors/0 can be
 * encoded as:
 *
 * specifier:   "/redfish/v1/Systems/{SystemId}/Processors/{ProcessorId}"
 * identifiers: {
 *   "SystemId": "izumi",
 *   "ProcessorId": "0"
 * }
 *
 * Optional filters:
 * When a resource identifier is a wildcard '*', the requester can use filters
 * to select a subset from the target resource set. Filters are associated with
 * resource types.
 *
 * For FQP_TYPE_REDFISH_RESOURCE, the FQP is an odata.type and the identifier
 * contains verbosity as a well-known string
 * specifier: "#ProcessorMetrics.v1_6_4.ProcessorMetrics"
 */
message Fqp {
  string specifier = 1;
  map<string, string> identifiers = 2;
  map<string, Filters> filters = 3;
  FqpType type = 4;
}

/*
 * google.protobuf.Struct also uses google.protobuf.Value and hence does not
 * encode all scalar Protobuf types. TypedStruct is a more complete variant
 * defined here.
 */
message TypedStruct {
  map<string, TypedValue> fields = 1;
}

message TypedValue {
  oneof value {
    bool bool_val = 1;
    int64 int_val = 2;
    uint64 uint_val = 3;
    double double_val = 4;
    string string_val = 5;
    bytes bytes_val = 6;
    TypedStruct struct_val = 7;
  }
}

/*
 * This Protobuf encodes a single instance of a Datapoint.  A datapoint is one
 * instance of a datum/related data (e.g., sensor values) sourced/sampled
 * together. Related data must resolve to the same timestamp at the source; any
 * variance must be within the sourcing system's timestamp precision.
 * timestamp: always overrides Update.timestamp if present.
 * res_fqp: is always the Resolved FQP.
 */
message DataPoint {
  Fqp res_fqp = 1;
  uint64 timestamp_ns = 2; /* Nanoseconds since Unix epoch */
  oneof data {
    TypedStruct key_value = 3;
    google.protobuf.Any native = 4;
    string json = 5;
  }
}

/*
 * Telemetry response is encapsulated in an Update message.
 * req_id: Request.req_id - reflected back in the Update message as context
 *         to the client.
 * epoch_ns: Typically the boot timestamp (since Unix epoch typically) of the
 *           producer. Usage: when data across different Updates need to be
 *           correlated, but keys for the same entity can change, correlations
 *           can be scoped to the same epoch.
 */
message Update {
  uint64 timestamp_ns = 1; /* Nanoseconds since Unix epoch */
  uint64 epoch_ns = 8;
  Fqp req_fqp = 2; /* Unused in Set RPCs */
  string req_id = 3;
  string hostname = 4; /* Useful when going through proxies */
  map<string, string> http_headers = 5;
  uint32 code = 6; /* Typically HTTP return code */
  /* Contains telemetry data for Get, Subscribe RPCs, response data for Set
   * RPCs */
  repeated DataPoint data_points = 7;
}

/*
 * Thresholds is a config for one Fqp.
 * It contains an array of Threshold for different sampling and exporting
 * rate. The array forms a logically consecutive threshold group, with array
 * index used to address an individual Threshold.  The active Threshold can
 * change dynamically to its adjacent Threshold node, based on metric values
 * crossing its cross_below_value or cross_above_value.
 */
message Thresholds {
  repeated Threshold threshold = 1;
}

/*
 * Threshold represents a single configuration level within a Thresholds group.
 *
 * index: the index of this Threshold in the container Thresholds array.
 *
 * sample_frequency_expect_ns, sample_frequency_max_ns and export_frequency_ns
 * follow same definition in RequestFqp.
 *
 * cross_above_value: upper threshold for metric value. If crossed, the next
 *   Threshold (index + 1) becomes active.
 * cross_below_value: lower threshold for metric value. If crossed, the previous
 *   Threshold (index - 1) becomes active.
 *
 * name: the name of this Threshold, can be used by the same fields in RequestFqp
 * to select one Threshold from Thresholds array.
 */
message Threshold {
  uint32 index = 1;
  uint64 sample_frequency_expect_ns = 2;
  uint64 sample_frequency_max_ns = 3;
  uint64 export_frequency_ns = 4;
  TypedValue cross_above_value = 5;
  TypedValue cross_below_value = 6;
  string name = 7;
}

enum SamplingMode {
  SAMPLING_MODE_UNSPECIFIED = 0;
  SAMPLING_MODE_ON_CHANGE = 1;  // == eventing
  SAMPLING_MODE_PERIODIC = 2;
  SAMPLING_MODE_THRESHOLD = 3;
  SAMPLING_MODE_ONCE = 4;  // Server will send the data set once and disconnect.
}

enum Encoding {
  ENCODING_UNSPECIFIED = 0;
  ENCODING_JSON = 1;
  ENCODING_PROTO_NATIVE = 2;
  ENCODING_PROTO_KEYVALUE = 3;
  ENCODING_BYTES = 4;   // encode unstructured/opaque data
  ENCODING_STRING = 5;  // encode unstructured strings
}

/*
 * Relative priorities for the producer per Fqp. Note that this is global across
 * all subscriptions
 */
enum FqpPriority {
  FQP_PRI_UNSPECIFIED = 0;
  FQP_PRI_HIG = 1;
  FQP_PRI_MED = 2;
  FQP_PRI_LOW = 3;
}

/*
 * Technically frequencies in this proto are periods between samples or (batch)
 * exports.
 * sample_frequency_ vs. export_frequency_: when RequestFqp is used in Subscribe
 * RPCs, separating the sampling frequency from the export frequency allows for
 * message batching efficiencies while retaining sampling resolution.
 * export_frequency_ns is ignored for other RPCs.
 *
 * If the following conditions are not met, a subscription is treated as an
 * error:
 * sample_frequency_max_ns > sample_frequency_expect_ns
 * (sample_frequency_max_ns == sample_frequency_expect_ns is treated as if
 *  sample_frequency_max_is not present)
 * export_frequency_ns > sample_frequency_expect_ns or sample_frequency_max_ns,
 *  whichever is greater)
 * (export_frequency_ns == sample_frequency_expect_ns or sample_frequency_max_ns
 *  is treated as if export_frequency_ns is not present)
 *
 * When the export_frequency_ns window expires, the producer packs and sends all
 * the collected samples and resets its samples buffer.
 *
 * sample_frequency_max_ns, sample_frequency_expect_ns, export_frequency_ns will
 * be ignored in SAMPLING_MODE_ON_CHANGE, server will try to send an Update
 * immediately when event happens.
 *
 * variant_fields_only: when true, the server sends only the fields that change
 * (sensors values, statistic counters, ...) associated with the Fqp. Otherwise,
 * all fields including the ones deemed invariant (sensor units in a Redfish
 * Resource for e.g.) are also included.
 *
 * threshold_policy: RequestFqp can also be used in server-side configuration.
 * In this case, either thresholds or threshold_config can be set. In a dynamic
 * subscription (where the Request message contains RequestFqp's), only
 * thresholds are legal. The configuration construct of threshold_config must
 * not be mixed with dynamic subscriptions.
 *
 * RequestFqp can be used in both config-based and inband (a.k.a. dynamic)
 * subscriptions. In the config-based subscription mode, a set of RequestFqp's
 * are grouped together in a named config group on the server. A config group
 * could be "repairability_basic" or "high_rate_thermal_sensors" or
 * "medium_rate_system_metrics" or equivalent useful groupings. The subscriber
 * passes the config group name in Request.req_config_group. Multiple such
 * config groups can exist on the server. Each config group needs a separate
 * subscription request. RequestFqp.req_fqp_name is used defined named
 * RequestFqp's for the server state management. In the dynamic subscription
 * mode, the subscriber sends the fully defined set of RequestFqp's in the
 * Request message. RequestFqp.req_fqp_name is optional in this mode (the
 * server may internally assign names if necessary in its implementation).
 */
message RequestFqp {
  Fqp fqp = 1;
  FqpPriority priority = 2;
  SamplingMode mode = 3;
  Encoding encoding = 4;
  uint64 sample_frequency_expect_ns = 5;
  uint64 sample_frequency_max_ns = 6;
  uint64 export_frequency_ns = 7;
  bool suppress_redundant = 8;
  bool variant_fields_only = 9;
  oneof config {
    Thresholds thresholds = 10;
    string threshold_config = 11;
  }
  string req_fqp_name = 12;
}

/*
 * - req_id can be used as context by the client and is reflected back in the
 *   Update message.
 * - Only one of req_fqp or req_config_group can be present. If both are
 *   present, req_config_group takes precedence.
 * - If req_config_group is present (non-NULL), the telemetry server looks up
 *   the req_fqp list on req_fqp_name from locally available state that is
 *   populated via other out-of-band methods (e.g., a configuration file), the
 *   out-of-band methods will provide equivalent result as passing the
 *   parameters over req_fqp, the inband method.
 * - hostname_fqdn - is useful when requests are proxied, and for debugging.
 */
message Request {
  string req_id = 1;
  string req_config_group = 2;
  repeated RequestFqp req_fqp = 3;
  // DSCP treatment for the subscription response messages
  uint32 req_qos = 4;
  map<string, string> http_headers = 5;
  string hostname_fqdn = 6;
}

/*
 * req_data is the HTTP request body, typically in req_data.json.
 * Only one instance of req_fqp is accepted if req_data is present. Typically,
 * multiple instances of req_fqp can be used for Delete() RPCs.
 *
 * No QoS to avoid out-of-order processing possibilities.
 *
 * hostname_fqdn - is useful when requests are proxied, and for debugging.
 */
message SetRequest {
  string req_id = 1;
  repeated RequestFqp req_fqp = 2;
  oneof req_data {
    TypedStruct key_value = 3;
    string json = 4;
    google.protobuf.Any native = 6;
  }
  map<string, string> http_headers = 5;
  string hostname_fqdn = 7;
}

service MachineTelemetry {
  rpc Subscribe(Request) returns (stream Update) {}
  rpc Get(Request) returns (Update) {}
  rpc Put(SetRequest) returns (Update) {}
  rpc Post(SetRequest) returns (Update) {}
  rpc Patch(SetRequest) returns (Update) {}
  rpc Delete(SetRequest) returns (Update) {}
  rpc SubscribeV2(stream Request) returns (stream Update) {}
}