Google Git Google Git
Sign in
gbmc/linux/cbbddb6b08cc9900a87cccaa9b22bbe06aa4f50a/./tools/perf/python/export-to-postgresql.py
blob: 9cac64dee361214de3d1f9907c5e6849327c8ed7 [file]
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
r"""
Export perf data to a postgresql database.
This script has been ported to use the modern perf Python module and
libpq via ctypes. It no longer requires PySide2 or QtSql for exporting.
The script assumes postgresql is running on the local machine and that the
user has postgresql permissions to create databases.
An example of using this script with Intel PT:
$ perf record -e intel_pt//u ls
$ python export-to-postgresql.py -i perf.data -o pt_example
To browse the database, psql can be used e.g.
$ psql pt_example
pt_example=# select * from samples_view where id < 100;
pt_example=# \d+
pt_example=# \d+ samples_view
pt_example=# \q
An example of using the database is provided by the script
exported-sql-viewer.py. Refer to that script for details.
Tables:
The tables largely correspond to perf tools' data structures. They are
largely self-explanatory.
samples
'samples' is the main table. It represents what instruction was
executing at a point in time when something (a selected event)
happened. The memory address is the instruction pointer or 'ip'.
branch_types
'branch_types' provides descriptions for each type of branch.
comm_threads
'comm_threads' shows how 'comms' relates to 'threads'.
comms
'comms' contains a record for each 'comm' - the name given to the
executable that is running.
dsos
'dsos' contains a record for each executable file or library.
machines
'machines' can be used to distinguish virtual machines if
virtualization is supported.
selected_events
'selected_events' contains a record for each kind of event that
has been sampled.
symbols
'symbols' contains a record for each symbol. Only symbols that
have samples are present.
threads
'threads' contains a record for each thread.
Views:
Most of the tables have views for more friendly display. The views are:
comm_threads_view
dsos_view
machines_view
samples_view
symbols_view
threads_view
Ported from tools/perf/scripts/python/export-to-postgresql.py
"""
from __future__ import annotations
import typing
import argparse
from ctypes import CDLL, c_char_p, c_int, c_void_p, c_ubyte
import ctypes.util
import os
import shutil
import struct
import tempfile
import sys
from typing import Any, Dict, Optional
import perf
# Need to access PostgreSQL C library directly to use COPY FROM STDIN
libpq_name = ctypes.util.find_library("pq")
if not libpq_name:
libpq_name = "libpq.so.5"
try:
libpq = CDLL(libpq_name)
except OSError as e:
print(f"Error loading {libpq_name}: {e}")
print("Please ensure PostgreSQL client library is installed.")
sys.exit(1)
PQconnectdb = libpq.PQconnectdb
PQconnectdb.restype = c_void_p
PQconnectdb.argtypes = [c_char_p]
PQfinish = libpq.PQfinish
PQfinish.argtypes = [c_void_p]
PQstatus = libpq.PQstatus
PQstatus.restype = c_int
PQstatus.argtypes = [c_void_p]
libpq.PQerrorMessage.restype = c_char_p
libpq.PQerrorMessage.argtypes = [c_void_p]
PQexec = libpq.PQexec
PQexec.restype = c_void_p
PQexec.argtypes = [c_void_p, c_char_p]
PQresultStatus = libpq.PQresultStatus
PQresultStatus.restype = c_int
PQresultStatus.argtypes = [c_void_p]
PQputCopyData = libpq.PQputCopyData
PQputCopyData.restype = c_int
PQputCopyData.argtypes = [c_void_p, c_void_p, c_int]
PQputCopyEnd = libpq.PQputCopyEnd
PQputCopyEnd.restype = c_int
PQputCopyEnd.argtypes = [c_void_p, c_void_p]
PQgetResult = libpq.PQgetResult
PQgetResult.restype = c_void_p
PQgetResult.argtypes = [c_void_p]
PQclear = libpq.PQclear
PQclear.argtypes = [c_void_p]
def toserverstr(s: str) -> bytes:
"""Convert string to server encoding (UTF-8)."""
return bytes(s, "UTF_8")
def toclientstr(s: str) -> bytes:
"""Convert string to client encoding (UTF-8)."""
return bytes(s, "UTF_8")
PERF_IP_FLAG_BRANCH = 1 << 0
PERF_IP_FLAG_CALL = 1 << 1
PERF_IP_FLAG_RETURN = 1 << 2
PERF_IP_FLAG_CONDITIONAL = 1 << 3
PERF_IP_FLAG_SYSCALLRET = 1 << 4
PERF_IP_FLAG_ASYNC = 1 << 5
PERF_IP_FLAG_INTERRUPT = 1 << 6
PERF_IP_FLAG_TX_ABORT = 1 << 7
PERF_IP_FLAG_TRACE_BEGIN = 1 << 8
PERF_IP_FLAG_TRACE_END = 1 << 9
PERF_IP_FLAG_IN_TX = 1 << 10
PERF_IP_FLAG_VMENTRY = 1 << 11
PERF_IP_FLAG_VMEXIT = 1 << 12
BRANCH_TYPES = [
(0, "no branch"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"),
(PERF_IP_FLAG_BRANCH, "unconditional jump"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, "software interrupt"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, "return from interrupt"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, "system call"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET,
"return from system call"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_INTERRUPT,
"hardware interrupt"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"),
(PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"),
]
for _b_type, _b_name in list(BRANCH_TYPES):
if (_b_type == PERF_IP_FLAG_BRANCH or
(_b_type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))):
continue
BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_BEGIN, f"trace begin / {_b_name}"))
BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_END, f"{_b_name} / trace end"))
class PostgresExporter:
"""Handles PostgreSQL connection and exporting of perf events."""
def __init__(self, dbname: str):
self.dbname = dbname
self.conn = None
self.session: Optional[perf.session] = None
self.output_dir_name = tempfile.mkdtemp(prefix=dbname + "-perf-data-")
self.created_output_dir = True
self.file_header = struct.pack("!11sii", b"PGCOPY\n\377\r\n\0", 0, 0)
self.file_trailer = b"\377\377"
# Caches and counters grouped to reduce instance attributes
self.caches: Dict[str, dict] = {
'machines': {},
'threads': {},
'comms': {},
'dsos': {},
'symbols': {},
'events': {},
'branch_types': {},
'call_paths': {}
}
self.symbol_rows: Dict[int, tuple[int, int, int, int, str]] = {}
self.next_id = {
'machine': 1,
'thread': 1,
'comm': 1,
'dso': 1,
'symbol': 1,
'event': 1,
'branch_type': 1,
'comm_thread': 1,
'call_path': 1,
'sample': 1,
'call': 1,
'context_switch': 1
}
self.files: Dict[str, Any] = {}
self.unhandled_count = 0
def connect(self, db_to_use: str) -> None:
"""Connect to database.
Reject URI schemes ('://') and key=value pairs ('=') to prevent
connection parameter injection, and escape backslashes and single
quotes per libpq single-quoted conninfo string rules.
"""
if "://" in db_to_use or "=" in db_to_use:
raise ValueError(f"Invalid database name: {db_to_use}")
safe_db = db_to_use.replace('\\', '\\\\').replace("'", "\\'")
conn_str = toclientstr(f"dbname='{safe_db}'")
self.conn = PQconnectdb(conn_str)
if PQstatus(self.conn) != 0:
PQfinish(self.conn)
self.conn = None
raise RuntimeError(f"PQconnectdb failed for {db_to_use}")
def disconnect(self) -> None:
"""Disconnect from database."""
if self.conn:
PQfinish(self.conn)
self.conn = None
def do_query(self, sql: str) -> None:
"""Execute a query and check status."""
res = PQexec(self.conn, toserverstr(sql))
status = PQresultStatus(res)
PQclear(res)
if status not in (1, 2): # PGRES_COMMAND_OK, PGRES_TUPLES_OK
error_msg = libpq.PQerrorMessage(self.conn).decode('utf-8')
raise RuntimeError(f"Query failed: {sql}. Error: {error_msg}")
def open_output_file(self, file_name: str):
"""Open intermediate binary file."""
path_name = self.output_dir_name + "/" + file_name
f = open(path_name, "wb+")
f.write(self.file_header)
return f
def close_output_file(self, f):
"""Close intermediate binary file."""
f.write(self.file_trailer)
f.close()
def copy_output_file(self, path_name: str, table_name: str):
"""Copy intermediate file to database."""
sql = f"COPY {table_name} FROM STDIN (FORMAT 'binary')"
res = PQexec(self.conn, toserverstr(sql))
if PQresultStatus(res) != 4: # PGRES_COPY_IN
PQclear(res)
err = libpq.PQerrorMessage(self.conn).decode()
raise RuntimeError(f"COPY FROM STDIN PQexec failed for {table_name}: {err}")
PQclear(res)
with open(path_name, "rb") as f:
data = f.read(65536)
while len(data) > 0:
c_data = (c_ubyte * len(data)).from_buffer_copy(data)
ret = PQputCopyData(self.conn, c_data, len(data))
if ret != 1:
raise RuntimeError(f"PQputCopyData failed for {table_name}")
data = f.read(65536)
ret = PQputCopyEnd(self.conn, None)
if ret != 1:
err = libpq.PQerrorMessage(self.conn).decode()
raise RuntimeError(f"PQputCopyEnd failed for {table_name}: {err}")
res = PQgetResult(self.conn)
while res:
status = PQresultStatus(res)
if status != 1: # PGRES_COMMAND_OK
error_msg = libpq.PQerrorMessage(self.conn).decode('utf-8')
PQclear(res)
raise RuntimeError(
f"COPY completion failed for {table_name}. Status {status}: {error_msg}"
)
PQclear(res)
res = PQgetResult(self.conn)
def setup_db(self) -> None:
"""Create database and tables. MUST be called after init."""
if "://" in self.dbname or "=" in self.dbname:
raise ValueError(f"Invalid database name: {self.dbname}")
self.created_output_dir = True
self.connect('postgres')
try:
# SQL identifiers are double-quoted; escape embedded double quotes by doubling them.
db_name = self.dbname.replace('"', '""')
self.do_query(f'CREATE DATABASE "{db_name}"')
except Exception as db_error:
shutil.rmtree(self.output_dir_name, ignore_errors=True)
raise db_error
self.disconnect()
self.connect(self.dbname)
self.do_query("SET client_min_messages TO WARNING")
self.do_query("""
CREATE TABLE selected_events (
id bigint NOT NULL,
name varchar(80))
""")
self.do_query("""
CREATE TABLE machines (
id bigint NOT NULL,
pid integer,
root_dir varchar(4096))
""")
self.do_query("""
CREATE TABLE threads (
id bigint NOT NULL,
machine_id bigint,
process_id bigint,
pid integer,
tid integer)
""")
self.do_query("""
CREATE TABLE comms (
id bigint NOT NULL,
comm varchar(16),
c_thread_id bigint,
c_time bigint,
exec_flag boolean)
""")
self.do_query("""
CREATE TABLE comm_threads (
id bigint NOT NULL,
comm_id bigint,
thread_id bigint)
""")
self.do_query("""
CREATE TABLE dsos (
id bigint NOT NULL,
machine_id bigint,
short_name varchar(256),
long_name varchar(4096),
build_id varchar(64))
""")
self.do_query("""
CREATE TABLE symbols (
id bigint NOT NULL,
dso_id bigint,
sym_start bigint,
sym_end bigint,
binding integer,
name varchar(2048))
""")
self.do_query("""
CREATE TABLE branch_types (
id integer NOT NULL,
name varchar(80))
""")
self.do_query("""
CREATE TABLE samples (
id bigint NOT NULL,
evsel_id bigint,
machine_id bigint,
thread_id bigint,
comm_id bigint,
dso_id bigint,
symbol_id bigint,
sym_offset bigint,
ip bigint,
time bigint,
cpu integer,
to_dso_id bigint,
to_symbol_id bigint,
to_sym_offset bigint,
to_ip bigint,
period bigint,
weight bigint,
transaction_ bigint,
data_src bigint,
branch_type integer,
in_tx boolean,
call_path_id bigint,
insn_count bigint,
cyc_count bigint,
flags integer)
""")
self.do_query('''
CREATE TABLE context_switches (
id bigint,
machine_id bigint,
time bigint,
cpu integer,
thread_out_id bigint,
comm_out_id bigint,
thread_in_id bigint,
comm_in_id bigint,
flags integer
)
''')
self.do_query('''
CREATE TABLE call_paths (
id bigint NOT NULL,
parent_id bigint,
symbol_id bigint,
ip bigint)
''')
self.do_query('''
CREATE TABLE calls (
id bigint NOT NULL,
thread_id bigint,
comm_id bigint,
call_path_id bigint,
call_time bigint,
return_time bigint,
branch_count bigint,
call_id bigint,
return_id bigint,
parent_call_path_id bigint,
flags integer,
parent_id bigint,
insn_count bigint,
cyc_count bigint
)
''')
self.do_query('''
CREATE TABLE ptwrite (
id bigint NOT NULL,
payload bigint,
exact_ip boolean
)
''')
self.do_query('''
CREATE TABLE cbr (
id bigint NOT NULL,
cbr integer,
mhz integer,
percent integer
)
''')
self.do_query('''
CREATE TABLE mwait (
id bigint NOT NULL,
hints integer,
extensions integer
)
''')
self.do_query('''
CREATE TABLE pwre (
id bigint NOT NULL,
cstate integer,
subcstate integer,
hw boolean
)
''')
self.do_query('''
CREATE TABLE exstop (
id bigint NOT NULL,
exact_ip boolean
)
''')
self.do_query('''
CREATE TABLE pwrx (
id bigint NOT NULL,
deepest_cstate integer,
last_cstate integer,
wake_reason integer
)
''')
self.files['evsel'] = self.open_output_file("evsel_table.bin")
self.files['machine'] = self.open_output_file("machine_table.bin")
self.files['thread'] = self.open_output_file("thread_table.bin")
self.files['comm'] = self.open_output_file("comm_table.bin")
self.files['comm_thread'] = self.open_output_file("comm_thread_table.bin")
self.files['dso'] = self.open_output_file("dso_table.bin")
self.files['symbol'] = self.open_output_file("symbol_table.bin")
self.files['branch_type'] = self.open_output_file("branch_type_table.bin")
self.files['sample'] = self.open_output_file("sample_table.bin")
self.files['context_switches'] = self.open_output_file("context_switches_table.bin")
self.files['call'] = self.open_output_file("call_table.bin")
self.files['call_path'] = self.open_output_file("call_path_table.bin")
self.files['ptwrite'] = self.open_output_file("ptwrite_table.bin")
self.files['cbr'] = self.open_output_file("cbr_table.bin")
self.files['mwait'] = self.open_output_file("mwait_table.bin")
self.files['pwre'] = self.open_output_file("pwre_table.bin")
self.files['exstop'] = self.open_output_file("exstop_table.bin")
self.files['pwrx'] = self.open_output_file("pwrx_table.bin")
self.write_evsel(0, "unknown")
self.write_machine(0, 0, "unknown")
self.write_thread(0, 0, 0, -1, -1)
self.write_comm(0, "unknown", 0, 0, 0)
self.write_dso(0, 0, "unknown", "unknown", "")
self.write_symbol(0, 0, 0, 0, 0, "unknown")
self.write_call_path(0, 0, 0, 0)
# Sentinel id = 0 rows in sample_table.bin and call_table.bin satisfy
# PostgreSQL FOREIGN KEY constraints (call_id/return_id -> samples.id
# and parent references) when a call or return event has no matching
# sample reference (0).
self.files['sample'].write(struct.pack(
"!hiqiqiqiqiqiqiqiQiQiQiIiqiqiQiQiQiQiQiQiiiBiqiQiQii",
25, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0,
8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, 1, 0, 8, 0, 8, 0, 8, 0, 4, 0
))
self.files['call'].write(struct.pack(
"!hiqiqiqiqiqiqiqiqiqiqiiiqiqiq",
14, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, 8, 0, 8, 0, 8, 0
))
for b_type, b_name in BRANCH_TYPES:
self.write_branch_type(b_type, b_name)
def write_branch_type(self, branch_type: int, name: str) -> None:
name_bytes = toserverstr(name)
n = len(name_bytes)
fmt = "!hiii" + str(n) + "s"
value = struct.pack(fmt, 2, 4, branch_type, n, name_bytes)
self.files['branch_type'].write(value)
def write_ptwrite(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<IQ", raw_buf)
flags, payload = data[0], data[1]
value = struct.pack("!hiqiQiB", 3, 8, id_, 8, payload, 1, flags & 1)
self.files['ptwrite'].write(value)
def write_cbr(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<BBBBII", raw_buf)
cbr_val, max_nonturbo, freq = data[0], data[2], data[4]
mhz = (freq + 500) // 1000
percent = ((cbr_val * 1000 // max_nonturbo) + 5) // 10 if max_nonturbo else 0
value = struct.pack("!hiqiiiiii", 4, 8, id_, 4, cbr_val, 4, int(mhz), 4, int(percent))
self.files['cbr'].write(value)
def write_mwait(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<IQ", raw_buf)
payload = data[1]
hints = payload & 0xff
extensions = (payload >> 32) & 0x3
value = struct.pack("!hiqiiii", 3, 8, id_, 4, hints, 4, extensions)
self.files['mwait'].write(value)
def write_pwre(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<IQ", raw_buf)
payload = data[1]
hw = (payload >> 7) & 1
cstate = (payload >> 12) & 0xf
subcstate = (payload >> 8) & 0xf
value = struct.pack("!hiqiiiiiB", 4, 8, id_, 4, cstate, 4, subcstate, 1, hw)
self.files['pwre'].write(value)
def write_exstop(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<I", raw_buf)
flags = data[0]
value = struct.pack("!hiqiB", 2, 8, id_, 1, flags & 1)
self.files['exstop'].write(value)
def write_pwrx(self, id_: int, raw_buf: bytes) -> None:
data = struct.unpack_from("<IQ", raw_buf)
payload = data[1]
deepest = payload & 0xf
last = (payload >> 4) & 0xf
wake = (payload >> 8) & 0xf
value = struct.pack("!hiqiiiiii", 4, 8, id_, 4, deepest, 4, last, 4, wake)
self.files['pwrx'].write(value)
def write_evsel(self, evsel_id: int, name: str) -> None:
"""Write event to binary file."""
name_bytes = toserverstr(name)
n = len(name_bytes)
fmt = "!hiqi" + str(n) + "s"
value = struct.pack(fmt, 2, 8, evsel_id, n, name_bytes)
self.files['evsel'].write(value)
def write_machine(self, machine_id: int, pid: int, root_dir: str) -> None:
"""Write machine to binary file."""
rd_bytes = toserverstr(root_dir)
n = len(rd_bytes)
fmt = "!hiqiii" + str(n) + "s"
value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, rd_bytes)
self.files['machine'].write(value)
def write_thread(self, thread_id: int, machine_id: int, process_id: int,
pid: int, tid: int) -> None:
"""Write thread to binary file."""
value = struct.pack("!hiqiqiqiIiI", 5, 8, thread_id, 8, machine_id,
8, process_id, 4, pid & 0xffffffff, 4, tid & 0xffffffff)
self.files['thread'].write(value)
def write_comm(self, comm_id: int, comm_str: str, thread_id: int,
time: int, exec_flag: int) -> None:
"""Write comm to binary file."""
comm_bytes = toserverstr(comm_str)
n = len(comm_bytes)
fmt = "!hiqi" + str(n) + "s" + "iqiqiB"
value = struct.pack(fmt, 5, 8, comm_id, n, comm_bytes, 8,
thread_id, 8, time, 1, exec_flag)
self.files['comm'].write(value)
def write_comm_thread(self, comm_thread_id: int, comm_id: int,
thread_id: int) -> None:
"""Write comm_thread to binary file."""
fmt = "!hiqiqiq"
value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id)
self.files['comm_thread'].write(value)
def write_dso(self, dso_id: int, machine_id: int, short_name: str,
long_name: str, build_id: str) -> None:
"""Write DSO to binary file."""
sn_bytes = toserverstr(short_name)
ln_bytes = toserverstr(long_name)
bi_bytes = toserverstr(build_id)
n1, n2, n3 = len(sn_bytes), len(ln_bytes), len(bi_bytes)
fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s"
value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1,
sn_bytes, n2, ln_bytes, n3, bi_bytes)
self.files['dso'].write(value)
def write_symbol(self, symbol_id: int, dso_id: int, sym_start: int,
sym_end: int, binding: int, symbol_name: str) -> None:
"""Write symbol to binary file."""
name_bytes = toserverstr(symbol_name)
n = len(name_bytes)
fmt = "!hiqiqiQiQiii" + str(n) + "s"
value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8,
sym_start, 8, sym_end, 4, binding, n, name_bytes)
self.files['symbol'].write(value)
def write_call_path(self, cp_id: int, parent_id: int, symbol_id: int,
ip: int) -> None:
"""Write call path to binary file."""
fmt = "!hiqiqiqiQ"
value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip)
self.files['call_path'].write(value)
def write_sample(self, sample_id: int, evsel_id: int, machine_id: int, thread_id: int,
comm_id: int, dso_id: int, symbol_id: int,
sample: perf.sample_event, call_path_id: int) -> None:
"""Write sample to binary file."""
addr_dso_id = self.get_dso_id(
sample.addr_dso or "Unknown_dso", sample.addr_dso or "Unknown_dso_long", "",
machine_id
)
addr_symbol_id = self.get_symbol_id(
addr_dso_id, sample.addr_symbol or "Unknown_symbol", 0, 0
)
value = struct.pack(
"!hiqiqiqiqiqiqiqiQiQiQiIiqiqiQiQiQiQiQiQiiiBiqiQiQii",
25, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id,
8, dso_id, 8, symbol_id, 8, getattr(sample, 'sym_offset', 0) or 0,
8, getattr(sample, 'sample_ip', 0) or 0,
8, getattr(sample, 'sample_time', 0) or 0,
4, getattr(sample, 'sample_cpu', 0) & 0xffffffff,
8, addr_dso_id,
8, addr_symbol_id,
8, getattr(sample, 'addr_sym_offset', 0) or 0,
8, getattr(sample, 'sample_addr', 0) or 0,
8, getattr(sample, 'sample_period', 0) or 0,
8, getattr(sample, 'sample_weight', 0) or 0,
8, getattr(sample, 'transaction', 0) or 0,
8, getattr(sample, 'sample_data_src', 0) or 0 or 0,
4, getattr(sample, 'branch_type', 0) or 0,
1, getattr(sample, 'in_tx', 0) or 0,
8, call_path_id,
8, getattr(sample, 'sample_insn_count', 0) or 0 or 0,
8, getattr(sample, 'sample_cyc_count', 0) or 0 or 0,
4, getattr(sample, 'flags', 0) or 0
)
self.files['sample'].write(value)
def get_machine_id(self, machine_pid: Optional[int]) -> int:
"""Get or create machine ID."""
if machine_pid is None or machine_pid <= 0 or machine_pid > 0x7fffffff:
machine_pid = -1
if machine_pid in self.caches['machines']:
return self.caches['machines'][machine_pid]
machine_id = self.next_id['machine']
self.next_id['machine'] += 1
self.write_machine(machine_id, machine_pid, "")
self.caches['machines'][machine_pid] = machine_id
return machine_id
def get_event_id(self, name: str) -> int:
"""Get or create event ID."""
if name in self.caches['events']:
return self.caches['events'][name]
event_id = self.next_id['event']
self.write_evsel(event_id, name)
self.caches['events'][name] = event_id
self.next_id['event'] += 1
return event_id
def get_thread_id(self, machine_id: int, pid: Optional[int], tid: Optional[int]) -> int:
"""Get or create thread ID."""
if pid is None:
pid = -1
elif pid > 0x7fffffff:
pid -= 0x100000000
if tid is None:
tid = -1
elif tid > 0x7fffffff:
tid -= 0x100000000
key = (machine_id, pid, tid)
if key in self.caches['threads']:
return self.caches['threads'][key]
process_id = self.get_thread_id(machine_id, pid, pid) if tid != pid else -1
thread_id = self.next_id['thread']
if process_id == -1: process_id = thread_id
self.write_thread(thread_id, machine_id, process_id, pid, tid)
self.caches['threads'][key] = thread_id
self.next_id['thread'] += 1
return thread_id
def get_comm_id(self, comm: str, thread_id: int) -> int:
"""Get or create comm ID."""
c_key = (comm, thread_id)
if c_key in self.caches['comms']:
comm_id = self.caches['comms'][c_key]
else:
comm_id = self.next_id['comm']
self.write_comm(comm_id, comm, thread_id, 0, 0)
self.caches['comms'][c_key] = comm_id
self.next_id['comm'] += 1
key = (comm_id, thread_id)
if 'comm_threads' not in self.caches:
self.caches['comm_threads'] = {}
if key not in self.caches['comm_threads']:
comm_thread_id = self.next_id['comm_thread']
self.write_comm_thread(comm_thread_id, comm_id, thread_id)
self.caches['comm_threads'][key] = True
self.next_id['comm_thread'] += 1
return comm_id
def get_dso_id(self, short_name: str, long_name: str,
build_id: str, machine_id: int = 0) -> int:
"""Get or create DSO ID."""
key = (machine_id, short_name, long_name, build_id)
if key in self.caches['dsos']:
return self.caches['dsos'][key]
short_key = (machine_id, short_name)
if not build_id and short_key in self.caches['dsos']:
return self.caches['dsos'][short_key]
dso_id = self.next_id['dso']
self.write_dso(dso_id, machine_id, short_name, long_name, build_id)
self.caches['dsos'][key] = dso_id
self.caches['dsos'][short_key] = dso_id
self.next_id['dso'] += 1
return dso_id
def get_symbol_id(self, dso_id: int, name: str, start: int,
end: int) -> int:
"""Get or create symbol ID."""
key = (dso_id, name)
if key in self.caches['symbols']:
symbol_id, cached_start, cached_end = self.caches['symbols'][key]
if cached_start == 0 and cached_end == 0 and (start != 0 or end != 0):
self.caches['symbols'][key] = (symbol_id, start, end)
self.symbol_rows[symbol_id] = (dso_id, start, end, 0, name)
return symbol_id
symbol_id = self.next_id['symbol']
self.caches['symbols'][key] = (symbol_id, start, end)
self.symbol_rows[symbol_id] = (dso_id, start, end, 0, name)
self.next_id['symbol'] += 1
return symbol_id
def get_call_path_id(self, parent_id: int, symbol_id: int,
ip: int) -> int:
"""Get or create call path ID."""
key = (parent_id, symbol_id, ip)
if key in self.caches['call_paths']:
return self.caches['call_paths'][key]
call_path_id = self.next_id['call_path']
self.write_call_path(call_path_id, parent_id, symbol_id, ip)
self.caches['call_paths'][key] = call_path_id
self.next_id['call_path'] += 1
return call_path_id
def process_context_switch(self, event: typing.Any) -> None:
"""Callback for processing context switch events."""
misc = getattr(event, 'misc', 0)
out = bool(misc & (1 << 13))
out_preempt = bool(misc & (1 << 14))
flags = (1 if out else 0) | ((1 if out_preempt else 0) << 1)
machine_id = self.get_machine_id(getattr(event, 'machine_pid', None))
sample_pid = getattr(event, 'sample_pid', None)
if sample_pid is None:
sample_pid = -1
elif sample_pid > 0x7fffffff:
sample_pid -= 0x100000000
sample_tid = getattr(event, 'sample_tid', None)
if sample_tid is None:
sample_tid = -1
elif sample_tid > 0x7fffffff:
sample_tid -= 0x100000000
next_prev_pid = getattr(event, 'next_prev_pid', None)
if next_prev_pid is None:
next_prev_pid = -1
elif next_prev_pid > 0x7fffffff:
next_prev_pid -= 0x100000000
next_prev_tid = getattr(event, 'next_prev_tid', None)
if next_prev_tid is None:
next_prev_tid = -1
elif next_prev_tid > 0x7fffffff:
next_prev_tid -= 0x100000000
th_a_id = self.get_thread_id(machine_id, sample_pid, sample_tid)
comm_a = "Unknown_comm"
if self.session:
try:
proc = self.session.find_thread(sample_pid, sample_tid)
if proc:
comm_a_name = proc.comm() or "Unknown_comm"
if comm_a_name:
comm_a = comm_a_name
except TypeError:
pass
comm_a_id = self.get_comm_id(comm_a, th_a_id)
th_b_id = self.get_thread_id(machine_id, next_prev_pid, next_prev_tid)
comm_b = "Unknown_comm"
if self.session:
try:
proc = self.session.find_thread(next_prev_pid, next_prev_tid)
if proc:
comm_b_name = proc.comm() or "Unknown_comm"
if comm_b_name:
comm_b = comm_b_name
except TypeError:
pass
comm_b_id = self.get_comm_id(comm_b, th_b_id)
if out:
th_out_id = th_a_id
comm_out_id = comm_a_id
th_in_id = th_b_id
comm_in_id = comm_b_id
else:
th_out_id = th_b_id
comm_out_id = comm_b_id
th_in_id = th_a_id
comm_in_id = comm_a_id
cs_id = self.next_id['context_switch']
self.next_id['context_switch'] += 1
time = getattr(event, "time", getattr(event, "sample_time", 0))
cpu = getattr(event, "sample_cpu", 0) or 0
if cpu > 0x7fffffff:
cpu -= 0x100000000
# 9 columns total:
fmt = "!hiQiqiQiiiqiqiqiqii"
value = struct.pack(
fmt, 9, 8, cs_id, 8, machine_id, 8, time, 4, cpu,
8, th_out_id, 8, comm_out_id, 8, th_in_id, 8, comm_in_id, 4, flags
)
self.files["context_switches"].write(value)
def get_call_path_ids(self, call_path: typing.Any, machine_id: int = 0) -> tuple[int, int]:
"""Add a perf.callchain as call_path rows, return its id and parent's."""
parent_id = 0
call_path_id = 0
for node in call_path:
dso_name = node.dso or "Unknown_dso"
symbol_name = node.symbol or "Unknown_symbol"
dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id)
symbol_id = self.get_symbol_id(dso_id, symbol_name, 0, 0)
parent_id = call_path_id
call_path_id = self.get_call_path_id(parent_id, symbol_id, node.ip)
return call_path_id, parent_id
def process_call_return(self, cr: perf.call_return) -> None:
"""Callback for processing call_return events."""
machine_id = self.get_machine_id(getattr(cr, "machine_pid", 0) or 0)
thread_id = self.get_thread_id(machine_id,
cr.pid if cr.pid is not None else -1,
cr.tid if cr.tid is not None else -1)
comm_id = self.get_comm_id(cr.comm or "Unknown_comm", thread_id)
call_path_id, parent_call_path_id = self.get_call_path_ids(cr.call_path or [], machine_id)
fmt = "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiq"
# call.id and call.parent_id use the db_id of the perf module, as a call
# is given an id before it returns its parent cannot be numbered here.
value = struct.pack(
fmt, 14, 8, cr.db_id, 8, thread_id, 8, comm_id, 8, call_path_id,
8, cr.call_time, 8, cr.return_time, 8, cr.branch_count,
8, cr.call_ref, 8, cr.return_ref,
8, parent_call_path_id, 4, cr.flags, 8, cr.parent_id,
8, cr.insn_count, 8, cr.cyc_count
)
self.files['call'].write(value)
def process_event(self, sample: typing.Any) -> None:
"""Callback for processing events."""
machine_db_id = getattr(sample, 'machine_pid', 0)
machine_id = self.get_machine_id(machine_db_id)
thread_id = self.get_thread_id(machine_id, sample.sample_pid, sample.sample_tid)
comm = "Unknown_comm"
try:
if self.session is not None:
proc = self.session.find_thread(sample.sample_pid, sample.sample_tid)
if proc:
comm = proc.comm() or "Unknown_comm"
except TypeError:
pass
comm_id = self.get_comm_id(comm, thread_id)
dso_id = self.get_dso_id(
sample.dso or "Unknown_dso",
sample.dso_long_name or "Unknown_dso_long",
sample.dso_bid or "",
machine_id
)
symbol_id = self.get_symbol_id(
dso_id,
sample.symbol or "Unknown_symbol",
sample.sym_start or 0,
sample.sym_end or 0
)
call_path_id = 0
if hasattr(sample, 'callchain') and sample.callchain:
parent_id = 0
for node in reversed(sample.callchain):
node_dso = getattr(node, 'dso', None) or getattr(node, 'map', None)
node_symbol = getattr(node, 'symbol', None) or getattr(node, 'sym', None)
dso_name = "Unknown_dso"
if node_dso:
dso_name = (node_dso if isinstance(node_dso, str)
else getattr(node_dso, 'name', "Unknown_dso") or "Unknown_dso")
symbol_name = "Unknown_symbol"
if node_symbol:
symbol_name = (node_symbol if isinstance(node_symbol, str)
else getattr(node_symbol, 'name', "Unknown_symbol")
or "Unknown_symbol")
node_dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id)
node_symbol_id = self.get_symbol_id(node_dso_id, symbol_name, 0, 0)
parent_id = self.get_call_path_id(parent_id, node_symbol_id, node.ip)
call_path_id = parent_id
sample_id = self.next_id['sample']
evsel_name = str(sample.evsel)
if evsel_name.startswith("evsel(") and evsel_name.endswith(")"):
evsel_name = evsel_name[6:-1]
self.write_sample(sample_id,
self.get_event_id(evsel_name),
machine_id, thread_id, comm_id, dso_id, symbol_id, sample,
call_path_id)
event_name_str = evsel_name
if hasattr(sample, "raw_buf"):
if event_name_str == "ptwrite":
self.write_ptwrite(sample_id, sample.raw_buf)
elif event_name_str == "cbr":
self.write_cbr(sample_id, sample.raw_buf)
elif event_name_str == "mwait":
self.write_mwait(sample_id, sample.raw_buf)
elif event_name_str == "pwre":
self.write_pwre(sample_id, sample.raw_buf)
elif event_name_str == "exstop":
self.write_exstop(sample_id, sample.raw_buf)
elif event_name_str == "pwrx":
self.write_pwrx(sample_id, sample.raw_buf)
self.next_id['sample'] += 1
def finalize(self) -> None:
"""Copy files to database and add keys/views."""
print("Copying to database...")
for symbol_id, (dso_id, sym_start, sym_end, binding, name) in self.symbol_rows.items():
self.write_symbol(symbol_id, dso_id, sym_start, sym_end, binding, name)
for name, f in self.files.items():
self.close_output_file(f)
table_mapping = {
'evsel': 'selected_events',
'machine': 'machines',
'thread': 'threads',
'comm': 'comms',
'comm_thread': 'comm_threads',
'dso': 'dsos',
'symbol': 'symbols',
'branch_type': 'branch_types',
'sample': 'samples',
'call': 'calls',
'call_path': 'call_paths',
'context_switches': 'context_switches',
'ptwrite': 'ptwrite',
'cbr': 'cbr',
'mwait': 'mwait',
'pwre': 'pwre',
'exstop': 'exstop',
'pwrx': 'pwrx'
}
for name, f in self.files.items():
table_name = table_mapping.get(name, name + "s")
self.copy_output_file(f.name, table_name)
print("Removing intermediate files...")
for name, f in self.files.items():
os.unlink(f.name)
shutil.rmtree(self.output_dir_name, ignore_errors=True)
print("Adding primary keys")
self.do_query("ALTER TABLE selected_events ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE machines ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE threads ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE comms ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE comm_threads ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE dsos ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE symbols ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE branch_types ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE samples ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE call_paths ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE calls ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE ptwrite ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE cbr ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE mwait ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE pwre ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE exstop ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE pwrx ADD PRIMARY KEY (id)")
self.do_query("ALTER TABLE context_switches ADD PRIMARY KEY (id)")
print("Adding foreign keys")
self.do_query('ALTER TABLE threads '
'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
'ADD CONSTRAINT processfk FOREIGN KEY (process_id) REFERENCES threads (id)')
self.do_query('ALTER TABLE comms '
'ADD CONSTRAINT threadfk FOREIGN KEY (c_thread_id) REFERENCES threads (id)')
self.do_query('ALTER TABLE comm_threads '
'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id)')
self.do_query('ALTER TABLE dsos '
'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id)')
self.do_query('ALTER TABLE symbols '
'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id)')
self.do_query('ALTER TABLE samples '
'ADD CONSTRAINT evselfk FOREIGN KEY (evsel_id) '
'REFERENCES selected_events (id),'
'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),'
'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id),'
'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),'
'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),'
'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) '
'REFERENCES symbols (id)')
self.do_query('ALTER TABLE call_paths '
'ADD CONSTRAINT parentfk FOREIGN KEY (parent_id) REFERENCES call_paths (id),'
'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id)')
self.do_query('ALTER TABLE calls '
'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),'
'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
'ADD CONSTRAINT call_pathfk FOREIGN KEY (call_path_id) '
'REFERENCES call_paths (id),'
'ADD CONSTRAINT callfk FOREIGN KEY (call_id) REFERENCES samples (id),'
'ADD CONSTRAINT returnfk FOREIGN KEY (return_id) REFERENCES samples (id),'
'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) '
'REFERENCES call_paths (id)')
# pcpid_idx, pid_idx, and comms.has_calls are required by
# exported-sql-viewer.py to filter the command dropdown to threads with
# call graphs and to expand parent/child call trees efficiently.
self.do_query('CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
self.do_query('CREATE INDEX pid_idx ON calls (parent_id)')
self.do_query('ALTER TABLE comms ADD has_calls boolean')
self.do_query('UPDATE comms SET has_calls = TRUE WHERE comms.id IN '
'(SELECT DISTINCT comm_id FROM calls)')
self.do_query('ALTER TABLE context_switches '
'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
'ADD CONSTRAINT toutfk FOREIGN KEY (thread_out_id) REFERENCES threads (id),'
'ADD CONSTRAINT tinfk FOREIGN KEY (thread_in_id) REFERENCES threads (id),'
'ADD CONSTRAINT coutfk FOREIGN KEY (comm_out_id) REFERENCES comms (id),'
'ADD CONSTRAINT cinfk FOREIGN KEY (comm_in_id) REFERENCES comms (id)')
self.do_query('ALTER TABLE ptwrite '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
self.do_query('ALTER TABLE cbr '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
self.do_query('ALTER TABLE mwait '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
self.do_query('ALTER TABLE pwre '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
self.do_query('ALTER TABLE exstop '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
self.do_query('ALTER TABLE pwrx '
'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
print("Creating views...")
self.do_query(
"CREATE VIEW machines_view AS "
"SELECT id, pid, root_dir, "
"CASE WHEN id=0 THEN 'unknown' WHEN pid=-1 THEN 'host' ELSE 'guest' END "
"AS host_or_guest FROM machines"
)
self.do_query(
"CREATE VIEW dsos_view AS "
"SELECT id, machine_id, "
"(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, "
"short_name, long_name, build_id FROM dsos"
)
self.do_query(
"CREATE VIEW symbols_view AS "
"SELECT id, name, (SELECT short_name FROM dsos WHERE id=dso_id) AS dso, "
"dso_id, sym_start, sym_end, "
"CASE WHEN binding=0 THEN 'local' WHEN binding=1 THEN 'global' ELSE 'weak' END "
"AS binding FROM symbols"
)
self.do_query(
"CREATE VIEW threads_view AS "
"SELECT id, machine_id, "
"(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, "
"process_id, pid, tid FROM threads"
)
self.do_query(
"CREATE VIEW comm_threads_view AS "
"SELECT comm_id, (SELECT comm FROM comms WHERE id = comm_id) AS command, "
"thread_id, (SELECT pid FROM threads WHERE id = thread_id) AS pid, "
"(SELECT tid FROM threads WHERE id = thread_id) AS tid FROM comm_threads"
)
self.do_query(
"CREATE VIEW call_paths_view AS "
"SELECT c.id, to_hex(c.ip) AS ip, c.symbol_id, "
"(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol, "
"(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id, "
"(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name, "
"c.parent_id, to_hex(p.ip) AS parent_ip, p.symbol_id AS parent_symbol_id, "
"(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol, "
"(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id, "
"(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name "
"FROM call_paths c LEFT JOIN call_paths p ON p.id = c.parent_id"
)
self.do_query(
"CREATE VIEW calls_view AS "
"SELECT calls.id, thread_id, "
"(SELECT pid FROM threads WHERE id = thread_id) AS pid, "
"(SELECT tid FROM threads WHERE id = thread_id) AS tid, "
"(SELECT comm FROM comms WHERE id = comm_id) AS command, "
"call_path_id, to_hex(ip) AS ip, symbol_id, "
"(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, "
"call_time, return_time, return_time - call_time AS elapsed_time, "
"branch_count, insn_count, cyc_count, "
"CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) "
"ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, "
"call_id, return_id, "
"CASE WHEN flags=0 THEN '' WHEN flags=1 THEN 'no call' "
"WHEN flags=2 THEN 'no return' WHEN flags=3 THEN 'no call/return' "
"WHEN flags IN (4, 6) THEN 'jmp' ELSE CAST(flags AS VARCHAR(6)) END AS flags, "
"parent_call_path_id, calls.parent_id "
"FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id"
)
self.do_query(
"CREATE VIEW samples_view AS "
"SELECT id, time, cpu, "
"(SELECT pid FROM threads WHERE id = thread_id) AS pid, "
"(SELECT tid FROM threads WHERE id = thread_id) AS tid, "
"(SELECT comm FROM comms WHERE id = comm_id) AS command, "
"(SELECT name FROM selected_events WHERE id = evsel_id) AS event, "
"to_hex(ip) AS ip_hex, "
"(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, sym_offset, "
"(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name, "
"to_hex(to_ip) AS to_ip_hex, "
"(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol, to_sym_offset, "
"(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name, "
"(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name, "
"in_tx, call_path_id, insn_count, cyc_count, "
"CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) "
"ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, flags FROM samples"
)
self.do_query(
"CREATE VIEW context_switches_view AS "
"SELECT context_switches.id, context_switches.machine_id, "
"context_switches.time, context_switches.cpu, "
"th_out.pid AS pid_out, th_out.tid AS tid_out, comm_out.comm AS comm_out, "
"th_in.pid AS pid_in, th_in.tid AS tid_in, comm_in.comm AS comm_in, "
"CASE WHEN flags=0 THEN 'in' WHEN flags=1 THEN 'out' "
"WHEN flags=3 THEN 'out preempt' ELSE CAST(flags AS VARCHAR(6)) END AS flags "
"FROM context_switches "
"INNER JOIN threads AS th_out ON th_out.id = context_switches.thread_out_id "
"INNER JOIN threads AS th_in ON th_in.id = context_switches.thread_in_id "
"INNER JOIN comms AS comm_out ON comm_out.id = context_switches.comm_out_id "
"INNER JOIN comms AS comm_in ON comm_in.id = context_switches.comm_in_id"
)
self.do_query(
"CREATE VIEW ptwrite_view AS "
"SELECT ptwrite.id, time, cpu, to_hex(payload) AS payload_hex, "
"CASE WHEN exact_ip=FALSE THEN 'False' ELSE 'True' END AS exact_ip "
"FROM ptwrite INNER JOIN samples ON samples.id = ptwrite.id"
)
self.do_query(
"CREATE VIEW cbr_view AS "
"SELECT cbr.id, time, cpu, cbr, mhz, percent "
"FROM cbr INNER JOIN samples ON samples.id = cbr.id"
)
self.do_query(
"CREATE VIEW mwait_view AS "
"SELECT mwait.id, time, cpu, to_hex(hints) AS hints_hex, "
"to_hex(extensions) AS extensions_hex "
"FROM mwait INNER JOIN samples ON samples.id = mwait.id"
)
self.do_query(
"CREATE VIEW pwre_view AS "
"SELECT pwre.id, time, cpu, cstate, subcstate, "
"CASE WHEN hw=FALSE THEN 'False' ELSE 'True' END AS hw "
"FROM pwre INNER JOIN samples ON samples.id = pwre.id"
)
self.do_query(
"CREATE VIEW exstop_view AS "
"SELECT exstop.id, time, cpu, "
"CASE WHEN exact_ip=FALSE THEN 'False' ELSE 'True' END AS exact_ip "
"FROM exstop INNER JOIN samples ON samples.id = exstop.id"
)
self.do_query(
"CREATE VIEW pwrx_view AS "
"SELECT pwrx.id, time, cpu, deepest_cstate, last_cstate, "
"CASE WHEN wake_reason=1 THEN 'Interrupt' "
"WHEN wake_reason=2 THEN 'Timer Deadline' "
"WHEN wake_reason=4 THEN 'Monitored Address' "
"WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' "
"ELSE CAST ( wake_reason AS VARCHAR(2) ) END AS wake_reason "
"FROM pwrx INNER JOIN samples ON samples.id = pwrx.id"
)
self.do_query(
"CREATE VIEW power_events_view AS "
"SELECT samples.id, samples.time, samples.cpu, selected_events.name AS event, "
"FORMAT('%6s', cbr.cbr) AS cbr, FORMAT('%6s', cbr.mhz) AS MHz, "
"FORMAT('%5s', cbr.percent) AS percent, to_hex(mwait.hints) AS hints_hex, "
"to_hex(mwait.extensions) AS extensions_hex, FORMAT('%3s', pwre.cstate) AS cstate, "
"FORMAT('%3s', pwre.subcstate) AS subcstate, "
"CASE WHEN pwre.hw=FALSE THEN 'False' WHEN pwre.hw=TRUE THEN 'True' "
"ELSE NULL END AS hw, "
"CASE WHEN exstop.exact_ip=FALSE THEN 'False' WHEN exstop.exact_ip=TRUE THEN 'True' "
"ELSE NULL END AS exact_ip, FORMAT('%3s', pwrx.deepest_cstate) AS deepest_cstate, "
"FORMAT('%3s', pwrx.last_cstate) AS last_cstate, "
"CASE WHEN pwrx.wake_reason=1 THEN 'Interrupt' "
"WHEN wake_reason=2 THEN 'Timer Deadline' "
"WHEN wake_reason=4 THEN 'Monitored Address' "
"WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' "
"ELSE FORMAT('%2s', pwrx.wake_reason) END AS wake_reason "
"FROM cbr FULL JOIN mwait ON mwait.id = cbr.id "
"FULL JOIN pwre ON pwre.id = cbr.id FULL JOIN exstop ON exstop.id = cbr.id "
"FULL JOIN pwrx ON pwrx.id = cbr.id INNER JOIN samples ON samples.id = "
"coalesce(cbr.id, mwait.id, pwre.id, exstop.id, pwrx.id) "
"INNER JOIN selected_events ON selected_events.id = samples.evsel_id "
"ORDER BY samples.id"
)
if __name__ == "__main__":
ap = argparse.ArgumentParser(
description="Export perf data to a postgresql database")
ap.add_argument("-i", "--input", default="perf.data",
help="Input file name")
ap.add_argument("-o", "--output", required=True,
help="Output database name")
ap.add_argument("--itrace", default=None,
help="itrace options, e.g. cr")
args = ap.parse_args()
exporter = PostgresExporter(args.output)
session = None
succeeded = False
caught_error = False
try:
exporter.setup_db()
session = perf.session(perf.data(args.input),
context_switch=exporter.process_context_switch,
sample=exporter.process_event,
call_return=exporter.process_call_return,
itrace=args.itrace)
exporter.session = session
try:
session.process_events()
finally:
# Break the C perf.session -> bound method -> exporter reference cycle.
exporter.session = None
exporter.finalize()
print(f"Successfully exported to {args.output}")
succeeded = True
except (OSError, RuntimeError, ValueError, KeyboardInterrupt) as e:
if not isinstance(e, (KeyboardInterrupt, SystemExit)):
import traceback
traceback.print_exc()
caught_error = True
finally:
exporter.session = None
for out_file in exporter.files.values():
try:
if not out_file.closed:
out_file.close()
except OSError:
pass
exporter.disconnect()
if not succeeded:
if (getattr(exporter, 'created_output_dir', False) and
os.path.exists(exporter.output_dir_name)):
shutil.rmtree(exporter.output_dir_name, ignore_errors=True)
if caught_error:
sys.exit(1)