| # | |
| # Toaster helper class | |
| # | |
| # Copyright (C) 2013 Intel Corporation | |
| # | |
| # SPDX-License-Identifier: MIT | |
| # | |
| # This bbclass is designed to extract data used by OE-Core during the build process, | |
| # for recording in the Toaster system. | |
| # The data access is synchronous, preserving the build data integrity across | |
| # different builds. | |
| # | |
| # The data is transferred through the event system, using the MetadataEvent objects. | |
| # | |
| # The model is to enable the datadump functions as postfuncs, and have the dump | |
| # executed after the real taskfunc has been executed. This prevents task signature changing | |
| # is toaster is enabled or not. Build performance is not affected if Toaster is not enabled. | |
| # | |
| # To enable, use INHERIT in local.conf: | |
| # | |
| # INHERIT += "toaster" | |
| # | |
| # | |
| # | |
| # | |
| # Find and dump layer info when we got the layers parsed | |
| python toaster_layerinfo_dumpdata() { | |
| import subprocess | |
| def _get_git_branch(layer_path): | |
| branch = subprocess.run(['git', 'symbolic-ref', 'HEAD'], cwd=layer_path, text=True, stdout=subprocess.PIPE).stdout.strip() | |
| branch = branch.replace('refs/heads/', '').rstrip() | |
| return branch | |
| def _get_git_revision(layer_path): | |
| revision = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=layer_path, text=True, stdout=subprocess.PIPE).stdout.strip() | |
| return revision | |
| def _get_url_map_name(layer_name): | |
| """ Some layers have a different name on openembedded.org site, | |
| this method returns the correct name to use in the URL | |
| """ | |
| url_name = layer_name | |
| url_mapping = {'meta': 'openembedded-core'} | |
| for key in url_mapping.keys(): | |
| if key == layer_name: | |
| url_name = url_mapping[key] | |
| return url_name | |
| def _get_layer_version_information(layer_path): | |
| layer_version_info = {} | |
| layer_version_info['branch'] = _get_git_branch(layer_path) | |
| layer_version_info['commit'] = _get_git_revision(layer_path) | |
| layer_version_info['priority'] = 0 | |
| return layer_version_info | |
| def _get_layer_dict(layer_path): | |
| layer_info = {} | |
| layer_name = layer_path.split('/')[-1] | |
| layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' | |
| layer_url_name = _get_url_map_name(layer_name) | |
| layer_info['name'] = layer_url_name | |
| layer_info['local_path'] = layer_path | |
| layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) | |
| layer_info['version'] = _get_layer_version_information(layer_path) | |
| return layer_info | |
| bblayers = e.data.getVar("BBLAYERS") | |
| llayerinfo = {} | |
| for layer in { l for l in bblayers.strip().split() if len(l) }: | |
| llayerinfo[layer] = _get_layer_dict(layer) | |
| bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data) | |
| } | |
| # Dump package file info data | |
| def _toaster_load_pkgdatafile(dirpath, filepath): | |
| import json | |
| import re | |
| pkgdata = {} | |
| with open(os.path.join(dirpath, filepath), "r") as fin: | |
| for line in fin: | |
| try: | |
| kn, kv = line.strip().split(": ", 1) | |
| m = re.match(r"^PKG:([^A-Z:]*)", kn) | |
| if m: | |
| pkgdata['OPKGN'] = m.group(1) | |
| kn = kn.split(":")[0] | |
| pkgdata[kn] = kv | |
| if kn.startswith('FILES_INFO'): | |
| pkgdata[kn] = json.loads(kv) | |
| except ValueError: | |
| pass # ignore lines without valid key: value pairs | |
| return pkgdata | |
| def _toaster_dumpdata(pkgdatadir, d): | |
| """ | |
| Dumps the data about the packages created by a recipe | |
| """ | |
| # No need to try and dumpdata if the recipe isn't generating packages | |
| if not d.getVar('PACKAGES'): | |
| return | |
| lpkgdata = {} | |
| datadir = os.path.join(pkgdatadir, 'runtime') | |
| # scan and send data for each generated package | |
| if os.path.exists(datadir): | |
| for datafile in os.listdir(datadir): | |
| if not datafile.endswith('.packaged'): | |
| lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) | |
| # Fire an event containing the pkg data | |
| bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) | |
| python toaster_package_dumpdata() { | |
| _toaster_dumpdata(d.getVar('PKGDESTWORK'), d) | |
| } | |
| python toaster_packagedata_dumpdata() { | |
| # This path needs to match do_packagedata[sstate-inputdirs] | |
| _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d) | |
| } | |
| # 2. Dump output image files information | |
| python toaster_artifact_dumpdata() { | |
| """ | |
| Dump data about SDK variables | |
| """ | |
| event_data = { | |
| "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME") | |
| } | |
| bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d) | |
| } | |
| # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data | |
| python toaster_collect_task_stats() { | |
| import bb.build | |
| import bb.event | |
| import bb.data | |
| import bb.utils | |
| import os | |
| if not e.data.getVar('BUILDSTATS_BASE'): | |
| return # if we don't have buildstats, we cannot collect stats | |
| toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist") | |
| def stat_to_float(value): | |
| return float(value.strip('% \n\r')) | |
| def _append_read_list(v): | |
| lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) | |
| with open(toaster_statlist_file, "a") as fout: | |
| taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}") | |
| fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) | |
| bb.utils.unlockfile(lock) | |
| def _read_stats(filename): | |
| # seconds | |
| cpu_time_user = 0 | |
| cpu_time_system = 0 | |
| # bytes | |
| disk_io_read = 0 | |
| disk_io_write = 0 | |
| started = 0 | |
| ended = 0 | |
| taskname = '' | |
| statinfo = {} | |
| with open(filename, 'r') as task_bs: | |
| for line in task_bs.readlines(): | |
| k,v = line.strip().split(": ", 1) | |
| statinfo[k] = v | |
| if "Started" in statinfo: | |
| started = stat_to_float(statinfo["Started"]) | |
| if "Ended" in statinfo: | |
| ended = stat_to_float(statinfo["Ended"]) | |
| if "Child rusage ru_utime" in statinfo: | |
| cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"]) | |
| if "Child rusage ru_stime" in statinfo: | |
| cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"]) | |
| if "IO write_bytes" in statinfo: | |
| write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r')) | |
| disk_io_write = disk_io_write + write_bytes | |
| if "IO read_bytes" in statinfo: | |
| read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r')) | |
| disk_io_read = disk_io_read + read_bytes | |
| return { | |
| 'stat_file': filename, | |
| 'cpu_time_user': cpu_time_user, | |
| 'cpu_time_system': cpu_time_system, | |
| 'disk_io_read': disk_io_read, | |
| 'disk_io_write': disk_io_write, | |
| 'started': started, | |
| 'ended': ended | |
| } | |
| if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): | |
| _append_read_list(e) | |
| pass | |
| if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file): | |
| events = [] | |
| with open(toaster_statlist_file, "r") as fin: | |
| for line in fin: | |
| (taskfile, taskname, filename, recipename) = line.strip().split("::") | |
| stats = _read_stats(filename) | |
| events.append((taskfile, taskname, stats, recipename)) | |
| bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) | |
| os.unlink(toaster_statlist_file) | |
| } | |
| # dump relevant build history data as an event when the build is completed | |
| python toaster_buildhistory_dump() { | |
| import re | |
| BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") | |
| BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) | |
| pkgdata_dir = e.data.getVar("PKGDATA_DIR") | |
| # scan the build targets for this build | |
| images = {} | |
| allpkgs = {} | |
| files = {} | |
| for target in e._pkgs: | |
| target = target.split(':')[0] # strip ':<task>' suffix from the target | |
| installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) | |
| if os.path.exists(installed_img_path): | |
| images[target] = {} | |
| files[target] = {} | |
| files[target]['dirs'] = [] | |
| files[target]['syms'] = [] | |
| files[target]['files'] = [] | |
| with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: | |
| for line in fin: | |
| line = line.rstrip(";") | |
| psize, punit, pname = line.split() | |
| # this size is "installed-size" as it measures how much space it takes on disk | |
| images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} | |
| with open("%s/depends.dot" % installed_img_path, "r") as fin: | |
| p = re.compile(r'\s*"(?P<name>[^"]+)"\s*->\s*"(?P<dep>[^"]+)"(?P<rec>.*?\[style=dotted\])?') | |
| for line in fin: | |
| m = p.match(line) | |
| if not m: | |
| continue | |
| pname = m.group('name') | |
| dependsname = m.group('dep') | |
| deptype = 'recommends' if m.group('rec') else 'depends' | |
| # If RPM is used for packaging, then there may be | |
| # dependencies such as "/bin/sh", which will confuse | |
| # _toaster_load_pkgdatafile() later on. While at it, ignore | |
| # any dependencies that contain parentheses, e.g., | |
| # "libc.so.6(GLIBC_2.7)". | |
| if dependsname.startswith('/') or '(' in dependsname: | |
| continue | |
| if not pname in images[target]: | |
| images[target][pname] = {'size': 0, 'depends' : []} | |
| if not dependsname in images[target]: | |
| images[target][dependsname] = {'size': 0, 'depends' : []} | |
| images[target][pname]['depends'].append((dependsname, deptype)) | |
| # files-in-image.txt is only generated if an image file is created, | |
| # so the file entries ('syms', 'dirs', 'files') for a target will be | |
| # empty for rootfs builds and other "image" tasks which don't | |
| # produce image files | |
| # (e.g. "bitbake core-image-minimal -c populate_sdk") | |
| files_in_image_path = "%s/files-in-image.txt" % installed_img_path | |
| if os.path.exists(files_in_image_path): | |
| with open(files_in_image_path, "r") as fin: | |
| for line in fin: | |
| lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] | |
| if lc[0].startswith("l"): | |
| files[target]['syms'].append(lc) | |
| elif lc[0].startswith("d"): | |
| files[target]['dirs'].append(lc) | |
| else: | |
| files[target]['files'].append(lc) | |
| for pname in images[target]: | |
| if not pname in allpkgs: | |
| try: | |
| pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname) | |
| except IOError as err: | |
| if err.errno == 2: | |
| # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime | |
| continue | |
| else: | |
| raise | |
| allpkgs[pname] = pkgdata | |
| data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files } | |
| bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data) | |
| } | |
| # get list of artifacts from sstate manifest | |
| python toaster_artifacts() { | |
| if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]: | |
| d2 = d.createCopy() | |
| d2.setVar('FILE', e.taskfile) | |
| # Use 'stamp-extra-info' if present, else use workaround | |
| # to determine 'SSTATE_PKGARCH' | |
| extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info') | |
| if extrainf: | |
| d2.setVar('SSTATE_PKGARCH', extrainf) | |
| else: | |
| if "do_populate_sdk" == e.taskname: | |
| d2.setVar('SSTATE_PKGARCH', d2.expand("${MACHINE}${SDKMACHINE}")) | |
| else: | |
| d2.setVar('SSTATE_PKGARCH', d2.expand("${MACHINE}")) | |
| manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0] | |
| if os.access(manifest, os.R_OK): | |
| with open(manifest) as fmanifest: | |
| artifacts = [fname.strip() for fname in fmanifest] | |
| data = {"task": e.taskid, "artifacts": artifacts} | |
| bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2) | |
| } | |
| # set event handlers | |
| addhandler toaster_layerinfo_dumpdata | |
| toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted" | |
| addhandler toaster_collect_task_stats | |
| toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed" | |
| addhandler toaster_buildhistory_dump | |
| toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" | |
| addhandler toaster_artifacts | |
| toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted" | |
| do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata " | |
| do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata " | |
| do_package[postfuncs] += "toaster_package_dumpdata " | |
| do_package[vardepsexclude] += "toaster_package_dumpdata " | |
| #do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata " | |
| #do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata " | |
| #do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata " | |
| #do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata " | |