poky/meta/classes-global/retain.bbclass - openbmc - Git at Google

 # Creates a tarball of the work directory for a recipe when one of its
 # tasks fails, or any other nominated directories.
 # Useful in cases where the environment in which builds are run is
 # ephemeral or otherwise inaccessible for examination during
 # debugging.
 #
 # To enable, simply add the following to your configuration:
 #
 # INHERIT += "retain"
 #
 # You can specify the recipe-specific directories to save upon failure
 # or always (space-separated) e.g.:
 #
 # RETAIN_DIRS_FAILURE = "${WORKDIR};prefix=workdir"    # default
 # RETAIN_DIRS_ALWAYS = "${T}"
 #
 # Naturally you can use overrides to limit it to a specific recipe:
 # RETAIN_DIRS_ALWAYS:pn-somerecipe = "${T}"
 #
 # You can also specify global (non-recipe-specific) directories to save:
 #
 # RETAIN_DIRS_GLOBAL_FAILURE = "${LOG_DIR}"
 # RETAIN_DIRS_GLOBAL_ALWAYS = "${BUILDSTATS_BASE}"
 #
 # If you wish to use a different tarball name prefix than the default of
 # the directory name, you can do so by specifying a ;prefix= followed by
 # the desired prefix (no spaces) in any of the RETAIN_DIRS_* variables.
 # e.g. to always save the log files with a "recipelogs" as the prefix for
 # the tarball of ${T} you would do this:
 #
 # RETAIN_DIRS_ALWAYS = "${T};prefix=recipelogs"
 #
 # Notes:
 # * For this to be useful you also need corresponding logic in your build
 #   orchestration tool to pick up any files written out to RETAIN_OUTDIR
 #   (with the other assumption being that no files are present there at
 #   the start of the build, since there is no logic to purge old files).
 # * Work directories can be quite large, so saving them can take some time
 #   and of course space.
 # * Tarball creation is deferred to the end of the build, thus you will
 #   get the state at the end, not immediately upon failure.
 # * Extra directories must naturally be populated at the time the retain
 #   class goes to save them (build completion); to try ensure this for
 #   things that are also saved on build completion (e.g. buildstats), put
 #   the INHERIT += "retain" after the INHERIT += lines for the class that
 #   is writing out the data that you wish to save.
 # * The tarballs have the tarball name as a top-level directory so that
 #   multiple tarballs can be extracted side-by-side easily.
 #
 # Copyright (c) 2020, 2024 Microsoft Corporation
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #

 RETAIN_OUTDIR ?= "${TMPDIR}/retained"
 RETAIN_DIRS_FAILURE ?= "${WORKDIR};prefix=workdir"
 RETAIN_DIRS_ALWAYS ?= ""
 RETAIN_DIRS_GLOBAL_FAILURE ?= ""
 RETAIN_DIRS_GLOBAL_ALWAYS ?= ""
 RETAIN_TARBALL_SUFFIX ?= "${DATETIME}.tar.gz"
 RETAIN_ENABLED ?= "1"


 def retain_retain_dir(desc, tarprefix, path, tarbasepath, d):
     import datetime

     outdir = d.getVar('RETAIN_OUTDIR')
     bb.utils.mkdirhier(outdir)
     suffix = d.getVar('RETAIN_TARBALL_SUFFIX')
     tarname = '%s_%s' % (tarprefix, suffix)
     tarfp = os.path.join(outdir, '%s' % tarname)
     tardir = os.path.relpath(path, tarbasepath)
     cmdargs = ['tar', 'cfa', tarfp]
     # Prefix paths within the tarball with the tarball name so that
     # multiple tarballs can be extracted side-by-side
     tarname_noext = os.path.splitext(tarname)[0]
     if tarname_noext.endswith('.tar'):
         tarname_noext = tarname_noext[:-4]
     cmdargs += ['--transform', 's:^:%s/:' % tarname_noext]
     cmdargs += [tardir]
     try:
         bb.process.run(cmdargs, cwd=tarbasepath)
     except bb.process.ExecutionError as e:
         # It is possible for other tasks to be writing to the workdir
         # while we are tarring it up, in which case tar will return 1,
         # but we don't care in this situation (tar returns 2 for other
         # errors so we we will see those)
         if e.exitcode != 1:
             bb.warn('retain: error saving %s: %s' % (desc, str(e)))


 addhandler retain_task_handler
 retain_task_handler[eventmask] = "bb.build.TaskFailed bb.build.TaskSucceeded"

 addhandler retain_build_handler
 retain_build_handler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"

 python retain_task_handler() {
     if d.getVar('RETAIN_ENABLED') != '1':
         return

     dirs = d.getVar('RETAIN_DIRS_ALWAYS')
     if isinstance(e, bb.build.TaskFailed):
         dirs += ' ' + d.getVar('RETAIN_DIRS_FAILURE')

     dirs = dirs.strip().split()
     if dirs:
         outdir = d.getVar('RETAIN_OUTDIR')
         bb.utils.mkdirhier(outdir)
         dirlist_file = os.path.join(outdir, 'retain_dirs.list')
         pn = d.getVar('PN')
         taskname = d.getVar('BB_CURRENTTASK')
         with open(dirlist_file, 'a') as f:
             for entry in dirs:
                 f.write('%s %s %s\n' % (pn, taskname, entry))
 }

 python retain_build_handler() {
     outdir = d.getVar('RETAIN_OUTDIR')
     dirlist_file = os.path.join(outdir, 'retain_dirs.list')

     if isinstance(e, bb.event.BuildStarted):
         if os.path.exists(dirlist_file):
             os.remove(dirlist_file)
         return

     if d.getVar('RETAIN_ENABLED') != '1':
         return

     savedirs = {}
     try:
         with open(dirlist_file, 'r') as f:
             for line in f:
                 pn, _, path = line.rstrip().split()
                 if not path in savedirs:
                     savedirs[path] = pn
         os.remove(dirlist_file)
     except FileNotFoundError:
         pass

     if e.getFailures():
         for path in (d.getVar('RETAIN_DIRS_GLOBAL_FAILURE') or '').strip().split():
             savedirs[path] = ''

     for path in (d.getVar('RETAIN_DIRS_GLOBAL_ALWAYS') or '').strip().split():
         savedirs[path] = ''

     if savedirs:
         bb.plain('NOTE: retain: retaining build output...')
         count = 0
         for path, pn in savedirs.items():
             prefix = None
             if ';' in path:
                 pathsplit = path.split(';')
                 path = pathsplit[0]
                 for param in pathsplit[1:]:
                     if '=' in param:
                         name, value = param.split('=', 1)
                         if name == 'prefix':
                             prefix = value
                         else:
                             bb.error('retain: invalid parameter "%s" in RETAIN_* variable value' % param)
                             return
                     else:
                         bb.error('retain: parameter "%s" missing value in RETAIN_* variable value' % param)
                         return
             if prefix:
                 itemname = prefix
             else:
                 itemname = os.path.basename(path)
             if pn:
                 # Always add the recipe name in front
                 itemname = pn + '_' + itemname
             if os.path.exists(path):
                 retain_retain_dir(itemname, itemname, path, os.path.dirname(path), d)
                 count += 1
             else:
                 bb.warn('retain: path %s does not currently exist' % path)
         if count:
             item = 'archive' if count == 1 else 'archives'
             bb.plain('NOTE: retain: saved %d %s to %s' % (count, item, outdir))
 }
	# Creates a tarball of the work directory for a recipe when one of its
	# tasks fails, or any other nominated directories.
	# Useful in cases where the environment in which builds are run is
	# ephemeral or otherwise inaccessible for examination during
	# debugging.
	#
	# To enable, simply add the following to your configuration:
	#
	# INHERIT += "retain"
	#
	# You can specify the recipe-specific directories to save upon failure
	# or always (space-separated) e.g.:
	#
	# RETAIN_DIRS_FAILURE = "${WORKDIR};prefix=workdir" # default
	# RETAIN_DIRS_ALWAYS = "${T}"
	#
	# Naturally you can use overrides to limit it to a specific recipe:
	# RETAIN_DIRS_ALWAYS:pn-somerecipe = "${T}"
	#
	# You can also specify global (non-recipe-specific) directories to save:
	#
	# RETAIN_DIRS_GLOBAL_FAILURE = "${LOG_DIR}"
	# RETAIN_DIRS_GLOBAL_ALWAYS = "${BUILDSTATS_BASE}"
	#
	# If you wish to use a different tarball name prefix than the default of
	# the directory name, you can do so by specifying a ;prefix= followed by
	# the desired prefix (no spaces) in any of the RETAIN_DIRS_* variables.
	# e.g. to always save the log files with a "recipelogs" as the prefix for
	# the tarball of ${T} you would do this:
	#
	# RETAIN_DIRS_ALWAYS = "${T};prefix=recipelogs"
	#
	# Notes:
	# * For this to be useful you also need corresponding logic in your build
	# orchestration tool to pick up any files written out to RETAIN_OUTDIR
	# (with the other assumption being that no files are present there at
	# the start of the build, since there is no logic to purge old files).
	# * Work directories can be quite large, so saving them can take some time
	# and of course space.
	# * Tarball creation is deferred to the end of the build, thus you will
	# get the state at the end, not immediately upon failure.
	# * Extra directories must naturally be populated at the time the retain
	# class goes to save them (build completion); to try ensure this for
	# things that are also saved on build completion (e.g. buildstats), put
	# the INHERIT += "retain" after the INHERIT += lines for the class that
	# is writing out the data that you wish to save.
	# * The tarballs have the tarball name as a top-level directory so that
	# multiple tarballs can be extracted side-by-side easily.
	#
	# Copyright (c) 2020, 2024 Microsoft Corporation
	#
	# SPDX-License-Identifier: GPL-2.0-only
	#

	RETAIN_OUTDIR ?= "${TMPDIR}/retained"
	RETAIN_DIRS_FAILURE ?= "${WORKDIR};prefix=workdir"
	RETAIN_DIRS_ALWAYS ?= ""
	RETAIN_DIRS_GLOBAL_FAILURE ?= ""
	RETAIN_DIRS_GLOBAL_ALWAYS ?= ""
	RETAIN_TARBALL_SUFFIX ?= "${DATETIME}.tar.gz"
	RETAIN_ENABLED ?= "1"


	def retain_retain_dir(desc, tarprefix, path, tarbasepath, d):
	import datetime

	outdir = d.getVar('RETAIN_OUTDIR')
	bb.utils.mkdirhier(outdir)
	suffix = d.getVar('RETAIN_TARBALL_SUFFIX')
	tarname = '%s_%s' % (tarprefix, suffix)
	tarfp = os.path.join(outdir, '%s' % tarname)
	tardir = os.path.relpath(path, tarbasepath)
	cmdargs = ['tar', 'cfa', tarfp]
	# Prefix paths within the tarball with the tarball name so that
	# multiple tarballs can be extracted side-by-side
	tarname_noext = os.path.splitext(tarname)[0]
	if tarname_noext.endswith('.tar'):
	tarname_noext = tarname_noext[:-4]
	cmdargs += ['--transform', 's:^:%s/:' % tarname_noext]
	cmdargs += [tardir]
	try:
	bb.process.run(cmdargs, cwd=tarbasepath)
	except bb.process.ExecutionError as e:
	# It is possible for other tasks to be writing to the workdir
	# while we are tarring it up, in which case tar will return 1,
	# but we don't care in this situation (tar returns 2 for other
	# errors so we we will see those)
	if e.exitcode != 1:
	bb.warn('retain: error saving %s: %s' % (desc, str(e)))


	addhandler retain_task_handler
	retain_task_handler[eventmask] = "bb.build.TaskFailed bb.build.TaskSucceeded"

	addhandler retain_build_handler
	retain_build_handler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"

	python retain_task_handler() {
	if d.getVar('RETAIN_ENABLED') != '1':
	return

	dirs = d.getVar('RETAIN_DIRS_ALWAYS')
	if isinstance(e, bb.build.TaskFailed):
	dirs += ' ' + d.getVar('RETAIN_DIRS_FAILURE')

	dirs = dirs.strip().split()
	if dirs:
	outdir = d.getVar('RETAIN_OUTDIR')
	bb.utils.mkdirhier(outdir)
	dirlist_file = os.path.join(outdir, 'retain_dirs.list')
	pn = d.getVar('PN')
	taskname = d.getVar('BB_CURRENTTASK')
	with open(dirlist_file, 'a') as f:
	for entry in dirs:
	f.write('%s %s %s\n' % (pn, taskname, entry))
	}

	python retain_build_handler() {
	outdir = d.getVar('RETAIN_OUTDIR')
	dirlist_file = os.path.join(outdir, 'retain_dirs.list')

	if isinstance(e, bb.event.BuildStarted):
	if os.path.exists(dirlist_file):
	os.remove(dirlist_file)
	return

	if d.getVar('RETAIN_ENABLED') != '1':
	return

	savedirs = {}
	try:
	with open(dirlist_file, 'r') as f:
	for line in f:
	pn, _, path = line.rstrip().split()
	if not path in savedirs:
	savedirs[path] = pn
	os.remove(dirlist_file)
	except FileNotFoundError:
	pass

	if e.getFailures():
	for path in (d.getVar('RETAIN_DIRS_GLOBAL_FAILURE') or '').strip().split():
	savedirs[path] = ''

	for path in (d.getVar('RETAIN_DIRS_GLOBAL_ALWAYS') or '').strip().split():
	savedirs[path] = ''

	if savedirs:
	bb.plain('NOTE: retain: retaining build output...')
	count = 0
	for path, pn in savedirs.items():
	prefix = None
	if ';' in path:
	pathsplit = path.split(';')
	path = pathsplit[0]
	for param in pathsplit[1:]:
	if '=' in param:
	name, value = param.split('=', 1)
	if name == 'prefix':
	prefix = value
	else:
	bb.error('retain: invalid parameter "%s" in RETAIN_* variable value' % param)
	return
	else:
	bb.error('retain: parameter "%s" missing value in RETAIN_* variable value' % param)
	return
	if prefix:
	itemname = prefix
	else:
	itemname = os.path.basename(path)
	if pn:
	# Always add the recipe name in front
	itemname = pn + '_' + itemname
	if os.path.exists(path):
	retain_retain_dir(itemname, itemname, path, os.path.dirname(path), d)
	count += 1
	else:
	bb.warn('retain: path %s does not currently exist' % path)
	if count:
	item = 'archive' if count == 1 else 'archives'
	bb.plain('NOTE: retain: saved %d %s to %s' % (count, item, outdir))
	}