blob: 53a07ee3a68093816a780b43bc8eaabdc4a3fadb [file]
#
# Copyright BitBake Contributors
#
# SPDX-License-Identifier: GPL-2.0-only
#
from bb.tests.fetch import FetcherTest
import bb
import bb.process
import glob
import hashlib
import json
import os
import shutil
import stat
import subprocess
import sys
import tempfile
import unittest
import venv
import zipfile
from bb.tests.support.httpserver import HTTPService
class BitbakeSetupTest(FetcherTest):
def setUp(self):
super(BitbakeSetupTest, self).setUp()
self.registrypath = os.path.join(self.tempdir, "bitbake-setup-configurations")
os.makedirs(self.registrypath)
self.git_init(cwd=self.registrypath)
self.git(['commit', '--allow-empty', '-m', "Initial commit"], cwd=self.registrypath)
self.testrepopath = os.path.join(self.tempdir, "test-repo")
os.makedirs(self.testrepopath)
self.git_init(cwd=self.testrepopath)
self.git(['commit', '--allow-empty', '-m', "Initial commit"], cwd=self.testrepopath)
oeinitbuildenv = """BBPATH=$1
export BBPATH
PATH={}:$PATH
""".format(os.path.join(self.testrepopath, 'scripts'))
self.add_file_to_testrepo('oe-init-build-env',oeinitbuildenv, script=True)
oesetupbuild = """#!/usr/bin/env python3
import getopt
import sys
import os
import shutil
opts, args = getopt.getopt(sys.argv[2:], "c:b:", ["no-shell"])
for option, value in opts:
if option == '-c':
template = value
if option == '-b':
builddir = value
confdir = os.path.join(builddir, 'conf')
os.makedirs(confdir, exist_ok=True)
with open(os.path.join(confdir, 'conf-summary.txt'), 'w') as f:
f.write(template)
shutil.copy(os.path.join(os.path.dirname(__file__), 'test-repo/test-file'), confdir)
with open(os.path.join(builddir, 'init-build-env'), 'w') as f:
f.write("BBPATH={}\\nexport BBPATH\\nPATH={}:$PATH".format(builddir, os.path.join(os.path.dirname(__file__), 'test-repo/scripts')))
"""
self.add_file_to_testrepo('scripts/oe-setup-build', oesetupbuild, script=True)
installbuildtools = """#!/usr/bin/env python3
import getopt
import sys
import os
opts, args = getopt.getopt(sys.argv[1:], "d:", ["downloads-directory=", "local-file="])
installdir = None
local_file = None
for option, value in opts:
if option == '-d':
installdir = value
elif option == '--local-file':
local_file = value
print("install-buildtools local-file={}".format(value))
print("Buildtools installed into {}".format(installdir))
os.makedirs(installdir, exist_ok=True)
"""
self.add_file_to_testrepo('scripts/install-buildtools', installbuildtools, script=True)
# Dummy buildtools installer for bb.fetch testing
self.buildtools_dir = os.path.join(self.tempdir, "buildtools-dl")
os.makedirs(self.buildtools_dir)
self.buildtools_filename = "x86_64-buildtools-nativesdk-standalone-test.sh"
buildtools_filepath = os.path.join(self.buildtools_dir, self.buildtools_filename)
with open(buildtools_filepath, 'w') as f:
f.write("#!/bin/sh\necho dummy\n")
with open(buildtools_filepath, 'rb') as f:
self.buildtools_sha256 = hashlib.sha256(f.read()).hexdigest()
bitbakeconfigbuild = """#!/usr/bin/env python3
import os
import sys
confdir = os.path.join(os.environ['BBPATH'], 'conf')
fragment = sys.argv[2]
with open(os.path.join(confdir, fragment), 'w') as f:
f.write('')
"""
self.add_file_to_testrepo('scripts/bitbake-config-build', bitbakeconfigbuild, script=True)
sometargetexecutable_template = """#!/usr/bin/env python3
import os
print("This is {}")
print("BBPATH is {{}}".format(os.environ["BBPATH"]))
"""
for e_name in ("some-target-executable-1", "some-target-executable-2"):
sometargetexecutable = sometargetexecutable_template.format(e_name)
self.add_file_to_testrepo('scripts/{}'.format(e_name), sometargetexecutable, script=True)
def runbbsetup(self, cmd):
bbsetup = os.path.abspath(os.path.dirname(__file__) + "/../../../bin/bitbake-setup")
# Set PYTHONPATH so subprocess can find bb module instead of relying on the current directory
env = os.environ.copy()
libdir = os.path.abspath(os.path.dirname(__file__) + "/../..")
env["PYTHONPATH"] = libdir + ":" + env.get("PYTHONPATH", "")
return bb.process.run([bbsetup, '--global-settings', os.path.join(self.tempdir, 'global-config')] + cmd, env=env, cwd=self.tempdir)
def _add_json_config_to_registry_helper(self, name, sources):
config = """
{
"sources": {
%s
},
"description": "Test configuration",
"bitbake-setup": {
"configurations": [
{
"name": "gadget",
"description": "Gadget configuration",
"oe-template": "test-configuration-gadget",
"oe-fragments": ["test-fragment-1"],
"install-buildtools": {
"url": "file://%s/%s",
"sha256sum": "%s"
}
},
{
"name": "gizmo",
"description": "Gizmo configuration",
"oe-template": "test-configuration-gizmo",
"oe-fragments": ["test-fragment-2"],
"setup-dir-name": "this-is-a-custom-gizmo-build"
},
{
"name": "gizmo-env-passthrough",
"description": "Gizmo configuration with environment-passthrough",
"bb-layers": ["layerC","layerD/meta-layer"],
"oe-fragments": ["test-fragment-1"],
"bb-env-passthrough-additions": [
"BUILD_ID",
"BUILD_DATE",
"BUILD_SERVER"
]
},
{
"name": "gizmo-no-fragment",
"description": "Gizmo no-fragment template-only configuration",
"oe-template": "test-configuration-gizmo"
},
{
"name": "gadget-notemplate",
"description": "Gadget notemplate configuration",
"notes": [
"Gadget notemplate notes",
"Second line"
],
"bb-layers": ["layerA","layerB/meta-layer"],
"oe-fragments": ["test-fragment-1"]
},
{
"name": "gizmo-notemplate-fragments-one-of",
"description": "Gizmo notemplate configuration",
"bb-layers": ["layerC","layerD/meta-layer"],
"oe-fragments": ["test-fragment-2"],
"oe-fragments-one-of": {
"fragment-desc": {
"description": "Test fragments (with description)",
"options" : [
{ "name": "fragment-desc-1", "description": "fragment 1 desc" },
{ "name": "fragment-desc-2", "description": "fragment 2 desc" }
]
},
"fragment-nodesc": {
"description": "Test fragments (no description)",
"options" : [
"fragment-nodesc-1",
"fragment-nodesc-2"
]
}
}
},
{
"name": "gizmo-notemplate-with-filerelative-layers",
"description": "Gizmo notemplate configuration using filerelative layers",
"bb-layers": ["layerC","layerD/meta-layer"],
"bb-layers-file-relative": ["layerE/meta-layer"],
"oe-fragments": ["test-fragment-2"]
}
]
},
"version": "1.0"
}
""" % (sources, self.buildtools_dir, self.buildtools_filename, self.buildtools_sha256)
os.makedirs(os.path.join(self.registrypath, os.path.dirname(name)), exist_ok=True)
with open(os.path.join(self.registrypath, name), 'w') as f:
f.write(config)
self.git(['add', name], cwd=self.registrypath)
self.git(['commit', '-m', "Adding " + name], cwd=self.registrypath)
return json.loads(config)
def add_json_config_to_registry(self, name, rev, branch):
sources = """
"test-repo": {
"git-remote": {
"remotes": {
"origin": {
"uri": "file://%s"
}
},
"branch": "%s",
"rev": "%s"
}
}
""" % (self.testrepopath, branch, rev)
return self._add_json_config_to_registry_helper(name, sources)
def add_local_json_config_to_registry(self, name, path):
sources = """
"test-repo": {
"local": {
"path": "%s"
}
}
""" % (path)
return self._add_json_config_to_registry_helper(name, sources)
def add_file_to_testrepo(self, name, content, script=False):
fullname = os.path.join(self.testrepopath, name)
os.makedirs(os.path.join(self.testrepopath, os.path.dirname(name)), exist_ok=True)
with open(fullname, 'w') as f:
f.write(content)
if script:
st = os.stat(fullname)
os.chmod(fullname, st.st_mode | stat.S_IEXEC)
self.git(['add', name], cwd=self.testrepopath)
self.git(['commit', '-m', "Adding " + name], cwd=self.testrepopath)
def config_is_unchanged(self, setuppath):
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["status"])
self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
self.assertIn("Configuration in {} has not changed".format(setuppath), out[0])
del os.environ['BBPATH']
def check_setupdir_files(self, setuppath, test_file_content):
with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f:
config_upstream = json.load(f)
with open(os.path.join(setuppath, 'layers', 'test-repo', 'test-file')) as f:
self.assertEqual(f.read(), test_file_content)
bitbake_config = config_upstream["bitbake-config"]
bb_build_path = os.path.join(setuppath, 'build')
bb_conf_path = os.path.join(bb_build_path, 'conf')
self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env')))
with open(os.path.join(setuppath, 'config', "sources-fixed-revisions.json")) as f:
sources_fixed_revisions = json.load(f)
self.assertTrue('test-repo' in sources_fixed_revisions['sources'].keys())
revision = self.git(['rev-parse', 'HEAD'], cwd=self.testrepopath).strip()
self.assertEqual(revision, sources_fixed_revisions['sources']['test-repo']['git-remote']['rev'])
if "oe-template" in bitbake_config:
with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
self.assertEqual(f.read(), bitbake_config["oe-template"])
with open(os.path.join(bb_conf_path, 'test-file')) as f:
self.assertEqual(f.read(), test_file_content)
else:
with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
self.assertIn(bitbake_config["description"], f.read())
with open(os.path.join(bb_conf_path, 'conf-notes.txt')) as f:
expected_notes = bitbake_config.get("notes")
if isinstance(expected_notes, list):
expected_notes = "\n".join(expected_notes)
self.assertEqual(f.read(), expected_notes + "\n" if expected_notes else "")
with open(os.path.join(bb_conf_path, 'bblayers.conf')) as f:
bblayers = f.read()
for l in bitbake_config["bb-layers"]:
self.assertIn(os.path.join(setuppath, "layers", l), bblayers)
for l in bitbake_config.get("bb-layers-file-relative") or []:
filerelative_layer = os.path.join(
os.path.dirname(config_upstream["path"]),
l,
)
self.assertIn(filerelative_layer, bblayers)
if 'oe-fragments' in bitbake_config:
for f in bitbake_config["oe-fragments"]:
self.assertTrue(os.path.exists(os.path.join(bb_conf_path, f)))
if 'bb-env-passthrough-additions' in bitbake_config:
with open(os.path.join(bb_build_path, 'init-build-env'), 'r') as f:
init_build_env = f.read()
self.assertTrue('BB_ENV_PASSTHROUGH_ADDITIONS' in init_build_env)
self.assertTrue('BUILD_ID' in init_build_env)
self.assertTrue('BUILD_DATE' in init_build_env)
self.assertTrue('BUILD_SERVER' in init_build_env)
# a more throrough test could be to initialize a bitbake build-env, export FOO to the shell environment, set the env-passthrough on it and finally check against 'bitbake-getvar FOO'
self.config_is_unchanged(setuppath)
def get_setup_path(self, cf, c):
if c == 'gizmo':
return os.path.join(self.tempdir, 'bitbake-builds', 'this-is-a-custom-gizmo-build')
return os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, "-".join(c.split())))
def test_setup(self):
# unset BBPATH to ensure tests run in isolation from the existing bitbake environment
if 'BBPATH' in os.environ:
del os.environ['BBPATH']
# check that no arguments works
self.runbbsetup([])
# check that --help works
self.runbbsetup(["--help"])
# change to self.tempdir to work with cwd-based default settings
os.chdir(self.tempdir)
# check that the default top-dir-prefix is cwd (now self.tempdir) with no global settings
out = self.runbbsetup(["settings", "list"])
self.assertIn("default top-dir-prefix {}".format(os.getcwd()), out[0])
# set up global location for dl-dir
settings_path = "{}/global-config".format(self.tempdir)
out = self.runbbsetup(["settings", "set", "--global", "default", "dl-dir", os.path.join(self.tempdir, 'downloads')])
self.assertIn("From section 'default' the setting 'dl-dir' was changed to", out[0])
self.assertIn("Settings written to".format(settings_path), out[0])
# check that writing settings works and then adjust them to point to
# test registry repo
out = self.runbbsetup(["settings", "set", "default", "registry", "'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)])
settings_path = "{}/bitbake-builds/settings.conf".format(self.tempdir)
self.assertIn(settings_path, out[0])
self.assertIn("From section 'default' the setting 'registry' was changed to", out[0])
self.assertIn("Settings written to".format(settings_path), out[0])
# check that listing settings works
out = self.runbbsetup(["settings", "list"])
self.assertIn("default top-dir-prefix {}".format(self.tempdir), out[0])
self.assertIn("default dl-dir {}".format(os.path.join(self.tempdir, 'downloads')), out[0])
self.assertIn("default registry {}".format('git://{};protocol=file;branch=master;rev=master'.format(self.registrypath)), out[0])
# check that 'list' produces correct output with no configs, one config and two configs
out = self.runbbsetup(["list"])
self.assertNotIn("test-config-1", out[0])
self.assertNotIn("test-config-2", out[0])
json_1 = self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master')
out = self.runbbsetup(["list"])
self.assertIn("test-config-1", out[0])
self.assertNotIn("test-config-2", out[0])
json_2 = self.add_json_config_to_registry('config-2/test-config-2.conf.json', 'master', 'master')
out = self.runbbsetup(["list", "--write-json={}".format(os.path.join(self.tempdir, "test-configs.json"))])
self.assertIn("test-config-1", out[0])
self.assertIn("test-config-2", out[0])
with open(os.path.join(self.tempdir, "test-configs.json")) as f:
json_configs = json.load(f)
self.assertIn("test-config-1", json_configs)
self.assertIn("test-config-2", json_configs)
# check that init/status/update work
# (the latter two should do nothing and say that config hasn't changed)
test_file_content = 'initial\n'
self.add_file_to_testrepo('test-file', test_file_content)
# test-config-1 is tested as a registry config and over http, test-config-2 as a local file
server = HTTPService(self.registrypath, host="127.0.0.1")
server.start()
variants = (
'gadget',
'gizmo',
'gizmo-env-passthrough',
'gizmo-no-fragment',
'gadget-notemplate',
'gizmo-notemplate-fragments-one-of fragment-desc-1 fragment-nodesc-1',
)
variants_local = variants + ('gizmo-notemplate-with-filerelative-layers',)
test_configurations = ({'name':'test-config-1','cmdline': 'test-config-1',
'buildconfigs': variants},
{'name':'test-config-2','cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
'buildconfigs': variants_local},
{'name':'test-config-1','cmdline':'http://127.0.0.1:{}/test-config-1.conf.json'.format(server.port),
'buildconfigs': variants}
)
try:
for v in test_configurations:
for c in v['buildconfigs']:
out = self.runbbsetup(["init", "--non-interactive", v['cmdline']] + c.split())
setuppath = self.get_setup_path(v['name'], c)
self.check_setupdir_files(setuppath, test_file_content)
finally:
server.stop()
# install buildtools
out = self.runbbsetup(["install-buildtools", "--setup-dir", setuppath])
self.assertIn("Buildtools installed into", out[0])
self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
# change a file in the test layer repo, make a new commit and
# test that status/update correctly report the change and update the config
prev_test_file_content = test_file_content
test_file_content = 'modified\n'
self.add_file_to_testrepo('test-file', test_file_content)
for c in variants:
setuppath = self.get_setup_path('test-config-1', c)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["status"])
self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, setuppath), out[0])
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
if c in ('gadget', 'gizmo'):
self.assertIn("Leaving the previous configuration in {}/build/conf-backup.".format(setuppath), out[0])
self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
self.check_setupdir_files(setuppath, test_file_content)
# make a new branch in the test layer repo, change a file on that branch,
# make a new commit, update the top level json config to refer to that branch,
# and test that status/update correctly report the change and update the config
prev_test_file_content = test_file_content
test_file_content = 'modified-in-branch\n'
branch = "another-branch"
self.git(['checkout', '-b', branch], cwd=self.testrepopath)
self.add_file_to_testrepo('test-file', test_file_content)
json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
for c in variants:
setuppath = self.get_setup_path('test-config-1', c)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["status"])
self.assertIn("Configuration in {} has changed:".format(setuppath), out[0])
self.assertIn('- "rev": "master"\n+ "rev": "another-branch"', out[0])
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
if c in ('gadget', 'gizmo'):
self.assertIn("Leaving the previous configuration in {}/build/conf-backup.".format(setuppath), out[0])
self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
self.check_setupdir_files(setuppath, test_file_content)
# do the same as the previous test, but now without updating the bitbake configuration (--update-bb-conf=no)
# and check that files have not been modified
def _conf_chksum(confdir: str) -> list:
sums = []
for f in glob.glob(f'{confdir}/*'):
if os.path.isfile(f) and not os.path.islink(f):
with open(f, 'rb') as fd:
sha = hashlib.sha256()
sha.update(fd.read())
sums.append(os.path.basename(f) + '_' + sha.hexdigest())
return sums
prev_test_file_content = test_file_content
test_file_content = 'modified-in-branch-no-bb-conf-update\n'
branch = "another-branch-no-bb-conf-update"
self.git(['checkout', '-b', branch], cwd=self.testrepopath)
self.add_file_to_testrepo('test-file', test_file_content)
json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
for c in variants:
setuppath = self.get_setup_path('test-config-1', c)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
# write something in local.conf and bblayers.conf
for f in ["local.conf", "bblayers.conf"]:
with open(f"{setuppath}/build/conf/{f}", "w") as fd:
fd.write("deadbeef")
sums_before = _conf_chksum(f"{setuppath}/build/conf")
out = self.runbbsetup(["update", "--update-bb-conf=no"])
sums_after = _conf_chksum(f"{setuppath}/build/conf")
self.assertEqual(sums_before, sums_after)
def _check_local_sources(custom_setup_dir):
custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir)
custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo')
self.assertTrue(os.path.islink(custom_layer_path))
self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
self.config_is_unchanged(custom_setup_path)
def _check_layer_backups(layer_path, expected_backups):
files = os.listdir(layer_path)
backups = len([f for f in files if 'backup' in f])
self.assertEqual(backups, expected_backups, msg = "Expected {} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files))
# Change the configuration to refer to a local source, then to another local source, then back to a git remote
# Run status/update after each change and verify that nothing breaks
# Also check that layer backups happen when expected
c = 'gadget'
setuppath = self.get_setup_path('test-config-1', c)
self.config_is_unchanged(setuppath)
layers_path = os.path.join(setuppath, 'layers')
layer_path = os.path.join(layers_path, 'test-repo')
_check_layer_backups(layers_path, 0)
json_1 = self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
_check_local_sources(setuppath)
_check_layer_backups(layers_path, 1)
prev_path = self.testrepopath
self.testrepopath = prev_path + "-2"
self.git(['clone', prev_path, self.testrepopath], cwd=self.tempdir)
json_1 = self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
_check_local_sources(setuppath)
_check_layer_backups(layers_path, 1)
self.testrepopath = prev_path
json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
self.check_setupdir_files(setuppath, test_file_content)
_check_layer_backups(layers_path, 1)
## edit a file without making a commit
with open(os.path.join(layer_path, 'local-modification'), 'w') as f:
f.write('locally-modified\n')
test_file_content = "modified-again\n"
self.add_file_to_testrepo('test-file', test_file_content)
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
_check_layer_backups(layers_path, 1)
## edit a file and make a commit such that no rebase conflicts occur
with open(os.path.join(layer_path, 'local-modification'), 'w') as f:
f.write('locally-modified-again\n')
self.git(['add', '.'], cwd=layer_path)
self.git(['commit', '-m', 'Adding a local modification'], cwd=layer_path)
test_file_content = "modified-again-and-again\n"
self.add_file_to_testrepo('test-file', test_file_content)
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
_check_layer_backups(layers_path, 1)
## edit a file and make a commit in a way that causes a rebase conflict
with open(os.path.join(layer_path, 'test-file'), 'w') as f:
f.write('locally-modified\n')
self.git(['add', '.'], cwd=layer_path)
self.git(['commit', '-m', 'Adding a local modification'], cwd=layer_path)
test_file_content = "remotely-modified\n"
self.add_file_to_testrepo('test-file', test_file_content)
with self.assertRaisesRegex(bb.process.ExecutionError, "Merge conflict in test-file"):
out = self.runbbsetup(["update", "--update-bb-conf=yes"])
_check_layer_backups(layers_path, 1)
# check source overrides, local sources provided with symlinks, and custom setup dir name
source_override_content = """
{
"sources": {
"test-repo": {
"local": {
"path": "."
}
}
}
}"""
override_filename = 'source-overrides.json'
custom_setup_dir = 'special-setup-dir'
self.add_file_to_testrepo(override_filename, source_override_content)
out = self.runbbsetup(["init", "--non-interactive", "--source-overrides", os.path.join(self.testrepopath, override_filename), "--setup-dir-name", custom_setup_dir, "test-config-1", "gadget"])
_check_local_sources(custom_setup_dir)
# same but use command line options to specify local overrides
custom_setup_dir = 'special-setup-dir-with-cmdline-overrides'
out = self.runbbsetup(["init", "--non-interactive", "-L", "test-repo", self.testrepopath, "--setup-dir-name", custom_setup_dir, "test-config-1", "gadget"])
_check_local_sources(custom_setup_dir)
def test_vscode(self):
if 'BBPATH' in os.environ:
del os.environ['BBPATH']
os.chdir(self.tempdir)
self.runbbsetup(["settings", "set", "default", "registry", "'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)])
self.add_file_to_testrepo('test-file', 'initial\n')
self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master')
# --init-vscode should create bitbake.code-workspace
self.runbbsetup(["init", "--non-interactive", "--init-vscode", "test-config-1", "gadget"])
setuppath = self.get_setup_path('test-config-1', 'gadget')
workspace_file = os.path.join(setuppath, 'bitbake.code-workspace')
self.assertTrue(os.path.exists(workspace_file),
"bitbake.code-workspace should be created with --init-vscode")
with open(workspace_file) as f:
workspace = json.load(f)
# top-level structure
self.assertIn('folders', workspace)
self.assertIn('settings', workspace)
self.assertIn('extensions', workspace)
self.assertIn('yocto-project.yocto-bitbake',
workspace['extensions']['recommendations'])
# folders: conf dir + test-repo (symlinks like oe-init-build-env-dir are skipped)
folder_names = {f['name'] for f in workspace['folders']}
self.assertIn('conf', folder_names)
self.assertIn('test-repo', folder_names)
# folder paths must be relative so the workspace is portable
for f in workspace['folders']:
self.assertFalse(os.path.isabs(f['path']),
"Folder path should be relative, got: {}".format(f['path']))
# bitbake extension settings
settings = workspace['settings']
self.assertTrue(settings.get('bitbake.disableConfigModification'))
self.assertEqual(settings['bitbake.pathToBuildFolder'],
os.path.join(setuppath, 'build'))
self.assertEqual(settings['bitbake.pathToEnvScript'],
os.path.join(setuppath, 'build', 'init-build-env'))
# file associations
self.assertIn('*.conf', settings.get('files.associations', {}))
self.assertIn('*.inc', settings.get('files.associations', {}))
# python extra paths: test-repo/scripts/ exists and should be listed
extra_paths = settings.get('python.analysis.extraPaths', [])
self.assertTrue(any('scripts' in p for p in extra_paths),
"python.analysis.extraPaths should include the scripts dir")
self.assertEqual(settings.get('python.analysis.extraPaths'),
settings.get('python.autoComplete.extraPaths'))
# --no-init-vscode should NOT create a workspace file
self.runbbsetup(["init", "--non-interactive", "--no-init-vscode", "test-config-1", "gadget-notemplate"])
notemplate_path = self.get_setup_path('test-config-1', 'gadget-notemplate')
self.assertFalse(
os.path.exists(os.path.join(notemplate_path, 'bitbake.code-workspace')),
"bitbake.code-workspace should not be created with --no-init-vscode")
# update with --init-vscode after a layer change should preserve
# user-added folders and settings while still rewriting managed ones
workspace['folders'].append({"name": "user-folder", "path": "user/custom"})
workspace['settings']['my.user.setting'] = 'preserved'
with open(workspace_file, 'w') as f:
json.dump(workspace, f, indent=4)
self.add_file_to_testrepo('test-file', 'updated\n')
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
self.runbbsetup(["update", "--update-bb-conf=no"])
del os.environ['BBPATH']
with open(workspace_file) as f:
updated = json.load(f)
self.assertIn('user/custom', {f['path'] for f in updated['folders']},
"User-added folder was removed during update")
self.assertIn('my.user.setting', updated['settings'],
"User-added setting was removed during update")
# update with a corrupt workspace file should log an error and leave it unchanged
self.add_file_to_testrepo('test-file', 'updated-again\n')
with open(workspace_file, 'w') as f:
f.write('{invalid json')
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
self.runbbsetup(["update", "--update-bb-conf=no"])
del os.environ['BBPATH']
with open(workspace_file) as f:
content = f.read()
self.assertEqual(content, '{invalid json',
"Corrupt workspace file should not be modified")
def _count_layer_backups(self, layers_path):
return len([f for f in os.listdir(layers_path) if 'backup' in f])
def test_install_buildtools_fetch(self):
"""Test that install-buildtools uses bb.fetch with sha256 and passes --local-file"""
import shutil
if 'BBPATH' in os.environ:
del os.environ['BBPATH']
os.chdir(self.tempdir)
registry_uri = "git://{};protocol=file;branch=master;rev=master".format(
self.registrypath)
self.runbbsetup(["settings", "set", "default", "registry", registry_uri])
self.add_file_to_testrepo('test-file', 'initial\n')
self.add_json_config_to_registry('test-config-bt.conf.json', 'master', 'master')
self.runbbsetup(["init", "--non-interactive", "test-config-bt", "gadget"])
setuppath = self.get_setup_path('test-config-bt', 'gadget')
# test config-driven install (url + sha256sum from config)
out = self.runbbsetup(["install-buildtools", "--setup-dir", setuppath])
self.assertIn("Buildtools installed into", out[0])
self.assertIn("install-buildtools local-file=", out[0])
self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
# test CLI overrides (use a different file to prove precedence)
shutil.rmtree(os.path.join(setuppath, 'buildtools'))
alt_filename = "alt-buildtools-test.sh"
alt_filepath = os.path.join(self.buildtools_dir, alt_filename)
with open(alt_filepath, 'w') as f:
f.write("#!/bin/sh\necho alt\n")
with open(alt_filepath, 'rb') as f:
alt_sha256 = hashlib.sha256(f.read()).hexdigest()
alt_url = "file://{}/{}".format(self.buildtools_dir, alt_filename)
out = self.runbbsetup(["install-buildtools", "--setup-dir", setuppath,
"--url", alt_url,
"--sha256", alt_sha256])
self.assertIn("Buildtools installed into", out[0])
self.assertIn("install-buildtools local-file=", out[0])
self.assertIn(alt_filename, out[0])
# test missing sha256sum is a hard error
shutil.rmtree(os.path.join(setuppath, 'buildtools'), ignore_errors=True)
with open(os.path.join(setuppath, 'config', 'config-upstream.json')) as f:
config_upstream = json.load(f)
config_upstream['bitbake-config']['install-buildtools'] = {
'url': 'file://{}/{}'.format(self.buildtools_dir, self.buildtools_filename)
}
with open(os.path.join(setuppath, 'config', 'config-upstream.json'), 'w') as f:
json.dump(config_upstream, f)
with self.assertRaises(bb.process.ExecutionError):
self.runbbsetup(["install-buildtools", "--setup-dir", setuppath, "--force"])
def test_update_rebase_conflicts_strategy(self):
"""Test the --rebase-conflicts-strategy option for the update command.
Covers three scenarios not exercised by test_setup:
1. Uncommitted tracked-file change (LocalModificationsError) + default 'abort'
strategy → clean error message containing 'has uncommitted changes' and a
hint at --rebase-conflicts-strategy=backup; no backup directory is created.
2. Same uncommitted change + 'backup' strategy → directory is renamed to a
timestamped backup and the layer is re-cloned cleanly.
3. Committed local change that conflicts with an incoming upstream commit
(RebaseError):
a. Default 'abort' strategy → error containing 'Merge conflict' and the
--rebase-conflicts-strategy=backup hint; no backup directory is created.
b. 'backup' strategy → backup + re-clone instead of a hard failure.
"""
if 'BBPATH' in os.environ:
del os.environ['BBPATH']
os.chdir(self.tempdir)
self.runbbsetup(["settings", "set", "default", "registry", "'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)])
self.add_file_to_testrepo('test-file', 'initial\n')
self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master')
self.runbbsetup(["init", "--non-interactive", "test-config-1", "gadget"])
setuppath = self.get_setup_path('test-config-1', 'gadget')
layer_path = os.path.join(setuppath, 'layers', 'test-repo')
layers_path = os.path.join(setuppath, 'layers')
# Scenario 1: uncommitted tracked change, default 'abort' strategy
# Advance upstream so an update is required.
self.add_file_to_testrepo('test-file', 'upstream-v2\n')
# Modify the same tracked file in the layer without committing.
with open(os.path.join(layer_path, 'test-file'), 'w') as f:
f.write('locally-modified\n')
os.environ['BBPATH'] = os.path.join(setuppath, 'build')
with self.assertRaises(bb.process.ExecutionError) as ctx:
self.runbbsetup(["update", "--update-bb-conf=no"])
self.assertIn('has uncommitted changes', str(ctx.exception))
self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception))
# No backup directory must have been created.
self.assertEqual(self._count_layer_backups(layers_path), 0,
"abort strategy must not create any backup")
# Scenario 2: same uncommitted change, 'backup' strategy
out = self.runbbsetup(["update", "--update-bb-conf=no", "--rebase-conflicts-strategy=backup"])
# One backup directory must now exist.
self.assertEqual(self._count_layer_backups(layers_path), 1,
"backup strategy must create exactly one backup")
# The re-cloned layer must be clean and at the upstream revision.
with open(os.path.join(layer_path, 'test-file')) as f:
self.assertEqual(f.read(), 'upstream-v2\n',
"re-cloned layer must contain the upstream content")
status = self.git(['status', '--porcelain'], cwd=layer_path).strip()
self.assertEqual(status, '',
"re-cloned layer must have no local modifications")
del os.environ['BBPATH']
# Scenario 3: committed conflicting change, 'backup' strategy
# Re-initialise a fresh setup so we start from a clean state.
self.runbbsetup(["init", "--non-interactive", "--setup-dir-name", "rebase-conflict-setup", "test-config-1", "gadget"])
conflict_setup = os.path.join(self.tempdir, 'bitbake-builds', 'rebase-conflict-setup')
conflict_layer = os.path.join(conflict_setup, 'layers', 'test-repo')
conflict_layers = os.path.join(conflict_setup, 'layers')
# Commit a local change that touches the same file as the next upstream commit.
with open(os.path.join(conflict_layer, 'test-file'), 'w') as f:
f.write('conflicting-local\n')
self.git(['add', 'test-file'], cwd=conflict_layer)
self.git(['commit', '-m', 'Local conflicting change'], cwd=conflict_layer)
# Advance upstream with a conflicting edit.
self.add_file_to_testrepo('test-file', 'conflicting-upstream\n')
os.environ['BBPATH'] = os.path.join(conflict_setup, 'build')
# Default stop strategy must still fail with a conflict error and include
# the --rebase-conflicts-strategy=backup hint (same handler as LocalModificationsError).
with self.assertRaises(bb.process.ExecutionError) as ctx:
self.runbbsetup(["update", "--update-bb-conf=no"])
self.assertIn('Merge conflict in test-file', str(ctx.exception))
self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception))
self.assertEqual(self._count_layer_backups(conflict_layers), 0)
# Backup strategy must succeed: backup the conflicted dir and re-clone.
self.runbbsetup(["update", "--update-bb-conf=no", "--rebase-conflicts-strategy=backup"])
self.assertEqual(self._count_layer_backups(conflict_layers), 1,
"backup strategy must create exactly one backup after a conflict")
with open(os.path.join(conflict_layer, 'test-file')) as f:
self.assertEqual(f.read(), 'conflicting-upstream\n',
"re-cloned layer must contain the upstream content after conflict backup")
del os.environ['BBPATH']
@unittest.skipIf(os.environ.get("BB_SKIP_PYPI_TESTS", "yes") != "no",
"PyPI packaging test (set BB_SKIP_PYPI_TESTS=no to run)")
class PyPIPackagingTest(unittest.TestCase):
"""Tests for PyPI packaging of bitbake-setup.
These tests build a wheel from source, install it in an isolated venv,
and verify the package works correctly. Skipped by default unless
BB_SKIP_PYPI_TESTS=no is set.
"""
wheel_path = None
build_tempdir = None
workspace_dir = None
@classmethod
def setUpClass(cls):
"""Build wheel once for all tests in this class."""
# Locate packaging-pypi/bitbake-setup directory
cls.pypi_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', 'packaging-pypi', 'bitbake-setup')
)
# Check for required build tools
cls._check_build_tools()
# Create temporary directory for packaging workspace
cls.build_tempdir = tempfile.mkdtemp(prefix="bitbake-pypi-build-")
# Run package-bitbake-setup.py to create packaging workspace
cls._create_packaging_workspace()
# Build the wheel
cls._build_wheel()
@classmethod
def _check_build_tools(cls):
"""Verify build tools are available, skip if not."""
try:
result = subprocess.run(
[sys.executable, "-m", "build", "--version"],
capture_output=True, check=True
)
except (subprocess.CalledProcessError, FileNotFoundError):
raise unittest.SkipTest("'build' package not installed (pip install build)")
@classmethod
def _create_packaging_workspace(cls):
"""Create packaging workspace using package-bitbake-setup.py."""
script = os.path.join(cls.pypi_dir, 'package-bitbake-setup.py')
cls.workspace_dir = os.path.join(cls.build_tempdir, 'workspace')
result = subprocess.run(
[sys.executable, script, '-d', cls.workspace_dir],
capture_output=True, text=True
)
if result.returncode != 0:
raise unittest.SkipTest(f"Packaging workspace creation failed: {result.stderr}")
@classmethod
def _build_wheel(cls):
"""Build the wheel in the packaging workspace."""
result = subprocess.run(
[sys.executable, "-m", "build", "--wheel"],
cwd=cls.workspace_dir,
capture_output=True, text=True
)
if result.returncode != 0:
raise unittest.SkipTest(f"Wheel build failed: {result.stderr}")
# Find the built wheel
dist_dir = os.path.join(cls.workspace_dir, 'dist')
wheels = glob.glob(os.path.join(dist_dir, '*.whl'))
if not wheels:
raise unittest.SkipTest("No wheel file found after build")
cls.wheel_path = wheels[0]
@classmethod
def tearDownClass(cls):
"""Clean up the shared wheel build artifacts."""
if cls.build_tempdir and os.environ.get("BB_TMPDIR_NOCLEAN") != "yes":
shutil.rmtree(cls.build_tempdir, ignore_errors=True)
elif cls.build_tempdir:
print(f"Not cleaning up {cls.build_tempdir}. Please remove manually.")
def setUp(self):
"""Create isolated venv for testing the installed package."""
self.venv_dir = tempfile.mkdtemp(prefix="bitbake-pypi-venv-")
# Create venv without pip (faster, no network needed)
venv.create(self.venv_dir, with_pip=False, symlinks=True)
# Get paths to venv python and bin directory
self.venv_python = os.path.join(self.venv_dir, 'bin', 'python')
self.venv_bin = os.path.join(self.venv_dir, 'bin')
# Install wheel using pip from the outer environment (offline, no-deps)
site_packages = self._get_site_packages()
result = subprocess.run(
[sys.executable, "-m", "pip", "install",
"--target", site_packages,
"--no-deps", "--no-index",
self.wheel_path],
capture_output=True, text=True
)
if result.returncode != 0:
self.fail(f"Failed to install wheel: {result.stderr}")
# Install console script entry point manually
self._install_console_script()
def _get_site_packages(self):
"""Get the site-packages directory in the venv."""
lib_dir = os.path.join(self.venv_dir, 'lib')
# Find python version directory
for d in os.listdir(lib_dir):
if d.startswith('python'):
return os.path.join(lib_dir, d, 'site-packages')
raise RuntimeError("Could not find site-packages in venv")
def _install_console_script(self):
"""Create console script wrapper in venv bin directory."""
site_packages = self._get_site_packages()
script_path = os.path.join(self.venv_bin, 'bitbake-setup')
script_content = f'''#!{self.venv_python}
import sys
sys.path.insert(0, "{site_packages}")
from bitbake_setup.__main__ import main
sys.exit(main())
'''
with open(script_path, 'w') as f:
f.write(script_content)
os.chmod(script_path, 0o755)
def tearDown(self):
"""Remove venv after test."""
if os.environ.get("BB_TMPDIR_NOCLEAN") != "yes":
shutil.rmtree(self.venv_dir, ignore_errors=True)
else:
print(f"Not cleaning up {self.venv_dir}. Please remove manually.")
def test_imports(self):
"""Verify all expected modules can be imported from installed package."""
site_packages = self._get_site_packages()
imports = ['bb', 'bitbake_setup']
for module in imports:
result = subprocess.run(
[self.venv_python, '-c', f'import sys; sys.path.insert(0, "{site_packages}"); import {module}'],
capture_output=True, text=True
)
self.assertEqual(result.returncode, 0,
f"Failed to import {module}: {result.stderr}")
def test_console_script_help(self):
"""Verify bitbake-setup --help runs successfully."""
script = os.path.join(self.venv_bin, 'bitbake-setup')
result = subprocess.run(
[script, '--help'],
capture_output=True, text=True
)
self.assertEqual(result.returncode, 0,
f"bitbake-setup --help failed: {result.stderr}")
self.assertIn('usage:', result.stdout.lower())
def test_console_script_list(self):
"""Verify bitbake-setup list runs successfully."""
script = os.path.join(self.venv_bin, 'bitbake-setup')
result = subprocess.run(
[script, 'list'],
capture_output=True, text=True
)
# List may return 0 even with no configurations
self.assertEqual(result.returncode, 0,
f"bitbake-setup list failed: {result.stderr}")
def test_package_metadata(self):
"""Verify package metadata is correctly set."""
site_packages = self._get_site_packages()
code = '''
import json
import sys
sys.path.insert(0, "{}")
from importlib.metadata import metadata
m = metadata("bitbake-setup")
print(json.dumps({{
"name": m["Name"],
"version": m["Version"],
"requires_python": m.get("Requires-Python", ""),
"license": m.get("License", ""),
}}))
'''.format(site_packages)
result = subprocess.run(
[self.venv_python, '-c', code],
capture_output=True, text=True
)
self.assertEqual(result.returncode, 0,
f"Failed to get metadata: {result.stderr}")
meta = json.loads(result.stdout)
self.assertEqual(meta['name'], 'bitbake-setup')
self.assertIn('>=3.9', meta['requires_python'])
def test_vendored_dependencies(self):
"""Verify vendored dependencies (bs4, ply, progressbar, simplediff) are not bundled in package."""
# Check that vendored packages do not exist in root of wheel
with zipfile.ZipFile(self.wheel_path, 'r') as whl:
names = whl.namelist()
# Check for expected package directories
expected = ['bs4/', 'ply/', 'progressbar/', 'simplediff/']
for pkg in expected:
found = any(n.startswith(pkg) for n in names)
self.assertFalse(found,
f"Unexpected vendored package '{pkg}' found in wheel")
def test_version_from_wheel(self):
"""Verify version is set correctly (not fallback 0.0.0 unless expected)."""
import re
# Extract version from wheel filename
wheel_name = os.path.basename(self.wheel_path)
# Wheel format: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
parts = wheel_name.split('-')
version = parts[1]
# Check version format (should be semver-like or contain git info)
version_pattern = r'^\d+\.\d+\.\d+.*$'
self.assertTrue(re.match(version_pattern, version),
f"Version '{version}' doesn't match expected pattern")
print(f"Extracted version from wheel: {version}")
self.assertNotEqual(version, '0.0.0',
"Version is fallback 0.0.0 - no git tags found")
def test_wheel_metadata_file(self):
"""Verify wheel METADATA file contains required fields."""
with zipfile.ZipFile(self.wheel_path, 'r') as whl:
# Find METADATA file in dist-info
metadata_path = None
for name in whl.namelist():
if name.endswith('.dist-info/METADATA'):
metadata_path = name
break
self.assertIsNotNone(metadata_path, "METADATA file not found in wheel")
# Parse metadata
metadata_content = whl.read(metadata_path).decode('utf-8')
# Check required fields
self.assertIn('Metadata-Version:', metadata_content)
self.assertIn('Name: bitbake-setup', metadata_content)
self.assertIn('Version:', metadata_content)
self.assertIn('Requires-Python:', metadata_content)
def test_entry_points(self):
"""Verify console script entry points are correctly defined."""
with zipfile.ZipFile(self.wheel_path, 'r') as whl:
# Find entry_points.txt in dist-info
entry_points_path = None
for name in whl.namelist():
if name.endswith('.dist-info/entry_points.txt'):
entry_points_path = name
break
self.assertIsNotNone(entry_points_path,
"entry_points.txt not found in wheel")
content = whl.read(entry_points_path).decode('utf-8')
self.assertIn('[console_scripts]', content)
self.assertIn('bitbake-setup', content)
self.assertIn('bitbake_setup.__main__:main', content)