| #!/usr/bin/env python3 |
| |
| # |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| |
| import argparse |
| import configparser |
| import copy |
| import datetime |
| import functools |
| import glob |
| import json |
| import logging |
| import os |
| import shutil |
| import signal |
| import string |
| import subprocess |
| import sys |
| import textwrap |
| import time |
| |
| __version__ = "2.19.1" |
| |
| bindir = os.path.abspath(os.path.dirname(__file__)) |
| sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')] |
| |
| import bb.msg # noqa: E402 |
| import bb.fetch # noqa: E402 |
| import bb.process # noqa: E402 |
| from bb import __version__ as libbb_version # noqa: E402 |
| |
| if __name__ == "__main__": |
| if __version__ != libbb_version: |
| sys.exit("Bitbake core library version and program version mismatch!") |
| |
| logger = bb.msg.logger_create('bitbake-setup', sys.stdout) |
| |
| # These settings can only be set in the global context |
| GLOBAL_ONLY_SETTINGS = ( |
| "top-dir-prefix", |
| "top-dir-name", |
| ) |
| |
| def color_enabled() -> bool: |
| """ |
| Our logger has a BBLogFormatter formatter which holds whether color is |
| enabled or not. Return this value. |
| """ |
| return logger.handlers[0].formatter.color_enabled |
| |
| def get_diff_color_param() -> str: |
| return "--color=always" if color_enabled() else "--color=never" |
| |
| def print_configs(prompt: str, choices: list[str], descriptions: list[str] = []): |
| """ |
| Helper function to print a list of choices and align the output. |
| Each option name is made bold to stand out, unless color is not enabled in |
| our logger. |
| """ |
| if not prompt.endswith(':'): |
| prompt += ":" |
| logger.plain(prompt) |
| |
| if not descriptions: |
| descriptions = ["" for _ in choices] |
| |
| # maximum size of all choices, for alignment |
| cmax = max([len(c) for c in choices]) + 1 |
| |
| for n, c in enumerate(choices): |
| msg = f"{n + 1}. " |
| if color_enabled(): |
| # make it bold |
| msg += "\033[1m" |
| msg += f"{c:<{cmax}}" |
| if color_enabled(): |
| msg += "\033[0m" |
| msg += f" {descriptions[n]}" |
| logger.plain(msg) |
| |
| # If bitbake is from a release tarball or somewhere like pypi where |
| # updates may not be straightforward, prefer to use the git repo as the |
| # default registry |
| def get_default_registry(): |
| internal_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry") |
| git_registry = "git://git.openembedded.org/bitbake;protocol=https;branch=master;rev=master" |
| if os.path.exists(os.path.dirname(__file__) + "/../.git"): |
| return internal_registry |
| else: |
| return git_registry |
| |
| def cache_dir(top_dir): |
| return os.path.join(top_dir, '.bitbake-setup-cache') |
| |
| def init_bb_cache(top_dir, settings, args): |
| dldir = settings["default"]["dl-dir"] |
| bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache') |
| os.makedirs(bb_cachedir, exist_ok=True) |
| |
| d = bb.data.init() |
| d.setVar("DL_DIR", dldir) |
| d.setVar("BB_CACHEDIR", bb_cachedir) |
| d.setVar("__BBSRCREV_SEEN", "1") |
| if args.no_network: |
| d.setVar("BB_SRCREV_POLICY", "cache") |
| bb.fetch.fetcher_init(d) |
| return d |
| |
| def save_bb_cache(): |
| bb.fetch.fetcher_parse_save() |
| bb.fetch.fetcher_parse_done() |
| |
| def get_config_name(config): |
| suffix = '.conf.json' |
| config_file = os.path.basename(config) |
| if config_file.endswith(suffix): |
| return config_file[:-len(suffix)] |
| else: |
| raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix)) |
| |
| def write_upstream_config(config_dir, config_data): |
| with open(os.path.join(config_dir, "config-upstream.json"),'w') as s: |
| json.dump(config_data, s, sort_keys=True, indent=4) |
| |
| def write_sources_fixed_revisions(config_dir, layer_dir, config_data): |
| json_path = os.path.join(config_dir, "sources-fixed-revisions.json") |
| json_link = os.path.join(layer_dir, "sources-fixed-revisions.json") |
| sources = {} |
| sources['sources'] = config_data |
| with open(os.path.join(config_dir, "sources-fixed-revisions.json"),'w') as s: |
| json.dump(sources, s, sort_keys=True, indent=4) |
| if not os.path.lexists(json_link): |
| os.symlink(os.path.relpath(json_path ,layer_dir), json_link) |
| |
| def commit_config(config_dir): |
| bb.process.run(['git', '-C', config_dir, 'add', '.']) |
| bb.process.run(['git', '-C', config_dir, 'commit', '--allow-empty', '--no-verify', '-a', '-m', 'Configuration at ' + time.asctime()]) |
| |
| def _write_layer_list(dest, repodirs): |
| layers = [] |
| for r in repodirs: |
| for root, _, files in os.walk(os.path.join(dest,r)): |
| if os.path.basename(root) == 'conf' and 'layer.conf' in files: |
| layers.append(os.path.relpath(os.path.dirname(root), dest)) |
| layers_f = os.path.join(dest, ".oe-layers.json") |
| with open(layers_f, 'w') as f: |
| json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4) |
| |
| def add_unique_timestamp_to_path(path): |
| timestamp = time.strftime("%Y%m%d%H%M%S") |
| path_unique = "{}.{}".format(path, timestamp) |
| if os.path.exists(path_unique): |
| import itertools |
| for i in itertools.count(start=1): |
| path_unique = "{}.{}.{}".format(path, timestamp, i) |
| if not os.path.exists(path_unique): |
| break |
| return path_unique |
| |
| def _get_remotes(r_remote): |
| remotes = [] |
| |
| if 'remotes' not in r_remote and 'uri' not in r_remote: |
| raise Exception("Expected key(s): 'remotes', 'uri'") |
| |
| if 'remotes' in r_remote: |
| for remote in r_remote['remotes']: |
| remotes.append(r_remote['remotes'][remote]['uri']) |
| |
| if 'uri' in r_remote: |
| remotes.append(r_remote['uri']) |
| |
| return remotes |
| |
| def checkout_layers(layers, confdir, layerdir, d, rebase_conflicts_strategy='abort'): |
| def _checkout_git_remote(r_remote, repodir, layers_fixed_revisions): |
| rev = r_remote['rev'] |
| branch = r_remote.get('branch', None) |
| |
| remotes = _get_remotes(r_remote) |
| |
| for remote in remotes: |
| prot,host,path,user,pswd,params = bb.fetch.decodeurl(remote) |
| fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) |
| logger.plain(" {}".format(r_name)) |
| if branch: |
| src_uri = f"{fetchuri};protocol={prot};rev={rev};branch={branch};destsuffix={repodir}" |
| else: |
| src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}" |
| fetcher = bb.fetch.Fetch([src_uri], d) |
| repodir_path = os.path.join(layerdir, repodir) |
| try: |
| do_fetch(fetcher, layerdir) |
| except (bb.fetch.LocalModificationsError, bb.fetch.RebaseError) as e: |
| if rebase_conflicts_strategy != 'backup': |
| e.msg += ("\nUse 'bitbake-setup update --rebase-conflicts-strategy=backup'" |
| " to automatically back up the directory and re-clone from upstream," |
| " or use 'bitbake-setup init -L %s /path/to/local/checkout'" |
| " to work with a local checkout instead." % r_name) |
| raise |
| backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') |
| logger.warning( |
| "%s\n" |
| "Renaming %s to %s to preserve your work, then re-cloning from upstream.", |
| e, repodir_path, backup_path) |
| os.rename(repodir_path, backup_path) |
| fetcher.unpack(layerdir) |
| urldata = fetcher.ud[src_uri] |
| revision = urldata.revision |
| layers_fixed_revisions[r_name]['git-remote']['rev'] = revision |
| |
| def _symlink_local(src, dst): |
| logger.plain("Making a symbolic link {} pointing to {}".format(dst, src)) |
| os.symlink(src, dst) |
| |
| layers_fixed_revisions = copy.deepcopy(layers) |
| repodirs = [] |
| oesetupbuild = None |
| logger.plain("Fetching layer/tool repositories into {}".format(layerdir)) |
| for r_name in layers: |
| r_data = layers[r_name] |
| repodir = r_data.get("path", r_name) |
| repodirs.append(repodir) |
| |
| r_remote = r_data.get('git-remote') |
| r_local = r_data.get('local') |
| if r_remote and r_local: |
| raise Exception("Source {} contains both git-remote and local properties.".format(r_name)) |
| |
| repodir_path = os.path.join(layerdir, repodir) |
| if os.path.lexists(repodir_path): |
| if os.path.islink(repodir_path): |
| os.remove(repodir_path) |
| elif r_local: |
| backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') |
| logger.warning("""Source {} in {} contains local modifications. Renaming to {} to preserve them. |
| For local development work it is recommended to clone the needed layers separately and re-initialize using -L option: |
| bitbake-setup init -L {} /path/to/repo/checkout""".format( |
| r_name, repodir_path, backup_path, r_name)) |
| os.rename(repodir_path, backup_path) |
| |
| if r_remote: |
| _checkout_git_remote(r_remote, repodir, layers_fixed_revisions) |
| if r_local: |
| _symlink_local(os.path.expanduser(r_local["path"]), repodir_path) |
| |
| if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')): |
| oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build') |
| oeinitbuildenvdir = os.path.join(layerdir, repodir) |
| |
| logger.plain(" ") |
| _write_layer_list(layerdir, repodirs) |
| |
| if oesetupbuild: |
| links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir} |
| for link,item in links.items(): |
| symlink = os.path.join(layerdir, link) |
| if os.path.lexists(symlink): |
| os.remove(symlink) |
| os.symlink(os.path.relpath(item,layerdir),symlink) |
| |
| return layers_fixed_revisions |
| |
| def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode=False): |
| def _setup_build_conf(layers, filerelative_layers, build_conf_dir): |
| os.makedirs(build_conf_dir) |
| layers_s = [] |
| |
| for layer in layers: |
| layer = os.path.join(layerdir, layer) |
| layers_s.append(" {} \\".format(layer)) |
| |
| for layer in filerelative_layers: |
| if thisdir: |
| layer = os.path.join(thisdir, layer) |
| else: |
| raise Exception("Configuration is using bb-layers-file-relative to specify " \ |
| "a layer path relative to itself. This can be done only " \ |
| "when the configuration is specified by its path on local " \ |
| "disk, not when it's in a registry or is fetched over http.") |
| layers_s.append(" {} \\".format(layer)) |
| |
| layers_s = "\n".join(layers_s) |
| bblayers_conf = """BBLAYERS ?= " \\ |
| {} |
| " |
| """.format(layers_s) |
| with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f: |
| f.write(bblayers_conf) |
| |
| local_conf = """# |
| # This file is intended for local configuration tweaks. |
| # |
| # If you would like to publish and share changes made to this file, |
| # it is recommended to put them into a distro config, or to create |
| # layer fragments from changes made here. |
| # |
| """ |
| with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f: |
| f.write(local_conf) |
| |
| with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f: |
| f.write("") |
| |
| with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f: |
| f.write(bitbake_config["description"] + "\n") |
| |
| with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f: |
| notes = bitbake_config.get("notes") |
| if isinstance(notes, list): |
| notes = "\n".join(notes) |
| f.write(notes + "\n" if notes else "") |
| |
| def _make_init_build_env(builddir, oeinitbuildenvdir): |
| builddir = os.path.realpath(builddir) |
| cmd = 'cd {}\nset {}\n. ./oe-init-build-env\nset --\n'.format(oeinitbuildenvdir, builddir) |
| initbuild_in_builddir = os.path.join(builddir, 'init-build-env') |
| |
| with open(initbuild_in_builddir, 'w') as f: |
| f.write("# init-build-env wrapper created by bitbake-setup\n") |
| f.write(cmd + '\n') |
| |
| def _prepend_passthrough_to_init_build_env(builddir): |
| env = bitbake_config.get("bb-env-passthrough-additions") |
| if not env: |
| return |
| |
| initbuild_in_builddir = os.path.join(builddir, 'init-build-env') |
| with open(initbuild_in_builddir) as f: |
| content = f.read() |
| |
| joined = " \\\n".join(env) |
| env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n" |
| env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n" |
| env += joined |
| env += '"' |
| |
| with open(initbuild_in_builddir, 'w') as f: |
| f.write("# environment passthrough added by bitbake-setup\n") |
| f.write(env + '\n') |
| f.write('\n') |
| f.write(content) |
| |
| bitbake_builddir = os.path.join(setupdir, "build") |
| logger.plain("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir)) |
| |
| template = bitbake_config.get("oe-template") |
| layers = bitbake_config.get("bb-layers") |
| if not template and not layers: |
| logger.error("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.") |
| return |
| oesetupbuild = os.path.join(layerdir, 'setup-build') |
| if template and not os.path.exists(oesetupbuild): |
| raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template)) |
| |
| bitbake_confdir = os.path.join(bitbake_builddir, 'conf') |
| backup_bitbake_confdir = add_unique_timestamp_to_path(os.path.join(bitbake_builddir, 'conf-backup')) |
| upstream_bitbake_confdir = add_unique_timestamp_to_path(os.path.join(bitbake_builddir, 'conf-upstream')) |
| |
| if os.path.exists(bitbake_confdir): |
| os.rename(bitbake_confdir, backup_bitbake_confdir) |
| |
| if layers: |
| filerelative_layers = bitbake_config.get("bb-layers-file-relative") or [] |
| _setup_build_conf(layers, filerelative_layers, bitbake_confdir) |
| |
| if template: |
| bb.process.run([oesetupbuild, "setup", "-c", template, "-b", bitbake_builddir, "--no-shell"]) |
| else: |
| oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir') |
| if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")): |
| logger.error("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment") |
| return |
| _make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir)) |
| |
| _prepend_passthrough_to_init_build_env(bitbake_builddir) |
| |
| siteconf_symlink = os.path.join(bitbake_confdir, "site.conf") |
| siteconf = os.path.normpath(os.path.join(setupdir, '..', "site.conf")) |
| if os.path.lexists(siteconf_symlink): |
| os.remove(siteconf_symlink) |
| os.symlink(os.path.relpath(siteconf, bitbake_confdir), siteconf_symlink) |
| |
| |
| init_script = os.path.join(bitbake_builddir, "init-build-env") |
| workspace_file = os.path.join(setupdir, "bitbake.code-workspace") |
| shell = "bash" |
| # oe-fragments entries can be strings or dicts with 'name'/'description' keys |
| oe_fragments = [f["name"] if isinstance(f, dict) else f for f in bitbake_config.get("oe-fragments", [])] |
| fragments = oe_fragments + sorted(bitbake_config.get("oe-fragment-choices",{}).values()) |
| if fragments: |
| bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments))) |
| |
| if os.path.exists(backup_bitbake_confdir): |
| conf_diff = get_diff(backup_bitbake_confdir, bitbake_confdir) |
| if not conf_diff: |
| logger.plain('New bitbake configuration from upstream is the same as the current one, no need to update it.') |
| shutil.rmtree(bitbake_confdir) |
| os.rename(backup_bitbake_confdir, bitbake_confdir) |
| if init_vscode or os.path.exists(workspace_file): |
| configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) |
| return |
| |
| logger.plain('Upstream bitbake configuration changes were found:') |
| logger.plain(conf_diff) |
| |
| if update_bb_conf == 'prompt': |
| y_or_n = input('Apply these changes to the current configuration? (y/N): ') |
| if y_or_n != 'y': |
| update_bb_conf = 'no' |
| |
| if update_bb_conf == 'no': |
| logger.plain('Ignoring upstream bitbake configuration changes') |
| logger.plain(f'Leaving the upstream configuration in {upstream_bitbake_confdir}') |
| os.rename(bitbake_confdir, upstream_bitbake_confdir) |
| os.rename(backup_bitbake_confdir, bitbake_confdir) |
| if init_vscode or os.path.exists(workspace_file): |
| configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) |
| return |
| |
| logger.plain('Applying upstream bitbake configuration changes') |
| logger.plain(f'Leaving the previous configuration in {backup_bitbake_confdir}') |
| |
| fragment_note = "Run 'bitbake-config-build enable-fragment <fragment-name>' to enable additional fragments or replace built-in ones (e.g. machine/<name> or distro/<name> to change MACHINE or DISTRO)." |
| |
| readme_extra = "" |
| if init_vscode: |
| readme_extra = "\nTo edit the code in VSCode, open the workspace: code {}\n".format(workspace_file) |
| |
| readme = """{}\n\nAdditional information is in {} and {}\n |
| Source the environment using '. {}' to run builds from the command line.\n |
| {}\n |
| The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf |
| {}""".format( |
| bitbake_config["description"], |
| os.path.join(bitbake_builddir,'conf/conf-summary.txt'), |
| os.path.join(bitbake_builddir,'conf/conf-notes.txt'), |
| init_script, |
| fragment_note, |
| bitbake_builddir, |
| readme_extra |
| ) |
| readme_file = os.path.join(bitbake_builddir, "README") |
| with open(readme_file, 'w') as f: |
| f.write(readme) |
| |
| logger.plain("This bitbake configuration provides:\n {}\n".format(bitbake_config["description"])) |
| logger.plain("Usage instructions and additional information are in\n {}\n".format(readme_file)) |
| logger.plain("To run builds, source the environment using\n . {}\n".format(init_script)) |
| logger.plain("{}\n".format(fragment_note)) |
| logger.plain("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n {}/conf\n".format(bitbake_builddir)) |
| if init_vscode: |
| logger.plain("To edit the code in VSCode, open the workspace:\n code {}\n".format(workspace_file)) |
| |
| if init_vscode or os.path.exists(workspace_file): |
| configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) |
| |
| def get_registry_config(registry_path, id): |
| for root, _, files in os.walk(registry_path): |
| for file in files: |
| if file.endswith('.conf.json') and id == get_config_name(file): |
| return os.path.join(root, file) |
| raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id)) |
| |
| def merge_overrides_into_sources(sources, overrides): |
| layers = copy.deepcopy(sources) |
| for k,v in overrides.items(): |
| if k in layers: |
| layers[k] = v |
| return layers |
| |
| def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt", init_vscode=False, rebase_conflicts_strategy='abort'): |
| layer_config = merge_overrides_into_sources(config["data"]["sources"], config["source-overrides"]["sources"]) |
| sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d, rebase_conflicts_strategy=rebase_conflicts_strategy) |
| bitbake_config = config["bitbake-config"] |
| thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None |
| setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode) |
| write_sources_fixed_revisions(confdir, layerdir, sources_fixed_revisions) |
| commit_config(confdir) |
| |
| def int_input(allowed_values, prompt=''): |
| n = None |
| while n is None: |
| try: |
| n = int(input(prompt)) |
| except ValueError: |
| prompt = 'Not a valid number, please try again: ' |
| continue |
| if n not in allowed_values: |
| prompt = 'Number {} not one of {}, please try again: '.format(n, allowed_values) |
| n = None |
| return n |
| |
| def flatten_bitbake_configs(configs): |
| def merge_configs(c1,c2): |
| c_merged = {} |
| for k,v in c2.items(): |
| if k not in c1.keys(): |
| c_merged[k] = v |
| for k,v in c1.items(): |
| if k not in c2.keys(): |
| c_merged[k] = v |
| else: |
| c_merged[k] = c1[k] + c2[k] |
| del c_merged['configurations'] |
| return c_merged |
| |
| flattened_configs = [] |
| for c in configs: |
| if 'configurations' not in c: |
| flattened_configs.append(c) |
| else: |
| for sub_c in flatten_bitbake_configs(c['configurations']): |
| flattened_configs.append(merge_configs(c, sub_c)) |
| return flattened_configs |
| |
| def choose_bitbake_config(configs, parameters, non_interactive): |
| flattened_configs = flatten_bitbake_configs(configs) |
| configs_dict = {i["name"]:i for i in flattened_configs} |
| |
| if parameters: |
| config_id = parameters[0] |
| if config_id not in configs_dict: |
| raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict)) |
| return configs_dict[config_id] |
| |
| enumerated_configs = list(enumerate(flattened_configs, 1)) |
| if len(enumerated_configs) == 1: |
| only_config = flattened_configs[0] |
| logger.plain("\nSelecting the only available bitbake configuration {}".format(only_config["name"])) |
| return only_config |
| |
| if non_interactive: |
| raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict)) |
| |
| logger.plain("") |
| print_configs("Available bitbake configurations", |
| [c["name"] for c in flattened_configs], |
| [c["description"] for c in flattened_configs]) |
| config_n = int_input([i[0] for i in enumerated_configs], |
| "\nPlease select one of the above bitbake configurations by its number: ") - 1 |
| return flattened_configs[config_n] |
| |
| def choose_config(configs, non_interactive): |
| not_expired_configs = [k for k in sorted(configs.keys()) if not has_expired(configs[k].get("expires", None))] |
| if len(not_expired_configs) == 1: |
| only_config = not_expired_configs[0] |
| logger.plain("\nSelecting the only available configuration {}\n".format(only_config)) |
| return only_config |
| |
| if non_interactive: |
| raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs)) |
| |
| descs = [] |
| for c in not_expired_configs: |
| d = configs[c]["description"] |
| expiry_date = configs[c].get("expires", None) |
| if expiry_date: |
| d += f" (supported until {expiry_date})" |
| descs.append(d) |
| |
| logger.plain("") |
| print_configs("Available Configuration Templates", |
| [c for c in not_expired_configs], |
| descs) |
| config_n = int_input([i[0] for i in list(enumerate(not_expired_configs, 1))], |
| "\nPlease select one of the above configurations by its number: ") - 1 |
| return not_expired_configs[config_n] |
| |
| def choose_fragments(possibilities, parameters, non_interactive, skip_selection): |
| choices = {} |
| for k,v in possibilities.items(): |
| if skip_selection and k in skip_selection: |
| logger.info("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k)) |
| continue |
| |
| # options can be a list of strings or a list of dicts |
| options = v["options"] |
| if len(options) > 0 and isinstance(v["options"][0], str): |
| options = [{"name": o, "description": ""} for o in v["options"]] |
| |
| choice = [o["name"] for o in options if o["name"] in parameters] |
| if len(choice) > 1: |
| raise Exception("Options specified on command line do not allow a single selection " |
| f"from possibilities {[o['name'] for o in options]}, please " |
| f"remove one or more from {parameters}") |
| if len(choice) == 1: |
| choices[k] = choice[0] |
| continue |
| |
| if non_interactive: |
| raise Exception(f"Unable to choose from options in non-interactive mode: {[o['name'] for o in options]}") |
| |
| logger.plain("") |
| print_configs(v["description"], |
| [o['name'] for o in options], |
| [o['description'] for o in options]) |
| options_enumerated = list(enumerate(options, 1)) |
| option_n = int_input([i[0] for i in options_enumerated], |
| "\nPlease select one of the above options by its number: ") - 1 |
| choices[k] = options_enumerated[option_n][1]["name"] |
| return choices |
| |
| def obtain_config(top_dir, registry, args, source_overrides, d): |
| if args.config: |
| config_id = args.config[0] |
| config_parameters = args.config[1:] |
| if os.path.exists(config_id): |
| config_id = os.path.abspath(config_id) |
| logger.info("Reading configuration from local file\n {}".format(config_id)) |
| upstream_config = {'type':'local', |
| 'path':config_id, |
| 'name':get_config_name(config_id), |
| 'data':json.load(open(config_id)) |
| } |
| elif config_id.startswith("http://") or config_id.startswith("https://"): |
| logger.info("Reading configuration from network URI\n {}".format(config_id)) |
| import urllib.request |
| try: |
| with urllib.request.urlopen(config_id) as f: |
| json_data = json.load(f) |
| upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json_data} |
| except json.JSONDecodeError as e: |
| raise Exception ("Invalid JSON from {}. Are you pointing to an HTML page? {}".format(config_id, e)) from e |
| else: |
| logger.info("Looking up config {} in configuration registry".format(config_id)) |
| registry_path = update_registry(registry, cache_dir(top_dir), d) |
| registry_configs = list_registry(registry_path, with_expired=True) |
| if config_id not in registry_configs: |
| raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id)) |
| upstream_config = {'type':'registry','registry':registry,'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))} |
| expiry_date = upstream_config['data'].get("expires", None) |
| if has_expired(expiry_date): |
| logger.warning("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date)) |
| else: |
| registry_path = update_registry(registry, cache_dir(top_dir), d) |
| registry_configs = list_registry(registry_path, with_expired=True) |
| config_id = choose_config(registry_configs, args.non_interactive) |
| config_parameters = [] |
| upstream_config = {'type':'registry','registry':registry,'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))} |
| |
| upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive) |
| upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection) |
| upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values()) |
| upstream_config['source-overrides'] = source_overrides |
| upstream_config['skip-selection'] = args.skip_selection |
| return upstream_config |
| |
| def obtain_overrides(args): |
| overrides = {'sources':{}} |
| if args.source_overrides: |
| overrides = json.load(open(args.source_overrides)) |
| overrides_dir = os.path.dirname(os.path.abspath(args.source_overrides)) |
| for s,v in overrides['sources'].items(): |
| local = v.get('local') |
| if local: |
| path = os.path.expanduser(local['path']) |
| if not os.path.isabs(path): |
| overrides['sources'][s]['local']['path'] = os.path.join(overrides_dir, path) |
| |
| for local_name, local_path in args.use_local_source: |
| overrides['sources'][local_name] = {'local':{'path':os.path.abspath(os.path.expanduser(local_path))}} |
| |
| return overrides |
| |
| def configure_vscode(setupdir, layerdir, builddir, init_script): |
| """ |
| Configure the VSCode environment by creating or updating a workspace file. |
| |
| Create or update a bitbake.code-workspace file with folders for the layers and build/conf. |
| Managed folders are regenerated; user-added folders are kept. Settings are merged, with |
| managed keys (bitbake.*, python extra paths) always overwritten. |
| """ |
| logger.debug("configure_vscode: setupdir={}, layerdir={}, builddir={}, init_script={}".format( |
| setupdir, layerdir, builddir, init_script)) |
| |
| # Get git repository directories |
| git_repos = [] |
| if os.path.exists(layerdir): |
| for entry in os.listdir(layerdir): |
| entry_path = os.path.join(layerdir, entry) |
| if os.path.isdir(entry_path) and not os.path.islink(entry_path): |
| # Check if it's a git repository |
| if os.path.exists(os.path.join(entry_path, '.git')): |
| git_repos.append(entry) |
| logger.debug("configure_vscode: found {} git repos: {}".format(len(git_repos), git_repos)) |
| |
| conf_path = os.path.relpath(os.path.join(builddir, "conf"), setupdir) |
| repo_paths = [os.path.relpath(os.path.join(layerdir, repo), setupdir) for repo in git_repos] |
| logger.debug("configure_vscode: conf_path={}, repo_paths={}".format(conf_path, repo_paths)) |
| |
| # Load existing workspace |
| workspace_file = os.path.join(setupdir, "bitbake.code-workspace") |
| workspace = { |
| "extensions": { |
| "recommendations": [ |
| "yocto-project.yocto-bitbake" |
| ] |
| } |
| } |
| if os.path.exists(workspace_file): |
| logger.debug("configure_vscode: loading existing workspace file: {}".format(workspace_file)) |
| try: |
| with open(workspace_file, 'r') as f: |
| workspace = json.load(f) |
| logger.debug("configure_vscode: loaded workspace with {} folders, {} settings".format( |
| len(workspace.get("folders", [])), len(workspace.get("settings", {})))) |
| except (json.JSONDecodeError, OSError) as e: |
| logger.error( |
| "Unable to read existing workspace file {}: {}. Skipping update.".format( |
| workspace_file, str(e) |
| ) |
| ) |
| return |
| else: |
| logger.debug("configure_vscode: creating new workspace file: {}".format(workspace_file)) |
| |
| # Update folders |
| existing_folders = workspace.get("folders", []) |
| new_folders = [{"name": "conf", "path": conf_path}] |
| for rp in repo_paths: |
| repo_name = os.path.basename(rp) |
| new_folders.append({"name": repo_name, "path": rp}) |
| # Keep any user-added folders that are not managed |
| managed_paths = {f["path"] for f in new_folders} |
| for f in existing_folders: |
| if f["path"] not in managed_paths: |
| new_folders.append(f) |
| logger.debug("configure_vscode: keeping user-added folder: {}".format(f["path"])) |
| workspace["folders"] = new_folders |
| logger.debug("configure_vscode: updated workspace with {} folders".format(len(new_folders))) |
| |
| # Build Python extra paths for each layer - only check top level of each repo |
| extra_paths = [] |
| subdirs_to_check = ['lib', 'scripts'] |
| for repo in git_repos: |
| repo_path_abs = os.path.join(layerdir, repo) |
| for subdir in subdirs_to_check: |
| sub_path = os.path.join(repo_path_abs, subdir) |
| if os.path.isdir(sub_path): |
| extra_paths.append(sub_path) |
| |
| # Update settings |
| existing_settings = workspace.get("settings", {}) |
| new_settings = { |
| "bitbake.disableConfigModification": True, |
| "bitbake.pathToBitbakeFolder": os.path.join(layerdir, "bitbake"), |
| "bitbake.pathToBuildFolder": builddir, |
| "bitbake.pathToEnvScript": init_script, |
| "bitbake.workingDirectory": builddir, |
| "files.associations": { |
| "*.conf": "bitbake", |
| "*.inc": "bitbake" |
| }, |
| "files.exclude": { |
| "**/.git/**": True |
| }, |
| "search.exclude": { |
| "**/.git/**": True, |
| "**/logs/**": True |
| }, |
| "files.watcherExclude": { |
| "**/.git/**": True, |
| "**/logs/**": True |
| }, |
| "python.analysis.exclude": [ |
| "**/.git/**", |
| "**/logs/**" |
| ], |
| "python.autoComplete.extraPaths": extra_paths, |
| "python.analysis.extraPaths": extra_paths |
| } |
| |
| # Merge settings: add missing, always update bitbake paths and python extra paths |
| for key, value in new_settings.items(): |
| if key not in existing_settings: |
| existing_settings[key] = value |
| elif key.startswith("bitbake.") or key in [ |
| "python.autoComplete.extraPaths", |
| "python.analysis.extraPaths", |
| ]: |
| # Always replace - these are managed/machine-generated settings |
| existing_settings[key] = value |
| elif key in [ |
| "files.associations", |
| "files.exclude", |
| "search.exclude", |
| "files.watcherExclude", |
| "python.analysis.exclude", |
| ]: |
| # For dicts and lists, merge new values in without removing user additions |
| if isinstance(value, dict): |
| if not isinstance(existing_settings[key], dict): |
| existing_settings[key] = {} |
| for k, v in value.items(): |
| if k not in existing_settings[key]: |
| existing_settings[key][k] = v |
| elif isinstance(value, list): |
| if not isinstance(existing_settings[key], list): |
| existing_settings[key] = [] |
| for item in value: |
| if item not in existing_settings[key]: |
| existing_settings[key].append(item) |
| |
| workspace["settings"] = existing_settings |
| logger.debug("configure_vscode: merged settings, total {} keys".format(len(existing_settings))) |
| |
| with open(workspace_file, 'w') as f: |
| json.dump(workspace, f, indent=4) |
| logger.debug("configure_vscode: wrote workspace file: {}".format(workspace_file)) |
| |
| def init_config(top_dir, settings, args): |
| create_siteconf(top_dir, args.non_interactive, settings) |
| |
| d = init_bb_cache(top_dir, settings, args) |
| |
| def handle_task_progress(event, d): |
| rate = event.rate if event.rate else '' |
| progress = event.progress if event.progress > 0 else 0 |
| logger.handlers[0].terminator = '\r' |
| logger.plain("{}% {} ".format(progress, rate)) |
| logger.handlers[0].terminator = '\n' |
| |
| source_overrides = obtain_overrides(args) |
| upstream_config = obtain_config(top_dir, settings["default"]["registry"], args, source_overrides, d) |
| logger.info("Run 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options']))) |
| |
| if args.setup_dir_name: |
| setup_dir_name = args.setup_dir_name |
| else: |
| setup_dir_name = "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")) |
| if 'setup-dir-name' in upstream_config['bitbake-config']: |
| mapping = { |
| k: v.partition("/")[2].replace(" ", "-").replace("/", "_") |
| for k, v in upstream_config['bitbake-config']['oe-fragment-choices'].items() |
| } |
| config_setup_dir_name = string.Template(upstream_config['bitbake-config']['setup-dir-name']).substitute(mapping) |
| config_setup_dir = os.path.join(top_dir, config_setup_dir_name) |
| if os.path.exists(config_setup_dir): |
| logger.info("Setup directory {} (as suggested by configuration) already exists, using the full name instead.\n".format(config_setup_dir)) |
| elif settings['default']['use-full-setup-dir-name'] != 'no': |
| logger.info("Using the full setup directory name instead of {} suggested by configuration, as set in the settings.\n".format(config_setup_dir)) |
| else: |
| setup_dir_name = config_setup_dir_name |
| |
| if not args.non_interactive: |
| n = input(f"Enter setup directory name [{setup_dir_name}]: ") |
| if n: |
| setup_dir_name = n |
| |
| setupdir = os.path.join(os.path.abspath(top_dir), setup_dir_name) |
| if os.path.exists(os.path.join(setupdir, "layers")): |
| logger.info(f"Setup already initialized in:\n {setupdir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize in this directory, remove it, and run 'bitbake-setup init' again.") |
| return |
| |
| logger.plain("Initializing a setup directory in\n {}".format(setupdir)) |
| if not args.non_interactive: |
| y_or_n = input('Continue? (y/N): ') |
| if y_or_n != 'y': |
| exit() |
| logger.plain("") |
| |
| os.makedirs(setupdir, exist_ok=True) |
| |
| confdir = os.path.join(setupdir, "config") |
| layerdir = os.path.join(setupdir, "layers") |
| |
| os.makedirs(confdir) |
| os.makedirs(layerdir) |
| |
| bb.process.run(["git", "-C", confdir, "init", "-b", "main"]) |
| # Make sure commiting doesn't fail if no default git user is configured on the machine |
| bb.process.run(["git", "-C", confdir, "config", "user.name", "bitbake-setup"]) |
| bb.process.run(["git", "-C", confdir, "config", "user.email", "bitbake-setup@not.set"]) |
| bb.process.run(["git", "-C", confdir, "commit", "--no-verify", "--allow-empty", "-m", "Initial commit"]) |
| |
| bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d) |
| |
| write_upstream_config(confdir, upstream_config) |
| update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes", init_vscode=args.init_vscode) |
| |
| bb.event.remove("bb.build.TaskProgress", None) |
| |
| def get_diff(file1, file2): |
| try: |
| bb.process.run(["diff", get_diff_color_param(), "-uNr", file1, file2]) |
| except bb.process.ExecutionError as e: |
| if e.exitcode == 1: |
| return e.stdout |
| else: |
| raise e |
| return None |
| |
| def are_layers_changed(layers, layerdir, d): |
| def _is_git_remote_changed(r_remote, repodir): |
| from bb.fetch.git import git_hash_re |
| |
| rev = r_remote['rev'] |
| branch = r_remote.get('branch', None) |
| |
| rev_parse_result = bb.process.run(['git', '-C', os.path.join(layerdir, repodir), "rev-parse", "HEAD"]) |
| local_revision = rev_parse_result[0].strip() |
| if git_hash_re.match(rev): |
| if rev != local_revision: |
| logger.info('Layer repository checked out into {} is at revision {} but should be at {}'.format(os.path.join(layerdir, repodir),local_revision, rev)) |
| return True |
| return False |
| |
| remotes = _get_remotes(r_remote) |
| changed = False |
| for remote in remotes: |
| type,host,path,user,pswd,params = bb.fetch.decodeurl(remote) |
| fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) |
| if branch: |
| fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d) |
| else: |
| fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d) |
| upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default') |
| if upstream_revision != local_revision: |
| changed = True |
| logger.info('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remote, os.path.join(layerdir, repodir), rev, local_revision, upstream_revision)) |
| return changed |
| |
| changed = False |
| for r_name in layers: |
| r_data = layers[r_name] |
| repodir = r_data.get("path", r_name) |
| |
| git_remote = r_data.get('git-remote') |
| if git_remote: |
| changed = changed | _is_git_remote_changed(git_remote, repodir) |
| |
| return changed |
| |
| def build_status(top_dir, settings, args, d, update=False): |
| setupdir = os.path.abspath(args.setup_dir) |
| |
| confdir = os.path.join(setupdir, "config") |
| layerdir = os.path.join(setupdir, "layers") |
| |
| current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json"))) |
| |
| args.config = current_upstream_config['non-interactive-cmdline-options'] |
| args.non_interactive = True |
| args.skip_selection = current_upstream_config['skip-selection'] |
| source_overrides = current_upstream_config["source-overrides"] |
| registry = current_upstream_config.get("registry") |
| new_upstream_config = obtain_config(top_dir, registry, args, source_overrides, d) |
| |
| write_upstream_config(confdir, new_upstream_config) |
| config_diff = bb.process.run(["git", "-C" , confdir, "diff", get_diff_color_param()])[0] |
| |
| if config_diff: |
| logger.plain('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff)) |
| if update: |
| update_build(new_upstream_config, confdir, setupdir, layerdir, d, |
| update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) |
| else: |
| bb.process.run(["git", "-C", confdir, "restore", "config-upstream.json"]) |
| return |
| |
| layer_config = merge_overrides_into_sources(current_upstream_config["data"]["sources"], current_upstream_config["source-overrides"]["sources"]) |
| if are_layers_changed(layer_config, layerdir, d): |
| if update: |
| update_build(current_upstream_config, confdir, setupdir, layerdir, |
| d, update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) |
| return |
| |
| logger.plain("\nConfiguration in {} has not changed.".format(setupdir)) |
| |
| def build_update(top_dir, settings, args, d): |
| build_status(top_dir, settings, args, d, update=True) |
| |
| def do_fetch(fetcher, dir): |
| # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch |
| # and we need to set up smth similar here |
| fetchlogdir = os.path.join(dir, 'logs') |
| os.makedirs(fetchlogdir, exist_ok=True) |
| fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))) |
| with open(fetchlog, 'a') as f: |
| oldstdout = sys.stdout |
| sys.stdout = f |
| try: |
| fetcher.download() |
| fetcher.unpack_update(dir) |
| except: |
| if os.path.exists(fetchlog) and os.path.getsize(fetchlog) > 0: |
| logger.error("Check {} for additional information that may help diagnoze the issue.".format(fetchlog)) |
| raise |
| finally: |
| sys.stdout = oldstdout |
| |
| def update_registry(registry, cachedir, d): |
| registrydir = 'configurations' |
| if registry.startswith("."): |
| full_registrydir = os.path.join(os.getcwd(), registry, registrydir) |
| elif registry.startswith("/"): |
| full_registrydir = os.path.join(registry, registrydir) |
| else: |
| full_registrydir = os.path.join(cachedir, registrydir) |
| logger.info("Fetching configuration registry\n {}\ninto\n {}".format(registry, full_registrydir)) |
| fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d) |
| do_fetch(fetcher, cachedir) |
| return full_registrydir |
| |
| def has_expired(expiry_date): |
| if expiry_date: |
| return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date) |
| return False |
| |
| def list_registry(registry_path, with_expired): |
| json_data = {} |
| |
| for root, _, files in os.walk(registry_path): |
| for file in files: |
| if file.endswith('.conf.json'): |
| config_name = get_config_name(file) |
| config_data = json.load(open(os.path.join(root, file))) |
| config_desc = config_data["description"] |
| expiry_date = config_data.get("expires", None) |
| if expiry_date: |
| if with_expired or not has_expired(expiry_date): |
| json_data[config_name] = {"description": config_desc, "expires": expiry_date} |
| else: |
| json_data[config_name] = {"description": config_desc} |
| return json_data |
| |
| def list_configs(settings, args): |
| import tempfile |
| top_dir = tempfile.mkdtemp(prefix="bitbake-setup-list-") |
| settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads') |
| d = init_bb_cache(top_dir, settings, args) |
| registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d) |
| json_data = list_registry(registry_path, args.with_expired) |
| shutil.rmtree(top_dir) |
| |
| logger.plain("Available configurations:") |
| for config_name, config_data in sorted(json_data.items()): |
| expiry_date = config_data.get("expires", None) |
| config_desc = config_data["description"] |
| if expiry_date: |
| if args.with_expired or not has_expired(expiry_date): |
| logger.plain("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date)) |
| else: |
| logger.plain("{}\t{}".format(config_name, config_desc)) |
| logger.plain("\nRun 'init' with one of the above configuration identifiers to set up a build.") |
| |
| if args.write_json: |
| with open(args.write_json, 'w') as f: |
| json.dump(json_data, f, sort_keys=True, indent=4) |
| logger.plain("Available configurations written into {}".format(args.write_json)) |
| |
| def install_buildtools(top_dir, settings, args, d): |
| buildtools_install_dir = os.path.join(args.setup_dir, 'buildtools') |
| if os.path.exists(buildtools_install_dir): |
| if not args.force: |
| logger.plain("Buildtools are already installed in {}.".format(buildtools_install_dir)) |
| env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*')) |
| if env_scripts: |
| logger.plain("If you wish to use them, you need to source the environment setup script e.g.") |
| for s in env_scripts: |
| logger.plain("$ . {}".format(s)) |
| logger.plain("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.") |
| return |
| shutil.rmtree(buildtools_install_dir) |
| |
| install_buildtools_script = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools') |
| |
| buildtools_config = {} |
| upstream_config_path = os.path.join(args.setup_dir, 'config', 'config-upstream.json') |
| try: |
| with open(upstream_config_path) as f: |
| upstream_config = json.load(f) |
| buildtools_config = upstream_config.get('bitbake-config', {}).get('install-buildtools', {}) |
| except FileNotFoundError: |
| pass |
| except json.JSONDecodeError as e: |
| logger.error("Failed to parse %s: %s" % (upstream_config_path, e)) |
| sys.exit(1) |
| |
| url = args.url if args.url is not None else buildtools_config.get('url') |
| sha256 = args.sha256 if args.sha256 is not None else buildtools_config.get('sha256sum') |
| |
| buildtools_download_dir = add_unique_timestamp_to_path( |
| os.path.join(args.setup_dir, 'buildtools-downloads/buildtools')) |
| |
| if args.default or (not url and not sha256): |
| # No config or --default: fall back to install-buildtools defaults |
| logger.plain("Using install-buildtools script defaults") |
| subprocess.check_call([install_buildtools_script, |
| '-d', buildtools_install_dir, |
| '--downloads-directory', buildtools_download_dir]) |
| return |
| |
| if not url: |
| logger.error("No buildtools URL configured. " |
| "Set 'url' in the install-buildtools config section or pass --url.") |
| sys.exit(1) |
| |
| if not sha256: |
| logger.error("No sha256sum configured for buildtools installer. " |
| "Add 'sha256sum' to the install-buildtools config section or pass --sha256.\n" |
| "You can obtain it with: sha256sum <installer-file>") |
| sys.exit(1) |
| |
| # Download using bb.fetch with sha256 validation |
| src_uri = "{};sha256sum={}".format(url, sha256) |
| |
| logger.plain("Fetching buildtools installer using bitbake fetcher") |
| logger.plain(" {}".format(url)) |
| |
| fetcher = bb.fetch.Fetch([src_uri], d) |
| try: |
| fetcher.download() |
| except bb.fetch.ChecksumError as e: |
| logger.error("Checksum mismatch for buildtools installer. " |
| "Verify the URL and downloaded file.\n%s" % str(e)) |
| sys.exit(1) |
| except bb.fetch.FetchError as e: |
| logger.error("Failed to download buildtools installer: %s" % str(e)) |
| sys.exit(1) |
| |
| # Unpack into the standard buildtools download location |
| os.makedirs(buildtools_download_dir, exist_ok=True) |
| fetcher.unpack(buildtools_download_dir) |
| |
| filename = os.path.basename(url.rstrip('/')) |
| matches = glob.glob(os.path.join(buildtools_download_dir, '**', filename), recursive=True) |
| if not matches: |
| logger.error("Unpacked file not found: %s" % filename) |
| sys.exit(1) |
| local_path = matches[0] |
| |
| logger.plain("Buildtools installer will be installed into {}".format( |
| buildtools_install_dir)) |
| |
| cmd = [ |
| install_buildtools_script, |
| '--local-file', local_path, |
| '-d', buildtools_install_dir, |
| ] |
| |
| try: |
| subprocess.check_call(cmd) |
| except subprocess.CalledProcessError as e: |
| logger.error("install-buildtools failed with exit code %d" % e.returncode) |
| sys.exit(1) |
| |
| def create_siteconf(top_dir, non_interactive, settings): |
| siteconfpath = os.path.join(top_dir, 'site.conf') |
| if os.path.exists(siteconfpath): |
| logger.info('A site.conf file already exists. Please remove it if you would like to replace it with a default one') |
| else: |
| logger.plain(f'{top_dir} looks like a new top directory. If you would like to use a different directory, answer "n" below and either:') |
| logger.plain('\t1) Change the default bitbake-setup settings:') |
| logger.plain('\t\tbitbake-setup settings set default top-dir-prefix <PATH>') |
| logger.plain('\t\tbitbake-setup settings set default top-dir-name <NAME>') |
| logger.plain('\t2) Pass one or more options on the command line to change the top level directory in that invocation only:') |
| logger.plain('\t\tbitbake-setup --setting default top-dir-prefix <PATH> ...') |
| logger.plain('\t\tbitbake-setup --setting default top-dir-name <NAME> ...') |
| logger.plain('') |
| |
| logger.plain('A common site.conf file will be created, please check it is correct before running builds\n {}\n'.format(siteconfpath)) |
| if not non_interactive: |
| y_or_n = input('Proceed? (y/N): ') |
| if y_or_n != 'y': |
| exit() |
| |
| os.makedirs(top_dir, exist_ok=True) |
| with open(siteconfpath, 'w') as siteconffile: |
| sstate_settings = textwrap.dedent( |
| """ |
| # |
| # Where to place shared-state files |
| # |
| # BitBake has the capability to accelerate builds based on previously built output. |
| # This is done using "shared state" files which can be thought of as cache objects |
| # and this option determines where those files are placed. |
| # |
| # You can wipe out TMPDIR leaving this directory intact and the build would regenerate |
| # from these files if no changes were made to the configuration. If changes were made |
| # to the configuration, only shared state files where the state was still valid would |
| # be used (done using checksums). |
| SSTATE_DIR ?= "{sstate_dir}" |
| # |
| # Hash Equivalence database location |
| # |
| # Hash equivalence improves reuse of sstate by detecting when a given sstate |
| # artifact can be reused as equivalent, even if the current task hash doesn't |
| # match the one that generated the artifact. This variable controls where the |
| # Hash Equivalence database ("hashserv.db") is stored and can be shared between |
| # concurrent builds. |
| BB_HASHSERVE_DB_DIR ?= "${{SSTATE_DIR}}" |
| """.format(sstate_dir=os.path.join(top_dir, ".sstate-cache")) |
| ) |
| siteconffile.write( |
| textwrap.dedent( |
| """\ |
| # This file is intended for build host-specific bitbake settings |
| |
| # Where to place downloads |
| # |
| # During a first build the system will download many different source code |
| # tarballs from various upstream projects. This can take a while, particularly |
| # if your network connection is slow. These are all stored in DL_DIR. When |
| # wiping and rebuilding you can preserve this directory to speed up this part of |
| # subsequent builds. This directory is safe to share between multiple builds on |
| # the same machine too. |
| DL_DIR ?= "{dl_dir}" |
| """.format( |
| dl_dir=settings["default"]["dl-dir"], |
| ) |
| ) + (sstate_settings if settings["default"]["common-sstate"] == 'yes' else "") |
| ) |
| |
| |
| def topdir_settings_path(top_dir): |
| return os.path.join(top_dir, 'settings.conf') |
| |
| def global_settings_path(args): |
| return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf') |
| |
| def load_settings(settings_path): |
| settings = configparser.ConfigParser() |
| if os.path.exists(settings_path): |
| logger.info('Loading settings from {}'.format(settings_path)) |
| settings.read_file(open(settings_path)) |
| |
| for section in settings.sections(): |
| for key, value in settings[section].items(): |
| settings[section][key] = value.strip('\"\'') |
| |
| return settings |
| |
| def change_setting(top_dir, args): |
| if vars(args)['global']: |
| settings_path = global_settings_path(args) |
| elif args.setting in GLOBAL_ONLY_SETTINGS: |
| logger.info(f"{args.setting} can only be set in the global config; '--global' is implied") |
| settings_path = global_settings_path(args) |
| else: |
| settings_path = topdir_settings_path(top_dir) |
| settings = load_settings(settings_path) |
| |
| if args.subcommand == 'set': |
| if args.section not in settings.keys(): |
| settings[args.section] = {} |
| settings[args.section][args.setting] = args.value |
| logger.plain(f"From section '{args.section}' the setting '{args.setting}' was changed to '{args.value}'") |
| if args.subcommand == 'unset': |
| if args.section in settings.keys() and args.setting in settings[args.section].keys(): |
| del settings[args.section][args.setting] |
| logger.plain(f"From section '{args.section}' the setting '{args.setting}' has been removed") |
| |
| os.makedirs(os.path.dirname(settings_path), exist_ok=True) |
| with open(settings_path, 'w') as settingsfile: |
| settings.write(settingsfile) |
| logger.info(f"Settings written to {settings_path}") |
| |
| def list_settings(all_settings): |
| for section, section_settings in all_settings.items(): |
| for key, value in section_settings.items(): |
| logger.plain("{} {} {}".format(section, key, value)) |
| |
| def settings_func(top_dir, all_settings, args): |
| if args.subcommand == 'list': |
| list_settings(all_settings) |
| elif args.subcommand == 'set' or args.subcommand == 'unset': |
| change_setting(top_dir, args) |
| |
| def get_setup_dir_via_bbpath(): |
| bbpath = os.environ.get('BBPATH') |
| if bbpath: |
| bitbake_dir = os.path.normpath(bbpath.split(':')[0]) |
| if os.path.exists(os.path.join(bitbake_dir,'init-build-env')): |
| setup_dir = os.path.dirname(bitbake_dir) |
| return setup_dir |
| return None |
| |
| def get_top_dir(args, settings): |
| setup_dir_via_bbpath = get_setup_dir_via_bbpath() |
| if setup_dir_via_bbpath: |
| top_dir = os.path.dirname(setup_dir_via_bbpath) |
| if os.path.exists(cache_dir(top_dir)): |
| return top_dir |
| |
| if hasattr(args, 'setup_dir'): |
| top_dir = os.path.dirname(os.path.normpath(args.setup_dir)) |
| return top_dir |
| |
| top_dir_prefix = settings['default']['top-dir-prefix'] |
| top_dir_name = settings['default']['top-dir-name'] |
| return os.path.join(top_dir_prefix, top_dir_name) |
| |
| def merge_settings(builtin_settings, global_settings, topdir_settings, cmdline_settings): |
| all_settings = builtin_settings |
| |
| for s in (global_settings, topdir_settings): |
| for section, section_settings in s.items(): |
| for setting, value in section_settings.items(): |
| if section not in all_settings.keys(): |
| all_settings[section] = {} |
| all_settings[section][setting] = value |
| |
| for (section, setting, value) in cmdline_settings: |
| if section not in all_settings.keys(): |
| all_settings[section] = {} |
| all_settings[section][setting] = value |
| |
| return all_settings |
| |
| def sigint_handler(sig, frame, func, top_dir): |
| logger.plain('\nShutting down...') |
| if isinstance(top_dir, str) and os.path.exists(top_dir): |
| if func in [init_config, build_update]: |
| logger.warning(f'{top_dir} may contain an incomplete setup!') |
| elif func == install_buildtools: |
| logger.warning(f'{top_dir} may contain an incomplete buildtools installation!') |
| exit() |
| |
| def main(): |
| def add_setup_dir_arg(parser): |
| setup_dir = get_setup_dir_via_bbpath() |
| if setup_dir: |
| parser.add_argument('--setup-dir', default=setup_dir, help="Path to the setup, default is %(default)s via BBPATH") |
| else: |
| parser.add_argument('--setup-dir', required=True, help="Path to the setup") |
| |
| parser = argparse.ArgumentParser( |
| description="BitBake setup utility. Run with 'init' argument to get started.", |
| epilog="Use %(prog)s <subcommand> --help to get help on a specific command" |
| ) |
| parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') |
| parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') |
| parser.add_argument('--version', action='version', version='%(prog)s ' + __version__) |
| parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') |
| parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.') |
| parser.add_argument('--global-settings', action='store', metavar='PATH', help='Path to the global settings file.') |
| parser.add_argument('--setting', default=[], action='append', dest='cmdline_settings', |
| nargs=3, metavar=('SECTION', 'SETTING', 'VALUE'), |
| help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".') |
| |
| subparsers = parser.add_subparsers() |
| |
| parser_list = subparsers.add_parser('list', help='List available configurations') |
| parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.') |
| parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.') |
| parser_list.set_defaults(func=list_configs) |
| |
| parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it') |
| parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.") |
| parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.') |
| parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.') |
| parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.') |
| parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.') |
| parser_init.add_argument('-L', '--use-local-source', default=[], action='append', nargs=2, metavar=('SOURCE_NAME', 'PATH'), |
| help='Symlink local source into a build, instead of getting it as prescribed by a configuration (useful for local development).') |
| parser_init.add_argument('--init-vscode', action=argparse.BooleanOptionalAction, default=bool(shutil.which('code')), |
| help='Generate VSCode workspace configuration (default: %(default)s)') |
| parser_init.set_defaults(func=init_config) |
| |
| parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration') |
| add_setup_dir_arg(parser_status) |
| parser_status.set_defaults(func=build_status) |
| |
| parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration') |
| add_setup_dir_arg(parser_update) |
| parser_update.add_argument('--update-bb-conf', choices=['prompt', 'yes', 'no'], default='prompt', help='Update bitbake configuration files (bblayers.conf, local.conf) (default: prompt)') |
| parser_update.add_argument('--rebase-conflicts-strategy', choices=['abort', 'backup'], default='abort', |
| help="What to do when a layer repository has local modifications that prevent " |
| "an in-place update: 'abort' (default) aborts with an error message; " |
| "'backup' renames the directory to a timestamped backup and re-clones from upstream.") |
| parser_update.set_defaults(func=build_update) |
| |
| parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') |
| add_setup_dir_arg(parser_install_buildtools) |
| parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.') |
| parser_install_buildtools.add_argument('--default', action='store_true', help='Use install-buildtools script defaults, ignoring any config.') |
| parser_install_buildtools.add_argument('--url', help='Full URL to the buildtools SDK installer. Overrides config value.') |
| parser_install_buildtools.add_argument('--sha256', help='SHA256 checksum of the buildtools installer. Overrides config value.') |
| parser_install_buildtools.set_defaults(func=install_buildtools) |
| |
| parser_settings_arg_global = argparse.ArgumentParser(add_help=False) |
| parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory") |
| |
| parser_settings = subparsers.add_parser('settings', |
| help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of configuration registry, downloads directory and other settings specific to a top directory)') |
| parser_settings.set_defaults(func=settings_func) |
| |
| subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file") |
| |
| subparser_settings.add_parser('list', help="List all settings with their values") |
| |
| parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global], |
| help="In a Section, set a setting to a certain value") |
| parser_settings_set.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'") |
| parser_settings_set.add_argument("setting", metavar="<setting>", help="Name of a setting") |
| parser_settings_set.add_argument("value", metavar="<value>", help="The setting value") |
| |
| parser_settings_unset = subparser_settings.add_parser('unset', parents=[parser_settings_arg_global], |
| help="Unset a setting, e.g. 'bitbake-setup settings unset default registry' would revert to the registry setting in a global settings file") |
| parser_settings_unset.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'") |
| parser_settings_unset.add_argument("setting", metavar="<setting>", help="The setting to remove") |
| |
| args = parser.parse_args() |
| |
| if args.debug: |
| logger.setLevel(logging.DEBUG) |
| elif args.quiet: |
| logger.setLevel(logging.ERROR) |
| |
| # Need to re-run logger_create with color argument |
| # (will be the same logger since it has the same name) |
| bb.msg.logger_create('bitbake-setup', output=sys.stdout, |
| color=args.color, |
| level=logger.getEffectiveLevel()) |
| |
| if 'func' in args: |
| if hasattr(args, 'setup_dir'): |
| if not os.path.exists(os.path.join(args.setup_dir,'build', 'init-build-env')): |
| logger.error("Not a valid setup directory: build/init-build-env does not exist in {}".format(args.setup_dir)) |
| return |
| |
| if not hasattr(args, 'non_interactive'): |
| args.non_interactive = True |
| |
| builtin_settings = {} |
| builtin_settings['default'] = { |
| 'top-dir-prefix':os.getcwd(), |
| 'top-dir-name':'bitbake-builds', |
| 'registry':get_default_registry(), |
| 'use-full-setup-dir-name':'no', |
| 'common-sstate':'yes', |
| } |
| |
| global_settings = load_settings(global_settings_path(args)) |
| top_dir = get_top_dir(args, merge_settings(builtin_settings, global_settings, {}, args.cmdline_settings)) |
| |
| # register handler now to pass top_dir |
| _handler = functools.partial(sigint_handler, |
| func=args.func, |
| top_dir=os.path.abspath(top_dir)) |
| signal.signal(signal.SIGINT, _handler) |
| |
| # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first |
| builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads') |
| |
| topdir_settings = load_settings(topdir_settings_path(top_dir)) |
| all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings) |
| |
| if args.func == settings_func: |
| settings_func(top_dir, all_settings, args) |
| return |
| if args.func == list_configs: |
| list_configs(all_settings, args) |
| return |
| |
| logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir)) |
| |
| try: |
| if args.func == init_config: |
| init_config(top_dir, all_settings, args) |
| else: |
| d = init_bb_cache(top_dir, all_settings, args) |
| args.func(top_dir, all_settings, args, d) |
| |
| save_bb_cache() |
| except (SystemExit, KeyboardInterrupt): |
| raise |
| except Exception as e: |
| if args.debug: |
| raise |
| logger.error(str(e)) |
| sys.exit(1) |
| else: |
| parser.print_help() |
| |
| if __name__ == '__main__': |
| main() |
