upstream-layers/openembedded-core/scripts/oe-setup-layers

#!/usr/bin/env python3
#
# Copyright OpenEmbedded Contributors
#
# SPDX-License-Identifier: MIT
#

# This file was copied from poky(or oe-core)/scripts/oe-setup-layers by running
#
# bitbake-layers create-layers-setup destdir
#
# It is recommended that you do not modify this file directly, but rather re-run the above command to get the freshest upstream copy.
#
# This script is idempotent. Subsequent runs only change what is necessary to
# ensure your layers match your configuration.

import argparse
import json
import os
import subprocess

def _is_repo_git_repo(repodir):
    try:
        curr_toplevel = subprocess.check_output(['git', '-C', repodir, 'rev-parse', '--show-toplevel'], stderr=subprocess.DEVNULL)
        if curr_toplevel.strip().decode("utf-8") == repodir:
            return True
    except subprocess.CalledProcessError:
        pass
    return False

def _is_repo_at_rev(repodir, rev):
    try:
        curr_rev = subprocess.check_output(['git', '-C', repodir, 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL)
        if curr_rev.strip().decode("utf-8") == rev:
            return True
    except subprocess.CalledProcessError:
        pass
    return False

def _is_repo_at_remote_uri(repodir, remote, uri):
    try:
        curr_uri = subprocess.check_output(['git', '-C', repodir, 'remote', 'get-url', remote], stderr=subprocess.DEVNULL)
        if curr_uri.strip().decode("utf-8") == uri:
            return True
    except subprocess.CalledProcessError:
        pass
    return False

def _contains_submodules(repodir):
    return os.path.exists(os.path.join(repodir,".gitmodules"))

def _write_layer_list(dest, repodirs):
    layers = []
    for r in repodirs:
        for root, dirs, files in os.walk(r):
            if os.path.basename(root) == 'conf' and 'layer.conf' in files:
                layers.append(os.path.relpath(os.path.dirname(root), dest))
    layers_f = os.path.join(dest, ".oe-layers.json")
    print("Writing list of layers into {}".format(layers_f))
    with open(layers_f, 'w') as f:
        json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)

def _get_remotes(r_remote):
    remotes = {}

    if not 'remotes' in r_remote and not 'uri' in r_remote:
        raise Exception("Expected key(s): 'remotes', 'uri'")

    if 'remotes' in r_remote:
        remotes = r_remote['remotes'].copy()

    if 'uri' in r_remote:
        r_name = ''

        if 'remotes' in r_remote:
            if not 'origin' in r_remote['remotes']:
                r_name = 'origin'
            else:
                import itertools
                for i in itertools.count(start=1):
                    if not 'origin-{}'.format(i) in r_remote['remotes']:
                        r_name = 'origin-{}'.format(i)
                        break
        else:
            r_name = 'origin'

        remotes.update({r_name: {'uri': r_remote['uri']}})

    return remotes

def _do_checkout(args, json):
    repos = json['sources']
    repodirs = []
    oesetupbuild = None
    for r_name in repos:
        r_data = repos[r_name]
        repodir = os.path.abspath(os.path.join(args['destdir'], r_data.get('path', r_name)))
        repodirs.append(repodir)

        if 'contains_this_file' in r_data.keys():
            force_arg = 'force_bootstraplayer_checkout'
            if not args[force_arg]:
                print('Note: not checking out source {repo}, use {repoflag} to override.'.format(repo=r_name, repoflag='--force-bootstraplayer-checkout'))
                continue
        r_remote = r_data['git-remote']
        rev = r_remote['rev']
        desc = r_remote['describe']
        if not desc:
            desc = rev[:10]
        branch = r_remote['branch']

        remotes = _get_remotes(r_remote)

        print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch))
        if not _is_repo_git_repo(repodir):
            cmd = ['git', 'init', '-q', repodir]
            print("Running '{}'".format(cmd))
            subprocess.check_output(cmd)

        for remote in remotes:
            if not _is_repo_at_remote_uri(repodir, remote, remotes[remote]['uri']):
                cmd = ['git', 'remote', 'remove', remote]
                # Ignore errors
                subprocess.run(cmd, cwd=repodir)
                cmd = ['git', 'remote', 'add', remote, remotes[remote]['uri']]
                print("Running '{}' in {}".format(cmd, repodir))
                subprocess.check_output(cmd, cwd=repodir)

                cmd = ['git', 'fetch', '-q', remote]
                print("Running '{}' in {}".format(cmd, repodir))
                # Ignore errors
                subprocess.run(cmd, cwd=repodir)

        if not _is_repo_at_rev(repodir, rev):
            cmd = ['git', 'fetch', '-q', '--all']
            print("Running '{}' in {}".format(cmd, repodir))
            # Ignore errors
            subprocess.run(cmd, cwd=repodir)

            cmd = ['git', 'checkout', '-q', rev]
            print("Running '{}' in {}".format(cmd, repodir))
            subprocess.check_output(cmd, cwd=repodir)

            if _contains_submodules(repodir):
                print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir))
        if os.path.exists(os.path.join(repodir, 'scripts/oe-setup-build')):
            oesetupbuild = os.path.join(repodir, 'scripts/oe-setup-build')

    _write_layer_list(args['destdir'], repodirs)

    if oesetupbuild:
        oesetupbuild_symlink = os.path.join(args['destdir'], 'setup-build')
        if os.path.exists(oesetupbuild_symlink):
            os.remove(oesetupbuild_symlink)
        os.symlink(os.path.relpath(oesetupbuild,args['destdir']),oesetupbuild_symlink)
        print("\nRun '{}' to list available build configuration templates and set up a build from one of them.".format(oesetupbuild_symlink))

parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/")

parser.add_argument('--force-bootstraplayer-checkout', action='store_true',
        help='Force the checkout of the layer containing this file (by default it is presumed that as this script is in it, the layer is already in place).')

try:
    defaultdest = os.path.dirname(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], universal_newlines=True, cwd=os.path.dirname(__file__)))
except subprocess.CalledProcessError as e:
    defaultdest = os.path.abspath(".")

parser.add_argument('--destdir', default=defaultdest, help='Where to check out the layers (default is {defaultdest}).'.format(defaultdest=defaultdest))
parser.add_argument('--jsondata', default=__file__+".json", help='File containing the layer data in json format (default is {defaultjson}).'.format(defaultjson=__file__+".json"))

args = parser.parse_args()

with open(args.jsondata) as f:
    json_f = json.load(f)

supported_versions = ["1.0"]
if json_f["version"] not in supported_versions:
    raise Exception("File {} has version {}, which is not in supported versions: {}".format(args.jsondata, json_f["version"], supported_versions))

_do_checkout(vars(args), json_f)