Refactor git_metadata step and yaml config. Add machine_setup and branch support. This adds the implementation of the machine setup step. Implements directory location and branch support for git clone. Refactors git_metadata step to be test driven. Adds tests for git_metadata step. Combines multiple yaml steps into one step for prebuild. Tested: I ran unit tests with pytest to test for function behaviors of git_metadata step. All test cases passed. Signed-off-by: Simon-Lii <thesimonli@google.com> Change-Id: Ia3524b15313c08140b9a55cbe3d2aa4fbf578744
diff --git a/cloudbuild.yaml b/cloudbuild.yaml index dfeda55..9c560b0 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml
@@ -1,34 +1,22 @@ steps: - name: ${_GIT_CLONE_TOOLCHAIN_IMAGE} - dir: ${_BUILD_SCRIPT_DIR} + entrypoint: /bin/python3 + env: + - REPOS=${_REPOS} + - FIRMWARE_MACHINE=${_FIRMWARE_MACHINE} + - MAIN_REPO_NAME=${_MAIN_REPO_NAME} + - BUILD_SCRIPT_DIR=${_BUILD_SCRIPT_DIR} args: - - clone - - '-b' - - ${_BRANCH} - - ${_MAIN_REPO} + - -c + - | + import prebuild - - name: ${_GIT_CLONE_TOOLCHAIN_IMAGE} - entrypoint: /bin/bash - dir: ${_BUILD_SCRIPT_DIR} - args: - - '-c' - - > - - list_of_repos=( ${_REPOS} ) - - for repo in "${list_of_repos[@]}" - - do - git clone "$repo" - done - - - - name: ${_BUILDER_TOOLCHAIN_IMAGE} - args: - - python3 - - prebuild.py + prebuild.main() timeout: 10000s options: machineType: E2_HIGHCPU_32 logging: CLOUD_LOGGING_ONLY + volumes: + - name: vol1 + path: /workspace/gcb/bmc
diff --git a/prebuild.py b/prebuild.py index dc2e011..c949266 100644 --- a/prebuild.py +++ b/prebuild.py
@@ -1,13 +1,13 @@ -from steps import git_metadata -import traceback +from steps import git_metadata, machine_setup class Prebuild(): - ''' Driver class that runs all the steps indicated in STEP_NAMES ''' def __init__(self): self.git_metadata_step = git_metadata.GitMetadata() + self.machine_setup_step = machine_setup.MachineSetup() def run_prebuild(self): self.git_metadata_step.run_script() + self.machine_setup_step.run_script() def main(): prebuild = Prebuild()
diff --git a/steps/git_metadata.py b/steps/git_metadata.py index c71ad6b..54b7518 100644 --- a/steps/git_metadata.py +++ b/steps/git_metadata.py
@@ -1,19 +1,20 @@ -from steps import step import sh -import os -import subprocess +from steps import step class GitMetadata(step.Step): - def __init__(self): - super().__init__() - repo_string = os.getenv('REPO_NAMES') - self.repos = [x.strip() for x in repo_string.split(',')] + def git_clone_with_branch(self, repo_url, branch, dir): + """Git clones a repo_url with a specific branch and clones into dir""" + sh.git("-C", dir, "clone", "-b", branch, repo_url) - def move_meta_repos_in_main_repo(self): - '''Moves the meta repos into the gbmc folder''' - sh.cd(self.build_dir) - for i in range(1, len(self.repos)): - sh.mv(self.repos[i], self.repos[0]) + def git_clone(self, repo_url, dir): + """Git clones a repo_url into dir""" + sh.git("-C", dir, "clone", repo_url) def run_script(self): - self.move_meta_repos_in_main_repo() + sh.cd(self.build_script_dir) + for repo in self.parse_repos(self.repos): + repo_url, branch, dir = repo + if branch: + self.git_clone_with_branch(repo_url, branch, dir) + else: + self.git_clone(repo_url, dir)
diff --git a/steps/machine_setup.py b/steps/machine_setup.py new file mode 100644 index 0000000..49d6062 --- /dev/null +++ b/steps/machine_setup.py
@@ -0,0 +1,13 @@ +from steps import step +import sh +import subprocess + +class MachineSetup(step.Step): + def run_script(self): + """Sets up the firmware machine environment used by the build""" + sh.cd(self.main_repo_name) + + # command = "export MACHINE=machine0 && source setup machine0 && umask002 " + command = " && ".join([self.export_machine(), self.source_setup(), self.umask()]) + + subprocess.run(["bash", "-c", command])
diff --git a/steps/step.py b/steps/step.py index f8c049f..820d8c6 100644 --- a/steps/step.py +++ b/steps/step.py
@@ -3,9 +3,16 @@ class Step(): def __init__(self): - self.build_dir = os.getenv('MAIN_PATH') - self.home_dir = str(sh.pwd())[0:-1] - self.machine_name = os.getenv('MACHINE_NAME') + self.firmware_machine = os.getenv("FIRMWARE_MACHINE") + self.main_repo_name = os.getenv("MAIN_REPO_NAME") + self.repos = os.getenv('REPOS') + self.build_script_dir = os.getenv('BUILD_SCRIPT_DIR') + + sh.cd(self.build_script_dir) + + def parse_repos(self, repos_string): + '''Parses a string representation of an array of triples''' + return eval(repos_string) def run_script(self): pass
diff --git a/unit_tests/test_git_metadata.py b/unit_tests/test_git_metadata.py new file mode 100644 index 0000000..f21e0ec --- /dev/null +++ b/unit_tests/test_git_metadata.py
@@ -0,0 +1,46 @@ +import unittest +from unittest.mock import MagicMock +from steps import git_metadata + +class TestGitMetadata(unittest.TestCase): + def setUp(self): + self.step = git_metadata.GitMetadata() + + self.step.repos = " [('repo0', 'branch0', 'dir0'), \ + ('repo1', 'branch1', ''), \ + ('repo2', '', 'dir1'), \ + ('repo3', '', 'dir1'), \ + ('repo4', '', 'dir1')]" + + self.step.git_clone_with_branch = MagicMock() + self.step.git_clone = MagicMock() + + self.step.run_script() + + def test_git_clone_with_branch(self): + """Tests git_clone_with_branch method is called with the right arguments""" + self.step.git_clone_with_branch.assert_any_call("repo0", "branch0", "dir0") + self.step.git_clone_with_branch.assert_called_with("repo1", "branch1", "") + + def test_git_clone(self): + """Tests git_clone method is called with the right arguments""" + self.step.git_clone.assert_any_call("repo2", "dir1") + self.step.git_clone.assert_any_call("repo3", "dir1") + self.step.git_clone.assert_called_with("repo4", "dir1") + + def test_git_clone_with_branch_order(self): + """Tests that git_clone_with_branch method calls are in the right order""" + actual_calls = self.step.git_clone_with_branch.call_args_list + expected_calls = [(("repo0", "branch0", "dir0"),), + (("repo1", "branch1", ""),)] + + self.assertEqual(actual_calls, expected_calls) + + def test_git_clone_order(self): + """Tests the order in which git_clone method are called""" + actual_calls = self.step.git_clone.call_args_list + expected_calls = [(("repo2", "dir1"),), + (("repo3", "dir1"),), + (("repo4", "dir1"),)] + + self.assertEqual(actual_calls, expected_calls)