| # SPDX-License-Identifier: GPL-2.0+ | |
| # | |
| # Copyright 2020 Google LLC | |
| # | |
| """Talks to the patchwork service to figure out what patches have been reviewed | |
| and commented on. Provides a way to display review tags and comments. | |
| Allows creation of a new branch based on the old but with the review tags | |
| collected from patchwork. | |
| """ | |
| import asyncio | |
| from collections import defaultdict | |
| import concurrent.futures | |
| from itertools import repeat | |
| import aiohttp | |
| import pygit2 | |
| from u_boot_pylib import terminal | |
| from u_boot_pylib import tout | |
| from patman import patchstream | |
| from patman import patchwork | |
| def process_reviews(content, comment_data, base_rtags): | |
| """Process and return review data | |
| Args: | |
| content (str): Content text of the patch itself - see pwork.get_patch() | |
| comment_data (list of dict): Comments for the patch - see | |
| pwork._get_patch_comments() | |
| base_rtags (dict): base review tags (before any comments) | |
| key: Response tag (e.g. 'Reviewed-by') | |
| value: Set of people who gave that response, each a name/email | |
| string | |
| Return: tuple: | |
| dict: new review tags (noticed since the base_rtags) | |
| key: Response tag (e.g. 'Reviewed-by') | |
| value: Set of people who gave that response, each a name/email | |
| string | |
| list of patchwork.Review: reviews received on the patch | |
| """ | |
| pstrm = patchstream.PatchStream.process_text(content, True) | |
| rtags = defaultdict(set) | |
| for response, people in pstrm.commit.rtags.items(): | |
| rtags[response].update(people) | |
| reviews = [] | |
| for comment in comment_data: | |
| pstrm = patchstream.PatchStream.process_text(comment['content'], True) | |
| if pstrm.snippets: | |
| submitter = comment['submitter'] | |
| person = f"{submitter['name']} <{submitter['email']}>" | |
| reviews.append(patchwork.Review(person, pstrm.snippets)) | |
| for response, people in pstrm.commit.rtags.items(): | |
| rtags[response].update(people) | |
| # Find the tags that are not in the commit | |
| new_rtags = defaultdict(set) | |
| for tag, people in rtags.items(): | |
| for who in people: | |
| is_new = (tag not in base_rtags or | |
| who not in base_rtags[tag]) | |
| if is_new: | |
| new_rtags[tag].add(who) | |
| return new_rtags, reviews | |
| def compare_with_series(series, patches): | |
| """Compare a list of patches with a series it came from | |
| This prints any problems as warnings | |
| Args: | |
| series (Series): Series to compare against | |
| patches (list of Patch): list of Patch objects to compare with | |
| Returns: | |
| tuple | |
| dict: | |
| key: Commit number (0...n-1) | |
| value: Patch object for that commit | |
| dict: | |
| key: Patch number (0...n-1) | |
| value: Commit object for that patch | |
| """ | |
| # Check the names match | |
| warnings = [] | |
| patch_for_commit = {} | |
| all_patches = set(patches) | |
| for seq, cmt in enumerate(series.commits): | |
| pmatch = [p for p in all_patches if p.subject == cmt.subject] | |
| if len(pmatch) == 1: | |
| patch_for_commit[seq] = pmatch[0] | |
| all_patches.remove(pmatch[0]) | |
| elif len(pmatch) > 1: | |
| warnings.append("Multiple patches match commit %d ('%s'):\n %s" % | |
| (seq + 1, cmt.subject, | |
| '\n '.join([p.subject for p in pmatch]))) | |
| else: | |
| warnings.append("Cannot find patch for commit %d ('%s')" % | |
| (seq + 1, cmt.subject)) | |
| # Check the names match | |
| commit_for_patch = {} | |
| all_commits = set(series.commits) | |
| for seq, patch in enumerate(patches): | |
| cmatch = [c for c in all_commits if c.subject == patch.subject] | |
| if len(cmatch) == 1: | |
| commit_for_patch[seq] = cmatch[0] | |
| all_commits.remove(cmatch[0]) | |
| elif len(cmatch) > 1: | |
| warnings.append("Multiple commits match patch %d ('%s'):\n %s" % | |
| (seq + 1, patch.subject, | |
| '\n '.join([c.subject for c in cmatch]))) | |
| else: | |
| warnings.append("Cannot find commit for patch %d ('%s')" % | |
| (seq + 1, patch.subject)) | |
| return patch_for_commit, commit_for_patch, warnings | |
| def show_responses(col, rtags, indent, is_new): | |
| """Show rtags collected | |
| Args: | |
| col (terminal.Colour): Colour object to use | |
| rtags (dict): review tags to show | |
| key: Response tag (e.g. 'Reviewed-by') | |
| value: Set of people who gave that response, each a name/email string | |
| indent (str): Indentation string to write before each line | |
| is_new (bool): True if this output should be highlighted | |
| Returns: | |
| int: Number of review tags displayed | |
| """ | |
| count = 0 | |
| for tag in sorted(rtags.keys()): | |
| people = rtags[tag] | |
| for who in sorted(people): | |
| terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag), | |
| newline=False, colour=col.GREEN, bright=is_new, | |
| col=col) | |
| terminal.tprint(who, colour=col.WHITE, bright=is_new, col=col) | |
| count += 1 | |
| return count | |
| def create_branch(series, new_rtag_list, branch, dest_branch, overwrite, | |
| repo=None): | |
| """Create a new branch with review tags added | |
| Args: | |
| series (Series): Series object for the existing branch | |
| new_rtag_list (list): List of review tags to add, one for each commit, | |
| each a dict: | |
| key: Response tag (e.g. 'Reviewed-by') | |
| value: Set of people who gave that response, each a name/email | |
| string | |
| branch (str): Existing branch to update | |
| dest_branch (str): Name of new branch to create | |
| overwrite (bool): True to force overwriting dest_branch if it exists | |
| repo (pygit2.Repository): Repo to use (use None unless testing) | |
| Returns: | |
| int: Total number of review tags added across all commits | |
| Raises: | |
| ValueError: if the destination branch name is the same as the original | |
| branch, or it already exists and @overwrite is False | |
| """ | |
| if branch == dest_branch: | |
| raise ValueError( | |
| 'Destination branch must not be the same as the original branch') | |
| if not repo: | |
| repo = pygit2.Repository('.') | |
| count = len(series.commits) | |
| new_br = repo.branches.get(dest_branch) | |
| if new_br: | |
| if not overwrite: | |
| raise ValueError("Branch '%s' already exists (-f to overwrite)" % | |
| dest_branch) | |
| new_br.delete() | |
| if not branch: | |
| branch = 'HEAD' | |
| target = repo.revparse_single('%s~%d' % (branch, count)) | |
| repo.branches.local.create(dest_branch, target) | |
| num_added = 0 | |
| for seq in range(count): | |
| parent = repo.branches.get(dest_branch) | |
| cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1)) | |
| repo.merge_base(cherry.oid, parent.target) | |
| base_tree = cherry.parents[0].tree | |
| index = repo.merge_trees(base_tree, parent, cherry) | |
| tree_id = index.write_tree(repo) | |
| lines = [] | |
| if new_rtag_list[seq]: | |
| for tag, people in new_rtag_list[seq].items(): | |
| for who in people: | |
| lines.append('%s: %s' % (tag, who)) | |
| num_added += 1 | |
| message = patchstream.insert_tags(cherry.message.rstrip(), | |
| sorted(lines)) | |
| repo.create_commit( | |
| parent.name, cherry.author, cherry.committer, message, tree_id, | |
| [parent.target]) | |
| return num_added | |
| def check_patch_count(num_commits, num_patches): | |
| """Check the number of commits and patches agree | |
| Args: | |
| num_commits (int): Number of commits | |
| num_patches (int): Number of patches | |
| """ | |
| if num_patches != num_commits: | |
| tout.warning(f'Warning: Patchwork reports {num_patches} patches, ' | |
| f'series has {num_commits}') | |
| def do_show_status(series, cover, patches, show_comments, show_cover_comments, | |
| col, warnings_on_stderr=True): | |
| """Check the status of a series on Patchwork | |
| This finds review tags and comments for a series in Patchwork, displaying | |
| them to show what is new compared to the local series. | |
| Args: | |
| series (Series): Series object for the existing branch | |
| cover (COVER): Cover letter info, or None if none | |
| patches (list of Patch): Patches sorted by sequence number | |
| show_comments (bool): True to show the comments on each patch | |
| show_cover_comments (bool): True to show the comments on the | |
| letter | |
| col (terminal.Colour): Colour object | |
| Return: tuple: | |
| int: Number of new review tags to add | |
| list: List of review tags to add, one item for each commit, each a | |
| dict: | |
| key: Response tag (e.g. 'Reviewed-by') | |
| value: Set of people who gave that response, each a name/email | |
| string | |
| """ | |
| compare = [] | |
| for pw_patch in patches: | |
| patch = patchwork.Patch(pw_patch.id) | |
| patch.parse_subject(pw_patch.series_data['name']) | |
| compare.append(patch) | |
| count = len(series.commits) | |
| new_rtag_list = [None] * count | |
| review_list = [None] * count | |
| with terminal.pager(): | |
| patch_for_commit, _, warnings = compare_with_series(series, compare) | |
| for warn in warnings: | |
| tout.do_output(tout.WARNING if warnings_on_stderr else tout.INFO, | |
| warn) | |
| for seq, pw_patch in enumerate(patches): | |
| compare[seq].patch = pw_patch | |
| for i in range(count): | |
| pat = patch_for_commit.get(i) | |
| if pat: | |
| patch_data = pat.patch.data | |
| comment_data = pat.patch.comments | |
| new_rtag_list[i], review_list[i] = process_reviews( | |
| patch_data['content'], comment_data, | |
| series.commits[i].rtags) | |
| num_to_add = _do_show_status( | |
| series, cover, patch_for_commit, show_comments, | |
| show_cover_comments, new_rtag_list, review_list, col) | |
| return num_to_add, new_rtag_list | |
| def _do_show_status(series, cover, patch_for_commit, show_comments, | |
| show_cover_comments, new_rtag_list, review_list, col): | |
| if cover and show_cover_comments: | |
| terminal.tprint(f'Cov {cover.name}', colour=col.BLACK, col=col, | |
| bright=False, back=col.YELLOW) | |
| for seq, comment in enumerate(cover.comments): | |
| submitter = comment['submitter'] | |
| person = '%s <%s>' % (submitter['name'], submitter['email']) | |
| terminal.tprint(f"From: {person}: {comment['date']}", | |
| colour=col.RED, col=col) | |
| print(comment['content']) | |
| print() | |
| num_to_add = 0 | |
| for seq, cmt in enumerate(series.commits): | |
| patch = patch_for_commit.get(seq) | |
| if not patch: | |
| continue | |
| terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]), | |
| colour=col.YELLOW, col=col) | |
| cmt = series.commits[seq] | |
| base_rtags = cmt.rtags | |
| new_rtags = new_rtag_list[seq] | |
| indent = ' ' * 2 | |
| show_responses(col, base_rtags, indent, False) | |
| num_to_add += show_responses(col, new_rtags, indent, True) | |
| if show_comments: | |
| for review in review_list[seq]: | |
| terminal.tprint('Review: %s' % review.meta, colour=col.RED, | |
| col=col) | |
| for snippet in review.snippets: | |
| for line in snippet: | |
| quoted = line.startswith('>') | |
| terminal.tprint( | |
| f' {line}', | |
| colour=col.MAGENTA if quoted else None, col=col) | |
| terminal.tprint() | |
| return num_to_add | |
| def show_status(series, branch, dest_branch, force, cover, patches, | |
| show_comments, show_cover_comments, test_repo=None): | |
| """Check the status of a series on Patchwork | |
| This finds review tags and comments for a series in Patchwork, displaying | |
| them to show what is new compared to the local series. | |
| Args: | |
| client (aiohttp.ClientSession): Session to use | |
| series (Series): Series object for the existing branch | |
| branch (str): Existing branch to update, or None | |
| dest_branch (str): Name of new branch to create, or None | |
| force (bool): True to force overwriting dest_branch if it exists | |
| cover (COVER): Cover letter info, or None if none | |
| patches (list of Patch): Patches sorted by sequence number | |
| show_comments (bool): True to show the comments on each patch | |
| show_cover_comments (bool): True to show the comments on the letter | |
| test_repo (pygit2.Repository): Repo to use (use None unless testing) | |
| """ | |
| col = terminal.Color() | |
| check_patch_count(len(series.commits), len(patches)) | |
| num_to_add, new_rtag_list = do_show_status( | |
| series, cover, patches, show_comments, show_cover_comments, col) | |
| if not dest_branch and num_to_add: | |
| msg = ' (use -d to write them to a new branch)' | |
| else: | |
| msg = '' | |
| terminal.tprint( | |
| f"{num_to_add} new response{'s' if num_to_add != 1 else ''} " | |
| f'available in patchwork{msg}') | |
| if dest_branch: | |
| num_added = create_branch(series, new_rtag_list, branch, | |
| dest_branch, force, test_repo) | |
| terminal.tprint( | |
| f"{num_added} response{'s' if num_added != 1 else ''} added " | |
| f"from patchwork into new branch '{dest_branch}'") | |
| async def check_status(link, pwork, read_comments=False, | |
| read_cover_comments=False): | |
| """Set up an HTTP session and get the required state | |
| Args: | |
| link (str): Patch series ID number | |
| pwork (Patchwork): Patchwork object to use for reading | |
| read_comments (bool): True to read comments and state for each patch | |
| Return: tuple: | |
| COVER object, or None if none or not read_cover_comments | |
| list of PATCH objects | |
| """ | |
| async with aiohttp.ClientSession() as client: | |
| return await pwork.series_get_state(client, link, read_comments, | |
| read_cover_comments) | |
| def check_and_show_status(series, link, branch, dest_branch, force, | |
| show_comments, show_cover_comments, pwork, | |
| test_repo=None): | |
| """Read the series status from patchwork and show it to the user | |
| Args: | |
| series (Series): Series object for the existing branch | |
| link (str): Patch series ID number | |
| branch (str): Existing branch to update, or None | |
| dest_branch (str): Name of new branch to create, or None | |
| force (bool): True to force overwriting dest_branch if it exists | |
| show_comments (bool): True to show the comments on each patch | |
| show_cover_comments (bool): True to show the comments on the letter | |
| pwork (Patchwork): Patchwork object to use for reading | |
| test_repo (pygit2.Repository): Repo to use (use None unless testing) | |
| """ | |
| loop = asyncio.get_event_loop() | |
| cover, patches = loop.run_until_complete(check_status( | |
| link, pwork, True, show_cover_comments)) | |
| show_status(series, branch, dest_branch, force, cover, patches, | |
| show_comments, show_cover_comments, test_repo=test_repo) |