488 lines
17 KiB
Python
488 lines
17 KiB
Python
# SPDX-License-Identifier: GPL-2.0+
|
|
#
|
|
# Copyright 2020 Google LLC
|
|
#
|
|
"""Talks to the patchwork service to figure out what patches have been reviewed
|
|
and commented on. Provides a way to display review tags and comments.
|
|
Allows creation of a new branch based on the old but with the review tags
|
|
collected from patchwork.
|
|
"""
|
|
|
|
import collections
|
|
import concurrent.futures
|
|
from itertools import repeat
|
|
import re
|
|
|
|
import pygit2
|
|
import requests
|
|
|
|
from patman import patchstream
|
|
from patman.patchstream import PatchStream
|
|
from patman import terminal
|
|
from patman import tout
|
|
|
|
# Patches which are part of a multi-patch series are shown with a prefix like
|
|
# [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
|
|
# part is optional. This decodes the string into groups. For single patches
|
|
# the [] part is not present:
|
|
# Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
|
|
RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
|
|
|
|
# This decodes the sequence string into a patch number and patch count
|
|
RE_SEQ = re.compile(r'(\d+)/(\d+)')
|
|
|
|
def to_int(vals):
|
|
"""Convert a list of strings into integers, using 0 if not an integer
|
|
|
|
Args:
|
|
vals (list): List of strings
|
|
|
|
Returns:
|
|
list: List of integers, one for each input string
|
|
"""
|
|
out = [int(val) if val.isdigit() else 0 for val in vals]
|
|
return out
|
|
|
|
|
|
class Patch(dict):
|
|
"""Models a patch in patchwork
|
|
|
|
This class records information obtained from patchwork
|
|
|
|
Some of this information comes from the 'Patch' column:
|
|
|
|
[RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
|
|
|
|
This shows the prefix, version, seq, count and subject.
|
|
|
|
The other properties come from other columns in the display.
|
|
|
|
Properties:
|
|
pid (str): ID of the patch (typically an integer)
|
|
seq (int): Sequence number within series (1=first) parsed from sequence
|
|
string
|
|
count (int): Number of patches in series, parsed from sequence string
|
|
raw_subject (str): Entire subject line, e.g.
|
|
"[1/2,v2] efi_loader: Sort header file ordering"
|
|
prefix (str): Prefix string or None (e.g. 'RFC')
|
|
version (str): Version string or None (e.g. 'v2')
|
|
raw_subject (str): Raw patch subject
|
|
subject (str): Patch subject with [..] part removed (same as commit
|
|
subject)
|
|
"""
|
|
def __init__(self, pid):
|
|
super().__init__()
|
|
self.id = pid # Use 'id' to match what the Rest API provides
|
|
self.seq = None
|
|
self.count = None
|
|
self.prefix = None
|
|
self.version = None
|
|
self.raw_subject = None
|
|
self.subject = None
|
|
|
|
# These make us more like a dictionary
|
|
def __setattr__(self, name, value):
|
|
self[name] = value
|
|
|
|
def __getattr__(self, name):
|
|
return self[name]
|
|
|
|
def __hash__(self):
|
|
return hash(frozenset(self.items()))
|
|
|
|
def __str__(self):
|
|
return self.raw_subject
|
|
|
|
def parse_subject(self, raw_subject):
|
|
"""Parse the subject of a patch into its component parts
|
|
|
|
See RE_PATCH for details. The parsed info is placed into seq, count,
|
|
prefix, version, subject
|
|
|
|
Args:
|
|
raw_subject (str): Subject string to parse
|
|
|
|
Raises:
|
|
ValueError: the subject cannot be parsed
|
|
"""
|
|
self.raw_subject = raw_subject.strip()
|
|
mat = RE_PATCH.search(raw_subject.strip())
|
|
if not mat:
|
|
raise ValueError("Cannot parse subject '%s'" % raw_subject)
|
|
self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
|
|
mat_seq = RE_SEQ.match(seq_info) if seq_info else False
|
|
if mat_seq is None:
|
|
self.version = seq_info
|
|
seq_info = None
|
|
if self.version and not self.version.startswith('v'):
|
|
self.prefix = self.version
|
|
self.version = None
|
|
if seq_info:
|
|
if mat_seq:
|
|
self.seq = int(mat_seq.group(1))
|
|
self.count = int(mat_seq.group(2))
|
|
else:
|
|
self.seq = 1
|
|
self.count = 1
|
|
|
|
|
|
class Review:
|
|
"""Represents a single review email collected in Patchwork
|
|
|
|
Patches can attract multiple reviews. Each consists of an author/date and
|
|
a variable number of 'snippets', which are groups of quoted and unquoted
|
|
text.
|
|
"""
|
|
def __init__(self, meta, snippets):
|
|
"""Create new Review object
|
|
|
|
Args:
|
|
meta (str): Text containing review author and date
|
|
snippets (list): List of snippets in th review, each a list of text
|
|
lines
|
|
"""
|
|
self.meta = ' : '.join([line for line in meta.splitlines() if line])
|
|
self.snippets = snippets
|
|
|
|
def compare_with_series(series, patches):
|
|
"""Compare a list of patches with a series it came from
|
|
|
|
This prints any problems as warnings
|
|
|
|
Args:
|
|
series (Series): Series to compare against
|
|
patches (:type: list of Patch): list of Patch objects to compare with
|
|
|
|
Returns:
|
|
tuple
|
|
dict:
|
|
key: Commit number (0...n-1)
|
|
value: Patch object for that commit
|
|
dict:
|
|
key: Patch number (0...n-1)
|
|
value: Commit object for that patch
|
|
"""
|
|
# Check the names match
|
|
warnings = []
|
|
patch_for_commit = {}
|
|
all_patches = set(patches)
|
|
for seq, cmt in enumerate(series.commits):
|
|
pmatch = [p for p in all_patches if p.subject == cmt.subject]
|
|
if len(pmatch) == 1:
|
|
patch_for_commit[seq] = pmatch[0]
|
|
all_patches.remove(pmatch[0])
|
|
elif len(pmatch) > 1:
|
|
warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
|
|
(seq + 1, cmt.subject,
|
|
'\n '.join([p.subject for p in pmatch])))
|
|
else:
|
|
warnings.append("Cannot find patch for commit %d ('%s')" %
|
|
(seq + 1, cmt.subject))
|
|
|
|
|
|
# Check the names match
|
|
commit_for_patch = {}
|
|
all_commits = set(series.commits)
|
|
for seq, patch in enumerate(patches):
|
|
cmatch = [c for c in all_commits if c.subject == patch.subject]
|
|
if len(cmatch) == 1:
|
|
commit_for_patch[seq] = cmatch[0]
|
|
all_commits.remove(cmatch[0])
|
|
elif len(cmatch) > 1:
|
|
warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
|
|
(seq + 1, patch.subject,
|
|
'\n '.join([c.subject for c in cmatch])))
|
|
else:
|
|
warnings.append("Cannot find commit for patch %d ('%s')" %
|
|
(seq + 1, patch.subject))
|
|
|
|
return patch_for_commit, commit_for_patch, warnings
|
|
|
|
def call_rest_api(url, subpath):
|
|
"""Call the patchwork API and return the result as JSON
|
|
|
|
Args:
|
|
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
|
|
subpath (str): URL subpath to use
|
|
|
|
Returns:
|
|
dict: Json result
|
|
|
|
Raises:
|
|
ValueError: the URL could not be read
|
|
"""
|
|
full_url = '%s/api/1.2/%s' % (url, subpath)
|
|
response = requests.get(full_url)
|
|
if response.status_code != 200:
|
|
raise ValueError("Could not read URL '%s'" % full_url)
|
|
return response.json()
|
|
|
|
def collect_patches(series, series_id, url, rest_api=call_rest_api):
|
|
"""Collect patch information about a series from patchwork
|
|
|
|
Uses the Patchwork REST API to collect information provided by patchwork
|
|
about the status of each patch.
|
|
|
|
Args:
|
|
series (Series): Series object corresponding to the local branch
|
|
containing the series
|
|
series_id (str): Patch series ID number
|
|
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
|
|
rest_api (function): API function to call to access Patchwork, for
|
|
testing
|
|
|
|
Returns:
|
|
list: List of patches sorted by sequence number, each a Patch object
|
|
|
|
Raises:
|
|
ValueError: if the URL could not be read or the web page does not follow
|
|
the expected structure
|
|
"""
|
|
data = rest_api(url, 'series/%s/' % series_id)
|
|
|
|
# Get all the rows, which are patches
|
|
patch_dict = data['patches']
|
|
count = len(patch_dict)
|
|
num_commits = len(series.commits)
|
|
if count != num_commits:
|
|
tout.warning('Warning: Patchwork reports %d patches, series has %d' %
|
|
(count, num_commits))
|
|
|
|
patches = []
|
|
|
|
# Work through each row (patch) one at a time, collecting the information
|
|
warn_count = 0
|
|
for pw_patch in patch_dict:
|
|
patch = Patch(pw_patch['id'])
|
|
patch.parse_subject(pw_patch['name'])
|
|
patches.append(patch)
|
|
if warn_count > 1:
|
|
tout.warning(' (total of %d warnings)' % warn_count)
|
|
|
|
# Sort patches by patch number
|
|
patches = sorted(patches, key=lambda x: x.seq)
|
|
return patches
|
|
|
|
def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, url,
|
|
rest_api=call_rest_api):
|
|
"""Find new rtags collected by patchwork that we don't know about
|
|
|
|
This is designed to be run in parallel, once for each commit/patch
|
|
|
|
Args:
|
|
new_rtag_list (list): New rtags are written to new_rtag_list[seq]
|
|
list, each a dict:
|
|
key: Response tag (e.g. 'Reviewed-by')
|
|
value: Set of people who gave that response, each a name/email
|
|
string
|
|
review_list (list): New reviews are written to review_list[seq]
|
|
list, each a
|
|
List of reviews for the patch, each a Review
|
|
seq (int): Position in new_rtag_list to update
|
|
cmt (Commit): Commit object for this commit
|
|
patch (Patch): Corresponding Patch object for this patch
|
|
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
|
|
rest_api (function): API function to call to access Patchwork, for
|
|
testing
|
|
"""
|
|
if not patch:
|
|
return
|
|
|
|
# Get the content for the patch email itself as well as all comments
|
|
data = rest_api(url, 'patches/%s/' % patch.id)
|
|
pstrm = PatchStream.process_text(data['content'], True)
|
|
|
|
rtags = collections.defaultdict(set)
|
|
for response, people in pstrm.commit.rtags.items():
|
|
rtags[response].update(people)
|
|
|
|
data = rest_api(url, 'patches/%s/comments/' % patch.id)
|
|
|
|
reviews = []
|
|
for comment in data:
|
|
pstrm = PatchStream.process_text(comment['content'], True)
|
|
if pstrm.snippets:
|
|
submitter = comment['submitter']
|
|
person = '%s <%s>' % (submitter['name'], submitter['email'])
|
|
reviews.append(Review(person, pstrm.snippets))
|
|
for response, people in pstrm.commit.rtags.items():
|
|
rtags[response].update(people)
|
|
|
|
# Find the tags that are not in the commit
|
|
new_rtags = collections.defaultdict(set)
|
|
base_rtags = cmt.rtags
|
|
for tag, people in rtags.items():
|
|
for who in people:
|
|
is_new = (tag not in base_rtags or
|
|
who not in base_rtags[tag])
|
|
if is_new:
|
|
new_rtags[tag].add(who)
|
|
new_rtag_list[seq] = new_rtags
|
|
review_list[seq] = reviews
|
|
|
|
def show_responses(rtags, indent, is_new):
|
|
"""Show rtags collected
|
|
|
|
Args:
|
|
rtags (dict): review tags to show
|
|
key: Response tag (e.g. 'Reviewed-by')
|
|
value: Set of people who gave that response, each a name/email string
|
|
indent (str): Indentation string to write before each line
|
|
is_new (bool): True if this output should be highlighted
|
|
|
|
Returns:
|
|
int: Number of review tags displayed
|
|
"""
|
|
col = terminal.Color()
|
|
count = 0
|
|
for tag in sorted(rtags.keys()):
|
|
people = rtags[tag]
|
|
for who in sorted(people):
|
|
terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
|
|
newline=False, colour=col.GREEN, bright=is_new)
|
|
terminal.tprint(who, colour=col.WHITE, bright=is_new)
|
|
count += 1
|
|
return count
|
|
|
|
def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
|
|
repo=None):
|
|
"""Create a new branch with review tags added
|
|
|
|
Args:
|
|
series (Series): Series object for the existing branch
|
|
new_rtag_list (list): List of review tags to add, one for each commit,
|
|
each a dict:
|
|
key: Response tag (e.g. 'Reviewed-by')
|
|
value: Set of people who gave that response, each a name/email
|
|
string
|
|
branch (str): Existing branch to update
|
|
dest_branch (str): Name of new branch to create
|
|
overwrite (bool): True to force overwriting dest_branch if it exists
|
|
repo (pygit2.Repository): Repo to use (use None unless testing)
|
|
|
|
Returns:
|
|
int: Total number of review tags added across all commits
|
|
|
|
Raises:
|
|
ValueError: if the destination branch name is the same as the original
|
|
branch, or it already exists and @overwrite is False
|
|
"""
|
|
if branch == dest_branch:
|
|
raise ValueError(
|
|
'Destination branch must not be the same as the original branch')
|
|
if not repo:
|
|
repo = pygit2.Repository('.')
|
|
count = len(series.commits)
|
|
new_br = repo.branches.get(dest_branch)
|
|
if new_br:
|
|
if not overwrite:
|
|
raise ValueError("Branch '%s' already exists (-f to overwrite)" %
|
|
dest_branch)
|
|
new_br.delete()
|
|
if not branch:
|
|
branch = 'HEAD'
|
|
target = repo.revparse_single('%s~%d' % (branch, count))
|
|
repo.branches.local.create(dest_branch, target)
|
|
|
|
num_added = 0
|
|
for seq in range(count):
|
|
parent = repo.branches.get(dest_branch)
|
|
cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
|
|
|
|
repo.merge_base(cherry.oid, parent.target)
|
|
base_tree = cherry.parents[0].tree
|
|
|
|
index = repo.merge_trees(base_tree, parent, cherry)
|
|
tree_id = index.write_tree(repo)
|
|
|
|
lines = []
|
|
if new_rtag_list[seq]:
|
|
for tag, people in new_rtag_list[seq].items():
|
|
for who in people:
|
|
lines.append('%s: %s' % (tag, who))
|
|
num_added += 1
|
|
message = patchstream.insert_tags(cherry.message.rstrip(),
|
|
sorted(lines))
|
|
|
|
repo.create_commit(
|
|
parent.name, cherry.author, cherry.committer, message, tree_id,
|
|
[parent.target])
|
|
return num_added
|
|
|
|
def check_patchwork_status(series, series_id, branch, dest_branch, force,
|
|
show_comments, url, rest_api=call_rest_api,
|
|
test_repo=None):
|
|
"""Check the status of a series on Patchwork
|
|
|
|
This finds review tags and comments for a series in Patchwork, displaying
|
|
them to show what is new compared to the local series.
|
|
|
|
Args:
|
|
series (Series): Series object for the existing branch
|
|
series_id (str): Patch series ID number
|
|
branch (str): Existing branch to update, or None
|
|
dest_branch (str): Name of new branch to create, or None
|
|
force (bool): True to force overwriting dest_branch if it exists
|
|
show_comments (bool): True to show the comments on each patch
|
|
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
|
|
rest_api (function): API function to call to access Patchwork, for
|
|
testing
|
|
test_repo (pygit2.Repository): Repo to use (use None unless testing)
|
|
"""
|
|
patches = collect_patches(series, series_id, url, rest_api)
|
|
col = terminal.Color()
|
|
count = len(series.commits)
|
|
new_rtag_list = [None] * count
|
|
review_list = [None] * count
|
|
|
|
patch_for_commit, _, warnings = compare_with_series(series, patches)
|
|
for warn in warnings:
|
|
tout.warning(warn)
|
|
|
|
patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
|
|
futures = executor.map(
|
|
find_new_responses, repeat(new_rtag_list), repeat(review_list),
|
|
range(count), series.commits, patch_list, repeat(url),
|
|
repeat(rest_api))
|
|
for fresponse in futures:
|
|
if fresponse:
|
|
raise fresponse.exception()
|
|
|
|
num_to_add = 0
|
|
for seq, cmt in enumerate(series.commits):
|
|
patch = patch_for_commit.get(seq)
|
|
if not patch:
|
|
continue
|
|
terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]),
|
|
colour=col.BLUE)
|
|
cmt = series.commits[seq]
|
|
base_rtags = cmt.rtags
|
|
new_rtags = new_rtag_list[seq]
|
|
|
|
indent = ' ' * 2
|
|
show_responses(base_rtags, indent, False)
|
|
num_to_add += show_responses(new_rtags, indent, True)
|
|
if show_comments:
|
|
for review in review_list[seq]:
|
|
terminal.tprint('Review: %s' % review.meta, colour=col.RED)
|
|
for snippet in review.snippets:
|
|
for line in snippet:
|
|
quoted = line.startswith('>')
|
|
terminal.tprint(' %s' % line,
|
|
colour=col.MAGENTA if quoted else None)
|
|
terminal.tprint()
|
|
|
|
terminal.tprint("%d new response%s available in patchwork%s" %
|
|
(num_to_add, 's' if num_to_add != 1 else '',
|
|
'' if dest_branch
|
|
else ' (use -d to write them to a new branch)'))
|
|
|
|
if dest_branch:
|
|
num_added = create_branch(series, new_rtag_list, branch,
|
|
dest_branch, force, test_repo)
|
|
terminal.tprint(
|
|
"%d response%s added from patchwork into new branch '%s'" %
|
|
(num_added, 's' if num_added != 1 else '', dest_branch))
|