1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2011 The Chromium OS Authors.
12 from patman import command
13 from patman import commit
14 from patman import gitutil
15 from patman.series import Series
17 # Tags that we detect and remove
18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
19 '|Reviewed-on:|Commit-\w*:')
21 # Lines which are allowed after a TEST= line
22 re_allowed_after_test = re.compile('^Signed-off-by:')
25 re_signoff = re.compile('^Signed-off-by: *(.*)')
28 re_cover = re.compile('^Cover-([a-z-]*): *(.*)')
31 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
33 # Change-Id will be used to generate the Message-Id and then be stripped
34 re_change_id = re.compile('^Change-Id: *(.*)')
37 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
39 # Commit tags that we want to collect and keep
40 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
42 # The start of a new commit in the git log
43 re_commit = re.compile('^commit ([0-9a-f]*)$')
45 # We detect these since checkpatch doesn't always do it
46 re_space_before_tab = re.compile('^[+].* \t')
48 # Match indented lines for changes
49 re_leading_whitespace = re.compile('^\s')
51 # States we can be in - can we use range() and still have comments?
52 STATE_MSG_HEADER = 0 # Still in the message header
53 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
54 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
55 STATE_DIFFS = 3 # In the diff part (past --- line)
58 """Class for detecting/injecting tags in a patch or series of patches
60 We support processing the output of 'git log' to read out the tags we
61 are interested in. We can also process a patch file in order to remove
62 unwanted tags or inject additional ones. These correspond to the two
65 def __init__(self, series, name=None, is_log=False):
66 self.skip_blank = False # True to skip a single blank line
67 self.found_test = False # Found a TEST= line
68 self.lines_after_test = 0 # Number of lines found after TEST=
69 self.warn = [] # List of warnings we have collected
70 self.linenum = 1 # Output line number we are up to
71 self.in_section = None # Name of start...END section we are in
72 self.notes = [] # Series notes
73 self.section = [] # The current section...END section
74 self.series = series # Info about the patch series
75 self.is_log = is_log # True if indent like git log
76 self.in_change = None # Name of the change list we are in
77 self.change_version = 0 # Non-zero if we are in a change list
78 self.change_lines = [] # Lines of the current change
79 self.blank_count = 0 # Number of blank lines stored up
80 self.state = STATE_MSG_HEADER # What state are we in?
81 self.signoff = [] # Contents of signoff line
82 self.commit = None # Current commit
84 def AddToSeries(self, line, name, value):
85 """Add a new Series-xxx tag.
87 When a Series-xxx tag is detected, we come here to record it, if we
88 are scanning a 'git log'.
91 line: Source line containing tag (useful for debug/error messages)
92 name: Tag name (part after 'Series-')
93 value: Tag value (part after 'Series-xxx: ')
96 self.in_section = name
97 self.skip_blank = False
99 self.series.AddTag(self.commit, line, name, value)
101 def AddToCommit(self, line, name, value):
102 """Add a new Commit-xxx tag.
104 When a Commit-xxx tag is detected, we come here to record it.
107 line: Source line containing tag (useful for debug/error messages)
108 name: Tag name (part after 'Commit-')
109 value: Tag value (part after 'Commit-xxx: ')
112 self.in_section = 'commit-' + name
113 self.skip_blank = False
115 def CloseCommit(self):
116 """Save the current commit into our commit list, and reset our state"""
117 if self.commit and self.is_log:
118 self.series.AddCommit(self.commit)
120 # If 'END' is missing in a 'Cover-letter' section, and that section
121 # happens to show up at the very end of the commit message, this is
122 # the chance for us to fix it up.
123 if self.in_section == 'cover' and self.is_log:
124 self.series.cover = self.section
125 self.in_section = None
126 self.skip_blank = True
129 def ParseVersion(self, value, line):
130 """Parse a version from a *-changes tag
133 value: Tag value (part after 'xxx-changes: '
134 line: Source line containing tag
137 The version as an integer
141 except ValueError as str:
142 raise ValueError("%s: Cannot decode version info '%s'" %
143 (self.commit.hash, line))
145 def FinalizeChange(self):
146 """Finalize a (multi-line) change and add it to the series or commit"""
147 if not self.change_lines:
149 change = '\n'.join(self.change_lines)
151 if self.in_change == 'Series':
152 self.series.AddChange(self.change_version, self.commit, change)
153 elif self.in_change == 'Cover':
154 self.series.AddChange(self.change_version, None, change)
155 elif self.in_change == 'Commit':
156 self.commit.AddChange(self.change_version, change)
157 self.change_lines = []
159 def ProcessLine(self, line):
160 """Process a single line of a patch file or commit log
162 This process a line and returns a list of lines to output. The list
163 may be empty or may contain multiple output lines.
165 This is where all the complicated logic is located. The class's
166 state is used to move between different states and detect things
169 We can be in one of two modes:
170 self.is_log == True: This is 'git log' mode, where most output is
171 indented by 4 characters and we are scanning for tags
173 self.is_log == False: This is 'patch' mode, where we already have
174 all the tags, and are processing patches to remove junk we
175 don't want, and add things we think are required.
178 line: text line to process
181 list of output lines, or [] if nothing should be output
183 # Initially we have no output. Prepare the input line string
185 line = line.rstrip('\n')
187 commit_match = re_commit.match(line) if self.is_log else None
193 # Handle state transition and skipping blank lines
194 series_tag_match = re_series_tag.match(line)
195 change_id_match = re_change_id.match(line)
196 commit_tag_match = re_commit_tag.match(line)
197 cover_match = re_cover.match(line)
198 signoff_match = re_signoff.match(line)
199 leading_whitespace_match = re_leading_whitespace.match(line)
201 if self.state == STATE_PATCH_HEADER:
202 tag_match = re_tag.match(line)
203 is_blank = not line.strip()
205 if (self.state == STATE_MSG_HEADER
206 or self.state == STATE_PATCH_SUBJECT):
209 # We don't have a subject in the text stream of patch files
210 # It has its own line with a Subject: tag
211 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
214 self.state = STATE_MSG_HEADER
216 # If a tag is detected, or a new commit starts
217 if series_tag_match or commit_tag_match or change_id_match or \
218 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
219 # but we are already in a section, this means 'END' is missing
220 # for that section, fix it up.
222 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
223 if self.in_section == 'cover':
224 self.series.cover = self.section
225 elif self.in_section == 'notes':
227 self.series.notes += self.section
228 elif self.in_section == 'commit-notes':
230 self.commit.notes += self.section
232 self.warn.append("Unknown section '%s'" % self.in_section)
233 self.in_section = None
234 self.skip_blank = True
236 # but we are already in a change list, that means a blank line
237 # is missing, fix it up.
239 self.warn.append("Missing 'blank line' in section '%s-changes'" % self.in_change)
240 self.FinalizeChange()
241 self.in_change = None
242 self.change_version = 0
244 # If we are in a section, keep collecting lines until we see END
247 if self.in_section == 'cover':
248 self.series.cover = self.section
249 elif self.in_section == 'notes':
251 self.series.notes += self.section
252 elif self.in_section == 'commit-notes':
254 self.commit.notes += self.section
256 self.warn.append("Unknown section '%s'" % self.in_section)
257 self.in_section = None
258 self.skip_blank = True
261 self.section.append(line)
263 # Detect the commit subject
264 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
265 self.commit.subject = line
267 # Detect the tags we want to remove, and skip blank lines
268 elif re_remove.match(line) and not commit_tag_match:
269 self.skip_blank = True
271 # TEST= should be the last thing in the commit, so remove
272 # everything after it
273 if line.startswith('TEST='):
274 self.found_test = True
275 elif self.skip_blank and is_blank:
276 self.skip_blank = False
278 # Detect Cover-xxx tags
280 name = cover_match.group(1)
281 value = cover_match.group(2)
283 self.in_section = 'cover'
284 self.skip_blank = False
285 elif name == 'letter-cc':
286 self.AddToSeries(line, 'cover-cc', value)
287 elif name == 'changes':
288 self.in_change = 'Cover'
289 self.change_version = self.ParseVersion(value, line)
291 # If we are in a change list, key collected lines until a blank one
294 # Blank line ends this change list
295 self.FinalizeChange()
296 self.in_change = None
297 self.change_version = 0
299 self.FinalizeChange()
300 self.in_change = None
301 self.change_version = 0
302 out = self.ProcessLine(line)
304 if not leading_whitespace_match:
305 self.FinalizeChange()
306 self.change_lines.append(line)
307 self.skip_blank = False
309 # Detect Series-xxx tags
310 elif series_tag_match:
311 name = series_tag_match.group(1)
312 value = series_tag_match.group(2)
313 if name == 'changes':
314 # value is the version number: e.g. 1, or 2
315 self.in_change = 'Series'
316 self.change_version = self.ParseVersion(value, line)
318 self.AddToSeries(line, name, value)
319 self.skip_blank = True
321 # Detect Change-Id tags
322 elif change_id_match:
323 value = change_id_match.group(1)
325 if self.commit.change_id:
326 raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
327 (self.commit.hash, self.commit.change_id, value))
328 self.commit.change_id = value
329 self.skip_blank = True
331 # Detect Commit-xxx tags
332 elif commit_tag_match:
333 name = commit_tag_match.group(1)
334 value = commit_tag_match.group(2)
336 self.AddToCommit(line, name, value)
337 self.skip_blank = True
338 elif name == 'changes':
339 self.in_change = 'Commit'
340 self.change_version = self.ParseVersion(value, line)
342 # Detect the start of a new commit
345 self.commit = commit.Commit(commit_match.group(1))
347 # Detect tags in the commit message
349 # Remove Tested-by self, since few will take much notice
350 if (tag_match.group(1) == 'Tested-by' and
351 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
352 self.warn.append("Ignoring %s" % line)
353 elif tag_match.group(1) == 'Patch-cc':
354 self.commit.AddCc(tag_match.group(2).split(','))
358 # Suppress duplicate signoffs
360 if (self.is_log or not self.commit or
361 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
364 # Well that means this is an ordinary line
366 # Look for space before tab
367 m = re_space_before_tab.match(line)
369 self.warn.append('Line %d/%d has space before tab' %
370 (self.linenum, m.start()))
372 # OK, we have a valid non-blank line
375 self.skip_blank = False
376 if self.state == STATE_DIFFS:
379 # If this is the start of the diffs section, emit our tags and
382 self.state = STATE_DIFFS
384 # Output the tags (signoff first), then change list
386 log = self.series.MakeChangeLog(self.commit)
389 out += self.commit.notes
391 elif self.found_test:
392 if not re_allowed_after_test.match(line):
393 self.lines_after_test += 1
398 """Close out processing of this patch stream"""
399 self.FinalizeChange()
401 if self.lines_after_test:
402 self.warn.append('Found %d lines after TEST=' %
403 self.lines_after_test)
405 def WriteMessageId(self, outfd):
406 """Write the Message-Id into the output.
408 This is based on the Change-Id in the original patch, the version,
412 outfd: Output stream file object
414 if not self.commit.change_id:
417 # If the count is -1 we're testing, so use a fixed time
418 if self.commit.count == -1:
419 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
421 time_now = datetime.datetime.now()
423 # In theory there is email.utils.make_msgid() which would be nice
424 # to use, but it already produces something way too long and thus
425 # will produce ugly commit lines if someone throws this into
426 # a "Link:" tag in the final commit. So (sigh) roll our own.
428 # Start with the time; presumably we wouldn't send the same series
429 # with the same Change-Id at the exact same second.
430 parts = [time_now.strftime("%Y%m%d%H%M%S")]
432 # These seem like they would be nice to include.
433 if 'prefix' in self.series:
434 parts.append(self.series['prefix'])
435 if 'version' in self.series:
436 parts.append("v%s" % self.series['version'])
438 parts.append(str(self.commit.count + 1))
440 # The Change-Id must be last, right before the @
441 parts.append(self.commit.change_id)
443 # Join parts together with "." and write it out.
444 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
446 def ProcessStream(self, infd, outfd):
447 """Copy a stream from infd to outfd, filtering out unwanting things.
449 This is used to process patch files one at a time.
452 infd: Input stream file object
453 outfd: Output stream file object
455 # Extract the filename from each diff, for nice warnings
458 re_fname = re.compile('diff --git a/(.*) b/.*')
460 self.WriteMessageId(outfd)
463 line = infd.readline()
466 out = self.ProcessLine(line)
468 # Try to detect blank lines at EOF
470 match = re_fname.match(line)
473 fname = match.group(1)
475 self.blank_count += 1
477 if self.blank_count and (line == '-- ' or match):
478 self.warn.append("Found possible blank line(s) at "
479 "end of file '%s'" % last_fname)
480 outfd.write('+\n' * self.blank_count)
481 outfd.write(line + '\n')
486 def GetMetaDataForList(commit_range, git_dir=None, count=None,
487 series = None, allow_overwrite=False):
488 """Reads out patch series metadata from the commits
490 This does a 'git log' on the relevant commits and pulls out the tags we
494 commit_range: Range of commits to count (e.g. 'HEAD..base')
495 git_dir: Path to git repositiory (None to use default)
496 count: Number of commits to list, or None for no limit
497 series: Series object to add information into. By default a new series
499 allow_overwrite: Allow tags to overwrite an existing tag
501 A Series object containing information about the commits.
505 series.allow_overwrite = allow_overwrite
506 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
508 stdout = command.RunPipe([params], capture=True).stdout
509 ps = PatchStream(series, is_log=True)
510 for line in stdout.splitlines():
515 def GetMetaData(start, count):
516 """Reads out patch series metadata from the commits
518 This does a 'git log' on the relevant commits and pulls out the tags we
522 start: Commit to start from: 0=HEAD, 1=next one, etc.
523 count: Number of commits to list
525 return GetMetaDataForList('HEAD~%d' % start, None, count)
527 def GetMetaDataForTest(text):
528 """Process metadata from a file containing a git log. Used for tests
534 ps = PatchStream(series, is_log=True)
535 for line in text.splitlines():
540 def FixPatch(backup_dir, fname, series, commit):
541 """Fix up a patch file, by adding/removing as required.
543 We remove our tags from the patch file, insert changes lists, etc.
544 The patch file is processed in place, and overwritten.
546 A backup file is put into backup_dir (if not None).
549 fname: Filename to patch file to process
550 series: Series information about this patch set
551 commit: Commit object for this patch file
553 A list of errors, or [] if all ok.
555 handle, tmpname = tempfile.mkstemp()
556 outfd = os.fdopen(handle, 'w', encoding='utf-8')
557 infd = open(fname, 'r', encoding='utf-8')
558 ps = PatchStream(series)
560 ps.ProcessStream(infd, outfd)
564 # Create a backup file if required
566 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
567 shutil.move(tmpname, fname)
570 def FixPatches(series, fnames):
571 """Fix up a list of patches identified by filenames
573 The patch files are processed in place, and overwritten.
576 series: The series object
577 fnames: List of patch files to process
579 # Current workflow creates patches, so we shouldn't need a backup
580 backup_dir = None #tempfile.mkdtemp('clean-patch')
583 commit = series.commits[count]
586 result = FixPatch(backup_dir, fname, series, commit)
588 print('%d warnings for %s:' % (len(result), fname))
593 print('Cleaned %d patches' % count)
595 def InsertCoverLetter(fname, series, count):
596 """Inserts a cover letter with the required info into patch 0
599 fname: Input / output filename of the cover letter file
600 series: Series object
601 count: Number of patches in the series
603 fd = open(fname, 'r')
604 lines = fd.readlines()
607 fd = open(fname, 'w')
609 prefix = series.GetPatchPrefix()
611 if line.startswith('Subject:'):
612 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
613 zero_repeat = int(math.log10(count)) + 1
614 zero = '0' * zero_repeat
615 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
617 # Insert our cover letter
618 elif line.startswith('*** BLURB HERE ***'):
619 # First the blurb test
620 line = '\n'.join(text[1:]) + '\n'
621 if series.get('notes'):
622 line += '\n'.join(series.notes) + '\n'
624 # Now the change list
625 out = series.MakeChangeLog(None)
626 line += '\n' + '\n'.join(out)