1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2011 The Chromium OS Authors.
15 from series import Series
17 # Tags that we detect and remove
18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
19 '|Reviewed-on:|Commit-\w*:')
21 # Lines which are allowed after a TEST= line
22 re_allowed_after_test = re.compile('^Signed-off-by:')
25 re_signoff = re.compile('^Signed-off-by: *(.*)')
27 # The start of the cover letter
28 re_cover = re.compile('^Cover-letter:')
31 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
34 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
36 # Change-Id will be used to generate the Message-Id and then be stripped
37 re_change_id = re.compile('^Change-Id: *(.*)')
40 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
42 # Commit tags that we want to collect and keep
43 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
45 # The start of a new commit in the git log
46 re_commit = re.compile('^commit ([0-9a-f]*)$')
48 # We detect these since checkpatch doesn't always do it
49 re_space_before_tab = re.compile('^[+].* \t')
51 # States we can be in - can we use range() and still have comments?
52 STATE_MSG_HEADER = 0 # Still in the message header
53 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
54 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
55 STATE_DIFFS = 3 # In the diff part (past --- line)
58 """Class for detecting/injecting tags in a patch or series of patches
60 We support processing the output of 'git log' to read out the tags we
61 are interested in. We can also process a patch file in order to remove
62 unwanted tags or inject additional ones. These correspond to the two
65 def __init__(self, series, name=None, is_log=False):
66 self.skip_blank = False # True to skip a single blank line
67 self.found_test = False # Found a TEST= line
68 self.lines_after_test = 0 # MNumber of lines found after TEST=
69 self.warn = [] # List of warnings we have collected
70 self.linenum = 1 # Output line number we are up to
71 self.in_section = None # Name of start...END section we are in
72 self.notes = [] # Series notes
73 self.section = [] # The current section...END section
74 self.series = series # Info about the patch series
75 self.is_log = is_log # True if indent like git log
76 self.in_change = 0 # Non-zero if we are in a change list
77 self.blank_count = 0 # Number of blank lines stored up
78 self.state = STATE_MSG_HEADER # What state are we in?
79 self.signoff = [] # Contents of signoff line
80 self.commit = None # Current commit
82 def AddToSeries(self, line, name, value):
83 """Add a new Series-xxx tag.
85 When a Series-xxx tag is detected, we come here to record it, if we
86 are scanning a 'git log'.
89 line: Source line containing tag (useful for debug/error messages)
90 name: Tag name (part after 'Series-')
91 value: Tag value (part after 'Series-xxx: ')
94 self.in_section = name
95 self.skip_blank = False
97 self.series.AddTag(self.commit, line, name, value)
99 def AddToCommit(self, line, name, value):
100 """Add a new Commit-xxx tag.
102 When a Commit-xxx tag is detected, we come here to record it.
105 line: Source line containing tag (useful for debug/error messages)
106 name: Tag name (part after 'Commit-')
107 value: Tag value (part after 'Commit-xxx: ')
110 self.in_section = 'commit-' + name
111 self.skip_blank = False
113 def CloseCommit(self):
114 """Save the current commit into our commit list, and reset our state"""
115 if self.commit and self.is_log:
116 self.series.AddCommit(self.commit)
118 # If 'END' is missing in a 'Cover-letter' section, and that section
119 # happens to show up at the very end of the commit message, this is
120 # the chance for us to fix it up.
121 if self.in_section == 'cover' and self.is_log:
122 self.series.cover = self.section
123 self.in_section = None
124 self.skip_blank = True
127 def ProcessLine(self, line):
128 """Process a single line of a patch file or commit log
130 This process a line and returns a list of lines to output. The list
131 may be empty or may contain multiple output lines.
133 This is where all the complicated logic is located. The class's
134 state is used to move between different states and detect things
137 We can be in one of two modes:
138 self.is_log == True: This is 'git log' mode, where most output is
139 indented by 4 characters and we are scanning for tags
141 self.is_log == False: This is 'patch' mode, where we already have
142 all the tags, and are processing patches to remove junk we
143 don't want, and add things we think are required.
146 line: text line to process
149 list of output lines, or [] if nothing should be output
151 # Initially we have no output. Prepare the input line string
153 line = line.rstrip('\n')
155 commit_match = re_commit.match(line) if self.is_log else None
161 # Handle state transition and skipping blank lines
162 series_tag_match = re_series_tag.match(line)
163 change_id_match = re_change_id.match(line)
164 commit_tag_match = re_commit_tag.match(line)
165 cover_match = re_cover.match(line)
166 cover_cc_match = re_cover_cc.match(line)
167 signoff_match = re_signoff.match(line)
169 if self.state == STATE_PATCH_HEADER:
170 tag_match = re_tag.match(line)
171 is_blank = not line.strip()
173 if (self.state == STATE_MSG_HEADER
174 or self.state == STATE_PATCH_SUBJECT):
177 # We don't have a subject in the text stream of patch files
178 # It has its own line with a Subject: tag
179 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
182 self.state = STATE_MSG_HEADER
184 # If a tag is detected, or a new commit starts
185 if series_tag_match or commit_tag_match or change_id_match or \
186 cover_match or cover_cc_match or signoff_match or \
187 self.state == STATE_MSG_HEADER:
188 # but we are already in a section, this means 'END' is missing
189 # for that section, fix it up.
191 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
192 if self.in_section == 'cover':
193 self.series.cover = self.section
194 elif self.in_section == 'notes':
196 self.series.notes += self.section
197 elif self.in_section == 'commit-notes':
199 self.commit.notes += self.section
201 self.warn.append("Unknown section '%s'" % self.in_section)
202 self.in_section = None
203 self.skip_blank = True
205 # but we are already in a change list, that means a blank line
206 # is missing, fix it up.
208 self.warn.append("Missing 'blank line' in section 'Series-changes'")
211 # If we are in a section, keep collecting lines until we see END
214 if self.in_section == 'cover':
215 self.series.cover = self.section
216 elif self.in_section == 'notes':
218 self.series.notes += self.section
219 elif self.in_section == 'commit-notes':
221 self.commit.notes += self.section
223 self.warn.append("Unknown section '%s'" % self.in_section)
224 self.in_section = None
225 self.skip_blank = True
228 self.section.append(line)
230 # Detect the commit subject
231 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
232 self.commit.subject = line
234 # Detect the tags we want to remove, and skip blank lines
235 elif re_remove.match(line) and not commit_tag_match:
236 self.skip_blank = True
238 # TEST= should be the last thing in the commit, so remove
239 # everything after it
240 if line.startswith('TEST='):
241 self.found_test = True
242 elif self.skip_blank and is_blank:
243 self.skip_blank = False
245 # Detect the start of a cover letter section
247 self.in_section = 'cover'
248 self.skip_blank = False
251 value = cover_cc_match.group(1)
252 self.AddToSeries(line, 'cover-cc', value)
254 # If we are in a change list, key collected lines until a blank one
257 # Blank line ends this change list
261 out = self.ProcessLine(line)
264 self.series.AddChange(self.in_change, self.commit, line)
265 self.skip_blank = False
267 # Detect Series-xxx tags
268 elif series_tag_match:
269 name = series_tag_match.group(1)
270 value = series_tag_match.group(2)
271 if name == 'changes':
272 # value is the version number: e.g. 1, or 2
275 except ValueError as str:
276 raise ValueError("%s: Cannot decode version info '%s'" %
277 (self.commit.hash, line))
278 self.in_change = int(value)
280 self.AddToSeries(line, name, value)
281 self.skip_blank = True
283 # Detect Change-Id tags
284 elif change_id_match:
285 value = change_id_match.group(1)
287 if self.commit.change_id:
288 raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
289 (self.commit.hash, self.commit.change_id, value))
290 self.commit.change_id = value
291 self.skip_blank = True
293 # Detect Commit-xxx tags
294 elif commit_tag_match:
295 name = commit_tag_match.group(1)
296 value = commit_tag_match.group(2)
298 self.AddToCommit(line, name, value)
299 self.skip_blank = True
301 # Detect the start of a new commit
304 self.commit = commit.Commit(commit_match.group(1))
306 # Detect tags in the commit message
308 # Remove Tested-by self, since few will take much notice
309 if (tag_match.group(1) == 'Tested-by' and
310 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
311 self.warn.append("Ignoring %s" % line)
312 elif tag_match.group(1) == 'Patch-cc':
313 self.commit.AddCc(tag_match.group(2).split(','))
317 # Suppress duplicate signoffs
319 if (self.is_log or not self.commit or
320 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
323 # Well that means this is an ordinary line
325 # Look for space before tab
326 m = re_space_before_tab.match(line)
328 self.warn.append('Line %d/%d has space before tab' %
329 (self.linenum, m.start()))
331 # OK, we have a valid non-blank line
334 self.skip_blank = False
335 if self.state == STATE_DIFFS:
338 # If this is the start of the diffs section, emit our tags and
341 self.state = STATE_DIFFS
343 # Output the tags (signeoff first), then change list
345 log = self.series.MakeChangeLog(self.commit)
348 out += self.commit.notes
350 elif self.found_test:
351 if not re_allowed_after_test.match(line):
352 self.lines_after_test += 1
357 """Close out processing of this patch stream"""
359 if self.lines_after_test:
360 self.warn.append('Found %d lines after TEST=' %
361 self.lines_after_test)
363 def WriteMessageId(self, outfd):
364 """Write the Message-Id into the output.
366 This is based on the Change-Id in the original patch, the version,
370 outfd: Output stream file object
372 if not self.commit.change_id:
375 # If the count is -1 we're testing, so use a fixed time
376 if self.commit.count == -1:
377 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
379 time_now = datetime.datetime.now()
381 # In theory there is email.utils.make_msgid() which would be nice
382 # to use, but it already produces something way too long and thus
383 # will produce ugly commit lines if someone throws this into
384 # a "Link:" tag in the final commit. So (sigh) roll our own.
386 # Start with the time; presumably we wouldn't send the same series
387 # with the same Change-Id at the exact same second.
388 parts = [time_now.strftime("%Y%m%d%H%M%S")]
390 # These seem like they would be nice to include.
391 if 'prefix' in self.series:
392 parts.append(self.series['prefix'])
393 if 'version' in self.series:
394 parts.append("v%s" % self.series['version'])
396 parts.append(str(self.commit.count + 1))
398 # The Change-Id must be last, right before the @
399 parts.append(self.commit.change_id)
401 # Join parts together with "." and write it out.
402 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
404 def ProcessStream(self, infd, outfd):
405 """Copy a stream from infd to outfd, filtering out unwanting things.
407 This is used to process patch files one at a time.
410 infd: Input stream file object
411 outfd: Output stream file object
413 # Extract the filename from each diff, for nice warnings
416 re_fname = re.compile('diff --git a/(.*) b/.*')
418 self.WriteMessageId(outfd)
421 line = infd.readline()
424 out = self.ProcessLine(line)
426 # Try to detect blank lines at EOF
428 match = re_fname.match(line)
431 fname = match.group(1)
433 self.blank_count += 1
435 if self.blank_count and (line == '-- ' or match):
436 self.warn.append("Found possible blank line(s) at "
437 "end of file '%s'" % last_fname)
438 outfd.write('+\n' * self.blank_count)
439 outfd.write(line + '\n')
444 def GetMetaDataForList(commit_range, git_dir=None, count=None,
445 series = None, allow_overwrite=False):
446 """Reads out patch series metadata from the commits
448 This does a 'git log' on the relevant commits and pulls out the tags we
452 commit_range: Range of commits to count (e.g. 'HEAD..base')
453 git_dir: Path to git repositiory (None to use default)
454 count: Number of commits to list, or None for no limit
455 series: Series object to add information into. By default a new series
457 allow_overwrite: Allow tags to overwrite an existing tag
459 A Series object containing information about the commits.
463 series.allow_overwrite = allow_overwrite
464 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
466 stdout = command.RunPipe([params], capture=True).stdout
467 ps = PatchStream(series, is_log=True)
468 for line in stdout.splitlines():
473 def GetMetaData(start, count):
474 """Reads out patch series metadata from the commits
476 This does a 'git log' on the relevant commits and pulls out the tags we
480 start: Commit to start from: 0=HEAD, 1=next one, etc.
481 count: Number of commits to list
483 return GetMetaDataForList('HEAD~%d' % start, None, count)
485 def GetMetaDataForTest(text):
486 """Process metadata from a file containing a git log. Used for tests
492 ps = PatchStream(series, is_log=True)
493 for line in text.splitlines():
498 def FixPatch(backup_dir, fname, series, commit):
499 """Fix up a patch file, by adding/removing as required.
501 We remove our tags from the patch file, insert changes lists, etc.
502 The patch file is processed in place, and overwritten.
504 A backup file is put into backup_dir (if not None).
507 fname: Filename to patch file to process
508 series: Series information about this patch set
509 commit: Commit object for this patch file
511 A list of errors, or [] if all ok.
513 handle, tmpname = tempfile.mkstemp()
514 outfd = os.fdopen(handle, 'w', encoding='utf-8')
515 infd = open(fname, 'r', encoding='utf-8')
516 ps = PatchStream(series)
518 ps.ProcessStream(infd, outfd)
522 # Create a backup file if required
524 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
525 shutil.move(tmpname, fname)
528 def FixPatches(series, fnames):
529 """Fix up a list of patches identified by filenames
531 The patch files are processed in place, and overwritten.
534 series: The series object
535 fnames: List of patch files to process
537 # Current workflow creates patches, so we shouldn't need a backup
538 backup_dir = None #tempfile.mkdtemp('clean-patch')
541 commit = series.commits[count]
544 result = FixPatch(backup_dir, fname, series, commit)
546 print('%d warnings for %s:' % (len(result), fname))
551 print('Cleaned %d patches' % count)
553 def InsertCoverLetter(fname, series, count):
554 """Inserts a cover letter with the required info into patch 0
557 fname: Input / output filename of the cover letter file
558 series: Series object
559 count: Number of patches in the series
561 fd = open(fname, 'r')
562 lines = fd.readlines()
565 fd = open(fname, 'w')
567 prefix = series.GetPatchPrefix()
569 if line.startswith('Subject:'):
570 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
571 zero_repeat = int(math.log10(count)) + 1
572 zero = '0' * zero_repeat
573 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
575 # Insert our cover letter
576 elif line.startswith('*** BLURB HERE ***'):
577 # First the blurb test
578 line = '\n'.join(text[1:]) + '\n'
579 if series.get('notes'):
580 line += '\n'.join(series.notes) + '\n'
582 # Now the change list
583 out = series.MakeChangeLog(None)
584 line += '\n' + '\n'.join(out)