1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2011 The Chromium OS Authors.
14 from series import Series
16 # Tags that we detect and remove
17 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
18 '|Reviewed-on:|Commit-\w*:')
20 # Lines which are allowed after a TEST= line
21 re_allowed_after_test = re.compile('^Signed-off-by:')
24 re_signoff = re.compile('^Signed-off-by: *(.*)')
26 # The start of the cover letter
27 re_cover = re.compile('^Cover-letter:')
30 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
33 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
36 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
38 # Commit tags that we want to collect and keep
39 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
41 # The start of a new commit in the git log
42 re_commit = re.compile('^commit ([0-9a-f]*)$')
44 # We detect these since checkpatch doesn't always do it
45 re_space_before_tab = re.compile('^[+].* \t')
47 # States we can be in - can we use range() and still have comments?
48 STATE_MSG_HEADER = 0 # Still in the message header
49 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
50 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
51 STATE_DIFFS = 3 # In the diff part (past --- line)
54 """Class for detecting/injecting tags in a patch or series of patches
56 We support processing the output of 'git log' to read out the tags we
57 are interested in. We can also process a patch file in order to remove
58 unwanted tags or inject additional ones. These correspond to the two
61 def __init__(self, series, name=None, is_log=False):
62 self.skip_blank = False # True to skip a single blank line
63 self.found_test = False # Found a TEST= line
64 self.lines_after_test = 0 # MNumber of lines found after TEST=
65 self.warn = [] # List of warnings we have collected
66 self.linenum = 1 # Output line number we are up to
67 self.in_section = None # Name of start...END section we are in
68 self.notes = [] # Series notes
69 self.section = [] # The current section...END section
70 self.series = series # Info about the patch series
71 self.is_log = is_log # True if indent like git log
72 self.in_change = 0 # Non-zero if we are in a change list
73 self.blank_count = 0 # Number of blank lines stored up
74 self.state = STATE_MSG_HEADER # What state are we in?
75 self.signoff = [] # Contents of signoff line
76 self.commit = None # Current commit
78 def AddToSeries(self, line, name, value):
79 """Add a new Series-xxx tag.
81 When a Series-xxx tag is detected, we come here to record it, if we
82 are scanning a 'git log'.
85 line: Source line containing tag (useful for debug/error messages)
86 name: Tag name (part after 'Series-')
87 value: Tag value (part after 'Series-xxx: ')
90 self.in_section = name
91 self.skip_blank = False
93 self.series.AddTag(self.commit, line, name, value)
95 def AddToCommit(self, line, name, value):
96 """Add a new Commit-xxx tag.
98 When a Commit-xxx tag is detected, we come here to record it.
101 line: Source line containing tag (useful for debug/error messages)
102 name: Tag name (part after 'Commit-')
103 value: Tag value (part after 'Commit-xxx: ')
106 self.in_section = 'commit-' + name
107 self.skip_blank = False
109 def CloseCommit(self):
110 """Save the current commit into our commit list, and reset our state"""
111 if self.commit and self.is_log:
112 self.series.AddCommit(self.commit)
114 # If 'END' is missing in a 'Cover-letter' section, and that section
115 # happens to show up at the very end of the commit message, this is
116 # the chance for us to fix it up.
117 if self.in_section == 'cover' and self.is_log:
118 self.series.cover = self.section
119 self.in_section = None
120 self.skip_blank = True
123 def ProcessLine(self, line):
124 """Process a single line of a patch file or commit log
126 This process a line and returns a list of lines to output. The list
127 may be empty or may contain multiple output lines.
129 This is where all the complicated logic is located. The class's
130 state is used to move between different states and detect things
133 We can be in one of two modes:
134 self.is_log == True: This is 'git log' mode, where most output is
135 indented by 4 characters and we are scanning for tags
137 self.is_log == False: This is 'patch' mode, where we already have
138 all the tags, and are processing patches to remove junk we
139 don't want, and add things we think are required.
142 line: text line to process
145 list of output lines, or [] if nothing should be output
147 # Initially we have no output. Prepare the input line string
149 line = line.rstrip('\n')
151 commit_match = re_commit.match(line) if self.is_log else None
157 # Handle state transition and skipping blank lines
158 series_tag_match = re_series_tag.match(line)
159 commit_tag_match = re_commit_tag.match(line)
160 cover_match = re_cover.match(line)
161 cover_cc_match = re_cover_cc.match(line)
162 signoff_match = re_signoff.match(line)
164 if self.state == STATE_PATCH_HEADER:
165 tag_match = re_tag.match(line)
166 is_blank = not line.strip()
168 if (self.state == STATE_MSG_HEADER
169 or self.state == STATE_PATCH_SUBJECT):
172 # We don't have a subject in the text stream of patch files
173 # It has its own line with a Subject: tag
174 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
177 self.state = STATE_MSG_HEADER
179 # If a tag is detected, or a new commit starts
180 if series_tag_match or commit_tag_match or \
181 cover_match or cover_cc_match or signoff_match or \
182 self.state == STATE_MSG_HEADER:
183 # but we are already in a section, this means 'END' is missing
184 # for that section, fix it up.
186 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
187 if self.in_section == 'cover':
188 self.series.cover = self.section
189 elif self.in_section == 'notes':
191 self.series.notes += self.section
192 elif self.in_section == 'commit-notes':
194 self.commit.notes += self.section
196 self.warn.append("Unknown section '%s'" % self.in_section)
197 self.in_section = None
198 self.skip_blank = True
200 # but we are already in a change list, that means a blank line
201 # is missing, fix it up.
203 self.warn.append("Missing 'blank line' in section 'Series-changes'")
206 # If we are in a section, keep collecting lines until we see END
209 if self.in_section == 'cover':
210 self.series.cover = self.section
211 elif self.in_section == 'notes':
213 self.series.notes += self.section
214 elif self.in_section == 'commit-notes':
216 self.commit.notes += self.section
218 self.warn.append("Unknown section '%s'" % self.in_section)
219 self.in_section = None
220 self.skip_blank = True
223 self.section.append(line)
225 # Detect the commit subject
226 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
227 self.commit.subject = line
229 # Detect the tags we want to remove, and skip blank lines
230 elif re_remove.match(line) and not commit_tag_match:
231 self.skip_blank = True
233 # TEST= should be the last thing in the commit, so remove
234 # everything after it
235 if line.startswith('TEST='):
236 self.found_test = True
237 elif self.skip_blank and is_blank:
238 self.skip_blank = False
240 # Detect the start of a cover letter section
242 self.in_section = 'cover'
243 self.skip_blank = False
246 value = cover_cc_match.group(1)
247 self.AddToSeries(line, 'cover-cc', value)
249 # If we are in a change list, key collected lines until a blank one
252 # Blank line ends this change list
256 out = self.ProcessLine(line)
259 self.series.AddChange(self.in_change, self.commit, line)
260 self.skip_blank = False
262 # Detect Series-xxx tags
263 elif series_tag_match:
264 name = series_tag_match.group(1)
265 value = series_tag_match.group(2)
266 if name == 'changes':
267 # value is the version number: e.g. 1, or 2
270 except ValueError as str:
271 raise ValueError("%s: Cannot decode version info '%s'" %
272 (self.commit.hash, line))
273 self.in_change = int(value)
275 self.AddToSeries(line, name, value)
276 self.skip_blank = True
278 # Detect Commit-xxx tags
279 elif commit_tag_match:
280 name = commit_tag_match.group(1)
281 value = commit_tag_match.group(2)
283 self.AddToCommit(line, name, value)
284 self.skip_blank = True
286 # Detect the start of a new commit
289 self.commit = commit.Commit(commit_match.group(1))
291 # Detect tags in the commit message
293 # Remove Tested-by self, since few will take much notice
294 if (tag_match.group(1) == 'Tested-by' and
295 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
296 self.warn.append("Ignoring %s" % line)
297 elif tag_match.group(1) == 'Patch-cc':
298 self.commit.AddCc(tag_match.group(2).split(','))
302 # Suppress duplicate signoffs
304 if (self.is_log or not self.commit or
305 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
308 # Well that means this is an ordinary line
310 # Look for space before tab
311 m = re_space_before_tab.match(line)
313 self.warn.append('Line %d/%d has space before tab' %
314 (self.linenum, m.start()))
316 # OK, we have a valid non-blank line
319 self.skip_blank = False
320 if self.state == STATE_DIFFS:
323 # If this is the start of the diffs section, emit our tags and
326 self.state = STATE_DIFFS
328 # Output the tags (signeoff first), then change list
330 log = self.series.MakeChangeLog(self.commit)
333 out += self.commit.notes
335 elif self.found_test:
336 if not re_allowed_after_test.match(line):
337 self.lines_after_test += 1
342 """Close out processing of this patch stream"""
344 if self.lines_after_test:
345 self.warn.append('Found %d lines after TEST=' %
346 self.lines_after_test)
348 def ProcessStream(self, infd, outfd):
349 """Copy a stream from infd to outfd, filtering out unwanting things.
351 This is used to process patch files one at a time.
354 infd: Input stream file object
355 outfd: Output stream file object
357 # Extract the filename from each diff, for nice warnings
360 re_fname = re.compile('diff --git a/(.*) b/.*')
362 line = infd.readline()
365 out = self.ProcessLine(line)
367 # Try to detect blank lines at EOF
369 match = re_fname.match(line)
372 fname = match.group(1)
374 self.blank_count += 1
376 if self.blank_count and (line == '-- ' or match):
377 self.warn.append("Found possible blank line(s) at "
378 "end of file '%s'" % last_fname)
379 outfd.write('+\n' * self.blank_count)
380 outfd.write(line + '\n')
385 def GetMetaDataForList(commit_range, git_dir=None, count=None,
386 series = None, allow_overwrite=False):
387 """Reads out patch series metadata from the commits
389 This does a 'git log' on the relevant commits and pulls out the tags we
393 commit_range: Range of commits to count (e.g. 'HEAD..base')
394 git_dir: Path to git repositiory (None to use default)
395 count: Number of commits to list, or None for no limit
396 series: Series object to add information into. By default a new series
398 allow_overwrite: Allow tags to overwrite an existing tag
400 A Series object containing information about the commits.
404 series.allow_overwrite = allow_overwrite
405 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
407 stdout = command.RunPipe([params], capture=True).stdout
408 ps = PatchStream(series, is_log=True)
409 for line in stdout.splitlines():
414 def GetMetaData(start, count):
415 """Reads out patch series metadata from the commits
417 This does a 'git log' on the relevant commits and pulls out the tags we
421 start: Commit to start from: 0=HEAD, 1=next one, etc.
422 count: Number of commits to list
424 return GetMetaDataForList('HEAD~%d' % start, None, count)
426 def GetMetaDataForTest(text):
427 """Process metadata from a file containing a git log. Used for tests
433 ps = PatchStream(series, is_log=True)
434 for line in text.splitlines():
439 def FixPatch(backup_dir, fname, series, commit):
440 """Fix up a patch file, by adding/removing as required.
442 We remove our tags from the patch file, insert changes lists, etc.
443 The patch file is processed in place, and overwritten.
445 A backup file is put into backup_dir (if not None).
448 fname: Filename to patch file to process
449 series: Series information about this patch set
450 commit: Commit object for this patch file
452 A list of errors, or [] if all ok.
454 handle, tmpname = tempfile.mkstemp()
455 outfd = os.fdopen(handle, 'w')
456 infd = open(fname, 'r')
457 ps = PatchStream(series)
459 ps.ProcessStream(infd, outfd)
463 # Create a backup file if required
465 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
466 shutil.move(tmpname, fname)
469 def FixPatches(series, fnames):
470 """Fix up a list of patches identified by filenames
472 The patch files are processed in place, and overwritten.
475 series: The series object
476 fnames: List of patch files to process
478 # Current workflow creates patches, so we shouldn't need a backup
479 backup_dir = None #tempfile.mkdtemp('clean-patch')
482 commit = series.commits[count]
484 result = FixPatch(backup_dir, fname, series, commit)
486 print('%d warnings for %s:' % (len(result), fname))
491 print('Cleaned %d patches' % count)
493 def InsertCoverLetter(fname, series, count):
494 """Inserts a cover letter with the required info into patch 0
497 fname: Input / output filename of the cover letter file
498 series: Series object
499 count: Number of patches in the series
501 fd = open(fname, 'r')
502 lines = fd.readlines()
505 fd = open(fname, 'w')
507 prefix = series.GetPatchPrefix()
509 if line.startswith('Subject:'):
510 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
511 zero_repeat = int(math.log10(count)) + 1
512 zero = '0' * zero_repeat
513 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
515 # Insert our cover letter
516 elif line.startswith('*** BLURB HERE ***'):
517 # First the blurb test
518 line = '\n'.join(text[1:]) + '\n'
519 if series.get('notes'):
520 line += '\n'.join(series.notes) + '\n'
522 # Now the change list
523 out = series.MakeChangeLog(None)
524 line += '\n' + '\n'.join(out)