1 # Copyright (c) 2015 Stephen Warren
2 # Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.
4 # SPDX-License-Identifier: GPL-2.0
6 # Generate an HTML-formatted log file containing multiple streams of data,
7 # each represented in a well-delineated/-structured fashion.
15 mod_dir = os.path.dirname(os.path.abspath(__file__))
17 class LogfileStream(object):
18 """A file-like object used to write a single logical stream of data into
19 a multiplexed log file. Objects of this type should be created by factory
20 functions in the Logfile class rather than directly."""
22 def __init__(self, logfile, name, chained_file):
23 """Initialize a new object.
26 logfile: The Logfile object to log to.
27 name: The name of this log stream.
28 chained_file: The file-like object to which all stream data should be
29 logged to in addition to logfile. Can be None.
35 self.logfile = logfile
37 self.chained_file = chained_file
40 """Dummy function so that this class is "file-like".
51 def write(self, data, implicit=False):
52 """Write data to the log stream.
55 data: The data to write tot he file.
56 implicit: Boolean indicating whether data actually appeared in the
57 stream, or was implicitly generated. A valid use-case is to
58 repeat a shell prompt at the start of each separate log
59 section, which makes the log sections more readable in
66 self.logfile.write(self, data, implicit)
68 self.chained_file.write(data)
71 """Flush the log stream, to ensure correct log interleaving.
82 self.chained_file.flush()
84 class RunAndLog(object):
85 """A utility object used to execute sub-processes and log their output to
86 a multiplexed log file. Objects of this type should be created by factory
87 functions in the Logfile class rather than directly."""
89 def __init__(self, logfile, name, chained_file):
90 """Initialize a new object.
93 logfile: The Logfile object to log to.
94 name: The name of this log stream or sub-process.
95 chained_file: The file-like object to which all stream data should
96 be logged to in addition to logfile. Can be None.
102 self.logfile = logfile
104 self.chained_file = chained_file
106 self.exit_status = None
109 """Clean up any resources managed by this object."""
112 def run(self, cmd, cwd=None, ignore_errors=False):
113 """Run a command as a sub-process, and log the results.
115 The output is available at self.output which can be useful if there is
119 cmd: The command to execute.
120 cwd: The directory to run the command in. Can be None to use the
122 ignore_errors: Indicate whether to ignore errors. If True, the
123 function will simply return if the command cannot be executed
124 or exits with an error code, otherwise an exception will be
125 raised if such problems occur.
128 The output as a string.
131 msg = '+' + ' '.join(cmd) + '\n'
132 if self.chained_file:
133 self.chained_file.write(msg)
134 self.logfile.write(self, msg)
137 p = subprocess.Popen(cmd, cwd=cwd,
138 stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
139 (stdout, stderr) = p.communicate()
143 output += 'stdout:\n'
147 output += 'stderr:\n'
149 exit_status = p.returncode
151 except subprocess.CalledProcessError as cpe:
153 exit_status = cpe.returncode
155 except Exception as e:
159 if output and not output.endswith('\n'):
161 if exit_status and not exception and not ignore_errors:
162 exception = Exception('Exit code: ' + str(exit_status))
164 output += str(exception) + '\n'
165 self.logfile.write(self, output)
166 if self.chained_file:
167 self.chained_file.write(output)
168 self.logfile.timestamp()
170 # Store the output so it can be accessed if we raise an exception.
172 self.exit_status = exit_status
177 class SectionCtxMgr(object):
178 """A context manager for Python's "with" statement, which allows a certain
179 portion of test code to be logged to a separate section of the log file.
180 Objects of this type should be created by factory functions in the Logfile
181 class rather than directly."""
183 def __init__(self, log, marker, anchor):
184 """Initialize a new object.
187 log: The Logfile object to log to.
188 marker: The name of the nested log section.
189 anchor: The anchor value to pass to start_section().
200 self.anchor = self.log.start_section(self.marker, self.anchor)
202 def __exit__(self, extype, value, traceback):
203 self.log.end_section(self.marker)
205 class Logfile(object):
206 """Generates an HTML-formatted log file containing multiple streams of
207 data, each represented in a well-delineated/-structured fashion."""
209 def __init__(self, fn):
210 """Initialize a new object.
213 fn: The filename to write to.
219 self.f = open(fn, 'wt')
220 self.last_stream = None
224 self.timestamp_start = self._get_time()
225 self.timestamp_prev = self.timestamp_start
226 self.timestamp_blocks = []
227 self.seen_warning = False
229 shutil.copy(mod_dir + '/multiplexed_log.css', os.path.dirname(fn))
233 <link rel="stylesheet" type="text/css" href="multiplexed_log.css">
234 <script src="http://code.jquery.com/jquery.min.js"></script>
236 $(document).ready(function () {
237 // Copy status report HTML to start of log for easy access
238 sts = $(".block#status_report")[0].outerHTML;
239 $("tt").prepend(sts);
241 // Add expand/contract buttons to all block headers
242 btns = "<span class=\\\"block-expand hidden\\\">[+] </span>" +
243 "<span class=\\\"block-contract\\\">[-] </span>";
244 $(".block-header").prepend(btns);
246 // Pre-contract all blocks which passed, leaving only problem cases
247 // expanded, to highlight issues the user should look at.
248 // Only top-level blocks (sections) should have any status
249 passed_bcs = $(".block-content:has(.status-pass)");
250 // Some blocks might have multiple status entries (e.g. the status
251 // report), so take care not to hide blocks with partial success.
252 passed_bcs = passed_bcs.not(":has(.status-fail)");
253 passed_bcs = passed_bcs.not(":has(.status-xfail)");
254 passed_bcs = passed_bcs.not(":has(.status-xpass)");
255 passed_bcs = passed_bcs.not(":has(.status-skipped)");
256 passed_bcs = passed_bcs.not(":has(.status-warning)");
257 // Hide the passed blocks
258 passed_bcs.addClass("hidden");
259 // Flip the expand/contract button hiding for those blocks.
260 bhs = passed_bcs.parent().children(".block-header")
261 bhs.children(".block-expand").removeClass("hidden");
262 bhs.children(".block-contract").addClass("hidden");
264 // Add click handler to block headers.
265 // The handler expands/contracts the block.
266 $(".block-header").on("click", function (e) {
267 var header = $(this);
268 var content = header.next(".block-content");
269 var expanded = !content.hasClass("hidden");
271 content.addClass("hidden");
272 header.children(".block-expand").first().removeClass("hidden");
273 header.children(".block-contract").first().addClass("hidden");
275 header.children(".block-contract").first().removeClass("hidden");
276 header.children(".block-expand").first().addClass("hidden");
277 content.removeClass("hidden");
281 // When clicking on a link, expand the target block
282 $("a").on("click", function (e) {
283 var block = $($(this).attr("href"));
284 var header = block.children(".block-header");
285 var content = block.children(".block-content").first();
286 header.children(".block-contract").first().removeClass("hidden");
287 header.children(".block-expand").first().addClass("hidden");
288 content.removeClass("hidden");
298 """Close the log file.
300 After calling this function, no more data may be written to the log.
316 # The set of characters that should be represented as hexadecimal codes in
318 _nonprint = ('%' + ''.join(chr(c) for c in range(0, 32) if c not in (9, 10)) +
319 ''.join(chr(c) for c in range(127, 256)))
321 def _escape(self, data):
322 """Render data format suitable for inclusion in an HTML document.
324 This includes HTML-escaping certain characters, and translating
325 control characters to a hexadecimal representation.
328 data: The raw string data to be escaped.
331 An escaped version of the data.
334 data = data.replace(chr(13), '')
335 data = ''.join((c in self._nonprint) and ('%%%02x' % ord(c)) or
337 data = cgi.escape(data)
340 def _terminate_stream(self):
341 """Write HTML to the log file to terminate the current stream's data.
351 if not self.last_stream:
353 self.f.write('</pre>\n')
354 self.f.write('<div class="stream-trailer block-trailer">End stream: ' +
355 self.last_stream.name + '</div>\n')
356 self.f.write('</div>\n')
357 self.f.write('</div>\n')
358 self.last_stream = None
360 def _note(self, note_type, msg, anchor=None):
361 """Write a note or one-off message to the log file.
364 note_type: The type of note. This must be a value supported by the
365 accompanying multiplexed_log.css.
366 msg: The note/message to log.
367 anchor: Optional internal link target.
373 self._terminate_stream()
374 self.f.write('<div class="' + note_type + '">\n')
375 self.f.write('<pre>')
377 self.f.write('<a href="#%s">' % anchor)
378 self.f.write(self._escape(msg))
381 self.f.write('\n</pre>\n')
382 self.f.write('</div>\n')
384 def start_section(self, marker, anchor=None):
385 """Begin a new nested section in the log file.
388 marker: The name of the section that is starting.
389 anchor: The value to use for the anchor. If None, a unique value
390 will be calculated and used
393 Name of the HTML anchor emitted before section.
396 self._terminate_stream()
397 self.blocks.append(marker)
398 self.timestamp_blocks.append(self._get_time())
401 anchor = str(self.anchor)
402 blk_path = '/'.join(self.blocks)
403 self.f.write('<div class="section block" id="' + anchor + '">\n')
404 self.f.write('<div class="section-header block-header">Section: ' +
405 blk_path + '</div>\n')
406 self.f.write('<div class="section-content block-content">\n')
411 def end_section(self, marker):
412 """Terminate the current nested section in the log file.
414 This function validates proper nesting of start_section() and
415 end_section() calls. If a mismatch is found, an exception is raised.
418 marker: The name of the section that is ending.
424 if (not self.blocks) or (marker != self.blocks[-1]):
425 raise Exception('Block nesting mismatch: "%s" "%s"' %
426 (marker, '/'.join(self.blocks)))
427 self._terminate_stream()
428 timestamp_now = self._get_time()
429 timestamp_section_start = self.timestamp_blocks.pop()
430 delta_section = timestamp_now - timestamp_section_start
431 self._note("timestamp",
432 "TIME: SINCE-SECTION: " + str(delta_section))
433 blk_path = '/'.join(self.blocks)
434 self.f.write('<div class="section-trailer block-trailer">' +
435 'End section: ' + blk_path + '</div>\n')
436 self.f.write('</div>\n')
437 self.f.write('</div>\n')
440 def section(self, marker, anchor=None):
441 """Create a temporary section in the log file.
443 This function creates a context manager for Python's "with" statement,
444 which allows a certain portion of test code to be logged to a separate
445 section of the log file.
448 with log.section("somename"):
452 marker: The name of the nested section.
453 anchor: The anchor value to pass to start_section().
456 A context manager object.
459 return SectionCtxMgr(self, marker, anchor)
461 def error(self, msg):
462 """Write an error note to the log file.
465 msg: A message describing the error.
471 self._note("error", msg)
473 def warning(self, msg):
474 """Write an warning note to the log file.
477 msg: A message describing the warning.
483 self.seen_warning = True
484 self._note("warning", msg)
486 def get_and_reset_warning(self):
487 """Get and reset the log warning flag.
493 Whether a warning was seen since the last call.
496 ret = self.seen_warning
497 self.seen_warning = False
501 """Write an informational note to the log file.
504 msg: An informational message.
510 self._note("info", msg)
512 def action(self, msg):
513 """Write an action note to the log file.
516 msg: A message describing the action that is being logged.
522 self._note("action", msg)
525 return datetime.datetime.now()
528 """Write a timestamp to the log file.
537 timestamp_now = self._get_time()
538 delta_prev = timestamp_now - self.timestamp_prev
539 delta_start = timestamp_now - self.timestamp_start
540 self.timestamp_prev = timestamp_now
542 self._note("timestamp",
543 "TIME: NOW: " + timestamp_now.strftime("%Y/%m/%d %H:%M:%S.%f"))
544 self._note("timestamp",
545 "TIME: SINCE-PREV: " + str(delta_prev))
546 self._note("timestamp",
547 "TIME: SINCE-START: " + str(delta_start))
549 def status_pass(self, msg, anchor=None):
550 """Write a note to the log file describing test(s) which passed.
553 msg: A message describing the passed test(s).
554 anchor: Optional internal link target.
560 self._note("status-pass", msg, anchor)
562 def status_warning(self, msg, anchor=None):
563 """Write a note to the log file describing test(s) which passed.
566 msg: A message describing the passed test(s).
567 anchor: Optional internal link target.
573 self._note("status-warning", msg, anchor)
575 def status_skipped(self, msg, anchor=None):
576 """Write a note to the log file describing skipped test(s).
579 msg: A message describing the skipped test(s).
580 anchor: Optional internal link target.
586 self._note("status-skipped", msg, anchor)
588 def status_xfail(self, msg, anchor=None):
589 """Write a note to the log file describing xfailed test(s).
592 msg: A message describing the xfailed test(s).
593 anchor: Optional internal link target.
599 self._note("status-xfail", msg, anchor)
601 def status_xpass(self, msg, anchor=None):
602 """Write a note to the log file describing xpassed test(s).
605 msg: A message describing the xpassed test(s).
606 anchor: Optional internal link target.
612 self._note("status-xpass", msg, anchor)
614 def status_fail(self, msg, anchor=None):
615 """Write a note to the log file describing failed test(s).
618 msg: A message describing the failed test(s).
619 anchor: Optional internal link target.
625 self._note("status-fail", msg, anchor)
627 def get_stream(self, name, chained_file=None):
628 """Create an object to log a single stream's data into the log file.
630 This creates a "file-like" object that can be written to in order to
631 write a single stream's data to the log file. The implementation will
632 handle any required interleaving of data (from multiple streams) in
633 the log, in a way that makes it obvious which stream each bit of data
637 name: The name of the stream.
638 chained_file: The file-like object to which all stream data should
639 be logged to in addition to this log. Can be None.
645 return LogfileStream(self, name, chained_file)
647 def get_runner(self, name, chained_file=None):
648 """Create an object that executes processes and logs their output.
651 name: The name of this sub-process.
652 chained_file: The file-like object to which all stream data should
653 be logged to in addition to logfile. Can be None.
659 return RunAndLog(self, name, chained_file)
661 def write(self, stream, data, implicit=False):
662 """Write stream data into the log file.
664 This function should only be used by instances of LogfileStream or
668 stream: The stream whose data is being logged.
669 data: The data to log.
670 implicit: Boolean indicating whether data actually appeared in the
671 stream, or was implicitly generated. A valid use-case is to
672 repeat a shell prompt at the start of each separate log
673 section, which makes the log sections more readable in
680 if stream != self.last_stream:
681 self._terminate_stream()
682 self.f.write('<div class="stream block">\n')
683 self.f.write('<div class="stream-header block-header">Stream: ' +
684 stream.name + '</div>\n')
685 self.f.write('<div class="stream-content block-content">\n')
686 self.f.write('<pre>')
688 self.f.write('<span class="implicit">')
689 self.f.write(self._escape(data))
691 self.f.write('</span>')
692 self.last_stream = stream
695 """Flush the log stream, to ensure correct log interleaving.