X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=tools%2Fmoveconfig.py;h=d362923b221b7ef5ed3c2b5f7a74f61f5551b705;hb=320e0570e67efbd093d7750655a758c66e9d5528;hp=e32104412e14e0b1c1f97a032be74f7aa6ac09dd;hpb=7fb0bacd389b9b041164965d31a8c50e8616f7a0;p=oweals%2Fu-boot.git diff --git a/tools/moveconfig.py b/tools/moveconfig.py index e32104412e..d362923b22 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -131,6 +131,11 @@ Available options Exit immediately if Make exits with a non-zero status while processing a defconfig file. + -s, --force-sync + Do "make savedefconfig" forcibly for all the defconfig files. + If not specified, "make savedefconfig" only occurs for cases + where at least one CONFIG was moved. + -H, --headers-only Only cleanup the headers; skip the defconfig processing @@ -138,6 +143,14 @@ Available options Specify the number of threads to run simultaneously. If not specified, the number of threads is the same as the number of CPU cores. + -r, --git-ref + Specify the git ref to clone for building the autoconf.mk. If unspecified + use the CWD. This is useful for when changes to the Kconfig affect the + default values and you want to capture the state of the defconfig from + before that change was in effect. If in doubt, specify a ref pre-Kconfig + changes (use HEAD if Kconfig changes are not committed). Worst case it will + take a bit longer to run, but will always do the right thing. + -v, --verbose Show any build errors as boards are built @@ -147,6 +160,7 @@ To see the complete list of supported options, run """ +import filecmp import fnmatch import multiprocessing import optparse @@ -407,6 +421,7 @@ class KconfigParser: self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') self.config_autoconf = os.path.join(build_dir, 'include', 'config', 'auto.conf') + self.defconfig = os.path.join(build_dir, 'defconfig') def get_cross_compile(self): """Parse .config file and return CROSS_COMPILE. @@ -472,9 +487,6 @@ class KconfigParser: else: new_val = not_set - if old_val == new_val: - return (ACTION_NO_CHANGE, new_val) - # If this CONFIG is neither bool nor trisate if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: # tools/scripts/define2mk.sed changes '1' to 'y'. @@ -483,7 +495,8 @@ class KconfigParser: if new_val[-2:] == '=y': new_val = new_val[:-1] + '1' - return (ACTION_MOVE, new_val) + return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, + new_val) def update_dotconfig(self): """Parse files for the config options and update the .config. @@ -539,11 +552,35 @@ class KconfigParser: f.write(value + '\n') updated = True + self.results = results os.remove(self.config_autoconf) os.remove(self.autoconf) return (updated, log) + def check_defconfig(self): + """Check the defconfig after savedefconfig + + Returns: + Return additional log if moved CONFIGs were removed again by + 'make savedefconfig'. + """ + + log = '' + + with open(self.defconfig) as f: + defconfig_lines = f.readlines() + + for (action, value) in self.results: + if action != ACTION_MOVE: + continue + if not value + '\n' in defconfig_lines: + log += color_text(self.options.color, COLOR_YELLOW, + "'%s' was removed by savedefconfig.\n" % + value) + + return log + class Slot: """A slot to store a subprocess. @@ -553,7 +590,7 @@ class Slot: for faster processing. """ - def __init__(self, configs, options, progress, devnull, make_cmd): + def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): """Create a new process slot. Arguments: @@ -562,22 +599,26 @@ class Slot: progress: A progress indicator. devnull: A file object of '/dev/null'. make_cmd: command name of GNU Make. + reference_src_dir: Determine the true starting config state from this + source tree. """ self.options = options self.progress = progress self.build_dir = tempfile.mkdtemp() self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) + self.reference_src_dir = reference_src_dir self.parser = KconfigParser(configs, options, self.build_dir) self.state = STATE_IDLE self.failed_boards = [] + self.suspicious_boards = [] def __del__(self): """Delete the working directory This function makes sure the temporary directory is cleaned away even if Python suddenly dies due to error. It should be done in here - because it is guranteed the destructor is always invoked when the + because it is guaranteed the destructor is always invoked when the instance of the class gets unreferenced. If the subprocess is still running, wait until it finishes. @@ -602,13 +643,11 @@ class Slot: """ if self.state != STATE_IDLE: return False - cmd = list(self.make_cmd) - cmd.append(defconfig) - self.ps = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) + self.defconfig = defconfig - self.state = STATE_DEFCONFIG self.log = '' + self.current_src_dir = self.reference_src_dir + self.do_defconfig() return True def poll(self): @@ -634,42 +673,54 @@ class Slot: return False if self.ps.poll() != 0: - self.log += color_text(self.options.color, COLOR_LIGHT_RED, - "Failed to process.\n") - if self.options.verbose: - self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, - self.ps.stderr.read()) - self.finish(False) - return True + self.handle_error() + elif self.state == STATE_DEFCONFIG: + if self.reference_src_dir and not self.current_src_dir: + self.do_savedefconfig() + else: + self.do_autoconf() + elif self.state == STATE_AUTOCONF: + if self.current_src_dir: + self.current_src_dir = None + self.do_defconfig() + else: + self.do_savedefconfig() + elif self.state == STATE_SAVEDEFCONFIG: + self.update_defconfig() + else: + sys.exit("Internal Error. This should not happen.") - if self.state == STATE_AUTOCONF: - (updated, log) = self.parser.update_dotconfig() - self.log += log + return True if self.state == STATE_IDLE else False - if not updated: - self.finish(True) - return True - """Save off the defconfig in a consistent way""" - cmd = list(self.make_cmd) - cmd.append('savedefconfig') - self.ps = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) - self.state = STATE_SAVEDEFCONFIG - return False + def handle_error(self): + """Handle error cases.""" - if self.state == STATE_SAVEDEFCONFIG: - if not self.options.dry_run: - shutil.move(os.path.join(self.build_dir, 'defconfig'), - os.path.join('configs', self.defconfig)) - self.finish(True) - return True + self.log += color_text(self.options.color, COLOR_LIGHT_RED, + "Failed to process.\n") + if self.options.verbose: + self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, + self.ps.stderr.read()) + self.finish(False) + + def do_defconfig(self): + """Run 'make _defconfig' to create the .config file.""" + + cmd = list(self.make_cmd) + cmd.append(self.defconfig) + self.ps = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE, + cwd=self.current_src_dir) + self.state = STATE_DEFCONFIG + + def do_autoconf(self): + """Run 'make include/config/auto.conf'.""" self.cross_compile = self.parser.get_cross_compile() if self.cross_compile is None: self.log += color_text(self.options.color, COLOR_YELLOW, "Compiler is missing. Do nothing.\n") self.finish(False) - return True + return cmd = list(self.make_cmd) if self.cross_compile: @@ -677,9 +728,49 @@ class Slot: cmd.append('KCONFIG_IGNORE_DUPLICATES=1') cmd.append('include/config/auto.conf') self.ps = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + cwd=self.current_src_dir) self.state = STATE_AUTOCONF - return False + + def do_savedefconfig(self): + """Update the .config and run 'make savedefconfig'.""" + + (updated, log) = self.parser.update_dotconfig() + self.log += log + + if not self.options.force_sync and not updated: + self.finish(True) + return + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "Syncing by savedefconfig...\n") + else: + self.log += "Syncing by savedefconfig (forced by option)...\n" + + cmd = list(self.make_cmd) + cmd.append('savedefconfig') + self.ps = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE) + self.state = STATE_SAVEDEFCONFIG + + def update_defconfig(self): + """Update the input defconfig and go back to the idle state.""" + + log = self.parser.check_defconfig() + if log: + self.suspicious_boards.append(self.defconfig) + self.log += log + orig_defconfig = os.path.join('configs', self.defconfig) + new_defconfig = os.path.join(self.build_dir, 'defconfig') + updated = not filecmp.cmp(orig_defconfig, new_defconfig) + + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, + "defconfig was updated.\n") + + if not self.options.dry_run and updated: + shutil.move(new_defconfig, orig_defconfig) + self.finish(True) def finish(self, success): """Display log along with progress and go to the idle state. @@ -712,17 +803,24 @@ class Slot: """ return self.failed_boards + def get_suspicious_boards(self): + """Returns a list of boards (defconfigs) with possible misconversion. + """ + return self.suspicious_boards + class Slots: """Controller of the array of subprocess slots.""" - def __init__(self, configs, options, progress): + def __init__(self, configs, options, progress, reference_src_dir): """Create a new slots controller. Arguments: configs: A list of CONFIGs to move. options: option flags. progress: A progress indicator. + reference_src_dir: Determine the true starting config state from this + source tree. """ self.options = options self.slots = [] @@ -730,7 +828,7 @@ class Slots: make_cmd = get_make_cmd() for i in range(options.jobs): self.slots.append(Slot(configs, options, progress, devnull, - make_cmd)) + make_cmd, reference_src_dir)) def add(self, defconfig): """Add a new subprocess if a vacant slot is found. @@ -771,21 +869,76 @@ class Slots: def show_failed_boards(self): """Display all of the failed boards (defconfigs).""" - failed_boards = [] + boards = [] + output_file = 'moveconfig.failed' + + for slot in self.slots: + boards += slot.get_failed_boards() + + if boards: + boards = '\n'.join(boards) + '\n' + msg = "The following boards were not processed due to error:\n" + msg += boards + msg += "(the list has been saved in %s)\n" % output_file + print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, + msg) + + with open(output_file, 'w') as f: + f.write(boards) + + def show_suspicious_boards(self): + """Display all boards (defconfigs) with possible misconversion.""" + boards = [] + output_file = 'moveconfig.suspicious' for slot in self.slots: - failed_boards += slot.get_failed_boards() + boards += slot.get_suspicious_boards() - if len(failed_boards) > 0: - msg = [ "The following boards were not processed due to error:" ] - msg += failed_boards - for line in msg: - print >> sys.stderr, color_text(self.options.color, - COLOR_LIGHT_RED, line) + if boards: + boards = '\n'.join(boards) + '\n' + msg = "The following boards might have been converted incorrectly.\n" + msg += "It is highly recommended to check them manually:\n" + msg += boards + msg += "(the list has been saved in %s)\n" % output_file + print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, + msg) - with open('moveconfig.failed', 'w') as f: - for board in failed_boards: - f.write(board + '\n') + with open(output_file, 'w') as f: + f.write(boards) + +class ReferenceSource: + + """Reference source against which original configs should be parsed.""" + + def __init__(self, commit): + """Create a reference source directory based on a specified commit. + + Arguments: + commit: commit to git-clone + """ + self.src_dir = tempfile.mkdtemp() + print "Cloning git repo to a separate work directory..." + subprocess.check_output(['git', 'clone', os.getcwd(), '.'], + cwd=self.src_dir) + print "Checkout '%s' to build the original autoconf.mk." % \ + subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() + subprocess.check_output(['git', 'checkout', commit], + stderr=subprocess.STDOUT, cwd=self.src_dir) + + def __del__(self): + """Delete the reference source directory + + This function makes sure the temporary directory is cleaned away + even if Python suddenly dies due to error. It should be done in here + because it is guaranteed the destructor is always invoked when the + instance of the class gets unreferenced. + """ + shutil.rmtree(self.src_dir) + + def get_dir(self): + """Return the absolute path to the reference source directory.""" + + return self.src_dir def move_config(configs, options): """Move config options to defconfig files. @@ -795,10 +948,19 @@ def move_config(configs, options): options: option flags """ if len(configs) == 0: - print 'Nothing to do. exit.' - sys.exit(0) + if options.force_sync: + print 'No CONFIG is specified. You are probably syncing defconfigs.', + else: + print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', + else: + print 'Move ' + ', '.join(configs), + print '(jobs: %d)\n' % options.jobs - print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs) + if options.git_ref: + reference_src = ReferenceSource(options.git_ref) + reference_src_dir = reference_src.get_dir() + else: + reference_src_dir = None if options.defconfigs: defconfigs = [line.strip() for line in open(options.defconfigs)] @@ -817,7 +979,7 @@ def move_config(configs, options): defconfigs.append(os.path.join(dirpath, filename)) progress = Progress(len(defconfigs)) - slots = Slots(configs, options, progress) + slots = Slots(configs, options, progress, reference_src_dir) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -834,6 +996,7 @@ def move_config(configs, options): print '' slots.show_failed_boards() + slots.show_suspicious_boards() def main(): try: @@ -852,18 +1015,22 @@ def main(): parser.add_option('-e', '--exit-on-error', action='store_true', default=False, help='exit immediately on any error') + parser.add_option('-s', '--force-sync', action='store_true', default=False, + help='force sync by savedefconfig') parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', action='store_true', default=False, help='only cleanup the headers') parser.add_option('-j', '--jobs', type='int', default=cpu_count, help='the number of jobs to run simultaneously') + parser.add_option('-r', '--git-ref', type='string', + help='the git ref to clone for building the autoconf.mk') parser.add_option('-v', '--verbose', action='store_true', default=False, help='show any build errors as boards are built') parser.usage += ' CONFIG ...' (options, configs) = parser.parse_args() - if len(configs) == 0: + if len(configs) == 0 and not options.force_sync: parser.print_usage() sys.exit(1) @@ -880,7 +1047,8 @@ def main(): if not options.cleanup_headers_only: move_config(configs, options) - cleanup_headers(configs, options.dry_run) + if configs: + cleanup_headers(configs, options.dry_run) if __name__ == '__main__': main()