From c39785d4e64587d3a07a8ce8f25046f407107e71 Mon Sep 17 00:00:00 2001 From: Richard Levitte Date: Thu, 15 Mar 2018 18:06:18 +0100 Subject: [PATCH] Move all dependency post-processing to util/add-depends.pl In the end, it's more efficient to only have one perl instance (that loads configdata.pm) dealing with dependency files than running one (that still loads configdata.pm) for each such file. Reviewed-by: Andy Polyakov (Merged from https://github.com/openssl/openssl/pull/5631) --- Configurations/descrip.mms.tmpl | 14 +- Configurations/unix-Makefile.tmpl | 11 +- Configurations/windows-makefile.tmpl | 6 +- util/add-depends.pl | 205 +++++++++++++++++++++++++-- util/postprocess-makedepend.pl | 160 --------------------- 5 files changed, 203 insertions(+), 193 deletions(-) delete mode 100644 util/postprocess-makedepend.pl diff --git a/Configurations/descrip.mms.tmpl b/Configurations/descrip.mms.tmpl index cf448aa74a..f6061f16f7 100644 --- a/Configurations/descrip.mms.tmpl +++ b/Configurations/descrip.mms.tmpl @@ -505,7 +505,7 @@ distclean : clean depend : descrip.mms descrip.mms : FORCE @ ! {- output_off() if $disabled{makedepend}; "" -} - @ $(PERL) {- sourcefile("util", "add-depends.pl") -} + @ $(PERL) {- sourcefile("util", "add-depends.pl") -} "VMS C" @ ! {- output_on() if $disabled{makedepend}; "" -} # Install helper targets ############################################# @@ -878,12 +878,9 @@ EOF my $incs_on = join("\n\t\@ ", @{$incs_cmds[0]}) || '!'; my $incs_off = join("\n\t\@ ", @{$incs_cmds[1]}) || '!'; my $depbuild = $disabled{makedepend} ? "" - : " /MMS=(FILE=${objd}${objn}.tmp-D,TARGET=$obj.OBJ)"; - my $postprocess_makedepend = - sourcefile("util", "postprocess-makedepend.pl"); - my $objdir = dirname($obj); + : " /MMS=(FILE=${objd}${objn}.D,TARGET=$obj.OBJ)"; - return <<"EOF" + return <<"EOF"; $obj.OBJ : $deps ${before} SET DEFAULT $forward @@ -894,11 +891,6 @@ $obj.OBJ : $deps ${after} - PURGE $obj.OBJ EOF - . ($disabled{makedepend} ? "" : <<"EOF" - \$(PERL) $postprocess_makedepend "VMS C" $objdir < $obj.tmp-D > $obj.d - - DELETE $obj.tmp-D;* -EOF - ); } sub libobj2shlib { my %args = @_; diff --git a/Configurations/unix-Makefile.tmpl b/Configurations/unix-Makefile.tmpl index e9d112ba69..cb134e7931 100644 --- a/Configurations/unix-Makefile.tmpl +++ b/Configurations/unix-Makefile.tmpl @@ -12,6 +12,7 @@ our $shlibextsimple = $target{shared_extension_simple} || ".so"; our $shlibextimport = $target{shared_import_extension} || ""; our $dsoext = $target{dso_extension} || ".so"; + our $makedepprog = $disabled{makedepend} ? undef : $config{makedepprog}; sub windowsdll { $config{target} =~ /^(?:Cygwin|mingw)/ } @@ -419,7 +420,9 @@ distclean: clean # concatenate only if that is true. depend: @: {- output_off() if $disabled{makedepend}; "" -} - @$(PERL) $(SRCDIR)/util/add-depends.pl + @$(PERL) $(SRCDIR)/util/add-depends.pl {- + defined $makedepprog && $makedepprog =~ /\/makedepend/ + ? 'makedepend' : 'gcc' -} @: {- output_on() if $disabled{makedepend}; "" -} # Install helper targets ############################################# @@ -970,7 +973,6 @@ EOF my $cmd; my $cmdflags; my $cmdcompile; - my $makedepprog = $disabled{makedepend} ? undef : $config{makedepprog}; if (grep /\.rc$/, @srcs) { $cmd = '$(RC)'; $cmdflags = '$(RCFLAGS)'; @@ -1032,9 +1034,8 @@ $obj$objext: $deps EOF if (defined $makedepprog && $makedepprog =~ /\/makedepend/) { $recipe .= <<"EOF"; - \$(MAKEDEPEND) -f- -o"|\$\@" -- $incs $cmdflags -- $srcs 2>/dev/null \\ - | \$(PERL) \$(SRCDIR)/util/postprocess-makedepend.pl \\ - 'makedepend' > $obj$depext + \$(MAKEDEPEND) -f- -Y -- $incs $cmdflags -- $srcs 2>/dev/null \\ + > $obj$depext EOF } } diff --git a/Configurations/windows-makefile.tmpl b/Configurations/windows-makefile.tmpl index dde22fb217..ca2e75e1f1 100644 --- a/Configurations/windows-makefile.tmpl +++ b/Configurations/windows-makefile.tmpl @@ -382,7 +382,7 @@ distclean: clean depend: @ rem {- output_off() if $disabled{makedepend}; "" -} - @ "$(PERL)" "$(SRCDIR)\util\add-depends.pl" + @ "$(PERL)" "$(SRCDIR)\util\add-depends.pl" "VC" @ rem {- output_on() if $disabled{makedepend}; "" -} # Install helper targets ############################################# @@ -599,9 +599,7 @@ EOF } return <<"EOF" if (!$disabled{makedepend}); $obj$depext: $deps - \$(CC) $cflags /Zs /showIncludes $srcs 2>&1 \\ - | "\$(PERL)" "\$(SRCDIR)\\util\\postprocess-makedepend.pl" \\ - "VC" "$obj$objext" > $obj$depext + \$(CC) $cflags /Zs /showIncludes $srcs 2>&1 > $obj$depext $obj$objext: $obj$depext \$(CC) $cflags -c \$(COUTFLAG)\$\@ $srcs EOF diff --git a/util/add-depends.pl b/util/add-depends.pl index d2bba5e192..31996c4edd 100644 --- a/util/add-depends.pl +++ b/util/add-depends.pl @@ -6,46 +6,225 @@ # in the file LICENSE in the source distribution or at # https://www.openssl.org/source/license.html +use strict; +use warnings; + use lib '.'; use configdata; +use File::Spec::Functions qw(canonpath rel2abs); use File::Compare qw(compare_text); +# When using stat() on Windows, we can get it to perform better by avoid some +# data. This doesn't affect the mtime field, so we're not losing anything... +${^WIN32_SLOPPY_STAT} = 1; + my $buildfile = $config{build_file}; -my $buildfile_new = "$buildfile-$$"; +my $build_mtime = (stat($buildfile))[9]; +my $rebuild = 0; my $depext = $target{dep_extension} || ".d"; -my @deps = +my @depfiles = sort - grep { -f $_ } + grep { + # This grep has side effects. Not only does if check the existence + # of the dependency file given in $_, but it also checks if it's + # newer than the build file, and if it is, sets $rebuild. + my @st = stat($_); + $rebuild = 1 if @st && $st[9] > $build_mtime; + scalar @st > 0; # Determines the grep result + } map { (my $x = $_) =~ s|\.o$|$depext|; $x; } grep { $unified_info{sources}->{$_}->[0] =~ /\.cc?$/ } keys %{$unified_info{sources}}; +exit 0 unless $rebuild; + +# Ok, primary checks are done, time to do some real work + +my $abs_srcdir = rel2abs($config{sourcedir}); +my $abs_blddir = rel2abs($config{builddir}); + +my $producer = shift @ARGV; +die "Producer not given\n" unless $producer; + +my %procedures = ( + 'gcc' => undef, # gcc style dependency files needs no mods + 'makedepend' => + sub { + # makedepend, in its infinite wisdom, wants to have the object file + # in the same directory as the source file. This doesn't work too + # well with out-of-source-tree builds, so we must resort to tricks + # to get things right. Fortunately, the .d files are always placed + # parallel with the object files, so all we need to do is construct + # the object file name from the dep file name. + (my $objfile = shift) =~ s|\.d$|.o|i; + my $line = shift; + + # Discard comments + return undef if $line =~ /^(#.*|\s*)$/; + + # Remove the original object file + $line =~ s|^.*\.o: | |; + # Also, remove any dependency that starts with a /, because those + # are typically system headers + $line =~ s/\s+\/(\\.|\S)*//g; + # Finally, discard all empty lines + return undef if $line =~ /^\s*$/; + + # All we got now is a dependency, just shave off surrounding spaces + $line =~ s/^\s+//; + $line =~ s/\s+$//; + return ($objfile, $line); + }, + 'VMS C' => + sub { + # current versions of DEC / Compaq / HP / VSI C strips away all + # directory information from the object file, so we must insert it + # back. To make life simpler, we simply replace it with the + # corresponding .D file that's had its extension changed. Since + # .D files are always written parallel to the object files, we + # thereby get the directory information for free. + (my $objfile = shift) =~ s|\.D$|.OBJ|i; + my $line = shift; + + # Shave off the target. + # + # The pattern for target and dependencies will always take this + # form: + # + # target SPACE : SPACE deps + # + # This is so a volume delimiter (a : without any spaces around it) + # won't get mixed up with the target / deps delimiter. We use this + # to easily identify what needs to be removed. + m|\s:\s|; $line = $'; + + # We know that VMS has system header files in text libraries, + # extension .TLB. We also know that our header files aren't stored + # in text libraries. Finally, we know that VMS C produces exactly + # one dependency per line, so we simply discard any line ending with + # .TLB. + return undef if /\.TLB\s*$/; + + # All we got now is a dependency, just shave off surrounding spaces + $line =~ s/^\s+//; + $line =~ s/\s+$//; + return ($objfile, $line); + }, + 'VC' => + sub { + # For the moment, we only support Visual C on native Windows, or + # compatible compilers. With those, the flags /Zs /showIncludes + # give us the necessary output to be able to create dependencies + # that nmake (or any 'make' implementation) should be able to read, + # with a bit of help. The output we're interested in looks like + # this (it always starts the same) + # + # Note: including file: {whatever header file} + # + # Since there's no object file name at all in that information, + # we must construct it ourselves. + + (my $objfile = shift) =~ s|\.d$|.obj|i; + my $line = shift; + + # There are also other lines mixed in, for example compiler + # warnings, so we simply discard anything that doesn't start with + # the Note: + + if (/^Note: including file: */) { + (my $tail = $') =~ s/\s*\R$//; + + # VC gives us absolute paths for all include files, so to + # remove system header dependencies, we need to check that + # they don't match $abs_srcdir or $abs_blddir + $tail = canonpath($tail); + if ($tail =~ m|^\Q$abs_srcdir\E|i + || $tail =~ m|^\Q$abs_blddir\E|i) { + return ($objfile, "\"$tail\""); + } + } + + return undef; + }, +); +my %continuations = ( + 'gcc' => undef, + 'makedepend' => "\\", + 'VMS C' => "-", + 'VC' => "\\", +); + +die "Producer unrecognised: $producer\n" + unless exists $procedures{$producer} && exists $continuations{$producer}; + +my $procedure = $procedures{$producer}; +my $continuation = $continuations{$producer}; + +my $buildfile_new = "$buildfile-$$"; + +my %collect = (); +if (defined $procedure) { + foreach my $depfile (@depfiles) { + open IDEP,$depfile or die "Trying to read $depfile: $!\n"; + while () { + s|\R$||; # The better chomp + my ($target, $deps) = $procedure->($depfile, $_); + $collect{$target}->{$deps} = 1 if defined $target; + } + close IDEP; + } +} + open IBF, $buildfile or die "Trying to read $buildfile: $!\n"; open OBF, '>', $buildfile_new or die "Trying to write $buildfile_new: $!\n"; while () { - $force_rewrite = 0; last if /^# DO NOT DELETE THIS LINE/; print OBF or die "$!\n"; - $force_rewrite = 1; } close IBF; print OBF "# DO NOT DELETE THIS LINE -- make depend depends on it.\n"; -foreach (@deps) { - open IBF,$_ or die "Trying to read $_: $!\n"; - while () { - print OBF or die "$!\n"; +if (defined $procedure) { + foreach my $target (sort keys %collect) { + my $prefix = $target . ' :'; + my @deps = sort keys %{$collect{$target}}; + + while (@deps) { + my $buf = $prefix; + $prefix = ''; + + while (@deps && ($buf eq '' + || length($buf) + length($deps[0]) <= 77)) { + $buf .= ' ' . shift @deps; + } + $buf .= ' '.$continuation if @deps; + + print OBF $buf,"\n" or die "Trying to print: $!\n" + } + } +} else { + foreach my $depfile (@depfiles) { + open IDEP,$depfile or die "Trying to read $depfile: $!\n"; + while () { + print OBF or die "Trying to print: $!\n"; + } + close IDEP; } - close IBF; } + close OBF; if (compare_text($buildfile_new, $buildfile) != 0) { rename $buildfile_new, $buildfile or die "Trying to rename $buildfile_new -> $buildfile: $!\n"; } -# On VMS, we want to remove all generations of this file, in case there are -# more than one -while (unlink $buildfile_new) {} + +END { + # On VMS, we want to remove all generations of this file, in case there + # are more than one, so we loop. + if (defined $buildfile_new) { + while (unlink $buildfile_new) {} + } +} diff --git a/util/postprocess-makedepend.pl b/util/postprocess-makedepend.pl deleted file mode 100644 index 323ce9e0e4..0000000000 --- a/util/postprocess-makedepend.pl +++ /dev/null @@ -1,160 +0,0 @@ -#! /usr/bin/env perl -# Copyright 2018 The OpenSSL Project Authors. All Rights Reserved. -# -# Licensed under the OpenSSL license (the "License"). You may not use -# this file except in compliance with the License. You can obtain a copy -# in the file LICENSE in the source distribution or at -# https://www.openssl.org/source/license.html - -use strict; -use warnings; - -use lib '.'; -use configdata; - -use File::Spec::Functions qw(canonpath rel2abs); - -my $abs_srcdir = rel2abs($config{sourcedir}); -my $abs_blddir = rel2abs($config{builddir}); - -my $producer = shift @ARGV; -die "Producer not given\n" unless $producer; - -my $procedure = { - 'makedepend' => - sub { - my $line = shift; - - # makedepend, in its infinite wisdom, wants to have the object file - # in the same directory as the source file. This doesn't work too - # well with out-of-source-tree builds, so we must resort to tricks - # to get things right. The trick is to call makedepend with an - # extra suffix that contains the desired object file path, like - # this: - # - # makedepend -f- -o"|dir/foo.o" -- $(CFLAGS) -- ../some/foo.c - # - # The result will look something like this: - # - # ../somewhere/foo|dir/foo.o: deps... - # - # Which is easy to massage by removing everything up to the first | - - # Remove everything up to the first | - $line =~ s/^.*\|//; - # Also, remove any dependency that starts with a /, because those - # are typically system headers - $line =~ s/\s+\/(\\.|\S)*//g; - # Finally, discard all empty lines or comment lines - return undef if $line =~ /:\s*$/ || $line =~ /^(#.*|\s*)$/; - - my ($target, $deps) = $line =~ /^((?:\\.|[^:])*):(.*)/; - $deps =~ s/^\s+//; - $deps =~ s/\s+$//; - return ($target, $deps); - }, - 'VMS C' => - sub { - my $line = shift; - - # current versions of DEC / Compaq / HP / VSI C strips away all - # directory information from the object file, so we must insert it - # back. Just to be safe against future changes, we check that there - # really is no directory information. - my $directory = shift; - - # The pattern for target and dependencies will always take this - # form: - # - # target SPACE : SPACE deps - # - # This is so a volume delimiter (a : without any spaces around it) - # won't get mixed up with the target / deps delimiter. We use this - # fact in the regexp below to make sure we do look at the target. - $line =~ s/^/$directory/ unless /^\S+[:>\]]\S+\s+:/; - - # We know that VMS has system header files in text libraries, - # extension .TLB. We also know that our header files aren't stored - # in text libraries. Finally, we know that VMS C produces exactly - # one dependency per line, so we simply discard any line ending with - # .TLB. - return undef if /\.TLB\s*$/; - - my ($target, $deps) = $line =~ /^(.*)\s:\s(.*)/; - $deps =~ s/^\s+//; - $deps =~ s/\s+$//; - return ($target, $deps); - }, - 'VC' => - sub { - my $line = shift; - my $object = shift; - - # For the moment, we only support Visual C on native Windows, or - # compatible compilers. With those, the flags /Zs /showIncludes - # give us the necessary output to be able to create dependencies - # that nmake (or any 'make' implementation) should be able to read, - # with a bit of help. The output we're interested in looks like - # this (it always starts the same) - # - # Note: including file: {whatever header file} - # - # So all we really have to do is to is to replace the start of the - # line with an object file specification, given to us as an extra - # argument (passed from $ARGV[1]); - # - # There are also other lines mixed in, for example compiler - # warnings, so we simply discard anything that doesn't start with - # the Note: - - if (/^Note: including file: */) { - (my $tail = $') =~ s/\s*\R$//; - - # VC gives us absolute paths for all include files, so to - # remove system header dependencies, we need to check that - # they don't match $abs_srcdir or $abs_blddir - $tail = canonpath($tail); - if ($tail =~ m|^\Q$abs_srcdir\E|i - || $tail =~ m|^\Q$abs_blddir\E|i) { - return ($object, "\"$tail\""); - } - } - - return undef; - }, -} -> {$producer}; - -die "Producer unrecognised: $producer\n" unless defined $procedure; - -my %collect = (); -while () { - s|\R$||; # The better chomp - my ($target, $deps) = $procedure->($_, @ARGV); - $collect{$target}->{$deps} = 1 - if defined $target; -} - -my $continuation = { - 'makedepend' => "\\", - 'VMS C' => "-", - 'VC' => "\\", -} -> {$producer}; - -die "Producer unrecognised: $producer\n" unless defined $continuation; - -foreach my $target (sort keys %collect) { - my $prefix = $target . ' :'; - my @deps = sort keys %{$collect{$target}}; - - while (@deps) { - my $buf = $prefix; - $prefix = ''; - - while (@deps && ($buf eq '' || length($buf) + length($deps[0]) <= 77)) { - $buf .= ' ' . shift @deps; - } - $buf .= ' '.$continuation if @deps; - - print $buf,"\n"; - } -} -- 2.25.1