From cbd27d5e18f3996e9c421481e64504600a659f3a Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Wed, 30 Jun 2010 08:22:59 -0700 Subject: [PATCH] Update support/rrsync and some packaging/* scripts. --- packaging/cull_options | 4 +- packaging/extern-squish | 18 ------- packaging/git-status.pl | 48 ++++++++++++++++++ packaging/nightly-rsync | 56 ++++++++++++++++++++- packaging/patch-update | 69 ++++++++++++++----------- packaging/release-rsync | 108 ++++++++++++++++++++++++++++++---------- packaging/var-checker | 83 ++++++++++++++++++++++++++++++ support/rrsync | 16 +++--- 8 files changed, 316 insertions(+), 86 deletions(-) delete mode 100755 packaging/extern-squish create mode 100644 packaging/git-status.pl create mode 100755 packaging/var-checker diff --git a/packaging/cull_options b/packaging/cull_options index 3ef470df..add2b988 100755 --- a/packaging/cull_options +++ b/packaging/cull_options @@ -7,7 +7,7 @@ use strict; our %short_no_arg; our %short_with_num; our %long_opt = ( - 'no-i-r' => 0, + 'daemon' => -1, 'fake-super' => 0, 'log-file' => 3, ); @@ -24,7 +24,7 @@ while () { undef $last_long_opt; } elsif (/\Qargs[ac++]\E = "--([^"=]+)"/) { $last_long_opt = $1; - $long_opt{$1} = 0; + $long_opt{$1} = 0 unless exists $long_opt{$1}; } elsif (defined($last_long_opt) && /\Qargs[ac++]\E = ([^["\s]+);/ && $1 ne 'dest_option') { $long_opt{$last_long_opt} = 2; diff --git a/packaging/extern-squish b/packaging/extern-squish deleted file mode 100755 index eb8b32e1..00000000 --- a/packaging/extern-squish +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/perl -# This script finds extraneous "extern" variables in the *.c files. -# Run it from inside the main rsync directory. - -use strict; - -my @files = glob('*.c'); - -foreach my $fn (@files) { - open(IN, '<', $fn) or die; - undef $/; $_ = ; $/ = "\n"; - close IN; - my @externs = /^extern .*?([^[\s(*;&.]+)(?:\[.*?\])?;/mg; - foreach my $find (@externs) { - my @matches = /(?; + exit 1 unless /^y/i; + $_[0] = $master_branch = $cur_branch; # Updates caller's $master_branch too. + } + + if ($check_patches_dir && -d 'patches/.git') { + ($cur_branch) = check_git_status($fatal_unless_clean, 'patches'); + if ($cur_branch ne $master_branch) { + print "The *patches* checkout is on branch $cur_branch, not branch $master_branch.\n"; + print "Do you want to change it to branch $master_branch? [n] "; + $_ = ; + exit 1 unless /^y/i; + system "cd patches && git checkout '$master_branch'"; + } + } +} + +sub check_git_status +{ + my($fatal_unless_clean, $subdir) = @_; + $subdir = '.' unless defined $subdir; + my $status = `cd '$subdir' && git status`; + my $is_clean = $status =~ /\nnothing to commit \(working directory clean\)/; + my($cur_branch) = $status =~ /^# On branch (.+)\n/; + if ($fatal_unless_clean && !$is_clean) { + if ($subdir eq '.') { + $subdir = ''; + } else { + $subdir = " *$subdir*"; + } + die "The$subdir checkout is not clean:\n", $status; + } + ($cur_branch, $is_clean, $status); +} + +1; diff --git a/packaging/nightly-rsync b/packaging/nightly-rsync index af9f53a3..3aaabcb2 100755 --- a/packaging/nightly-rsync +++ b/packaging/nightly-rsync @@ -3,8 +3,8 @@ use strict; # This script expects the directory ~/samba-rsync-ftp to exist and to be a # copy of the /home/ftp/pub/rsync dir on samba.org. It also requires a -# pristine CVS checkout of rsync (don't use your normal rsync build dir -# unless you're 100% sure that there are not unchecked-in changes). +# git checkout of rsync (feel free to use your normal rsync build dir as +# long as it doesn't have any uncommitted changes). # # If this is run with -ctu, it will make an updated "nightly" tar file in # the nightly dir. It will also remove any old tar files, regenerate the @@ -56,6 +56,58 @@ if ($make_tar) { } close IN; + my $confversion; + open(IN, '<', 'configure.ac') or die "Unable to open configure.ac: $!\n"; + while () { + if (/^RSYNC_VERSION=(.*)/) { + $confversion = $1; + last; + } + } + close IN; + die "Unable to find RSYNC_VERSION in configure.ac\n" unless defined $confversion; + + open(IN, '<', 'OLDNEWS') or die "Unable to open OLDNEWS: $!\n"; + $_ = ; + my($lastversion) = /(\d+\.\d+\.\d+)/; + my $last_protocol_version; + while () { + if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { + $last_protocol_version = $pver if $ver eq $lastversion; + } + } + close IN; + die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; + + my($protocol_version,$subprotocol_version); + open(IN, '<', 'rsync.h') or die "Unable to open rsync.h: $!\n"; + while () { + if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { + $protocol_version = $1; + } elsif (/^#define\s+SUBPROTOCOL_VERSION\s+(\d+)/) { + $subprotocol_version = $1; + } + } + close IN; + die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; + die "Unable to determine the current SUBPROTOCOL_VERSION.\n" unless defined $subprotocol_version; + + if ($confversion =~ /dev|pre/) { + if ($last_protocol_version ne $protocol_version) { + if ($subprotocol_version == 0) { + die "SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.\n"; + } + } else { + if ($subprotocol_version != 0) { + die "SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.\n"; + } + } + } else { + if ($subprotocol_version != 0) { + die "SUBPROTOCOL_VERSION must be 0 for a final release.\n"; + } + } + print "Creating $name.tar.gz\n"; system "rsync -a @extra_files $name/"; system "git archive --format=tar --prefix=$name/ HEAD | tar xf -"; diff --git a/packaging/patch-update b/packaging/patch-update index 15d1b6c3..dba8e472 100755 --- a/packaging/patch-update +++ b/packaging/patch-update @@ -1,11 +1,12 @@ -#!/usr/bin/perl -w -# This script is used to turn one or more of the "patch/*" branches +#!/usr/bin/perl +# This script is used to turn one or more of the "patch/BASE/*" branches # into one or more diffs in the "patches" directory. Pass the option # --gen if you want generated files in the diffs. Pass the name of # one or more diffs if you want to just update a subset of all the # diffs. use strict; +use warnings; use Getopt::Long; my $patches_dir = 'patches'; @@ -30,10 +31,19 @@ if (defined $incl_generated_files) { die "No '$patches_dir' directory was found.\n" unless -d $patches_dir; die "No '.git' directory present in the current dir.\n" unless -d '.git'; -my($status, $is_clean, $starting_branch) = &check_git_status; -if (!$skip_branch_check && !$is_clean) { - die "The checkout is not clean:\n", $status; +require 'packaging/git-status.pl'; +check_git_state($master_branch, !$skip_branch_check, 1); + +my $master_commit; +open PIPE, '-|', "git log -1 --no-color $master_branch" or die $!; +while () { + if (/^commit (\S+)/) { + $master_commit = $1; + last; + } } +close PIPE; +die "Unable to determine commit hash for master branch: $master_branch\n" unless defined $master_commit; my @extra_files; open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; @@ -60,7 +70,7 @@ my %patches; # Start by finding all patches so that we can load all possible parents. open(PIPE, '-|', 'git', 'branch', '-l') or die $!; while () { - if (m# patch/(.*)#) { + if (m# patch/\Q$master_branch\E/(.*)#o) { $patches{$1} = 1; } } @@ -70,7 +80,7 @@ my @patches = sort keys %patches; my(%parent, %description); foreach my $patch (@patches) { - my $branch = "patch/$patch"; + my $branch = "patch/$master_branch/$patch"; my $desc = ''; open(PIPE, '-|', 'git', 'diff', '-U1000', "$master_branch...$branch", '--', "PATCH.$patch") or die $!; while () { @@ -114,7 +124,7 @@ if ($incl_generated_files) { } sleep 1 while $last_touch >= time; -system "git checkout $starting_branch" and exit 1; +system "git checkout $master_branch" and exit 1; exit; @@ -124,24 +134,27 @@ sub update_patch my($patch) = @_; my $parent = $parent{$patch}; + my $based_on; if (defined $parent) { unless ($completed{$parent}++) { update_patch($parent); } - $parent = "patch/$parent"; + $based_on = $parent = "patch/$master_branch/$parent"; } else { $parent = $master_branch; + $based_on = $master_commit; } print "======== $patch ========\n"; sleep 1 while $incl_generated_files && $last_touch >= time; - system "git checkout patch/$patch" and return 0; + system "git checkout patch/$master_branch/$patch" and return 0; - my $ok = system("git merge $parent") == 0; + my $ok = system("git merge $based_on") == 0; if (!$ok || $launch_shell) { - print qq|"git merge $parent" incomplete -- please fix.\n| if !$ok; - $ENV{PS1} = "[$parent] patch/$patch: "; + my($parent_dir) = $parent =~ m{([^/]+)$}; + print qq|"git merge $based_on" incomplete -- please fix.\n| if !$ok; + $ENV{PS1} = "[$parent_dir] $patch: "; while (1) { if (system($ENV{SHELL}) != 0) { print "Abort? [n/y] "; @@ -149,21 +162,21 @@ sub update_patch next unless /^y/i; return 0; } - ($status, $is_clean) = &check_git_status; + my($cur_branch, $is_clean, $status) = check_git_status(0); last if $is_clean; print $status; } } open(OUT, '>', "$patches_dir/$patch.diff") or die $!; - print OUT $description{$patch}, "\n"; + print OUT $description{$patch}, "\nbased-on: $based_on\n"; if ($incl_generated_files) { system "$make_gen_cmd && rsync -a @extra_files $tmp_dir/$patch/" and exit 1; } $last_touch = time; - open(PIPE, '-|', 'git', 'diff', $parent) or die $!; + open(PIPE, '-|', 'git', 'diff', $based_on) or die $!; DIFF: while () { while (m{^diff --git a/PATCH}) { while () { @@ -200,22 +213,18 @@ sub update_patch exit; -sub check_git_status -{ - open(IN, '-|', 'git status') or die $!; - my $status = join('', ); - close IN; - my $is_clean = $status =~ /\nnothing to commit \(working directory clean\)/; - my($starting_branch) = $status =~ /^# On branch (.+)\n/; - ($status, $is_clean, $starting_branch); -} - sub usage { die <); -close IN; -die "The checkout is not clean:\n", $status unless $status =~ /\nnothing to commit \(working directory clean\)/; -die "The checkout is not on the $master_branch branch.\n" unless $status =~ /^# On branch $master_branch\n/; +require 'packaging/git-status.pl'; +check_git_state($master_branch, 1, 1); my $confversion; open(IN, '<', 'configure.ac') or die $!; @@ -79,8 +77,27 @@ die "Unable to find RSYNC_VERSION in configure.ac\n" unless defined $confversion open(IN, '<', 'OLDNEWS') or die $!; $_ = ; -close IN; my($lastversion) = /(\d+\.\d+\.\d+)/; +my($last_protocol_version, %pdate); +while () { + if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { + $pdate{$ver} = $pdate if defined $pdate; + $last_protocol_version = $pver if $ver eq $lastversion; + } +} +close IN; +die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; + +my $protocol_version; +open(IN, '<', 'rsync.h') or die $!; +while () { + if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { + $protocol_version = $1; + last; + } +} +close IN; +die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; my $version = $confversion; $version =~ s/dev/pre1/ || $version =~ s/pre(\d+)/ 'pre' . ($1 + 1) /e; @@ -118,6 +135,23 @@ chomp($_ = ); $release = $_ if $_ ne ''; $release .= ".$pre" if $pre; +(my $finalversion = $version) =~ s/pre\d+//; +my($proto_changed,$proto_change_date); +if ($protocol_version eq $last_protocol_version) { + $proto_changed = 'unchanged'; + $proto_change_date = "\t\t"; +} else { + $proto_changed = 'changed'; + if (!defined($proto_change_date = $pdate{$finalversion})) { + while (1) { + print "On what date did the protocol change to $protocol_version get checked in? (dd Mmm yyyy) "; + chomp($_ = ); + last if /^\d\d \w\w\w \d\d\d\d$/; + } + $proto_change_date = "$_\t"; + } +} + my($srcdir,$srcdiffdir,$lastsrcdir,$skipping); if ($lastversion =~ /pre/) { if (!$pre) { @@ -157,7 +191,6 @@ EOT print " "; $_ = ; -(my $finalversion = $version) =~ s/pre\d+//; my %specvars = ( 'Version:' => $finalversion, 'Release:' => $release, '%define fullversion' => "\%{version}$pre", 'Released' => "$version.", '%define srcdir' => $srcdir ); @@ -184,18 +217,19 @@ foreach my $fn (@tweak_files) { s/^(This man ?page is current for version) \S+ (of rsync)/$1 $version $2/m or die "Unable to update current version info in $fn\n"; } elsif ($fn eq 'rsync.h') { - s/(#define\s+SUBPROTOCOL_VERSION)\s+\d+/$1 0/ + s{(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)} + { $1 . ' ' . get_subprotocol_version($2) }e or die "Unable to find SUBPROTOCOL_VERSION define in $fn\n"; - next if $pre; } elsif ($fn eq 'NEWS') { - s/^(NEWS for rsync \Q$finalversion\E) \(UNRELEASED\)\s*\n/$1 ($today)\n/mi - or die "The first line of $fn is not in the right format. It must be:\n" - . "NEWS for rsync $finalversion (UNRELEASED)\n"; - next if $pre; + s{^(NEWS for rsync \Q$finalversion\E )(\(UNRELEASED\))\s*(\nProtocol: )(\d+) (\([^)]+\))\n} + { $1 . ($pre ? $2 : "($today)") . "$3$protocol_version ($proto_changed)\n" }ei + or die "The first 2 lines of $fn are not in the right format. They must be:\n" + . "NEWS for rsync $finalversion (UNRELEASED)\n" + . "Protocol: $protocol_version ($proto_changed)\n"; } elsif ($fn eq 'OLDNEWS') { - s/^\t\S\S\s\S\S\S\s\d\d\d\d(\t\Q$finalversion\E)/\t$ztoday$1/m + s{^(\t\S\S\s\S\S\S\s\d\d\d\d)(\t\Q$finalversion\E\t).*} + { ($pre ? $1 : "\t$ztoday") . $2 . $proto_change_date . $protocol_version }em or die "Unable to find \"?? ??? $year\t$finalversion\" line in $fn\n"; - next if $pre; } elsif ($fn eq 'options.c') { if (s/(Copyright \(C\) 2002-)(\d+)( Wayne Davison)/$1$year$3/ && $2 ne $year) { @@ -229,7 +263,7 @@ print $break, < "; $_ = ; +# We want to use our passphrase-providing "gpg" script, so modify the PATH. +$ENV{PATH} = "$curdir/packaging/bin:$path"; + my $passphrase; while (1) { ReadMode('noecho'); @@ -281,17 +322,23 @@ while (1) { umask $oldmask; $ENV{'GPG_PASSFILE'} = $passfile; - # We want to use our passphrase-providing "gpg" script, so modify the PATH. - $ENV{PATH} = "packaging/bin:$path"; $_ = `git tag -s -m 'Version $version.' v$version 2>&1`; - $ENV{PATH} = $path; - unlink($passfile); print $_; next if /bad passphrase/; - last unless /failed/; - exit 1; + exit 1 if /failed/; + + if (-d 'patches/.git') { + $_ = `cd patches && git tag -s -m 'Version $version.' v$version 2>&1`; + print $_; + exit 1 if /bad passphrase|failed/; + } + + unlink($passfile); + last; } +$ENV{PATH} = $path; + # Extract the generated files from the old tar. @_ = @extra_files; map { s#^#rsync-$lastversion/# } @_; @@ -299,7 +346,7 @@ system "tar xzf $lasttar_file @_"; rename("rsync-$lastversion", 'a'); print "Creating $diff_file ...\n"; -system "./config.status Makefile; make gen; rsync -a @extra_files b/"; +system "$make_gen_cmd && rsync -a @extra_files b/" and exit 1; my $sed_script = 's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:'; system "(git diff v$lastversion v$version; diff -upN a b | sed -r '$sed_script') | gzip -9 >$diff_file"; system "rm -rf a"; @@ -354,6 +401,15 @@ EOT exit; +sub get_subprotocol_version +{ + my($subver) = @_; + if ($pre && $proto_changed eq 'changed') { + return $subver == 0 ? 1 : $subver; + } + 0; +} + sub usage { die < 1 } qw( t_stub.c t_unsafe.c tls.c trimslash.c ); +my %add_compat_c = map { $_ => 1 } qw( t_stub.c tls.c trimslash.c wildtest.c ); +my %add_util_c = map { $_ => 1 } qw( t_stub.c t_unsafe.c ); +my %sizes; + +open(IN, '<', 'syscall.c') or die $!; +undef $/; my $syscall_c = ; $/ = "\n"; +close IN; +$syscall_c =~ s/^extern\s.*//mg; + +open(IN, '<', 'lib/compat.c') or die $!; +undef $/; my $compat_c = ; $/ = "\n"; +close IN; +$compat_c =~ s/^extern\s.*//mg; + +open(IN, '<', 'util.c') or die $!; +undef $/; my $util_c = ; $/ = "\n"; +close IN; +$util_c =~ s/^extern\s.*//mg; + +my @files = glob('*.c'); + +foreach my $fn (@files) { + open(IN, '<', $fn) or die $!; + undef $/; $_ = ; $/ = "\n"; + close IN; + + my @vars = /^(?!(?:extern|enum)\s)([a-zA-Z]\S*\s+.*);/mg; + my @externs = /^extern\s+(.*);/mg; + + $_ .= $syscall_c if $add_syscall_c{$fn}; + $_ .= $compat_c if $add_compat_c{$fn}; + $_ .= $util_c if $add_util_c{$fn}; + s/INFO_GTE/info_levels/g; + s/DEBUG_GTE/debug_levels/g; + + check_vars($fn, 'var', @vars); + check_vars($fn, 'extern', @externs); +} + +exit; + +# The file's contents are in $_. +sub check_vars +{ + my $fn = shift; + my $type = shift; + + foreach my $line (@_) { + $line =~ s/\s*\{.*\}//; + $line =~ s/\s*\(.*\)//; + foreach my $item (split(/\s*,\s*/, $line)) { + $item =~ s/\s*=.*//; + my $sz = $item =~ s/(\[.*?\])// ? $1 : ''; + my($var) = $item =~ /([^*\s]+)$/; + if (!defined $var) { + print "Bogus match? ($item)\n"; + next; + } + if ($sz) { + if (defined $sizes{$var}) { + if ($sizes{$var} ne $sz) { + print $fn, ' has inconsistent size for "', $var, + "\": $sizes{$var} vs $sz\n"; + } + } else { + $sizes{$var} = $sz; + } + } + my @matches = /(? 1, 'min-size' => 1, 'modify-window' => 1, - 'no-i-r' => 0, 'no-implied-dirs' => 0, 'no-r' => 0, 'no-relative' => 0, @@ -109,6 +108,7 @@ our %long_opt = ( 'size-only' => 0, 'skip-compress' => 1, 'specials' => 0, + 'stats' => 0, 'suffix' => 1, 'super' => 0, 'temp-dir' => 2, @@ -175,10 +175,10 @@ while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) { } else { if ($subdir ne '/') { # Validate args to ensure they don't try to leave our restricted dir. - s#//+#/#g; - s#^/##; - s#^$#.#; - die "Do not use .. in any path!\n" if m#(^|/)\\?\.\\?\.(\\?/|$)#; + s{//+}{/}g; + s{^/}{}; + s{^$}{.}; + die "$0: do not use .. in any path!\n" if m{(^|/)\\?\.\\?\.(\\?/|$)}; } push(@args, bsd_glob($_, GLOB_LIMIT|GLOB_NOCHECK|GLOB_BRACE|GLOB_QUOTE)); } @@ -205,10 +205,10 @@ sub check_arg my($opt, $arg, $type) = @_; $arg =~ s/\\(.)/$1/g; if ($subdir ne '/' && ($type == 3 || ($type == 2 && !$am_sender))) { - $arg =~ s#//#/#g; + $arg =~ s{//}{/}g; die "Do not use .. in --$opt; anchor the path at the root of your restricted dir.\n" - if $arg =~ m#(^|/)\.\.(/|$)#; - $arg =~ s#^/#$subdir/#; + if $arg =~ m{(^|/)\.\.(/|$)}; + $arg =~ s{^/}{$subdir/}; } $arg; } -- 2.34.1