freebsd-ports/Tools/scripts/addport

#!/usr/bin/env perl
#
# addport - perl script that adds new ports to the
# FreeBSD Ports Collection.  Replaces easy-import.
#
# Copyright (c) 2000 Will Andrews and Michael Haro
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# Original shell script & idea: Will Andrews <will@FreeBSD.org>
# Original conversion to Perl : Michael Haro <mharo@FreeBSD.org>
# Maintainer alumni           : Renato Botelho <garga@FreeBSD.org>
#
# Id: addport,v 1.2 2000/04/02 06:21:13 will Exp (original shell script)
# Id: addport,v 1.5 2000/04/22 22:19:43 mharo Exp (perl conversion)
# $FreeBSD$
#
# MAINTAINER=	crees@FreeBSD.org
#

# Smartmatch operator (~~) used
require 5.10.1;

use Cwd "abs_path";
use Getopt::Std;
use Sys::Hostname;
use locale;
use strict;

# Subroutine prototypes
sub warnx($);
sub err($$);
sub errx($$);
sub prompt($);
sub query($);
sub usage();
sub contains($@);
sub lsports();
sub lastcomment();

my %opts;

getopts('ac:d:fh:il:L:mns:tu:y', \%opts);

my $autofill_l = $opts{'l'};
my $autofill_L = $opts{'L'};
my $autofill = ($autofill_l ? $autofill_l : $autofill_L);
my $c = $opts{'c'} if ($opts{'c'} ne "");
my $distdir = $opts{'s'} if ($opts{'s'} ne "");
my $dir = $opts{'d'};
my $h = "repo.FreeBSD.org";
$h = $opts{'h'} if ($opts{'h'} ne "");
my $u = $ENV{USER};
$u = $opts{'u'} if ($opts{'u'} ne "");
my $more_testing = $opts{'t'};
my $interactive = $opts{'i'};
my $nomkdir = $opts{'m'};
my $addlchk = $opts{'a'};
my $nofetch = $opts{'f'};
my $checkexist = $opts{'y'};
my $currentdir = abs_path(".");
my %l10nprefix = (
	'chinese'	=> 'zh-',
	'french'	=> 'fr-',
	'german'	=> 'de-',
	'hebrew'	=> 'iw-',
	'hungarian'	=> 'hu-',
	'japanese'	=> 'ja-',
	'korean'	=> 'ko-',
	'portuguese'	=> 'pt-',
	'russian'	=> 'ru-',
	'ukrainian'	=> 'uk-',
	'vietnamese'	=> 'vi-',
);

my $tmpdir;
my $repo;
my $portsdir = $ENV{PORTSDIR} ? $ENV{PORTSDIR} : '/usr/ports';
my $repo = $ENV{ADDPSVNROOT}? $ENV{ADDPSVNROOT} : "svn+ssh://$u\@$h/ports/head";
my $make = "make";
my $portlint = `which portlint`; chomp $portlint;
my $plint_args = "-N -a -c -t";
my $perl = "perl";
my $cp = "cp";
my $mv = "mv";
my $rm = "rm";
chomp(my $svnlite = `which svnlite`);
my $svn = ($svnlite ? $svnlite : "svn");
my $keyword = '\$FreeBSD\\\$';
# vars required for commitfile
my $descr; my $portversion; my $pkgcomment;
my $tmp; my $pkgcommentlen; my $comment; my $maintainer = "";
my $maintaineraddr; my $tmp2; my $offset; my $commitfile = "";
my $moved = "";
$tmp = $tmp2 = $offset = 0;

# Check the editor.
my $edit = "/usr/bin/vi";
$edit = $ENV{EDITOR} if ($ENV{EDITOR} ne "");

# Check svn version
my $svnversion = `$svn --version --quiet 2>/dev/null`;
chomp $svnversion;
if ($svnversion eq "") {
	errx(1, "Subversion binary not found in \$PATH, aborting.");
}
my @svnversion = split(/\./, $svnversion);
if ($svnversion[0] == 1 && $svnversion[1] < 7) {
	errx(1, "minimum Subversion version of 1.7 not met, aborting.");
}

# stuff that always happens when we start
BEGIN {
	$tmpdir=`mktemp -d -t ap`;
	chomp $tmpdir;
	if ($tmpdir eq "") {
		errx(1,"making random tmpdir didn't work, aborting.");
	}
}

# stuff that always happens when we exit
END {
	# only remove $tmpdir if it points to something in /tmp
	# this is a silly little security thing
	if (defined($rm) && defined($tmpdir)) {
		system("$rm -rf $tmpdir") if ($tmpdir =~ m,/tmp/,);
	}
}

# setup the list of commands to run on the new port(s).
my @commands;
my $passenv = "";
if ($addlchk && -f $portlint) {
	$passenv = "DISTDIR=\"$distdir\"" if -d $distdir;
	$passenv = $passenv . " PORTSDIR=\"$portsdir\"" if !$nomkdir;
	push(@commands, "$make $passenv clean check-categories");
	push(@commands, "$portlint $plint_args");
	push(@commands, "$make $passenv FETCH_BEFORE_ARGS='-A' checksum") if !$nofetch;
	if ($more_testing) {
		push(@commands, "$make $passenv distclean");
		push(@commands, "$make $passenv build");
		push(@commands, "$make $passenv distclean");
	}
	if (!$nomkdir) {
		chdir $tmpdir;
		print "Checking out Mk directory to ensure portlint correctness.\n";
		system("$svn co $repo/Mk Mk") && errx(1, "Could not checkout Mk directory");
		system("$svn co $repo/Templates Templates") && errx(1, "Could not checkout Templates directory");
		chdir $currentdir;
	}
}

if ($dir eq "") {
	warnx("Need to specify a directory with -d argument!");
	usage();
	exit 1;
}

# check the port doesn't exist already

if ($checkexist) {
	my $found = 0;
	print ">> Fetching INDEX to scan for duplicates...\n";
	my $indexfile = "$tmpdir/" . `$make -C $portsdir -V INDEXFILE`;
	system("$make -C /usr/ports INDEXDIR=$tmpdir fetchindex") && errx(1, "Could not fetch INDEX file.");
	my @namepart;
	foreach (split(/\,/, $dir)) {
		s/^.*\///;
		foreach (split(/[-_]/)) {
			next if length () <=2 or /^rubygem$/;
			push(@namepart, $_);
		}
	}
	open(INDEXFILE, "< $indexfile") or errx(1, "$indexfile unreadable.");
	while (my $line = <INDEXFILE>) {
		$line =~ m,^[^|]*\|/usr/ports/[^/]*/([^|]*)\|,;
		$line = $1;
		foreach my $dpart (@namepart) {
			if ($line =~ /^[^ ]*\b$dpart\b/i) {
				$line =~ s/\s+/ /g;
				print "$dpart matches $line\n";
				$found = 1;
			}
		}
	}
	if ($found) {
		prompt ("Possible duplicates found -- still OK?")
			and errx(1, "Investigating duplicates");
	}
}

# make sure we're in the right place.
chdir $currentdir;
my @dirs = split(/\,/, $dir);
foreach my $i (@dirs) { $i = abs_path($i); }
my $portname; my $wrapat;
foreach my $thisdir (@dirs) {
	# make double sure where we are..
	chdir $thisdir;
	# do some dir sanity checking first
	errx(1, "Please specify valid directories to import new ports from.") if $thisdir eq "";
	errx(1, "$thisdir is either not a directory or does not exist.") if (! -d $thisdir);

	print "Working with port directory $thisdir.\n";

	$portname = `basename $thisdir`;	# avoid problems with dirs containing `/' in cvs
	chomp $portname;
	warnx("Port directory contains upper-case character! Please try using an all lower-case name to make everybody's life a bit easier.") if ($portname =~ /[A-Z]/);
	if ($interactive) {
		if (prompt("Port directory name will be $portname in SVN Repo.  OK? ")) {
			do {
				$portname = query("Preferred name for port directory? ");
			} while (prompt("Is the new name $portname OK? "));
		}
	}

	chdir $thisdir or err(1, "$thisdir");

	# now run the tests on this baby.
	for (@commands) {
		system("$_") && errx(1, "'$_' had problems. aborting.");
	}

	# Get the category name and make it suitable for use with svn
	my @categories = split(/ /, `$make -VCATEGORIES`);
	my $category = $categories[0];
	if ($interactive) {
		if (prompt("Port $portname will be put in category $category.  OK? " )) {
			do {
				$category = query("Preferred category for $portname? ");
			} while (prompt("Is the new category $category OK? "));
		}
	}
	chomp $category;

	# Do commitfile checking but only if the user did not request automatic filling.
	if (!$autofill) {
		if (-f $c) {
			system("$mv $c $tmpdir/commitfile") && errx(1, "Oops, can't move commitfile!");
			print "\nRemember, you asked to use a commit file to read for the commit log.\n";
			print "This means you'll get a message saying the log message was unchanged or\n";
			print "not specified.  Just tell it to continue and it will be committed.\n\n";
			$commitfile = "--file $tmpdir/commitfile";
		}
	} else {
		## Set up the autofill file.
		# Read COMMENT for part of the commit message.
		if ($autofill_l) {
			chomp($pkgcomment = `$make $passenv -V COMMENT`);
			# Change the first character to lowercase to make it fit with the
			# rest of the commit message, only if the second is not upper case.
			$pkgcomment =~ s/(^.)(?![A-Z])/\l$1/;
			$pkgcomment .= ".";
			$pkgcomment .= "\n\n" if ($autofill != -1);
		} else {
			$pkgcomment = `cat pkg-descr`;
			$pkgcomment .= "\n" if ($autofill != -1);
		}
		chomp($maintaineraddr = `$make $passenv -V MAINTAINER`);
		chomp($portversion = `$make $passenv -V PORTVERSION`);
		# Read Makefile to find necessary variables.
		open(MAKEFILE, "Makefile") or die("Can't open Makefile for reading: $!");
		while(<MAKEFILE>) {
			chomp;
			die ("Old style Makefile headers detected") if (/^# (?:[Nn]ew )?[Pp]orts collection [Mm]akefile/);
			($maintainer) = (m/^# Created by:\s+(\w.*)$/) if (/^# Created by/);
		}
		close(MAKEFILE);
		$maintainer = $maintaineraddr unless ($maintainer);
		# Write out the data to the comment file.
		open(AUTOFILL, "> $tmpdir/commitfile") or die("Can't open $tmpdir/commitfile for writing: $!");
		if ($autofill_l) {
			# pretty print; wrap @ 72 characters
			$tmp = "Add $portname $portversion, $pkgcomment";
			$wrapat = 72;
			while($wrapat > 1) {
				$tmp2 = $tmp;
				$tmp =~ s/(.{$wrapat}([^ ]+)?) /$1\n/g;
				last unless $tmp =~ /^[^\n]{73}/;
				$wrapat--;
				$tmp = $tmp2;
			}
		} else {
			$tmp = $pkgcomment;
		}
		print AUTOFILL $tmp;
		print AUTOFILL "PR:		$autofill\n" if ($autofill != -1);
		print AUTOFILL "Submitted by:	$maintainer" if ($maintainer && $autofill != -1);
		close(AUTOFILL);
		print "Okay, a commit log message was automatically generated for you.\n";
		print "Now you will have a chance to edit it to make sure it's OK to use.\n";
		print "Here's the contents of the file:\n--start--\n";
		open(AUTOFILL, "$tmpdir/commitfile") or die("Can't open $tmpdir/commitfile for reading: $!");
		print while (<AUTOFILL>);
		close(AUTOFILL);
		$tmp = prompt("\n--end--\nDo you wish to edit the file before continuing? ");
		system("$edit $tmpdir/commitfile") if ($tmp == 0);
		print "\nRemember, you asked to use a commit file to read for the commit log.\n";
		print "This means you'll get a message saying the log message was unchanged or\n";
		print "not specified.  Just tell it to continue and it will be committed.\n\n";
		$commitfile = "--file $tmpdir/commitfile";
	}

	print "We're ready to commit.\n";
	print "Source directory: $thisdir\n";
	print "Target SVN Repo directory: ports/$category/$portname\n";
	prompt("Adding port $portname to $category OK? ") && errx(1, "user abort requested");
	
	chdir $tmpdir or err(1, "$tmpdir");

	# let's get our hands dirty.
	if (! -d "ports") {
		system("$svn co --depth empty $repo ports") && errx(1, "can't get ports root, aborting.");
		chdir "ports" or err(1,"ports");
		system("$svn up --depth files $category") && errx(1, "can't get temporary category directory, aborting.");
	}
	chdir $category or err(1,"$category");

	# check for previous existence of port -- on the way use filthy
	# home-made XML parser.
	# Until svn revs are in the database, we'll use dates.
	print "Checking for previous versions of $category/$portname... \n";
	my $previous_incarnation = "bogus";
	my $oldportlist = `/usr/bin/fetch -qo - http://people.FreeBSD.org/~crees/removed_ports/index.xml`;
	if ($oldportlist !=~ /^fetch:[^:]+: Not Found/) {
		foreach (split("\n", $oldportlist)) {
			if (/^ +\<port\>$category\/$portname(?:\<removed_revision\>r([0-9]*)\<\/removed_revision\>)?\<removed_date\>([^<]*)/) {
				print "Found one!\n";
				if ($1 == "") {
					$previous_incarnation = $2;
					$previous_incarnation =~ s,/,-,g;
					print "This port was last alive on $previous_incarnation.\n";
					$previous_incarnation = "\{$previous_incarnation\}";
				} else {
					$previous_incarnation = $1 - 1;
					print "The last living revision of this port was r$previous_incarnation.\n";
				}
				last;
			}
		}
	} else {
		print "Could not fetch list-- perhaps the site is down.";
	}
	if ($previous_incarnation ne "bogus") {
		print "Fetching older version... ";
		system("$svn cp -q '$repo/$category/$portname\@$previous_incarnation' .");
		print "[DONE]\n";
		print "Removing irrelevant files and directories... ";
		my @oldfiles = split("\0", `cd $portname && find . -type f -print0`);
		my @olddirs = split("\0", `cd $portname && find . -type d -print0 | sort -r`);
		my @newfiles = split("\0", `cd $thisdir && find . -type f -print0`);
		my @newdirs = split("\0", `cd $thisdir && find . -type d -print0| sort -r`);

		foreach my $file (@oldfiles) {
			system("cd $portname && $svn rm $file")
				if !($file ~~ @newfiles)
		}

		foreach my $directory (@olddirs) {
			system("cd $portname && $svn rm $directory")
				if !($directory ~~ @newdirs);
		}

		print "[DONE]\n";

		# Remove cvs2svn props if present
		print "Removing cvs2svn props...\n";
		system("$svn propdel -qR cvs2svn:cvs-rev $portname");

		$moved = "MOVED";
		print "Removing port's entry from $moved...\n";
		system("cd .. && $svn up -q $moved && sed -i '' -e '\\,^$category/$portname\|\|,d' $moved");

		# Add note to log about readdition
		system("echo '(Readdition of $category/$portname which was removed on $previous_incarnation)\n' > $tmpdir/commitfile.tmp && cat $tmpdir/commitfile >> $tmpdir/commitfile.tmp && mv $tmpdir/commitfile.tmp $tmpdir/commitfile") unless ($commitfile eq "");

	} else {
		print "[none found]\n";
	}
	system("$cp -PRp $thisdir .");
	system("$svn add --force --depth empty `find $portname -type d | grep -v '^$portname/work'`") && errx(1, "svn add for dirs failed, aborting.");

	system("$svn add --force `find $portname -type f | grep -v '^$portname/work'`") && errx(1, "svn add for files failed, aborting.");

	# Find keywords in old files and strip
	print "Stripping any keywords...\n";
	system("sed -i '' -e 's,\\\$Free" . "BSD:[^\$]*\\\$,\$Free" . "BSD\$,' \$(find $portname -type f)");

	# find files with keywords in and propset

	print "Setting correct properties on files...\n";

	my @portfiles = split("\0", `find $portname -type f -print0`);
	my $portfiles = join(" ", @portfiles);
	my @keywordfiles = split("\n", `grep -l $keyword $portfiles`);
	foreach (@portfiles) {
		if ($_ ~~ @keywordfiles) {
			system("$svn -q propset svn:keywords FreeBSD=%H $_");
			system("$svn -q propdel fbsd:nokeywords $_");
		} else {
			system("$svn -q propset fbsd:nokeywords on $_");
			system("$svn -q propdel svn:keywords $_");
		}
	}

	# strip svn:executable if added-- not allowed

	system("cd $portname && $svn -qR propdel svn:executable");

	# figure out where the port name belongs in category Makefile
	my ($spaces, @ports) = &lsports;
	errx(1, "Error: $portname already exists in $category\'s Makefile") if (&contains($portname, @ports));
	my $port = "";
	foreach my $tmp (sort(@ports)) {
		if ($tmp gt $portname) {
			$port = $tmp;
			last;
		}
	}

	# now let's insert it
	my $cmd;
	if (scalar @ports == 0) {
		# there were no previous SUBDIR += lines, so we're going to
		# put ourselves after the last comment (we can't be after a
		# .include <bsd.port.subdir.mk> for example).
		my $lastcommentnum = &lastcomment;
		$cmd = "$lastcommentnum\n+\ni\n";
	} else {
		if ($port eq "") {
			# there were previous SUBDIR lines, but none was greater than we are,
			# means, we're the last port, so, add ourselves after the last port
			$port = $ports[$#ports];
			$cmd = "/^" . $spaces . "SUBDIR += $port/\na\n";
		} else {
			# OK, append ourselves in the right place, so things *stay* sorted.
			$cmd = "/^" . $spaces . "SUBDIR += $port/\ni\n";
		}
	}
	print "Inserting new port into $category/Makefile...\n";
	open(ED, "|ed Makefile") || die "Cannot start ed to actually insert module\n";
	print ED "$cmd" . $spaces . "SUBDIR += $portname\n.\nw\nq\n";
		close(ED);

	# commit the actual port.
	chdir "$tmpdir/ports" or err(1, "$tmpdir/ports");
	if ($opts{'n'}) {
		print "Faking commit....\n";
	} else {
		system("$svn ci $commitfile $moved $category/Makefile $category/$portname") && errx(1, "svn commit failed, aborting.");
	}
}

print <<EOF;
You're done! The new port $portname has been completely imported in
the tree. Don't forget to add the creator's name and email address to
the Contributors' List if they are not already there. To do this, edit
doc/head/en_US.ISO8859-1/articles/contributors/contrib.additional.xml.
EOF

sub warnx($) {
	my ($msg) = @_;
	print STDERR $0 . ": " . $msg . "\n";
}

sub err($$) {
	my ($ex, $msg) = @_;

	warnx("WARNING: err called incorrectly") if (($ex !~ m/^\d+/) || ($msg eq ""));
	print STDERR $0 . ": " . $msg . ": $!\n";
	exit $ex;
}

sub errx($$) {
	my ($ex,$msg) = @_;

	warnx("WARNING: errx called incorrectly") if (($ex !~ m/^\d+/) || ($msg eq ""));
	print STDERR $0 . ": " . $msg . "\n";
	exit $ex;
}

sub prompt($) {
	my ($msg) = @_;
	my $reply = query($msg);
	return 0 if ($reply =~ m/^[Yy]/);
	return 1 if ($reply =~ m/^[Nn]/);
}

sub query($) {
	my ($msg) = @_;

	print "$msg";
	my $reply = <>;
	chomp $reply;
	return $reply;
}

sub usage() {
#addport,v \$Revision: 1.21 $
print <<EOF;
authors: <will\@FreeBSD.org>, <mharo\@FreeBSD.org>

SYNOPSIS
	$0 [-c commitfile] [-h host] [-l PR number | -L PR number]
	   [-s distdir] [-u user] [-afimnty] -d directory

	Where "directory" contains the comma-delimited list
	of root directories of new ports that you wish to
	add to the Ports Collection.  The name of this directory
	*WILL* matter in regards to the repository!

OPTIONS
	-a		Perform checks on the port to make sure
			there are no problems.  Recommended.
	-c file		Use file in place of normal log message.
	-f		Do not fetch the distfile.
	-h host		Use a svnhost besides repo.FreeBSD.org.
	-i		Interactive mode; allow more control over
			where things are placed.
	-l PR#		Attempts to autogenerate a commit message by
			reading the Makefile file.
			The PR number must be passed to -l.  If there is
			no PR (i.e., self-created or submitted in
			private email), use PR# -1.
	-L PR#		Like -l but it'll generate commit message based
			on pkg-descr
	-m		Do not checkout ports/Mk (needed for support
			of portlinting etc).
	-n		Do not actually commit anything.
	-s distdir	Use a different directory besides the default,
			for downloading distfiles.
	-t		Do more port testing.  Requires -a.
	-u user		Use a different username (default: $u).
	-y		Check for similarly named ports.

ENVIRONMENT VARIABLES
	$0 supports the following environment variables:

	ADDPSVNROOT	- Location of SVN repository.
	USER		- Username of user invoking $0.

EXAMPLES
	% addport -n -d greatgame,helpfuldev,shoot
		Will show what happens but not actually commit ports
	named "greatgame", "helpfuldev", and "shoot".

	% addport
		Displays this message.  :-)

EOF
}

sub contains($@) {
    # look if the first parameter is contained in the list following it
    my ($item, @list) = @_;

    foreach my $i (@list) {
	return 1 if $i eq $item;
    }
    return 0;
}

sub lsports() {
    my @rv = ();
    my $spaces;

    open(F, "Makefile") || die "can't open Makefile: $!";
    while(<F>) {
	chomp;
	chomp;
	next if $_ !~ m/SUBDIR/;

	if ( !defined($spaces) ) {
		m/^([\s\t]+)[^\s\t]/;
		$spaces = $1;
	}

	s/^[ \t]+SUBDIR[ \t]+\+?=[\ \t]+//;
	push(@rv, $_);
    }
    close(F);

    return ($spaces, @rv);
}

# this finds the last comment in the Makefile
sub lastcomment() {
	my $num = 0;
	my $diff = 0;

	open(F, "Makefile");
	while(<F>) {
		chomp;
		if ($_ =~ m/^#/) {
			$num += $diff;
			$num++;
			$diff = 0;
		} else {
			$diff += 1;
		}
		next;
	}
	return $num;
}