#!/usr/local/bin/perl
# newsstar is generated from newsstar.in by configure

#   newsstar - advanced news fetcher
#   Copyright (C) 2001-2002 Tony Houghton <h@realh.co.uk>

#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.

#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.

#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


use Fcntl ':flock';
use IO::Handle;

# Values set by configure, and dependents thereof; some may appear to be
# redundant but they are needed in case they are referenced by others
$CONF_DIR = '/usr/local/etc/newsstar';
$NEWS_USER = 'news:news';
$NEWS_PATH = '';
$prefix = "/usr/local";
$exec_prefix = "/usr/local";
$libexecdir = "${exec_prefix}/libexec";
$pkglibexecdir = "${exec_prefix}/libexec/newsstar";
$NEWSSTAR_BIN = "${exec_prefix}/libexec/newsstar/newsstar.bin";
$MAINCF = $CONF_DIR . "/main.cf";
$INCOMING_DIR = "/var/spool/newsstar/incoming";
$RC_DIR = "/var/spool/newsstar/lib";
$MASTER_IGNORE = "$CONF_DIR/master.ignore";
$SENDMAIL = "/usr/sbin/sendmail";
# Can't easily use /var/run for lockfile because we're not root
$LOCKFILE = "$RC_DIR/lock.pid";
$CURSES_ON = "$RC_DIR/curses_on";

# Compile-time defaults that override run-time detection but are overridden
# by run-time CLI options or main.cf
$DEFAULT_SPOOL_DIR = "";
$DEFAULT_SN_SPOOL_DIR = "";
$DEFAULT_OUTGOING_DIR = "";
$DEFAULT_SNEWS_OUTGOING_DIR = "";
$DEFAULT_ARTICLES_DIR = "";
$DEFAULT_SN_ARTICLES_DIR = "";
$DEFAULT_SNEWS_ARTICLES_DIR = "";

# Values to be read from main.cf
$SPOOL_DIR = "";
$OUTGOING_DIR = "";
$ARTICLES_DIR = "";
$ACTIVE_FILE = "";

# Check CLI options
$SORT_NEWSRCS = 0;
@NEWSSTAR_OPTIONS = ();
$FEED = "";
$MAILBACK = 0;
$USE_SN = 0;
$USE_SNEWS = 0;
$REBUILD_NEWSRCS = 0;
$REBUILD_NEWSRC_DEFAULT = "";
$SNSTORE = "";
$OVERVIEW="";
$PREPROC=0;
$POSTPROC=0;
$VERBOSE = 0;
$QUIET = 0;
$SPLIT_SINKS = 0;
$WRAPPER = "";
foreach (@ARGV)
{
	if (/^-A/)
	{
		s/-A//;
		$REBUILD_NEWSRCS = 1;
		$REBUILD_NEWSRC_DEFAULT = $_;
	}
	elsif (/^-a$/)
	{
		$SORT_NEWSRCS = 1;
	}
	elsif (/^-m$/)
	{
		$MAILBACK = 1;
	}
	elsif (/^-o/)
	{
		push(@NEWSSTAR_OPTIONS, $_);
		s/^-o//;
		$FEED = $_;
	}
	elsif (/^-sn$/)
	{
		push(@NEWSSTAR_OPTIONS, $_);
		$USE_SN = 1;
	}
	elsif (/^-snews$/)
	{
		push(@NEWSSTAR_OPTIONS, $_);
		$USE_SNEWS = 1;
	}
	elsif (/^--snstore=/)
	{
		$SNSTORE=$_;
		$SNSTORE =~ s/^--snstore=//;
	}
	elsif (/^--overview=/)
	{
		$OVERVIEW=$_;
		$OVERVIEW =~ s/^--overview=//;
	}
	elsif (/^--preprocess$/)
	{
		$PREPROC = 1;
	}
	elsif (/^--postprocess$/)
	{
		$POSTPROC = 1;
	}
	elsif (/^-v/) # Don't care whether -v or -vv
	{
		$VERBOSE = $_;
		push(@NEWSSTAR_OPTIONS, $_);
	}
	elsif (/^-b$/)
	{
		$QUIET = 1;
		push(@NEWSSTAR_OPTIONS, $_);
	}
	elsif (/^-s$/)
	{
		$SPLIT_SINKS = 1;
		push(@NEWSSTAR_OPTIONS, $_);
	}
	elsif (/^-w/)
	{
		$WRAPPER = $_;
		$WRAPPER =~ s/^-w//;
	}
	else
	{
		push(@NEWSSTAR_OPTIONS, $_);
	}
}
if ($USE_SN && $USE_SNEWS)
{
	die "Can\'t use -sn and -snews together\n";
}
if ($POSTPROCESS && $PREPROCESS)
{
	die "--postprocess and --preprocess options are mutually exclusive\n";
}
$SPLIT_SINKS = 1 if ($QUIET && $VERBOSE);

if ($VERBOSE)
{
	print STDERR "-----------------------------------------------------\n";
	print STDERR "Starting newsstar at ", `date`;
}

if (-f "$CONF_DIR/filter.pl")
{
	$external_filter = do "$CONF_DIR/filter.pl";
	if ($external_filter)
	{
		print_split("Using upload filter\n");
	}
	else
	{
		if ($!)
		{
			print_split("Not using upload filter:\n\t$!\n");
		}
		elsif ($@)
		{
			die "Errors in $CONF_DIR/filter.pl:\n$@";
		}
	}
}
elsif ($VERBOSE)
{
	print_split("No upload filter present\n");
}

# Get main.cf options
$rnews_server = "";
if (open MAINCFFD, $MAINCF)
{
	my @found_options;
	my @maincf_options = <MAINCFFD>;
	close MAINCFFD;
	@found_options = grep(/^local_server\s/, @maincf_options);
	$local_server = $found_options[0];
	$local_server =~ s/^local_server\s*//;
	$local_server =~ s/"//g;
	chomp($local_server);
	if ($local_server ne "")
	{
		$rnews_server = "-S " . $local_server;
	}
	if (@found_options = grep(/^spool_dir\s/, @maincf_options))
	{
		$SPOOL_DIR = $found_options[0];
		$SPOOL_DIR =~ s/^spool_dir\s*//;
		$SPOOL_DIR =~ s/"//g;
		chomp($SPOOL_DIR);
	}
	if (@found_options = grep(/^outgoing_dir\s/, @maincf_options))
	{
		$OUTGOING_DIR = $found_options[0];
		$OUTGOING_DIR =~ s/^outgoing_dir\s*//;
		$OUTGOING_DIR =~ s/"//g;
		chomp($OUTGOING_DIR);
	}
	if (@found_options = grep(/^active_file\s/, @maincf_options))
	{
		$ACTIVE_FILE = $found_options[0];
		$ACTIVE_FILE =~ s/^active_file\s*//;
		$ACTIVE_FILE =~ s/"//g;
		chomp($ACTIVE_FILE);
	}
	if (@found_options = grep(/^articles_dir\s/, @maincf_options))
	{
		$ARTICLES_DIR = $found_options[0];
		$ARTICLES_DIR =~ s/^articles_dir\s*//;
		$ARTICLES_DIR =~ s/"//g;
		chomp($ARTICLES_DIR);
	}
}

if ($USE_SN)
{
	$SPOOL_DIR = $DEFAULT_SN_SPOOL_DIR if ($SPOOL_DIR eq "");
	$SPOOL_DIR = "/var/spool/sn" if ($SPOOL_DIR eq "");
	$OUTGOING_DIR = $DEFAULT_OUTGOING_DIR if ($OUTGOING_DIR eq "");
	$OUTGOING_DIR = $SPOOL_DIR . "/.outgoing"  if ($OUTGOING_DIR eq "");
	$OUTGOING_NEWSSTAR_DIR = $OUTGOING_DIR . "/.newsstar";
	$ARTICLES_DIR = $DEFAULT_SN_ARTICLES_DIR if ($ARTICLES_DIR eq "");
	$ARTICLES_DIR = $SPOOL_DIR if ($ARTICLES_DIR eq "");
}
else
{
	$SPOOL_DIR = $DEFAULT_SPOOL_DIR if ($SPOOL_DIR eq "");
	$SPOOL_DIR = "/var/spool/news" if ($SPOOL_DIR eq "");
	if ($USE_SNEWS)
	{
		$OUTGOING_DIR = $DEFAULT_SNEWS_OUTGOING_DIR if ($OUTGOING_DIR eq "");
		$OUTGOING_DIR = "/var/lib/news/suck" if ($OUTGOING_DIR eq "");
	}
	elsif ($OUTGOING_DIR eq "")
	{
		$OUTGOING_DIR = $DEFAULT_OUTGOING_DIR;
		$OUTGOING_DIR = $SPOOL_DIR . "/out.going" if ($OUTGOING_DIR eq "");
		if (! -d $OUTGOING_DIR)
		{
			die "Can\'t find expected $OUTGOING_DIR\n" .
				"Try adding:\n" .
				"outgoing_dir " . $SPOOL_DIR . "/outgoing\n" .
				"to $MAINCF\n";
		}
	}
	$OUTGOING_NEWSSTAR_DIR = $OUTGOING_DIR . "/newsstar";
	$ACTIVE_FILE = "/var/lib/news/active" if ($ACTIVE_FILE eq "");
	if ($ARTICLES_DIR eq "")
	{
		if ($USE_SNEWS)
		{
			$ARTICLES_DIR = $DEFAULT_SNEWS_ARTICLES_DIR;
		}
		else
		{
			$ARTICLES_DIR = $DEFAULT_ARTICLES_DIR;
		}
		if ($ARTICLES_DIR eq "")
		{
			if (-d "$SPOOL_DIR/articles")
			{
				$ARTICLES_DIR = "$SPOOL_DIR/articles";
			}
			else
			{
				$ARTICLES_DIR = $SPOOL_DIR;
			}
		}
	}
}
$RNEWS_FILE = $INCOMING_DIR . "/rnews";
$FAILED_DIR = $OUTGOING_NEWSSTAR_DIR . "/failed";

if ($USE_SN)
{
	if ($SNSTORE eq "")
	{
		if (-x "/usr/sbin/snstore")
		{
			$SNSTORE = "/usr/sbin/snstore"
		}
		elsif (-x "/usr/local/sbin/snstore")
		{
			$SNSTORE = "/usr/local/sbin/snstore"
		}
		elsif (-x "/usr/bin/snstore")
		{
			$SNSTORE = "/usr/bin/snstore"
		}
		elsif (-x "/usr/local/bin/snstore")
		{
			$SNSTORE = "/usr/local/bin/snstore"
		}
		else
		{
			die "Can\'t find snstore program\n";
		}
	}
	elsif (! -x $SNSTORE)
	{
		die "$SNSTORE doesn\'t exist or is inaccessible\n";
	}
	push(@NEWSSTAR_OPTIONS, "--snstore=$SNSTORE");
}
elsif ($USE_SNEWS)
{
	if ($OVERVIEW eq "")
	{
		if (-x "/usr/sbin/overview")
		{
			$OVERVIEW = "/usr/sbin/overview"
		}
		elsif (-x "/usr/local/sbin/overview")
		{
			$OVERVIEW = "/usr/local/sbin/overview"
		}
		elsif (-x "/usr/bin/overview")
		{
			$OVERVIEW = "/usr/bin/overview"
		}
		elsif (-x "/usr/local/bin/overview")
		{
			$OVERVIEW = "/usr/local/bin/overview"
		}
		else
		{
			die "Can\'t find s-news' overview program\n";
		}
	}
	elsif (! -x $OVERVIEW)
	{
		die "$OVERVIEW doesn\'t exist or is inaccessible\n";
	}
}
if (!$ENV{"PATH"} !~ /$NEWS_PATH/)
{
	$ENV{"PATH"} .= ":" . "$NEWS_PATH";
}
if (!$USE_SN && !$USE_SNEWS && system("innconfval > /dev/null"))
{
	die "INN binaries not found or not working correctly";
}

# Make directories (use system mkdir because perl's might not support -p)
system("mkdir -p $INCOMING_DIR");
system("mkdir -p $FAILED_DIR");		# Also creates $OUTGOING_NEWSSTAR_DIR

# Handle locking
$locked = 0;
foreach ("HUP", "INT", "QUIT", "TERM")
{
	$SIG{"$_"} = \&remove_lock;
}
create_lock();

if (!$POSTPROCESS)
{
	# If $REBUILD_NEWSRCS is false, this just backs up newsrcs
	$warned_deprecated_ignore=0;
	rebuild_newsrcs($REBUILD_NEWSRC_DEFAULT, $SORT_NEWSRCS);

	# Prepare outgoing articles
	if ($FEED eq "")
	{
		foreach (<$OUTGOING_DIR/*>)
		{
			if (-f $_)
			{
				s/$OUTGOING_DIR\///;
				process_feed($_);
			}
		}
	}
	else
	{
		process_feed($FEED);
	}
}

if (!$PREPROCESS && !$POSTPROCESS)
{
	# Save tty opts in case newsstar.bin crashes hard with curses active
	my $tty_opts;

	$tty_opts = `stty -g` if $ENV{"TERM"} ;
	# Do the actual transfer
	if ($WRAPPER)
	{
		system($WRAPPER, $NEWSSTAR_BIN, @NEWSSTAR_OPTIONS);
	}
	else
	{
		system($NEWSSTAR_BIN, @NEWSSTAR_OPTIONS);
	}
	if (-f $CURSES_ON)
	{
		system("stty $tty_opts") if ($ENV{"TERM"});
		unlink($CURSES_ON);
	}
}

if (!$PREPROCESS)
{
	if (-s "$RNEWS_FILE")
	{
		print_split("Posting downloaded articles to local server via rnews\n");
		if ($USE_SN)
		{
			system("$SNSTORE -r -c < $RNEWS_FILE");
		}
		else
		{
			system("rnews $rnews_server -v $RNEWS_FILE");
		}
	}
	if ($USE_SNEWS)
	{
		print_split("Adding new articles to overviews\n");
		system($OVERVIEW);
	}
	print_split("Cleaning up\n");
	unlink <$INCOMING_DIR/*>;
	if ($MAILBACK)
	{
		foreach (<$FAILED_DIR/*>)
		{
			mailback_failed($_);
		}
	}
}
if ($VERBOSE)
{
	print STDERR "All done\n";
	print STDERR "-----------------------------------------------------\n";
}
remove_lock();

# Generates a unique numerical filename in a given directory
# arg0 = directory, arg1 = name/number to start from
sub temp_filename
{
	while (-e "$_[0]/$_[1]")
	{
		$_[1] += 1;
	}
	return $_[1];
}

# Loads ignore file from arg0: "cooked" patterns go into array ref'd by arg1,
# negate array is ref'd by arg2. arg3 is start index of the arrays, updated
# value is returned (to support multiple files)
sub load_ignore {
	my $ignore = $_[1];
	my $negate = $_[2];
	my $line = $_[3];
	if (open(IGNORE, $_[0]))
	{
		while (<IGNORE>)
		{
			# Skip blank lines and comments
			s/^\s*//;
			if (/^$/ or /^\#/)
			{
				next;
			}
			chomp;
			$negate->[$line] = 0;
			$negate->[$line] = 1 if (s/^\!//);
			push(@{$ignore}, $_);
			++$line;
		}
		close(IGNORE);
	}
	return $line;
}
# Adds newsgroups from hash ref'd by arg2 to newsrc file arg0, pruning groups
# matched by regexps in filename arg1, and any not present in arg2.  If group
# not present in existing newsrc file it's given last downloaded value in arg3.
# If arg4 is true, sort newsrc files by group name
sub process_active
{
	# Seem to have to copy parameters to stop perl randomly deleting them
	my $newsrcf = $_[0];
	my $ignoref = $_[1];
	my $active = $_[2];
	my $lastdl = $_[3];
	my $sort_wanted = $_[4];
	my %newsrc = ();
	my @newsrc_keys;
	my ($group, $last, $key);
	open RCFILE, $newsrcf or die "Can\'t read " . $newsrcf . "\n";
	while (<RCFILE>)
	{
		($group, $last) = split(/\s+/, $_);
		chomp($group);
		chomp($last);
		$newsrc{"$group"} = $last;
	}
	close(RCFILE);
	foreach (keys(%{$active}))
	{
		chomp($_);
		if (!defined($newsrc{$_}))
		{
			# Best not to print message, because we'll usually be removing
			# the group when processing ignore
			$newsrc{$_} = $lastdl;
		}
	}
	# Remove any groups present in newsrc but not in active
	foreach (keys(%newsrc))
	{
		if (!defined($active->{$_}))
		{
			print_split("  Removing group $_\n");
			delete $newsrc{$_};
		}
	}
	# Remove any groups matching patterns from ignore file. Patterns beginning
	# with ! are negated, meaning matching groups are preserved
	if (-r "$MASTER_IGNORE" || -r $ignoref)
	{
		my @ignore = ();
		my @negate = ();
		my $line = 0;
		$line = load_ignore("$ignoref", \@ignore, \@negate, $line);
		$line = load_ignore("$MASTER_IGNORE", \@ignore, \@negate, $line);
		foreach $key (keys(%newsrc))
		{
			$line = 0;
			foreach (@ignore)
			{
				if ($key =~ /$_/)
				{
					last if ($negate[$line]);
					delete $newsrc{$key};
				}
				++$line;
			}
		}
	}
	if (!open RCFILE, ">" . $newsrcf)
	{
		print STDERR "Can\'t write " . $newsrcf . "\n";
		return;
	}
	if ($sort_wanted)
	{
		@newsrc_keys = sort(keys(%newsrc));
	}
	else
	{
		@newsrc_keys = keys(%newsrc);
	}
	foreach (@newsrc_keys)
	{
		chomp($_);
		print RCFILE "$_" . " " . $newsrc{$_} . "\n";
	}
	close(RCFILE);
}

# Process a single newsrc file for feed $_[0], using active hash ref'd by $_[1]
# new groups have last downloaded value from arg2.
# If arg3 is true, sort newsrc files by group name
sub process_newsrc {
	my $oldsrc = $_[0];
	$oldsrc =~ s/\/newsrc\./\/oldsrc\./;
	system("cp", $_[0], $oldsrc);
	if ($REBUILD_NEWSRCS)
	{
		my $ignore = $_[0];
		my $new_ignore;
		print_split("Rebuilding $_[0]\n");
		$ignore =~ s/\/newsrc\./\/ignore\./;
		$new_ignore = $ignore;
		$new_ignore =~ s/$RC_DIR/$CONF_DIR/;
		if (-f $ignore && !$warned_deprecated_ignore)
		{
			print "Warning: ignore files in rc_dir ($RC_DIR) are deprecated;\n";
			print "		 please move them to conf_dir ($CONF_DIR)\n";
			$warned_deprecated_ignore = 1;
		}
		$ignore = $new_ignore if (-f $new_ignore);
		process_active($_[0], $ignore, $_[1], $_[2], $_[3]);
	}
}

# Loads active file and processes each newsrc file;
# new groups have last downloaded value from arg0.
# If arg1 is true, sort newsrc files by group name
sub rebuild_newsrcs
{
	my $newsrcfile;
	my %active_hash = ();
	if ($REBUILD_NEWSRCS)
	{
		my @active;
		if ($USE_SN)
		{
			@active = <$ARTICLES_DIR/*>;
		}
		else
		{
			if (!open ACTIVE, "$ACTIVE_FILE")
			{
				print STDERR "Can\'t open $ACTIVE_FILE";
				return;
			}
			@active = <ACTIVE>;
		}
		foreach (@active)
		{
			if ($USE_SN)
			{
				# Not sure whether sn allows any non-hidden directories that
				# aren't newsgroups, but better check for them just in case
				next if ($_ eq $RC_DIR || $_ eq $INCOMING_DIR ||
					$_ eq $OUTGOING_DIR || ! -d $_);
				s/.*\///g;
			}
			else
			{
				s/\s.*//;
			}
			chomp;
			$active_hash{$_} = " ";
		}
	}
	if ($FEED eq "")
	{
		foreach (<$RC_DIR/newsrc.*>)
		{
			if (/\/newsrc\.sample$/)
			{
				#print "Warning: newsrc.sample found; ";
				#print "you should delete or rename it\n";
				next;
			}
			process_newsrc($_, \%active_hash, $_[0], $_[1]);
		}
	}
	else
	{
		process_newsrc("$RC_DIR/newsrc.$FEED", \%active_hash, $_[0], $_[1]);
	}
}


sub create_lock {
	my $stale_lock = (-f $LOCKFILE);
	open(NSTARLOCKFD, ">>$LOCKFILE") or
		die "Can\'t create lockfile $LOCKFILE:\n\t$!\n";
	if (!flock(NSTARLOCKFD, LOCK_EX | LOCK_NB))
	{
		die "Can\'t get lock: $!\nIs newsstar running already?\n";
	}
	$locked = 1;
	autoflush NSTARLOCKFD 1;
	truncate(NSTARLOCKFD, 0);
	print NSTARLOCKFD "$$\n";
	if ($stale_lock)
	{
		print STDERR "Stale lock found, did newsstar crash last time?\n";
	}
}

sub remove_lock {
	if ($locked)
	{
		flock(NSTARLOCKFD, LOCK_UN | LOCK_NB);
		$locked = 0;
	}
	if (-f $LOCKFILE)
	{
		unlink($LOCKFILE);
	}
}

# Prepares outgoing articles for feed named by $_[0]
sub process_feed {
	my $destname = 0;
	my $srcname;
	my $destpath;
	my $feed = $_[0];
	return 0 if (!has_outgoing($feed));
	if ($USE_SN)
	{
		my $port = find_port($feed);
		my $feedpath = "$OUTGOING_DIR/$feed:$port";
		my @outfiles = ();
		if (-d $feedpath)
		{
			@outfiles = <$feedpath/*>;
		}
		if (@outfiles)
		{
			print "Preparing outgoing articles for $feed\n";
			mkdir("$OUTGOING_NEWSSTAR_DIR/$feed", 0755);
			foreach $srcname (@outfiles)
			{
				$destname = temp_filename("$OUTGOING_NEWSSTAR_DIR/$feed",
					$destname);
				$destpath = "$OUTGOING_NEWSSTAR_DIR/$feed/" . $destname;
				copy_and_filter($srcname, $destpath, $feed);
				unlink($srcname);
			}
		}
	}
	else
	{
		# INN or s-news
		my $feed_path;
		if (!$USE_SNEWS)
		{
			$feed_path = "$OUTGOING_DIR/$feed";
		}
		else
		{
			$feed_path = "$OUTGOING_DIR/$feed/outgoing";
		}
		# For INN we rename its feed file to our working file then flush
		# the feed.
		# For s-news we lock its feed file, process and unlink it and create
		# a new empty feed file before releasing the lock.
		if (-s $feed_path)
		{
			my $work_feed;
			print "Preparing outgoing articles for $feed\n";
			mkdir("$OUTGOING_NEWSSTAR_DIR/$feed", 0755);
			if (!$USE_SNEWS)
			{
				rename($feed_path, "$OUTGOING_DIR/newsstar.feed") or
					die "Unable to flush newsfeed";
				if (system("ctlinnd flush $feed > /dev/null"))
				{
					unlink("$OUTGOING_DIR/newsstar.feed");
					die "Can\'t flush newsfeed";
				}
				$work_feed = "$OUTGOING_DIR/newsstar.feed";
			}
			else
			{
				$work_feed = $feed_path;
			}
			open FEEDFILE, $work_feed;
			if ($USE_SNEWS && !flock(FEEDFILE, LOCK_EX))
			{
				die "Can\'t get lock for $work_feed\n";
			}
			while (<FEEDFILE>)
			{
				# Isolate first column
				s/^\s*(\S*).*/$1/;
				chomp;
				print_split("  $_\n");
				# How to get file depends on whether $_ is a sm token
				if (/^@/)
				{
					$srcname = "sm $_ |";
				}
				else
				{
					$srcname = $ARTICLES_DIR . "/" . $_;
				}
				$destname = temp_filename("$OUTGOING_NEWSSTAR_DIR/$feed",
					$destname);
				$destpath = "$OUTGOING_NEWSSTAR_DIR/$feed/" . $destname;
				copy_and_filter($srcname, $destpath, $feed);
			}
			unlink($work_feed);
			if ($USE_SNEWS)
			{
				system("touch", $feed_path);
			}
			close FEEDFILE;
		}
	}
}

# Arg 0 is source filename, arg 1 is dest filename, arg 2 is server name
sub copy_and_filter
{
	if (!open IFP, "$_[0]")
	{
		print_split("	Can't read article, probably cancelled/expired\n");
		return;
	}
	my @text = <IFP>;
	close IFP;
	my $already_warned = 0;
	if ($USE_SN)
	{
		# Get rid of CRs and leading .'s
		for ($_ = 0; $_ < @text; ++$_)
		{
			$text[$_] =~ s/\r$//;
			if ($text[$_] =~ /^\.$/)
			{
				$text[$_] = undef;
				last;
			}
			else
			{
				$text[$_] =~ s/^\.//;
			}
		}
	}
	if ($external_filter)
	{
		filter(\@text, $_[2]);
	}
	else
	{
		my $supersedes = -1;
		my $newsq = 0;
		my $cancel = 0;
		for ($_ = 0; $_ < @text; ++$_)
		{
			# NB some servers also object to Supersedes
			if ($text[$_] =~ /^NNTP-Posting|^Xref|^X-Trace|^X-Comp/)
			{
				$text[$_] = "";
			}
			elsif ($text[$_] =~ /^Message-I[Dd]:.*\.newsq\./)
			{
				$newsq = 1;
			}
			elsif ($text[$_] =~ /^Control: cancel/)
			{
				$cancel = 1;
			}
			elsif ($text[$_] =~ /^Supersedes:/)
			{
				$supersedes = $_;
			}
			elsif ($text[$_] =~ /^$/)
			{
				last;		# end of header, don't need to test anymore
			}
		}
		if ($newsq)
		{
			if ($cancel)
			{
				print_split("	Removing newsq cancel message\n");
				@text = ();
				$already_warned = 1;
			}
			elsif ($supersedes != -1)
			{
				print_split("	Removing Supersedes header from newsq post\n");
				$text[$supersedes] = "";
			}
		}
	}
	if (!open OFP, ">$_[1]")
	{
		print STDERR "Can\'t write to $_[1]\n";
		return;
	}
	print OFP @text;
	close OFP;
	# "Empty" articles get ignored
	if (! -s $_[1])
	{
		if (!$already_warned)
		{
			print_split("	Removing blank post\n");
		}
		unlink $_[1]
	}
}

# Mail failed posting from filename $_[0] back to sender
sub mailback_failed {
	my @fbody = ();
	my $sender = "";
	my $from = "";
	my $reply_to = "";
	my $server = "";
	my $reason = "";
	my $filename = $_[0];

	if (!open(FBODY, $filename))
	{
		print STDERR "Can't open " . $filename . " to mail to sender\n";
		return;
	}
	# Read headers
	while (<FBODY>)
	{
		if (/^X-Rejected-Server: /)
		{
			s/^X-Rejected-Server: //;
			$server = $_;
		}
		elsif (/^X-Rejected-Reason: /)
		{
			s/^X-Rejected-Reason: //;
			$reason = $_;
		}
		else
		{
			push(@fbody, $_);
		}
		if (/^$/)
		{
			last;
		}
		if (/^Reply-To: /)
		{
			s/^Reply-To: //;
			$reply_to = $_;
		}
		elsif (/^From: /)
		{
			s/^From: //;
			$from = $_;
		}
		elsif (/^Sender: /)
		{
			s/^Sender: //;
			$sender = $_;
		}
	}
	# Body
	while (<FBODY>)
	{
		push(@fbody, $_);
	}
	close(FBODY);
	# Choose recipient in order: Reply-To, Sender, From, postmaster
	if ($reply_to eq "")
	{
		$reply_to = $sender;
		if ($reply_to eq "")
		{
			$reply_to = $from;
			if ($reply_to eq "")
			{
				$reply_to = "postmaster\n";
			}
		}
	}
	# $NEWS_USER may also contain group
	my $news_from = $NEWS_USER;
	$news_from =~ s/[:.].*//;
	if (!open(SENDMAIL, "|$SENDMAIL -oi -t"))
	{
		print STDERR "Can't run sendmail to bounce rejected posting\n";
		return;
	}
	print SENDMAIL "From: " . $news_from . "\@" . `hostname -f`;
	print SENDMAIL "To: " . $reply_to;
	print SENDMAIL <<"ENDENDEND";
Subject: Rejected Usenet posting
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="NewsstarRejectedPosting"

This is a multi-part message in MIME format.
--NewsstarRejectedPosting
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit

ENDENDEND

	print SENDMAIL "The following Usenet posting was rejected by server ";
	print SENDMAIL $server;
	print SENDMAIL "\nThe reason it was rejected was:\n";
	print SENDMAIL $reason;
	print SENDMAIL  <<"ENDENDEND";

--NewsstarRejectedPosting
Content-Type: message/news
Content-Disposition: attachment

ENDENDEND

	print SENDMAIL @fbody;
	print SENDMAIL  <<"ENDENDEND";

--NewsstarRejectedPosting--
ENDENDEND

	close (SENDMAIL);
	unlink($filename);
}

# Given a server name, this tries to work out which port to use with sn
sub find_port {
	my $port = 119;
	my $server = $_[0];
	# First try to find port from server's cf file
	if (open CFFILE, "$CONF_DIR/cf.$server")
	{
		my @grepport = grep(/^port\s/, <CFFILE>);
		close CFFILE;
		if (@grepport)
		{
			$port = $grepport[0];
			$port =~ s/^port\s*//;
			$port =~ s/"//g;
			chomp($port);
		}
		# If cf file is present, but doesn't contain port, we should use 119
	}
	else
	{
		# Use the first one we find
		my @outgoing = <$OUTGOING_DIR/$server:*>;
		if (@outgoing == 1)
		{
			$port = $outgoing[0];
			$port =~ s/.*://;
			chomp($port);
		}
	}
	return $port;
}

# Whether config wants an outgoing feed for a given server ($_[0])
sub has_outgoing {
	my $has_feed = 1;
	open(SERVERCF, $CONF_DIR . "/cf." . $_[0]);
	$has_feed = 0 if (grep(/^feed\s+"?[nf0]/i, <SERVERCF>));
	close(SERVERCF);
	return $has_feed;
}

# Only print these messages if !$QUIET or $VERBOSE. If $SPLIT_SINKS print them
# to STDERR, otherwise to STDOUT
sub print_split {
	return if ($QUIET && !$VERBOSE);
	if ($SPLIT_SINKS)
	{
		print STDERR @_;
	}
	else
	{
		print @_;
	}
}
