Add option to add common headers to all projects.

Signed-off-by: Thomas Hochstein <thh@thh.name>

ChangeLog

@@ -1,3 +1,18 @@
+yapfaq 1.1.0 (unreleased)
+  * --test: Set Supersedes and don't modify Message-ID if -o is set.
+  * --test: Force -o if -n is not set.
+  * Add conversion script for old status files.
+  * Accept a posting-frequency of "never", too.
+  * Check for illegal headers and stop posting, if found.
+  * Disabled projects must be posted, if forced.
+  * Warn of unencoded 8bit characters in header or body if -d is set.
+  * Update POD.
+  * Add an option for a common headers file for all projects.
+
+yapfaq 1.0.1 (2025-01-24)
+  * Add %t placeholder for Message-ID (feature parity with 0.9).
+  * Remove debugging code.
+
 yapfaq 1.0.0 (2025-01-23)
   * Complete rewrite.
   * Add POD.

bin/yapfaq.pl

@@ -19,7 +19,7 @@
 # It can be redistributed and/or modified under the same terms under 
 # which Perl itself is published.
 
-my $VERSION = "1.0.0";
+my $VERSION = "1.1.0-pre";
 (my $NAME = $0) =~ s#^.*/##;
 
 use utf8;
@@ -32,8 +32,6 @@ use Path::Tiny; # CPAN
 use Getopt::Long qw(GetOptions);
 Getopt::Long::config ('bundling');
 
-use Data::Dumper;
-
 # configuration #######################
 # may be overwritten via ~/.yapfaqrc or command line
 my %Config;
@@ -47,6 +45,8 @@ $Config{'nntp-pass'}    = '';      # password for AUTHINFO
 $Config{'force-auth'}   = 0;       # set to 1 to force authentication
 $Config{'starttls'}     = 0;       # set to 1 to use STARTTLS if possible
 
+$Config{'xtraheaders'} = '';      # path to file with extra headers
+
 $Config{'verbose'}      = 0;       # set to 1 to get status messages
 $Config{'debug'}        = 0;       # set to 1 to get some debug output,
                                    # set to 2 for NNTP debug output
@@ -102,6 +102,7 @@ GetOptions ('p|project=s'   => \$OptProject,
             'nntp-pass=s'   => \$Config{'nntp-pass'},
             'starttls!'     => \$Config{'starttls'},
             'force-auth!'   => \$Config{'force-auth'},
+            'xtraheaders=s' => \$Config{'xtraheaders'},
             'v|verbose!'    => \$Config{'verbose'},
             'd|debug!'      => \$Config{'debug'},
             'c|config'      => \&ShowConf,
@@ -114,6 +115,9 @@ if ($OptSimulation) {
   $Config{'verbose'} = 1;
 }
 
+# -t implies -o if -n is not set
+$OptOutput = 1 if $OptTest && !$OptNewsgroup;
+
 ### create list of @Projects from $Config{'datadir'} unless -p is set
 my @Projects;
 if (!$OptProject) {
@@ -264,7 +268,7 @@ sub AddDuration {
 
 ### ------------------------------------------------------------------
 ### return a hash of all headers (ignoring duplicate headers)
-# taken and modified from tinews.pl
+# taken and modified from pgpverify.pl
 sub ParseHeaders {
   my @Headers = @_;
   my (%Header, $Label, $Value);
@@ -376,6 +380,11 @@ sub BuildPosting {
     warn "W: '$BodyFile' not found.\n";
     return '';
   }
+  my $XtraHeaderFile = $Config{'xtraheaders'} if $Config{'xtraheaders'};
+  if ($Config{'xtraheaders'} && not -r $XtraHeaderFile) {
+    warn "W: '$XtraHeaderFile' not found.\n";
+    return '';
+  }
 
   # today (TD)
   my $TD = DateTime->now->set_time_zone('local');
@@ -403,6 +412,10 @@ sub BuildPosting {
   print "- Reading headers ($Project.hdr) and body ($Project.txt).\n" if $Config{'debug'};
   my @Headers = path($HeaderFile)->lines;
   my @Body    = path($BodyFile)->lines;
+  if ($Config{'xtraheaders'}) {
+    print "- Reading extra headers ($XtraHeaderFile).\n" if $Config{'debug'};
+    push @Headers, path($XtraHeaderFile)->lines ;
+  }
   my %Header  = &ParseHeaders(@Headers);
 
   # check for mandatory headers
@@ -411,6 +424,17 @@ sub BuildPosting {
     return '';
   }
 
+  # check for illegal headers
+  my $FoundIllegalHeader = 0;
+  foreach (qw/Date User-Agent X-Newsreader X-Mailer Injection-Date
+    Injection-Info NNTP-Posting-Date NNTP-Posting-Host X-Trace/) {
+    if ($Header{lc($_)})  {
+      warn "W: $_ header may not be set in '$HeaderFile'.\n";
+      $FoundIllegalHeader = 1;
+    }
+  }
+  return '' if $FoundIllegalHeader;
+
   # add Date:
   push @Headers, 'Date: ' . $TD->strftime('%a, %d %b %Y %H:%M:%S %z') . "\n";
   # add missing Message-ID:
@@ -418,6 +442,16 @@ sub BuildPosting {
   # add User-Agent
   push @Headers, "User-Agent: $NAME/$VERSION\n";
 
+  # check for unencoded 8bit characters in header or body in --debug mode
+  # taken from tinews.pl
+  if ($Config{'debug'}) {
+    print "- Raw 8-bit data in headers.\n" if (grep {/[\x80-\xff]/} @Headers);
+     # check for MIME headers and warn for 8bit characters in body if missing
+    if (!defined($Header{'mime-version'}) || !defined($Header{'content-type'})) {
+      print "- 8bit data in body without MIME-headers.\n" if (grep {/[\x80-\xff]/} @Body);
+    }
+  }
+
   # parse pseudo headers from body
   my ($InRealBody,$LastModified,$PostingFrequency);
   foreach (@Body) {
@@ -464,17 +498,20 @@ sub BuildPosting {
     # %m  current month
     # %d  current day
     # %p  PID
+    # %t  timestamp (seconds since epoch)
     if (/^Message-ID: /i) {
       my $TDY = $TD->strftime('%Y');
       my $TDM = $TD->strftime('%m');
       my $TDD = $TD->strftime('%d');
+      my $TimeStamp = time;
       $_ =~ s/\%n/$Project/g;
       $_ =~ s/\%y/$TDY/g;
       $_ =~ s/\%m/$TDM/g;
       $_ =~ s/\%d/$TDD/g;
       $_ =~ s/\%p/$$/g;
+      $_ =~ s/\%t/$TimeStamp/g;
       # add random part in test mode
-      if ($OptTest) {
+      if ($OptTest && !$OptOutput) {
         my $random = sprintf("%08X", rand(0xFFFFFFFF));
         $_ =~ s/</<test-$random-/;
       }
@@ -490,7 +527,7 @@ sub BuildPosting {
     }
     # add Supersedes: if set
     if (/^Supersedes: /) {
-      if ($LastMID && !$OptTest) {
+      if ($LastMID && (!$OptTest or $OptOutput)) {
         $_= "Supersedes: $LastMID\n";
       } else {
         $_ = '';
@@ -511,8 +548,8 @@ sub BuildPosting {
     }
   }
 
-  # not due if Posting-Freqency is "none"
-  if ($PostingFrequency =~ /none/) {
+  # not due if Posting-Freqency is "none" or never
+  if ($PostingFrequency =~ /none|never/ && !$OptForce) {
     print "... is disabled.\n" if $Config{'verbose'} or $Config{'debug'};
     return '';
   }
@@ -723,6 +760,14 @@ Use a TLS encrypted connection (via STARTTLS) if available.
 You can override this option on the command line by using
 B<--starttls> or B<--nostarttls> accordingly.
 
+=item B<xtraheaders> = I<path>
+
+Path to a file with common headers for all project files. Those
+headers will be appended to each project.
+
+You can override this option on the command line by using
+B<--xtraheaders> = I<path>.
+
 =back
 
 =head2 Project files
@@ -736,7 +781,14 @@ files need to be in B<datadir>.
 Needs to have at least I<From:>, I<Subject:> and I<Newsgroups:> and
 can contain all other headers that the posting should have. Headers
 must conform to RFC 5536 and RFC 5322 and use MIME encoded words for
-8bit characters. B<yapfaq> won't convert headers.
+8bit characters. B<yapfaq> won't convert headers, but will warn of
+unencoded 8bit characters in B<--debug> mode. Longer headers should
+be folded; B<yapfaq> won't fold headers.
+
+The headers file must not contain any of the following headers:
+I<Date:>, I<User-Agent:>, I<X-Newsreader:>, I<X-Mailer:>,
+I<Injection-Date:>, I<Injection-Info:>, I<NNTP-Posting-Date:>,
+I<NNTP-Posting-Host:> or I<X-Trace:>.
 
 I<Subject:> may contain a I<%LM> placeholder that will be replaced
 with the I<Last-modified:> pseudo-header from the text file
@@ -747,11 +799,12 @@ curly brackets and spaces) is removed.
 If a I<Message-ID:> header is present, placeholders in that header
 will be replaced: I<%n> with the project name, I<%y> with the current
 year (YYYY), I<%m> with the current month (MM), I<%d> with the
-current day (DD) and I<%p> with the current process ID (PID) of
-B<yapfaq>. If no I<Message-ID:> header is present, the I<Message-ID>
-will be generated with the hostname of the system B<yapfaq> is
-running on and I<%n-%y-%m-%d> as template for the left hand side. If
-the I<Message-ID:> header in the headers file does not contain
+current day (DD), I<%p> with the current process ID (PID) of
+B<yapfaq> and I<%t> with a timestamp timestamp (seconds since epoch).
+If no I<Message-ID:> header is present, the I<Message-ID> will be
+generated with the hostname of the system B<yapfaq> is running on and
+I<%n-%y-%m-%d> as template for the left hand side. If the
+I<Message-ID:> header in the headers file does not contain
 placeholders, the next repost will most probably fail.
 
 If an I<Expires:> header is present, it must contain a time period of
@@ -797,7 +850,8 @@ I<Last-modified:> and I<Posting-frequency> will be evaluated by
 B<yapfaq>.
 
 If your content contains 8bit characters, you'll need suitable MIME
-headers in your headers file.
+headers in your headers file.  B<yapfaq> will warn of unencoded 8bit
+characters with missung MIME headers in B<--debug> mode.
 
 B<Example text file with pseudo-headers>
 
@@ -841,7 +895,7 @@ unique I<Message-ID:> (and no I<Supersedes:> header).
 
 Don't post via NNTP, but print to STDOUT.
 
-Combine with B<--test> to avoid updating project status.
+Use B<--test> instead to avoid updating project status.
 
 Intended for testing purposes or to pipe in another program like
 I<inews> or I<tinews.pl>. If you want to pipe the output to another
@@ -859,20 +913,18 @@ Display this man page and exit.
 =item B<-s>, B<--simulation>
 
 Simulation mode. Don't post, just show which projects would be due.
-Implies B<--test> and B<--verbose>.
+Implies B<--test> (without B<--output>) and B<--verbose>.
 
 Can be combined with B<--project> to show if just one project is due.
 
 =item B<-t>, B<--test>
 
 Test mode. Don't update project status (time and Message-ID of last
-posting), dont' add a I<Supersedes:> header and modify the
-I<Message-ID:> with a random part.
+posting); if project is posted to Usenet, dont' add a I<Supersedes:>
+header and modify the I<Message-ID:> with a random part.
 
-The text(s) will still be posted if due or forced by B<--force>.
-
-Combine with B<--output> to redirect output to STDOUT or with
-B<--newsgroup> to override the I<Newsgroups:> header.
+Implies B<--output> (to redirect output to STDOUT) as long as
+B<--newsgroup> (to override the I<Newsgroups:> header) is not set.
 
 =item B<-V>, B<--version>
 
@@ -909,16 +961,16 @@ that are not:
 Do a test run of your I<example> text and and print it on STDOUT
 (whether ist is due or not):
 
-    yapfaq.pl -t -f -o -p example
-    (or yapfaq.pl -tfop example)
+    yapfaq.pl -t -f -p example
+    (or yapfaq.pl -tfp example)
 
 The same, with debugging output (add "-d"):
 
-    yapfaq.pl -tfdop example
+    yapfaq.pl -tfdp example
 
 Force a test post of your I<example> text to I<alt.test>, even if
-the text is not due to be posted (same as before, just replace "-o"
-by "-n alt-test"):
+the text is not due to be posted (same as before, just add
+"-n alt-test"):
 
     yapfaq.pl -t -f -p example -n alt.test
 

contrib/convert.pl

@@ -0,0 +1,16 @@
+#! /usr/bin/perl -w
+#
+# Convert pre-1.0 yapfaq status file to new format
+# 
+# Usage: convert.pl < old.txt.cfg > new.cfg
+
+while (<>) {
+  if (/Lastpost/) {
+    $_ =~ /Lastpost:\s(\d\d?\.\d\d?\.\d\d\d\d)/;
+    print "Last-Posted: $1\n";
+  }
+  if (/LastMID/) {
+    $_ =~ /LastMID:\s(<[^>]+>)/;
+    print "Last-Message-ID: $1\n";
+  }
+}