Add addpost.pl.

Signed-off-by: Thomas Hochstein <thh@thh.name>

bin/addpost.pl

@@ -0,0 +1,250 @@
+#! /usr/bin/perl
+#
+# addpost.pl
+#
+# This script will add dropped feedlog data from a single posting.
+#
+# It is part of the NewsStats package.
+#
+# Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
+#
+# It can be redistributed and/or modified under the same terms under
+# which Perl itself is published.
+
+BEGIN {
+  use File::Basename;
+  # we're in .../bin, so our module is in ../lib
+  push(@INC, dirname($0).'/../lib');
+}
+use strict;
+use warnings;
+
+use NewsStats qw(:DEFAULT ParseHeaders);
+require '/usr/lib/news/innshellvars.pl';
+
+use Date::Format;
+use DBI;
+use Encode qw(encode);
+use Getopt::Long qw(GetOptions);
+Getopt::Long::config ('bundling');
+
+################################# Subroutines ##################################
+
+# taken from cancelwatch.pl by Ralf Doeblitz
+sub safe_pipe_fork($) {
+  my ($pid, $sleep_count);
+  my $name=shift;
+
+  do {
+    $pid = open(KID, $name);
+    unless (defined $pid) {
+      warn "cannot fork: $!";
+      die "bailing out" if $sleep_count++ > 6;
+      sleep 10;
+    }
+  } until defined $pid;
+
+  return ($pid);
+}
+
+# taken from cancelwatch.pl by Ralf Doeblitz
+sub safe_pipe_read(@) {
+  my @cmd = @_;
+  my $pid = safe_pipe_fork('-|');
+
+  if ($pid) { # parent
+    my @a=<KID>; 
+    close KID or die "child exited: $?";
+    return @a;
+  } else {
+    exec { $cmd[0] } @cmd;
+    exit -1;
+  }
+}
+################################# Main program #################################
+
+### read commandline options
+my ($OptRawDB,$OptConfFile);
+GetOptions ('rawdb=s'         => \$OptRawDB,
+            'conffile=s'      => \$OptConfFile,
+            'h|help'          => \&ShowPOD,
+            'V|version'       => \&ShowVersion) or exit 1;
+
+### read configuration
+my %Conf = %{ReadConfig($OptConfFile)};
+
+### override configuration via commandline options
+my %ConfOverride;
+$ConfOverride{'DBTableRaw'}   = $OptRawDB if $OptRawDB;
+&OverrideConfig(\%Conf,\%ConfOverride);
+
+### init database
+my $DBHandle   = InitDB(\%Conf,1);
+my $DBRaw      = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableRaw'});
+
+### parsing data ---------------------------------------------------------------
+# feedlog will get $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups and $Headers
+
+# get MID from @ARGV
+my $Mid = $ARGV[0];
+&Bleat(2, "No Message-ID given!\nUsage: addpost.pl <Message-ID>") if !$Mid;
+
+# get storage token from MID
+my ($Token) = safe_pipe_read("$inn::newsbin/grephistory", $Mid);
+chomp($Token);
+
+# get arrival time from history
+my $Timestamp;
+open(my $HISTORY, '<', $inn::history)
+    or die "Can't open $inn::history: $!";
+while (<$HISTORY>)
+{
+  next unless /$Token/;
+  $_ =~ /^\[.+\]\s+(\d+)~/;
+  $Timestamp = $1;
+}
+
+# get headers, $Path and $Newsgroups
+my @Headers = safe_pipe_read("$inn::newsbin/sm", '-H', $Token);
+my $Headers = join("", @Headers);
+my %Header = ParseHeaders(split(/\n/,$Headers));
+my $Path = $Header{'path'};
+my $Newsgroups = $Header{'newsgroups'};
+# get $Peer
+$Path =~ /^news.szaf.org!([^!]+)/;
+my $Peer = $1;
+# get size
+my @Article = safe_pipe_read("$inn::newsbin/sm", $Token);
+my $Article = join("", @Article);
+my $Size = length($Article);
+
+# convert headers to UTF-8, if necessary
+$Headers = encode('utf-8', $Headers) if ($Headers =~ /[\x80-\x{ffff}]/);
+
+# parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
+my $Day  = time2str("%Y-%m-%d", $Timestamp);
+my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
+
+### write to database
+my $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
+                                         timestamp,token,size,peer,path,
+                                         newsgroups,headers)
+                                         VALUES (?,?,?,?,?,?,?,?,?,?)",
+                                         $Conf{'DBDatabase'},
+                                         $Conf{'DBTableRaw'}));
+
+if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
+                       $Path, $Newsgroups, $Headers)) {
+  &Bleat(2, sprintf('Database error %s while processing %s: %s',
+         $DBI::err, $Mid, $DBI::errstr));
+};
+$DBQuery->finish;
+
+### close handles
+close $HISTORY;
+$DBHandle->disconnect;
+
+__END__
+
+################################ Documentation #################################
+
+=head1 NAME
+
+addpost - add data for a single posting to a database
+
+=head1 SYNOPSIS
+
+B<addlog> [B<-Vh>] [B<--conffile> I<filename>] [B<--rawdb> I<database table>] I<Message-ID>
+
+=head1 REQUIREMENTS
+
+See L<doc/README>.
+
+=head1 DESCRIPTION
+
+This script will log overview data and complete headers to a database
+table for further examination for a single posting, identified by its
+Message-ID. It's intended to add postings dropped by F<feedlog>.
+
+=head2 Configuration
+
+B<feedlog> will read its configuration from F<newsstats.conf> which
+should be present in etc/ via Config::Auto or from a configuration file
+submitted by the B<--conffile> option.
+
+See L<doc/INSTALL> for an overview of possible configuration options.
+
+You can override the database written to by using the B<--rawdb>
+option.
+
+=head1 OPTIONS
+
+=over 3
+
+=item B<-V>, B<--version>
+
+Display version and copyright information and exit.
+
+=item B<-h>, B<--help>
+
+Display this man page and exit.
+
+=item B<--conffile> I<filename>
+
+Read configuration from I<filename> instead of F<newsstats.conf>.
+
+=item B<--rawdb> I<table> (raw data table)
+
+Override I<DBTableRaw> from F<newsstats.conf>.
+
+=back
+
+=head1 INSTALLATION
+
+See L<doc/INSTALL>.
+
+=head1 EXAMPLES
+
+Add a posting with Message-ID: <mylocalpart@domain.example>
+
+    addposts -- '<mylocalpart@domain.example>'
+
+=head1 FILES
+
+=over 4
+
+=item F<bin/addlog.pl>
+
+The script itself.
+
+=item F<lib/NewsStats.pm>
+
+Library functions for the NewsStats package.
+
+=item F<etc/newsstats.conf>
+
+Runtime configuration file.
+
+=back
+
+=head1 BUGS
+
+Please report any bugs or feature requests to the author or use the
+bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
+
+=back
+
+This script is part of the B<NewsStats> package.
+
+=head1 AUTHOR
+
+Thomas Hochstein <thh@thh.name>
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
+
+This program is free software; you may redistribute it and/or modify it
+under the same terms as Perl itself.
+
+=cut

bin/dbcreate.pl

@@ -68,7 +68,7 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
   KEY `day` (`day`),
   KEY `mid` (`mid`),
   KEY `peer` (`peer`)
-) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COMMENT='Raw data';
+) ENGINE=InnoDB  DEFAULT CHARSET=utf8mb4 COMMENT='Raw data';
 RAW
 --
 -- Table structure for table DBTableGrps

bin/feedlog.pl

@@ -26,6 +26,7 @@ use Sys::Syslog qw(:standard :macros);
 
 use Date::Format;
 use DBI;
+use Encode qw(encode);
 use Getopt::Long qw(GetOptions);
 Getopt::Long::config ('bundling');
 
@@ -106,6 +107,9 @@ while (<>) {
     $Headers .= $_."\n" ;
   }
 
+  # convert headers to UTF-8, if necessary
+  $Headers = encode('utf-8', $Headers) if ($Headers =~ /[\x80-\x{ffff}]/);
+
   # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
   my $Day  = time2str("%Y-%m-%d", $Timestamp);
   my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);

bin/gatherstats.pl

@@ -486,6 +486,7 @@ sub ClientStats {
                            version => $Version);
           push @Clients, { %UserAgent };
         } else {
+          # won't be reached, as $Client is set to User-Agent: in any case
           &Bleat(1,sprintf("%s FAILED", $Header{'message-id'})) if !@Clients;
         }
       }

contrib/dopostingstats.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# installation path is /srv/newsstats/, please adjust accordingly
+# installation path is /opt/newsstats/, please adjust accordingly
 
 # get month
 MONTH=$1
@@ -8,6 +8,6 @@ if ! [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then
 fi
 
 # post stats
-/srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y
-/srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t server --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y
-/srv/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t client --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y
+/opt/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $MONTH | /opt/newsstats/bin/postingstats.pl --month $MONTH | /opt/newsstats/contrib/tinews.pl -X -Y
+/opt/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $MONTH | /opt/newsstats/bin/postingstats.pl -t server --month $MONTH | /opt/newsstats/contrib/tinews.pl -X -Y
+/opt/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $MONTH | /opt/newsstats/bin/postingstats.pl -t client --month $MONTH | /opt/newsstats/contrib/tinews.pl -X -Y

contrib/yearstats.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
-# installation path is /srv/newsstats/, please adjust accordingly
+# installation path is /opt/newsstats/, please adjust accordingly
 # $1: newsgroup
 echo "Stats for $1"
-cd /srv/newsstats/
+cd /opt/newsstats/
 for year in {2012..2022}
 do
  echo -n "${year}: "

doc/ChangeLog

@@ -1,5 +1,8 @@
 NewsStats 0.5.0 (unreleased)
-
+  * Change install path from /srv/ to /opt/.
+  * dbcreate: Change raw table to InnoDB and utf8mb4.
+  * feedlog: Fix errors when saving unencoded 8bit headers.
+  * Add addpost (to add post data dropped by feedlog).
 
 NewsStats 0.4.0 (2025-06-02)
   * Reformat $Conf{TLH} for GroupStats only.

doc/INSTALL

@@ -17,7 +17,7 @@ INSTALLATION INSTRUCTIONS
    * Download the current version of NewsStats from
      <https://th-h.de/net/software/newsstats/>.
 
-   * Untar it into a directory of your choice, i.e. /srv/newsstats:
+   * Untar it into a directory of your choice, i.e. /opt/newsstats:
 
      $ cd /srv
      $ tar -xzf newsstats-n.n.n.tar.gz