Release 0.4.0

Signed-off-by: Thomas Hochstein <thh@thh.name>

README.md

@@ -4,12 +4,14 @@
 
 ## Description
 
-**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in a database which in turn can be used to generate various reports.
+**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in databases which in turn can be used to generate various reports (postings per group, injection server or posting agent, per month).
 
-The software package is still under development.
+This software is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy.
-
-It is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy.
 
 ## More information
 
 Please see the [distribution page](https://th-h.de/net/software/newsstats/) (in German).
+
+* General overview and examples: [README](doc/README)
+* Installation instructions: [INSTALL](doc/INSTALL)
+* Changelog: [ChangeLog](doc/ChangeLog)

bin/clientstats.pl

@@ -0,0 +1,634 @@
+#! /usr/bin/perl
+#
+# clientstats.pl
+#
+# This script will get statistical data on newsreader (client) usage
+# from a database.
+#
+# It is part of the NewsStats package.
+#
+# Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
+#
+# It can be redistributed and/or modified under the same terms under
+# which Perl itself is published.
+
+BEGIN {
+  use File::Basename;
+  # we're in .../bin, so our module is in ../lib
+  push(@INC, dirname($0).'/../lib');
+}
+use strict;
+use warnings;
+
+use NewsStats qw(:DEFAULT :TimePeriods :Output :SQLHelper ReadGroupList);
+
+use DBI;
+use Getopt::Long qw(GetOptions);
+Getopt::Long::config ('bundling');
+
+################################# Main program #################################
+
+### read commandline options
+my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat,
+    $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy,
+    $OptReportType,$OptSums,$UppBound,$OptVersions,$OptConfFile);
+GetOptions ('c|captions!'    => \$OptCaptions,
+            'comments!'      => \$OptComments,
+            'db=s'           => \$OptDB,
+            'filetemplate=s' => \$OptFileTemplate,
+            'f|format=s'     => \$OptFormat,
+            'g|group-by=s'   => \$OptGroupBy,
+            'l|lower=i'      => \$LowBound,
+            'm|month=s'      => \$OptMonth,
+            'n|names=s'      => \$OptNames,
+            'o|order-by=s'   => \$OptOrderBy,
+            'r|report=s'     => \$OptReportType,
+            's|sums!'        => \$OptSums,
+            'u|upper=i'      => \$UppBound,
+            'v|versions!'    => \$OptVersions,
+            'conffile=s'     => \$OptConfFile,
+            'h|help'         => \&ShowPOD,
+            'V|version'      => \&ShowVersion) or exit 1;
+# parse parameters
+# $OptComments defaults to TRUE if --filetemplate is not used
+$OptComments = 1 if (!$OptFileTemplate && !defined($OptComments));
+# parse $OptReportType
+if ($OptReportType) {
+  if ($OptReportType =~ /sums?/i) {
+    $OptReportType = 'sum';
+  } else {
+    $OptReportType  = 'default';
+  }
+}
+
+### read configuration
+my %Conf = %{ReadConfig($OptConfFile)};
+
+### set DBTable
+$Conf{'DBTable'} = $Conf{'DBTableClnts'};
+$Conf{'DBTable'} = $OptDB if $OptDB;
+  
+### init database
+my $DBHandle = InitDB(\%Conf,1);
+
+### get time period and names, prepare SQL 'WHERE' clause
+# get time period
+# and set caption for output and expression for SQL 'WHERE' clause
+my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth);
+# bail out if --month is invalid
+&Bleat(2,"--month option has an invalid format - ".
+         "please use 'YYYY-MM', 'YYYY-MM:YYYY-MM' or 'ALL'!") if !$CaptionPeriod;
+# get list of clients and set expression for SQL 'WHERE' clause
+# with placeholders as well as a list of names to bind to them
+my ($SQLWhereNames,@SQLBindNames);
+if ($OptNames) {
+  ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,'client');
+  # bail out if --names is invalid
+  &Bleat(2,"--names option has an invalid format!")
+    if !$SQLWhereNames;
+}
+
+### build SQL WHERE clause
+my $ExcludeSums = $OptSums ? '' : "client != 'ALL'";
+my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
+                                    $ExcludeSums,"version = 'ALL'",
+                                    &SQLSetBounds('default',$LowBound,$UppBound));
+
+### get sort order and build SQL 'ORDER BY' clause
+# force to 'month' for $OptReportType 'sum'
+$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
+# default to 'name' if $OptGroupBy is not set and
+# just one name is requested, but more than one month
+$OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
+                         and $OptNames and $OptNames !~ /[:*%]/);
+# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
+# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
+my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, 'client, version');
+# $GroupBy will contain 'month' or 'client, version' (parsed result of $OptGroupBy)
+# set it to 'month' or 'key' for OutputData()
+$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key';
+
+### get report type and build SQL 'SELECT' query
+my $SQLSelect;
+my $SQLGroupClause = '';
+
+if ($OptReportType and $OptReportType ne 'default') {
+  $SQLGroupClause = "GROUP BY client, version";
+  # change $SQLOrderClause: replace everything before 'postings'
+  $SQLOrderClause =~ s/BY.+postings/BY postings/;
+  $SQLSelect = "'All months',LEFT(client,40),SUM(postings)";
+  # change $SQLOrderClause: replace 'postings' with 'SUM(postings)'
+  $SQLOrderClause =~ s/postings/SUM(postings)/;
+ } else {
+  $SQLSelect = "month,LEFT(client,40),postings";
+};
+
+### get length of longest name delivered by query
+### for formatting purposes
+my $Field = ($GroupBy eq 'month') ? 'LEFT(client,40)' : 'month';
+my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
+                                              $Field,'postings',$SQLWhereClause,
+                                              '',@SQLBindNames);
+
+### build and execute SQL query
+my ($DBQuery);
+# prepare query
+$DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s',
+                                      $SQLSelect,
+                                      $Conf{'DBDatabase'},$Conf{'DBTable'},
+                                      $SQLWhereClause,$SQLGroupClause,
+                                      $SQLOrderClause));
+# execute query
+$DBQuery->execute(@SQLBindNames)
+  or &Bleat(2,sprintf("Can't get client data for %s from %s.%s: %s\n",
+                      $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
+                      $DBI::errstr));
+
+### output results
+# set default to 'pretty'
+$OptFormat = 'pretty' if !$OptFormat;
+# print captions if --caption is set
+my $LeadIn;
+if ($OptCaptions && $OptComments) {
+  # print time period with report type
+  my $CaptionReportType = '(number of postings for each month)';
+  if ($OptReportType and $OptReportType ne 'default') {
+    $CaptionReportType  = '(number of all postings for that time period)';
+  }
+  $LeadIn .= sprintf("# ----- Report for %s %s\n",$CaptionPeriod,$CaptionReportType);
+  # print name list if --names is set
+  $LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames)))
+    if $OptNames;
+  # print boundaries, if set
+  my $CaptionBoundary= '(counting only months fulfilling this condition)';
+  $LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n",
+         $LowBound ? $LowBound : '',$LowBound ? '=>' : '',
+         $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary)
+    if ($LowBound or $UppBound);
+  # print primary and secondary sort order
+  $LeadIn .= sprintf("# ----- Grouped by %s (%s), sorted %s%s\n",
+         ($GroupBy eq 'month') ? 'Months' : 'Names',
+         ($OptGroupBy and $OptGroupBy =~ /-?desc$/i) ? 'descending' : 'ascending',
+         ($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
+         ($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
+}
+
+# output data
+# (changed code copy from NewsStats::OutputData)
+my ($LastIteration, $FileName, $Handle, $OUT);
+
+# define output types
+my %LegalOutput;
+@LegalOutput{('dump','list','pretty')} = ();
+# bail out if format is unknown
+&Bleat(2,"Unknown output type '$OptFormat'!") if !exists($LegalOutput{$OptFormat});
+
+while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) {
+  # save client for later use
+  my $Client = $Key;
+  # care for correct sorting order and abstract from month and keys:
+  # $Caption will be $Month or $Key, according to sorting order,
+  # and $Key will be $Key or $Month, respectively
+  my $Caption;
+  if ($GroupBy eq 'key') {
+    $Caption = $Key;
+    $Key     = $Month;
+  } else {
+    $Caption = $Month;
+  }
+  # set output file handle
+  if (!$OptFileTemplate) {
+    $Handle = *STDOUT{IO}; # set $Handle to a reference to STDOUT
+  } elsif (!defined($LastIteration) or $LastIteration ne $Caption) {
+    close $OUT if ($LastIteration);
+    # safeguards for filename creation:
+    # replace potential problem characters with '_'
+    $FileName = sprintf('%s-%s',$OptFileTemplate,$Caption);
+    $FileName =~ s/[^a-zA-Z0-9_-]+/_/g;
+    open ($OUT,">$FileName")
+      or &Bleat(2,sprintf("Cannot open output file '%s': $!",
+                          $FileName));
+    $Handle = $OUT;
+  };
+  print $Handle &FormatOutput($OptFormat, $OptComments, $LeadIn, $Caption,
+                              $Key, $Value, 0, $MaxLength, $MaxValLength, $LastIteration);
+  # output client versions
+  if ($OptVersions) {
+    ### get client versions
+    # $SQLWhereClause without 'ALL' version, with client and month set
+    $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
+                                      $ExcludeSums,"version != 'ALL'",
+                                      'client = ?','month = ?',
+                                      &SQLSetBounds('default',$LowBound,$UppBound));
+    # push client and month to @SQLVersBindNames
+    my @SQLVersBindNames = @SQLBindNames;
+    push (@SQLVersBindNames, ($Client, $Month));
+
+    # save length of longest client
+    my $ClientMaxLenght    = $MaxLength;
+    my $ClientMaxValLenght = $MaxValLength;
+    # get length of longest version delivered by query
+    # for formatting purposes
+    my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
+                                                  'version','postings',$SQLWhereClause,
+                                                  '',@SQLVersBindNames);
+    if ($MaxLength) {
+      # add lenght of '- '
+      $MaxLength += 2;
+      # set to length of longest client, if longer
+      $MaxLength    = $ClientMaxLenght    if $ClientMaxLenght    > $MaxLength;
+      $MaxValLength = $ClientMaxValLenght if $ClientMaxValLenght > $MaxValLength;
+    }
+
+    # prepare query
+    my $DBVersQuery = $DBHandle->prepare(sprintf('SELECT version,postings FROM %s.%s %s %s %s',
+                                                 $Conf{'DBDatabase'},$Conf{'DBTable'},
+                                                 $SQLWhereClause,$SQLGroupClause,
+                                                 $SQLOrderClause));
+    # execute query
+    $DBVersQuery->execute(@SQLVersBindNames)
+      or &Bleat(2,sprintf("Can't get version data for %s from %s.%s: %s\n",
+                          $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
+                          $DBI::errstr));
+    # output versions
+    while (my ($Version, $Postings) = $DBVersQuery->fetchrow_array) {
+      $Version = '- ' . $Version;
+      print $Handle &FormatOutput($OptFormat, $OptComments, $LeadIn, '',
+                                  $Version, $Postings, 0, $MaxLength, $MaxValLength,
+                                  '');
+    }
+  }
+  $LastIteration = $Caption;
+};
+close $OUT if ($OptFileTemplate);
+
+### close handles
+$DBHandle->disconnect;
+
+__END__
+
+################################ Documentation #################################
+
+=head1 NAME
+
+clientstats - create reports on client usage
+
+=head1 SYNOPSIS
+
+B<clientstats> [B<-Vhcsv> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<client(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
+
+=head1 REQUIREMENTS
+
+See L<doc/README>.
+
+=head1 DESCRIPTION
+
+This script create reports on newsgroup usage (number of postings
+using each client per month) taken from result tables created by
+B<gatherstats.pl>.
+
+=head2 Features and options
+
+=head3 Time period and names
+
+The time period to act on defaults to last month; you can assign another
+time period or a single month (or drop all time constraints) via the
+B<--month> option (see below).
+
+B<clientstats> will process all clients by default; you can limit
+processing to only some clients by supplying a list of those names by
+using the B<--names> option (see below).
+
+=head3 Report type
+
+You can choose between different B<--report> types: postings per month
+or all postings summed up; for details, see below.
+
+=head3 Upper and lower boundaries
+
+Furthermore you can set an upper and/or lower boundary to exclude some
+results from output via the B<--lower> and B<--upper> options,
+respectively. By default, all clients with more and/or less postings
+per month will be excluded from the result set (i.e. not shown and
+not considered for sum reports).
+
+=head3 Sorting and formatting the output
+
+By default, all results are grouped by month; you can group results by
+clients instead via the B<--group-by> option. Within those groups,
+the list of clients (or months) is sorted alphabetically
+(or chronologically, respectively) ascending. You can change that order
+(and sort by number of postings) with the B<--order-by> option. For
+details and exceptions, please see below.
+
+The results will be formatted as a kind of table; you can change the
+output format to a simple list or just a list of names and number of
+postings with the B<--format> option. Captions will be added by means
+of the B<--caption> option; all comments (and captions) can be
+supressed by using B<--nocomments>.
+
+Last but not least you can redirect all output to a number of files,
+e.g. one for each month, by submitting the B<--filetemplate> option,
+see below.
+
+=head2 Configuration
+
+B<clientstats> will read its configuration from F<newsstats.conf>
+which should be present in etc/ via Config::Auto or from a configuration
+file submitted by the B<--conffile> option.
+
+See doc/INSTALL for an overview of possible configuration options.
+
+You can override some configuration options via the B<--db> option.
+
+=head1 OPTIONS
+
+=over 3
+
+=item B<-V>, B<--version>
+
+Display version and copyright information and exit.
+
+=item B<-h>, B<--help>
+
+Display this man page and exit.
+
+=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
+
+Set processing period to a single month in YYYY-MM format or to a time
+period between two month in YYYY-MM:YYYY-MM format (two month, separated
+by a colon). By using the keyword I<all> instead, you can set no
+processing period to process the whole database. Defaults to last month.
+
+=item B<-n>, B<--names> I<name(s)>
+
+Limit processing to a certain set of client names. I<names(s)>
+can be a single name (Thunderbird), a group of names (Ice*) or a list
+of either of these, separated by colons, for example
+
+   Forte Agent:Thunderbird:Ice*
+
+Spaces or special characters like "*" need to be quoted from the shell,
+like
+
+   -n 'Forte Agent:Thunderbird:Ice*'
+
+There is no way to limit processing to a specific version, but you can
+alway grep through the output.
+
+=item B<-s>, B<--sums|--nosums> (sum per month)
+
+Include "virtual" clients named "ALL" for every month in output,
+containing the sum of all detected clients for that month. False
+by default.
+
+=item B<-v>, B<--versions|--noversions> (client versions)
+
+Include a list of all observed versions of each client in output.
+Version information will be displayed with indents ('-') below each
+client, sorted in the same way (by postings or alphanumeric). False
+by default.
+
+=item B<-r>, B<--report> I<default|sums>
+
+Choose the report type: I<default> or I<sums>
+
+By default, B<clientstats> will report the number of postings for each
+client in each month. But it can also report the total sum of postings
+per client for all months. Sums of B<--versions> can be included.
+
+For report type I<sums>, the B<group-by> option has no meaning and
+will be silently ignored (see below).
+
+=item B<-l>, B<--lower> I<lower boundary>
+
+Set the lower boundary. See below.
+
+=item B<-l>, B<--upper> I<upper boundary>
+
+Set the upper boundary.
+
+By default, all clients with more postings per month than the
+upper boundary and/or less postings per month than the lower boundary
+will be excluded from further processing. For the default report that
+means each month only clients with a number of postings between the
+boundaries will be displayed. For the sums report, clients with a
+number of postings exceeding the boundaries in all (!) months will
+not be considered.
+
+=item B<-g>, B<--group-by> I<month[-desc]|name[-desc]>
+
+By default, all results are grouped by month, sorted chronologically in
+ascending order, like this:
+
+    # ----- 2012-01:
+    40tude_Dialog:  5873
+    Forte Agent  :  7735
+    Thunderbird  : 20925
+    # ----- 2012-02:
+    40tude_Dialog:  4142
+    Forte Agent  :  5895
+    Thunderbird  : 19091
+
+The results can be grouped by client instead via
+B<--group-by> I<name>:
+
+    # ----- 40tude_Dialog:
+    2012-01:  5873
+    2012-02:  4142
+    # ----- Forte Agent:
+    2012-01:  7735
+    2012-02:  5895
+    # ----- Thunderbird:
+    2012-01: 20925
+    2012-02: 19091
+
+By appending I<-desc> to the group-by option parameter, you can reverse
+the sort order - e.g. B<--group-by> I<month-desc> will give:
+
+    # ----- 2012-02:
+    40tude_Dialog:  4142
+    Forte Agent  :  5895
+    Thunderbird  : 19091
+    # ----- 2012-01:
+    40tude_Dialog:  5873
+    Forte Agent  :  7735
+    Thunderbird  : 20925
+
+Sums reports (see above) will always be grouped by months; this option
+will therefore be ignored.
+
+=item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
+
+Within each group (a single month or single client, see above),
+the report will be sorted by name (or month) in ascending alphabetical
+order by default. You can change the sort order to descending or sort
+by number of postings instead.
+
+By default, output is sorted alphabetically:
+
+    # ----- 2012-01:
+    40tude_Dialog:  5873
+    Forte Agent  :  7735
+    Thunderbird  : 20925
+
+Using B<--order-by> I<postings[-desc]>, it will be sorted from most
+to least postings:
+
+    # ----- 2012-01:
+    Thunderbird  : 20925
+    Forte Agent  :  7735
+    40tude_Dialog:  5873
+
+=item B<-f>, B<--format> I<pretty|list|dump>
+
+Select the output format, I<pretty> (a kind of table) being the default:
+
+    # ----- 2012-01:
+    40tude_Dialog:  5873
+    Forte Agent  :  7735
+    # ----- 2012-02:
+    40tude_Dialog:  4142
+    Forte Agent  :  5895
+
+I<list> format looks like this (each client preceded by month):
+
+    2012-01 40tude_Dialog 5873
+    2012-01 Forte Agent 7735
+    2012-02 40tude_Dialog 4142
+    2012-02 Forte Agent 5895
+
+And I<dump> format looks like this:
+
+    # 2012-01:
+    40tude_Dialog 5873
+    Forte Agent 7735
+    # 2012-02:
+    40tude_Dialog 4142
+    Forte Agent 5895
+
+You can remove the comments (lines after '#') by using B<--nocomments>,
+see below.
+
+=item B<-c>, B<--captions|--nocaptions>
+
+Add captions to output, like this:
+
+    ----- Report for 2012-01 to 2012-02 (number of postings for each month)
+    ----- Names: Thunderbird
+    ----- Threshold: 8000 => x (counting only month fulfilling this condition)
+    ----- Grouped by Month (ascending), sorted by number of postings descending
+
+False by default.
+
+=item B<--comments|--nocomments>
+
+Add comments (group headers) to I<dump> and I<pretty> output. True by
+default as long as B<--filetemplate> is not set.
+
+Use I<--nocomments> to suppress anything except client names or months
+and numbers of postings.
+
+=item B<--filetemplate> I<filename template>
+
+Save output to file(s) instead of dumping it to STDOUT. B<clientstats>
+will create one file for each month (or each client, according to the
+setting of B<--group-by>, see above), with filenames composed by adding
+year and month (or client names) to the I<filename template>, for
+example with B<--filetemplate> I<stats>:
+
+    stats-2012-01
+    stats-2012-02
+    ... and so on
+
+=item B<--db> I<database table>
+
+Override I<DBTableClnts> or I<DBTableClnts> from F<newsstats.conf>.
+
+=item B<--conffile> I<filename>
+
+Read configuration from I<filename> instead of F<newsstats.conf>.
+
+=back
+
+=head1 INSTALLATION
+
+See L<doc/INSTALL>.
+
+=head1 EXAMPLES
+
+Show number of postings per client for lasth month in I<pretty> format:
+
+    clientstats 
+
+Show that report for January of 2010 and Thunderbird plus Ice*:
+
+    clientstats --month 2010-01 --names 'Thunderbird:Ice*'
+
+Only show clients with at least 30 postings last month and the versions
+of those clients, ordered each by number of postings, descending,
+in I<pretty> format:
+
+    clientstats --lower 30 --versions --order-by postings-desc
+
+List number of postings per client for each month of 2010 and redirect
+output to one file for each month, named hosts-2010-01 and so on, in
+machine-readable form (without formatting):
+
+    clientstats -m 2010-01:2010-12 -f dump --filetemplate hosts
+
+=head1 FILES
+
+=over 4
+
+=item F<bin/clientstats.pl>
+
+The script itself.
+
+=item F<lib/NewsStats.pm>
+
+Library functions for the NewsStats package.
+
+=item F<etc/newsstats.conf>
+
+Runtime configuration file.
+
+=back
+
+=head1 BUGS
+
+Please report any bugs or feature requests to the author or use the
+bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
+
+=head1 SEE ALSO
+
+=over 2
+
+=item -
+
+L<doc/README>
+
+=item -
+
+L<doc/INSTALL>
+
+=item -
+
+gatherstats -h
+
+=back
+
+This script is part of the B<NewsStats> package.
+
+=head1 AUTHOR
+
+Thomas Hochstein <thh@thh.name>
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
+
+This program is free software; you may redistribute it and/or modify it
+under the same terms as Perl itself.
+
+=cut

bin/dbcreate.pl

@@ -46,7 +46,7 @@ my $DBCreate = <<SQLDB;
 CREATE DATABASE IF NOT EXISTS `$Conf{'DBDatabase'}` DEFAULT CHARSET=utf8;
 SQLDB
 
-my %DBCreate = ('DBTableRaw'  => <<RAW, 'DBTableGrps' => <<GRPS, 'DBTableHosts' => <<HOSTS);
+my %DBCreate = ('DBTableRaw' => <<RAW, 'DBTableGrps' => <<GRPS, 'DBTableHosts' => <<HOSTS, 'DBTableClnts' => <<CLIENTS);
 --
 -- Table structure for table DBTableRaw
 --
@@ -102,6 +102,23 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableHosts'}` (
   KEY `host` (`host`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COMMENT='Postings per server';
 HOSTS
+--
+-- Table structure for table DBTableClnts
+--
+
+CREATE TABLE IF NOT EXISTS `$Conf{'DBTableClnts'}` (
+  `id` bigint(20) unsigned NOT NULL auto_increment,
+  `month` varchar(7) character set ascii NOT NULL,
+  `client` varchar(150) NOT NULL,
+  `version` varchar(50) NOT NULL,
+  `postings` int(11) NOT NULL,
+  `revision` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `month_client_version` (`month`,`client`,`version`),
+  KEY `month` (`month`),
+  KEY `client` (`client`)
+) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='Postings per client';
+CLIENTS
 
 ##### --------------------------------------------------------------------------
 ##### Installation / upgrade instructions
@@ -201,16 +218,7 @@ if (!$OptUpdate) {
 } else {
   ##### upgrade mode
   print "----------\nStarting upgrade process.\n";
-  my $PackageVersion = '0.03';
+  # TBD
-  if ($OptUpdate < $PackageVersion) {
-    if ($OptUpdate < 0.02) {
-      # 0.01 -> 0.02
-      # &DoMySQL('...;');
-      # print "v0.02: Database upgrades ...\n";
-      # &PrintInstructions('0.02',<<"      INSTRUCTIONS");
-      # INSTRUCTIONS
-    };
-  };
   # Display general upgrade instructions
   print $Upgrade;
 };
@@ -290,11 +298,11 @@ See L<doc/INSTALL> for an overview of possible configuration options.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
 
 =item B<-u>, B<--update> I<version>
 
@@ -302,7 +310,7 @@ Don't do a fresh install, but update from I<version>.
 
 =item B<--conffile> I<filename>
 
-Load configuration from I<filename> instead of F<newsstats.conf>.
+Read configuration from I<filename> instead of F<newsstats.conf>.
 
 =back
 

bin/feedlog.pl

@@ -167,8 +167,9 @@ time.
 
 All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
 fails to initiate a database connection at startup, it will log to
-I<syslog> with I<CRIT> priority and go in an endless loop, as
+I<syslog> with I<CRIT> priority and go in an endless loop, trying again
-terminating would only result in a rapid respawn.
+to connect every 5 seconds, as terminating would only result in a rapid
+respawn.
 
 =head2 Configuration
 
@@ -184,15 +185,15 @@ See L<doc/INSTALL> for an overview of possible configuration options.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
 
 =item B<-d>, B<--debug>
 
-Output debugging information to STDERR while parsing STDIN. You'll
+Print debugging information to STDERR while parsing STDIN. You'll
 find that information most probably in your B<INN> F<errlog> file.
 
 =item B<-q>, B<--quiet>
@@ -201,7 +202,7 @@ Suppress logging to syslog.
 
 =item B<--conffile> I<filename>
 
-Load configuration from I<filename> instead of F<newsstats.conf>.
+Read configuration from I<filename> instead of F<newsstats.conf>.
 
 =back
 

bin/gatherstats.pl

@@ -23,6 +23,7 @@ use warnings;
 use NewsStats qw(:DEFAULT :TimePeriods ListNewsgroups ParseHierarchies ReadGroupList ParseHeaders);
 
 use DBI;
+use Encode qw(decode encode);
 use Getopt::Long qw(GetOptions);
 Getopt::Long::config ('bundling');
 
@@ -31,7 +32,7 @@ Getopt::Long::config ('bundling');
 # define types of information that can be gathered
 # all / groups (/ clients / hosts)
 my %LegalStats;
-@LegalStats{('all','groups','hosts')} = ();
+@LegalStats{('all','groups','hosts','clients')} = ();
 
 ################################# Main program #################################
 
@@ -61,8 +62,8 @@ my %Conf = %{ReadConfig($OptConfFile)};
 my %ConfOverride;
 $ConfOverride{'DBTableRaw'}   = $OptRawDB if $OptRawDB;
 $ConfOverride{'DBTableGrps'}  = $OptGroupsDB if $OptGroupsDB;
-$ConfOverride{'DBTableClnts'} = $OptClientsDB if $OptClientsDB;
 $ConfOverride{'DBTableHosts'} = $OptHostsDB if $OptHostsDB;
+$ConfOverride{'DBTableClnts'} = $OptClientsDB if $OptClientsDB;
 $ConfOverride{'TLH'} = $OptTLH if $OptTLH;
 &OverrideConfig(\%Conf,\%ConfOverride);
 
@@ -84,6 +85,21 @@ my ($Period) = &GetTimePeriod($OptMonth);
 &Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ".
          "'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time');
 
+### init database
+my $DBHandle   = InitDB(\%Conf,1);
+my $DBRaw      = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableRaw'});
+my $DBGrps     = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableGrps'});
+my $DBHosts    = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableHosts'});
+my $DBClients  = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableClnts'});
+
+### get data for each month
+&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
+foreach my $Month (&ListMonth($Period)) {
+
+  print "---------- $Month ----------\n" if $OptDebug;
+
+  ### GroupStats
+  if ($OptStatsType eq 'all' or $OptStatsType eq 'groups') {
     ### reformat $Conf{'TLH'}
     my $TLH;
     if ($Conf{'TLH'}) {
@@ -112,21 +128,6 @@ if ($Conf{'TLH'}) {
         $TLH = '(' . $TLH . ')';
       };
     };
-
-### init database
-my $DBHandle = InitDB(\%Conf,1);
-my $DBRaw    = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableRaw'});
-my $DBGrps   = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableGrps'});
-my $DBHosts  = sprintf('%s.%s',$Conf{'DBDatabase'},$Conf{'DBTableHosts'});
-
-### get data for each month
-&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
-foreach my $Month (&ListMonth($Period)) {
-
-  print "---------- $Month ----------\n" if $OptDebug;
-
-  ### GroupStats
-  if ($OptStatsType eq 'all' or $OptStatsType eq 'groups') {
     &GroupStats($DBHandle,$DBRaw,$DBGrps,$Month,$TLH,$OptCheckgroupsFile,$OptMID,$OptTest,$OptDebug);
   };
 
@@ -138,7 +139,16 @@ foreach my $Month (&ListMonth($Period)) {
                         googlegroups.com heirich.name news.neostrada.pl netcologne.de newsdawg.com newscene.com
                         news-service.com octanews.com readnews.com wieslauf.sub.de highway.telekom.at
                         united-newsserver.de xennanews.com xlned.com xsnews.nl news.xs4all.nl);
-    &HostStats($DBHandle,$DBRaw,$DBHosts,$Month,$OptMID,$OptTest,$OptDebug,@KnownHosts);
+    &HostStats($DBHandle,$DBRaw,$DBHosts,$Month,$OptTLH,$OptMID,$OptTest,$OptDebug,@KnownHosts);
+  };
+
+  ### ClientStats
+  if ($OptStatsType eq 'all' or $OptStatsType eq 'clients') {
+    # define agents/clients that shouldn't be counted
+    my @DropAgents = qw(debian fedora firefox gecko gentoo lightning mandriva mnenhy mozilla
+                        pclinuxos perl php presto suse suse/opensuse thunderbrowse ubuntu version);
+    push(@DropAgents, 'red hat');
+    &ClientStats($DBHandle,$DBRaw,$DBClients,$Month,$OptTLH,$OptMID,$OptTest,$OptDebug,@DropAgents);
   };
 };
 
@@ -154,10 +164,10 @@ sub GroupStats {
 ###      $DBRaw            : database table for raw data (to read from)
 ###      $DBGrps           : database table for groups data (to write to)
 ###      $Month            : current month to do
-###      $MID              : specific Message-ID to fetch (testing purposes)
 ###      $TLH              : TLHs to collect
 ###      $Checkgroupsfile  : filename template for checkgroups file
 ###                          (expanded to $Checkgroupsfile-$Month)
+###      $MID              : specific Message-ID to fetch (testing purposes)
 ###      $Test             : test mode
 ###      $Debug            : debug mode
 ### OUT: (nothing)
@@ -252,46 +262,24 @@ sub HostStats {
 ###      $DBRaw            : database table for raw data (to read from)
 ###      $DBHosts          : database table for hosts data (to write to)
 ###      $Month            : current month to do
+###      $TLH              : TLHs to collect
 ###      $MID              : specific Message-ID to fetch (testing purposes)
 ###      $Test             : test mode
 ###      $Debug            : debug mode
 ###      @KnownHosts       : list of known hosts with subdomains
 ### OUT: (nothing)
-  my ($DBHandle,$DBRaw,$DBHosts,$Month,$MID,$Test,$Debug,@KnownHosts) = @_;
+  my ($DBHandle,$DBRaw,$DBHosts,$Month,$TLH,$MID,$Test,$Debug,@KnownHosts) = @_;
 
   my (%Postings,$DBQuery);
 
-  if (!$MID) {
+  $DBQuery = GetHeaders($DBHandle,$DBRaw,$Month,$MID);
-    # get raw header data from raw table for given month
-    $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ".
-                                             "WHERE day LIKE ? AND NOT disregard",
-                                             $DBRaw));
-    $DBQuery->execute($Month.'-%')
-      or &Bleat(2,sprintf("Can't get hosts data for %s from %s: ".
-                          "$DBI::errstr\n",$Month,$DBRaw));
-  } else {
-    $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ".
-                                             "WHERE mid = ?", $DBRaw));
-    $DBQuery->execute($MID)
-      or &Bleat(2,sprintf("Can't get hosts data for %s from %s: ".
-                          "$DBI::errstr\n",$MID,$DBRaw));
-  }
 
   ### ----------------------------------------------
   print "----- HostStats -----\n" if $Debug;
   ### parse headers
   while (my ($Newsgroups,$Headers) = $DBQuery->fetchrow_array) {
     ### skip postings with wrong TLH
-    # remove whitespace from contents of Newsgroups:
+    next if ($TLH && !CheckTLH($Newsgroups,$TLH));
-    chomp($Newsgroups);
-    $Newsgroups =~ s/\s//;
-    my $GroupCount;
-    for (split /,/, $Newsgroups) {
-      # don't count newsgroup/hierarchy in wrong TLH
-      next if($TLH and !/^$TLH/);
-      $GroupCount++;
-    };
-    next if !$GroupCount;
 
     my $Host;
     my %Header = ParseHeaders(split(/\n/,$Headers));
@@ -344,15 +332,13 @@ sub HostStats {
       }
     }
 
-    # lowercase
-    $Host = lc($Host);
-
     # count host
     if ($Host) {
+      $Host = lc($Host);
       $Postings{$Host}++;
       $Postings{'ALL'}++;
     } else {
-      &Bleat(2,sprintf("%s FAILED", $Header{'message-id'})) if !$Host;
+      &Bleat(1,sprintf("%s FAILED", $Header{'message-id'})) if !$Host;
     }
 
     printf("%s: %s\n", $Header{'message-id'}, $Host) if ($MID or $Debug && $Debug >1);
@@ -381,6 +367,363 @@ sub HostStats {
   };
 };
 
+sub ClientStats {
+### ----------------------------------------------------------------------------
+### collect number of postings per client (and version)
+### IN : $DBHandle         : database handle
+###      $DBRaw            : database table for raw data (to read from)
+###      $DBClients        : database table for clients data (to write to)
+###      $Month            : current month to do
+###      $TLH              : TLHs to collect
+###      $MID              : specific Message-ID to fetch (testing purposes)
+###      $Test             : test mode
+###      $Debug            : debug mode
+###      @DropAgents       : list of UserAgent "agents" that won't be counted
+### OUT: (nothing)
+  my ($DBHandle,$DBRaw,$DBClients,$Month,$TLH,$MID,$Test,$Debug,@DropAgents) = @_;
+
+  my (%Postings,$DBQuery);
+  my %DropAgent = map { $_ => 1 } @DropAgents;
+
+  $DBQuery = GetHeaders($DBHandle,$DBRaw,$Month,$MID);
+
+  ### ----------------------------------------------
+  print "----- ClientStats -----\n" if $Debug;
+  ### parse headers
+  while (my ($Newsgroups,$Headers) = $DBQuery->fetchrow_array) {
+    ### skip postings with wrong TLH
+    next if ($TLH && !CheckTLH($Newsgroups,$TLH));
+
+    my (@Clients, $Client, $Version);
+    my %Header = ParseHeaders(split(/\n/,$Headers));
+
+    ### X-Mailer
+    if ($Header{'x-mailer'}) {
+      # transfer to x-newsreader and parse from there
+      $Header{'x-newsreader'} = $Header{'x-mailer'};
+    }
+    ### X-Newsreader
+    if ($Header{'x-newsreader'}) {
+      $Header{'x-newsreader'} = RemoveComments($Header{'x-newsreader'});
+      # remove 'http://' and 'via' (CrossPoint)
+      $Header{'x-newsreader'} =~ s/https?:\/\///;
+      $Header{'x-newsreader'} =~ s/ ?via(.+)?$//;
+      # parse header
+      # User-Agent style
+      if ($Header{'x-newsreader'} =~ /^([^\/ ]+\/[^\/ ]+ ?)+$/) {
+        # transfer to user-agent and parse from there
+        $Header{'user-agent'} = $Header{'x-newsreader'};
+      # "client name version"
+      } elsif ($Header{'x-newsreader'} =~ / /) {
+        ($Client, $Version) = ParseXNewsreader($Header{'x-newsreader'});
+      } else {
+        $Client = $Header{'x-newsreader'};
+        $Version = '';
+      }
+      if ($Client) {
+        # special cases
+        $Client  = 'CrossPoint'     if $Client =~ /^CrossPoint\//;
+        $Client  = 'Virtual Access' if $Client =~ /^Virtual Access/;
+        my %UserAgent = (agent   => $Client,
+                         version => $Version);
+        push @Clients, { %UserAgent };
+      } else {
+        $Header{'user-agent'} = $Header{'x-newsreader'};
+      }
+    }
+    ### User-Agent
+    if(!@Clients && $Header{'user-agent'}) {
+      $Header{'user-agent'} = RemoveComments($Header{'user-agent'});
+      ### well-formed?
+      if ($Header{'user-agent'} =~ /^([^\/ ]+\/[^\/ ]+ ?)+$/) {
+        @Clients = ParseUserAgent($Header{'user-agent'});
+      } else {
+        # snip and add known well-formed agents from the trailing end
+        while ($Header{'user-agent'} =~ /(((Hamster)|(Hamster-Pg)|(KorrNews)|(OE-Tools)|(Mime-proxy))(\/[^\/ ]+))$/) {
+          push @Clients, ParseUserAgent($1);
+          $Header{'user-agent'} =~ s/ [^\/ ]+\/[^\/ ]+$//;
+        }
+        ### special cases
+        # remove 'http://open-news-network.org'
+        $Header{'user-agent'} =~ s/^https?:\/\/open-news-network.org(\S+)?//;
+        # Thunderbird
+        if ($Header{'user-agent'} =~ /((Mozilla[- ])?Thunderbird) ?([0-9.]+)?/) {
+          $Client  = 'Thunderbird';
+          $Version = $3;
+        # XP
+        } elsif ($Header{'user-agent'} =~ /((TrueXP|FreeXP|XP2(\/Agent)?)) \/(.+)$/) {
+          $Client  = $1;
+          $Version = $4;
+          $Client  = 'XP2' if $Client eq 'XP2/Agent';
+        ### most general case
+        # client version
+        # client/version
+        # client/32 version
+        # - version may end in one non-numeric character
+        # - including trailing beta/pre/...
+        # 1) client:   (([^0-9]+)|(\D+\/\d+))
+        # 2) version:  (\S+\d\D?)
+        # 3) trailing: (( alpha\d?)|( beta\d?)|( rc\d)| pre| trialware)?
+        } elsif ($Header{'user-agent'} =~ /^(([^0-9]+)|(\D+\/\d+))[\/ ]((\S+\d\D?)(( alpha\d?)|( beta\d?)|( rc\d)| pre| trialware)?)$/) {
+          $Client  = $1;
+          $Version = $4;
+        ### some very special cases
+        # SeaMonkey/nn
+        } elsif ($Header{'user-agent'} =~ /SeaMonkey\/([0-9.]+)/) {
+          $Client  = 'Seamonkey';
+          $Version = $1;
+        # Emacs nn/Gnus nn
+        } elsif ($Header{'user-agent'} =~ /Emacs [0-9.]+\/Gnus ([0-9.]+)/) {
+          $Client  = 'Gnus';
+          $Version = $1;
+        # failed to parse
+        } else {
+          $Client = $Header{'user-agent'};
+        }
+        # count client, if found
+        if ($Client) {
+          my %UserAgent = (agent   => $Client,
+                           version => $Version);
+          push @Clients, { %UserAgent };
+        } else {
+          &Bleat(1,sprintf("%s FAILED", $Header{'message-id'})) if !@Clients;
+        }
+      }
+    }
+
+    if (@Clients) {
+      $Postings{'ALL'}{'ALL'}++;
+      foreach (@Clients) {
+        # filter agents for User-Agent with multiple agents
+        next if $#Clients && exists($DropAgent{lc($_->{'agent'})});
+        # remove whitespace
+        $_->{'agent'}   =~ s/^\s+|\s+$//g;
+        $_->{'version'} =~ s/^\s+|\s+$//g if $_->{'version'};
+        # encode to utf-8, if necessary
+        $_->{'agent'}   = encode('UTF-8', $_->{'agent'})   if $_->{'agent'} =~ /[\x80-\x{ffff}]/;
+        $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/;
+        # truncate overlong clients or versions
+        $_->{'agent'}   = substr($_->{'agent'}, 0, 150)  if length($_->{'agent'}) > 150;
+        $_->{'version'} = substr($_->{'version'}, 0, 50) if $_->{'version'} and length($_->{'version'}) > 50;
+        # special cases
+        # Mozilla
+        $_->{'agent'} = 'Mozilla' if $_->{'agent'} eq '•Mozilla';
+        $_->{'agent'} =~ s/^Mozilla //;
+        # Forte Agent
+        $_->{'agent'} = 'Forte Agent' if $_->{'agent'} eq 'ForteAgent';
+        if ($_->{'agent'} eq 'Forte Agent') {
+          $_->{'version'} =~ s/-/\//;
+          $_->{'version'} = '' if $_->{'version'} eq '32Bit';
+        }
+        # count client ('ALL') and client/version (if version is present)
+        $Postings{$_->{'agent'}}{'ALL'}++;
+        $Postings{$_->{'agent'}}{$_->{'version'}}++ if $_->{'version'};
+
+        printf("%s: %s {%s}\n", $Header{'message-id'}, $_->{'agent'},
+                                $_->{'version'} ? $Postings{$_->{'agent'}}{$_->{'version'}} : '')
+                                if ($MID or $Debug && $Debug >1);
+      }
+    }
+  };
+
+  # delete old data for that month
+  if (!$Test) {
+    $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s WHERE month = ?",
+                                     $DBClients),undef,$Month)
+      or &Bleat(2,sprintf("Can't delete old client data for %s from %s: ".
+                          "$DBI::errstr\n",$Month,$DBClients));
+  };
+
+  foreach my $Client (sort keys %Postings) {
+    foreach my $Version (sort keys %{$Postings{$Client}}) {
+      printf ("%s {%s}: %d\n",$Client,$Version,$Postings{$Client}{$Version}) if $Debug;
+
+      if (!$Test) {
+        # write to database
+        $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s ".
+                                              "(month,client,version,postings) ".
+                                              "VALUES (?, ?, ?, ?)",$DBClients));
+        $DBQuery->execute($Month, $Client, $Version, $Postings{$Client}{$Version})
+          or &Bleat(2,sprintf("Can't write groups data for %s/%s/%s to %s: ".
+                              "$DBI::errstr\n",$Month,$Client,$Version,$DBClients));
+        $DBQuery->finish;
+      };
+    }
+  };
+
+};
+
+sub GetHeaders {
+### ----------------------------------------------------------------------------
+### get (newsgroups and) raw headers from database
+### IN : $DBHandle: database handle
+###      $DBRaw   : database table for raw data (to read from)
+###      $Month   : current month to do
+###      $MID     : specific Message-ID to fetch (testing purposes)
+### OUT: DBI statement handle
+  my ($DBHandle,$DBRaw,$Month,$MID) = @_;
+
+  my $DBQuery;
+
+  if (!$MID) {
+    # get raw header data from raw table for given month
+    $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ".
+                                             "WHERE day LIKE ? AND NOT disregard",
+                                             $DBRaw));
+    $DBQuery->execute($Month.'-%')
+      or &Bleat(2,sprintf("Can't get header data for %s from %s: ".
+                          "$DBI::errstr\n",$Month,$DBRaw));
+  } else {
+    $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ".
+                                             "WHERE mid = ?", $DBRaw));
+    $DBQuery->execute($MID)
+      or &Bleat(2,sprintf("Can't get header data for %s from %s: ".
+                          "$DBI::errstr\n",$MID,$DBRaw));
+  }
+  return $DBQuery;
+}
+
+sub CheckTLH {
+### ----------------------------------------------------------------------------
+### count newsgroups from legal TLH(s)
+### IN : $Newsgroups: comma separated list of newsgroups
+###      $TLH       : (reference to an array of) legal TLH(s)
+### OUT: number of newsgroups from legal TLH(s)
+  my ($Newsgroups,$TLH) = @_;
+
+  my (@TLH,$GroupCount);
+
+  # fill @TLH from $TLH, which can be an array reference or a scalar value
+  if (ref($TLH) eq 'ARRAY') {
+    @TLH = @{$TLH};
+  } else {
+    push @TLH, $TLH;
+  }
+
+  # remove whitespace from contents of Newsgroups:
+  chomp($Newsgroups);
+  $Newsgroups =~ s/\s//;
+  for (split /,/, $Newsgroups) {
+    my $Newsgroup = $_;
+    foreach (@TLH) {
+      # increment $GroupCount if $Newsgroup starts with $TLH
+      $GroupCount++ if $Newsgroup =~ /^$_/;
+    }
+  };
+
+  return $GroupCount;
+}
+
+sub RemoveComments {
+### ----------------------------------------------------------------------------
+### remove comments and other junk from header
+### IN : $Header: a header
+### OUT: the header, with comments and other junk removed
+  my $Header = shift;
+
+  # decode MIME encoded words
+  if ($Header =~ /=\?\S+\?[BQ]\?/) {
+    $Header = decode("MIME-Header",$Header);
+  }
+  
+  # remove nested comments from '(' to first ')'
+  while ($Header =~ /\([^)]+\)/) {
+    $Header =~ s/\([^()]+?\)//;
+  }
+
+  # remove dangling ')'
+  $Header =~ s/\S+\)//;
+
+  # remove from dangling '(' to end of header
+  $Header =~ s/\(.+$//;
+
+  # remove from '[' to first ']'
+  $Header =~ s/\[[^\[\]]+?\]//;
+
+  # remove 'Nr. ... lebt'
+  $Header =~ s/Nr\. \d+ lebt//;
+
+  # remove nn:nn:nn
+  $Header =~ s/\d\d:\d\d:\d\d//;
+
+  # remove 'mm/... '
+  $Header =~ s/\/mm\/\S+//;
+
+  # remove ' DE' / _DE'
+  $Header =~ s/[ _]DE//;
+
+  # remove trailing 'eol' or '-shl'
+  # or ml-inews[-sig]
+  $Header =~ s/(eol)|(-shl)|(ml-inews(-sig)?)$//;
+
+  # remove from ';' or ',' (CrossPoint)
+  # or '&' to end of header
+  $Header =~ s/[;,&].+$//;
+
+  # remove from 'by ' or 'unter Windows' or '@ Windows'
+  # to end of header
+  $Header =~ s/((by )|(unter +Windows)|(@ Windows)).+$//;
+
+  # remove superfluous whitespace in header
+  # and whitespace around header
+  $Header =~ s/\s+/ /g;
+  $Header =~ s/^\s+|\s+$//g;
+
+  return $Header;
+}
+
+sub ParseXNewsreader {
+### ----------------------------------------------------------------------------
+### parse X-Newsreader header (client and version, if present)
+### IN : $XNR: a X-Newsreader header
+### OUT: client and version, if present
+  my $XNR = shift;
+
+  my ($Client, $Version);
+
+  foreach (split(/ /,$XNR)) {
+    # add to client name if no digit present
+    if (!/\d[0-9.]/ or /\/\d$/) {
+      $Client .= $_ . ' ' ;
+    # otherwise, use as version and terminate parsing
+    } else {
+      $Version = $_;
+      last;
+    }
+  }
+
+  # remove trailing whitespace
+  $Client =~ s/\s+$// if $Client;
+
+  # set $Version
+  $Version = '' if !$Version;
+   
+  return $Client, $Version;
+}
+
+
+sub ParseUserAgent {
+### ----------------------------------------------------------------------------
+### parse User-Agent header (agent and version)
+### IN : $UserAgent: a User-Agent header
+### OUT: array of hashes (agent/version)
+  my $UserAgent = shift;
+
+  my @UserAgents;
+
+  # a well-formed User-Agent header will contain pairs of
+  # client/version, i.e. 'slrn/0.9.7.3'
+  foreach (split(/ /,$UserAgent)) {
+    my %UserAgent;
+    /^(.+)\/(.+)$/;
+    $UserAgent{'agent'}   = $1;
+    $UserAgent{'version'} = $2;
+    push @UserAgents, { %UserAgent };
+  }
+  
+  return @UserAgents;
+}
+
 __END__
 
 ################################ Documentation #################################
@@ -391,7 +734,7 @@ gatherstats - process statistical data from a raw source
 
 =head1 SYNOPSIS
 
-B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] [B<--conffile> I<filename>]
+B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--hostsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--conffile> I<filename>]
 
 =head1 REQUIREMENTS
 
@@ -438,23 +781,34 @@ override that default through the B<--groupsdb> option.
 =item B<hosts> (postings from host per month)
 
 B<gatherstats> will examine Injection-Info:, X-Trace: and Path:
-headers and try to normalize them. Groups not in I<TLH> will be
+headers and try to normalize them. The sum of all detected hosts will
-ignored. The sum of all detected hosts will also saved for each month.
+also be saved for each month. Groups not in I<TLH> will be ignored.
 
 Data is written to I<DBTableHosts> (see L<doc/INSTALL>); you can
 override that default through the B<--hostsdb> option.
 
+=item B<clients> (postings by client per month)
+
+B<gatherstats> will examine User-Agent:, X-Newsreader: and X-Mailer:
+headers and try to remove comments and non-standard contents. Clients
+and client versions are counted separately. The sum of all detected
+clients will also be saved for each month. Groups not in I<TLH> will
+be ignored. 
+
+Data is written to I<DBTableClnts> (see L<doc/INSTALL>); you can
+override that default through the B<--clientsdb> option.
+
 =back
 
 =head2 Configuration
 
 B<gatherstats> will read its configuration from F<newsstats.conf>
-which should be present in etc/ via Config::Auto or from a configuration file
+which should be present in etc/ via Config::Auto or from a configuration
-submitted by the B<--conffile> option.
+file submitted by the B<--conffile> option.
 
 See L<doc/INSTALL> for an overview of possible configuration options.
 
-You can override configuration options via the B<--hierarchy>,
+You can override configuration options by using the B<--hierarchy>,
 B<--rawdb>, B<--groupsdb>, B<--clientsdb> and B<--hostsdb> options,
 respectively.
 
@@ -464,15 +818,15 @@ respectively.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
 
 =item B<-d>, B<--debug>
 
-Output debugging information to STDOUT while processing (number of
+Print debugging information to STDOUT while processing (number of
 postings per group).
 
 =item B<-t>, B<--test>
@@ -484,15 +838,17 @@ conjunction with B<--test> ... everything else seems a bit pointless.
 
 Set processing period to a single month in YYYY-MM format or to a time
 period between two month in YYYY-MM:YYYY-MM format (two month, separated
-by a colon).
+by a colon). Defaults to last month.
 
 =item B<-s>, B<--stats> I<type>
 
-Set processing type to one of I<all>, I<groups> or I<hosts>. Defaults
+Set processing type to one of I<all>, I<groups>, I<hosts> or I<clients>.
-to all.
+Defaults to I<all>.
 
 =item B<-c>, B<--checkgroups> I<filename template>
 
+Relevant only for newsgroup stats (I<groups>).
+
 Check each group against a list of valid newsgroups read from a file,
 one group on each line and ignoring everything after the first
 whitespace (so you can use a file in checkgroups format or (part of)
@@ -511,10 +867,12 @@ Newsgroups not found in the checkgroups file will be dropped (and
 logged to STDERR), and newsgroups found there but having no postings
 will be added with a count of 0 (and logged to STDERR).
 
-=item B<--hierarchy> I<TLH> (newsgroup hierarchy)
+=item B<--hierarchy> I<TLH> (newsgroup hierarchy/hierarchies)
 
 Override I<TLH> from F<newsstats.conf>.
 
+I<TLH> can be a single word or a comma-separated list.
+
 =item B<--rawdb> I<table> (raw data table)
 
 Override I<DBTableRaw> from F<newsstats.conf>.
@@ -523,17 +881,17 @@ Override I<DBTableRaw> from F<newsstats.conf>.
 
 Override I<DBTableGrps> from F<newsstats.conf>.
 
-=item B<--clientsdb> I<table> (client data table)
-
-Override I<DBTableClnts> from F<newsstats.conf>.
-
 =item B<--hostsdb> I<table> (host data table)
 
 Override I<DBTableHosts> from F<newsstats.conf>.
 
+=item B<--clientsdb> I<table> (client data table)
+
+Override I<DBTableClnts> from F<newsstats.conf>.
+
 =item B<--conffile> I<filename>
 
-Load configuration from I<filename> instead of F<newsstats.conf>.
+Read configuration from I<filename> instead of F<newsstats.conf>.
 
 =back
 

bin/groupstats.pl

@@ -283,7 +283,7 @@ See L<doc/README>.
 
 =head1 DESCRIPTION
 
-This script create reports on newsgroup usage (number of postings per
+This script creates reports on newsgroup usage (number of postings per
 group per month) taken from result tables created by
 B<gatherstats.pl>.
 
@@ -291,16 +291,16 @@ B<gatherstats.pl>.
 
 =head3 Time period and newsgroups
 
-The time period to act on defaults to last month; you can assign another
+The time period to act on defaults to last month; you can assign
-time period or a single month (or drop all time constraints) via the
+another time period or a single month (or drop all time constraints)
-B<--month> option (see below).
+via the B<--month> option (see below).
 
 B<groupstats> will process all newsgroups by default; you can limit
-processing to only some newsgroups by supplying a list of those groups via
+processing to only some newsgroups by supplying a list of those groups
-B<--newsgroups> option (see below). You can include hierarchy levels in
+via B<--newsgroups> option (see below). You can include hierarchy
-the output by adding the B<--sums> switch (see below). Optionally
+levels in the output by adding the B<--sums> switch (see below).
-newsgroups not present in a checkgroups file can be excluded from output,
+Optionally newsgroups not present in a checkgroups file can be excluded
-sse B<--checkgroups> below.
+from output, sse B<--checkgroups> below.
 
 =head3 Report type
 
@@ -321,26 +321,27 @@ below.
 =head3 Sorting and formatting the output
 
 By default, all results are grouped by month; you can group results by
-newsgroup instead via the B<--groupy-by> option. Within those groups, the
+newsgroup instead via the B<--groupy-by> option. Within those groups,
-list of newsgroups (or months) is sorted alphabetically (or
+the list of newsgroups (or months) is sorted alphabetically (or
-chronologically, respectively) ascending. You can change that order (and
+chronologically, respectively) ascending. You can change that order
-sort by number of postings) with the B<--order-by> option. For details and
+(and sort by number of postings) with the B<--order-by> option. For
-exceptions, please see below.
+details and exceptions, please see below.
 
 The results will be formatted as a kind of table; you can change the
-output format to a simple list or just a list of newsgroups and number of
+output format to a simple list or just a list of newsgroups and number
-postings with the B<--format> option. Captions will be added by means of
+of postings with the B<--format> option. Captions will be added by means
-the B<--caption> option; all comments (and captions) can be supressed by
+of the B<--caption> option; all comments (and captions) can be supressed
-using B<--nocomments>.
+by using B<--nocomments>.
 
-Last but not least you can redirect all output to a number of files, e.g.
+Last but not least you can redirect all output to a number of files,
-one for each month, by submitting the B<--filetemplate> option, see below.
+e.g. one for each month, by submitting the B<--filetemplate> option,
+see below.
 
 =head2 Configuration
 
 B<groupstats> will read its configuration from F<newsstats.conf>
-which should be present in etc/ via Config::Auto or from a configuration file
+which should be present in etc/ via Config::Auto or from a configuration
-submitted by the B<--conffile> option.
+file submitted by the B<--conffile> option.
 
 See doc/INSTALL for an overview of possible configuration options.
 
@@ -352,18 +353,18 @@ You can override some configuration options via the B<--groupsdb> option.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
 
 =item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
 
 Set processing period to a single month in YYYY-MM format or to a time
 period between two month in YYYY-MM:YYYY-MM format (two month, separated
 by a colon). By using the keyword I<all> instead, you can set no
-processing period to process the whole database.
+processing period to process the whole database. Defaults to last month.
 
 =item B<-n>, B<--newsgroups> I<newsgroup(s)>
 
@@ -388,17 +389,20 @@ See the B<gatherstats> man page for details.
 This option does not work together with the B<--checkgroups> option as
 all "virtual" groups will not be present in the checkgroups file.
 
+False by default.
+
 =item B<--checkgroups> I<filename>
 
-Restrict output to those newgroups present in a file in checkgroups format
+Restrict output to those newgroups present in a file in checkgroups
-(one newgroup name per line; everything after the first whitespace on each
+format (one newgroup name per line; everything after the first
-line is ignored). All other newsgroups will be removed from output.
+whitespace on each line is ignored). All other newsgroups will be
+removed from output.
 
-Contrary to B<gatherstats>, I<filename> is not a template, but refers to
+Contrary to B<gatherstats>, I<filename> is not a template, but refers
-a single file in checkgroups format.
+to a single file in checkgroups format.
 
-The B<--sums> option will not work together with this option as "virtual"
+The B<--sums> option will not work together with this option as
-groups will not be present in the checkgroups file.
+"virtual" groups will not be present in the checkgroups file.
 
 =item B<-r>, B<--report> I<default|average|sums>
 
@@ -406,8 +410,8 @@ Choose the report type: I<default>, I<average> or I<sums>
 
 By default, B<groupstats> will report the number of postings for each
 newsgroup in each month. But it can also report the average number of
-postings per group for all months or the total sum of postings per group
+postings per group for all months or the total sum of postings per
-for all months.
+group for all months.
 
 For report types I<average> and I<sums>, the B<group-by> option has no
 meaning and will be silently ignored (see below).
@@ -426,12 +430,13 @@ Set the boundary type to one of I<default>, I<level>, I<average> or
 I<sums>.
 
 By default, all newsgroups with more postings per month than the upper
-boundary and/or less postings per month than the lower boundary will be
+boundary and/or less postings per month than the lower boundary will
-excluded from further processing. For the default report that means each
+be
-month only newsgroups with a number of postings between the boundaries
+excluded from further processing. For the default report that means
-will be displayed. For the other report types, newsgroups with a number of
+each month only newsgroups with a number of postings between the
-postings exceeding the boundaries in all (!) months will not be
+boundaries will be displayed. For the other report types, newsgroups
-considered.
+with a number of postings exceeding the boundaries in all (!) months
+will not be considered.
 
 For example, lets take a list of newsgroups like this:
 
@@ -461,22 +466,23 @@ month. If you want to list all newsgroups with more than 25 postings
 I<in total>, you'll have to set the boundary type to I<sum>, see below.
 
 A boundary type of I<level> will show only those newsgroups - at all -
-that satisfy the boundaries in each and every single month. With the above
+that satisfy the boundaries in each and every single month. With the
-list of newsgroups and
+above list of newsgroups and
 C<groupstats --month 2012-01:2012-03 --lower 25 --boundary level --report sums>,
 you'll get this result:
 
     ----- All months:
     de.comp.datenbanken.ms-access        293
 
-de.comp.datenbanken.mysql has not been considered because it had less than
+de.comp.datenbanken.mysql has not been considered because it had less
-25 postings in 2012-02 (only).
+than 25 postings in 2012-02 (only).
 
-You can use that to get a list of newsgroups that have more (or less) then
+You can use that to get a list of newsgroups that have more (or less)
-x postings in every month during the whole reporting period.
+then x postings in every month during the whole reporting period.
 
-A boundary type of I<average> will show only those newsgroups - at all -that
+A boundary type of I<average> will show only those newsgroups - at
-satisfy the boundaries on average. With the above list of newsgroups and
+all - that satisfy the boundaries on average. With the above list of
+newsgroups and
 C<groupstats --month 2012-01:2012-03 --lower 25 --boundary avg --report sums>,
 you'll get this result:
 
@@ -491,8 +497,8 @@ The average number of postings in the three groups is:
     de.comp.datenbanken.mysql          48.33
 
 Last but not least, a boundary type of I<sums> will show only those
-newsgroups - at all - that satisfy the boundaries with the total sum of
+newsgroups - at all - that satisfy the boundaries with the total sum
-all postings during the reporting period. With the above list of
+of all postings during the reporting period. With the above list of
 newsgroups and
 C<groupstats --month 2012-01:2012-03 --lower 25 --boundary sum --report sums>,
 you'll finally get this result:
@@ -505,8 +511,8 @@ you'll finally get this result:
 
 =item B<-g>, B<--group-by> I<month[-desc]|newsgroups[-desc]>
 
-By default, all results are grouped by month, sorted chronologically in
+By default, all results are grouped by month, sorted chronologically
-ascending order, like this:
+in ascending order, like this:
 
     ----- 2012-01:
     de.comp.datenbanken.ms-access         84
@@ -525,8 +531,8 @@ B<--group-by> I<newsgroup>:
     2012-01         88
     2012-02         21
 
-By appending I<-desc> to the group-by option parameter, you can reverse
+By appending I<-desc> to the group-by option parameter, you can
-the sort order - e.g. B<--group-by> I<month-desc> will give:
+reverse the sort order - e.g. B<--group-by> I<month-desc> will give:
 
     ----- 2012-02:
     de.comp.datenbanken.ms-access        126
@@ -541,9 +547,9 @@ this option will therefore be ignored.
 =item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
 
 Within each group (a single month or single newsgroup, see above), the
-report will be sorted by newsgroup names in ascending alphabetical order
+report will be sorted by newsgroup names in ascending alphabetical
-by default. You can change the sort order to descending or sort by number
+order by default. You can change the sort order to descending or sort
-of postings instead.
+by number of postings instead.
 
 =item B<-f>, B<--format> I<pretty|list|dump>
 
@@ -587,19 +593,19 @@ False by default.
 
 =item B<--comments|--nocomments>
 
-Add comments (group headers) to I<dump> and I<pretty> output. True by default
+Add comments (group headers) to I<dump> and I<pretty> output. True by
-as logn as B<--filetemplate> is not set.
+default as long as B<--filetemplate> is not set.
 
-Use I<--nocomments> to suppress anything except newsgroup names/months and
+Use I<--nocomments> to suppress anything except newsgroup names/months
-numbers of postings.
+and numbers of postings.
 
 =item B<--filetemplate> I<filename template>
 
-Save output to file(s) instead of dumping it to STDOUT. B<groupstats> will
+Save output to file(s) instead of dumping it to STDOUT. B<groupstats>
-create one file for each month (or each newsgroup, accordant to the
+will create one file for each month (or each newsgroup, according to
-setting of B<--group-by>, see above), with filenames composed by adding
+the setting of B<--group-by>, see above), with filenames composed by
-year and month (or newsgroup names) to the I<filename template>, for
+adding year and month (or newsgroup names) to the I<filename template>,
-example with B<--filetemplate> I<stats>:
+for example with B<--filetemplate> I<stats>:
 
     stats-2012-01
     stats-2012-02
@@ -611,7 +617,7 @@ Override I<DBTableGrps> from F<newsstats.conf>.
 
 =item B<--conffile> I<filename>
 
-Load configuration from I<filename> instead of F<newsstats.conf>.
+Read configuration from I<filename> instead of F<newsstats.conf>.
 
 =back
 
@@ -635,9 +641,9 @@ by number of postings, descending, in I<pretty> format:
 
     groupstats --upper 30 --order-by postings-desc
 
-Show the total of all postings for the year of 2010 for all groups that
+Show the total of all postings for the year of 2010 for all groups
-had 30 postings or less in every single month in that year, ordered by
+that had 30 postings or less in every single month in that year,
-number of postings in descending order:
+ordered by number of postings in descending order:
 
     groupstats -m 2010-01:2010-12 -u 30 -b level -r sums -o postings-desc
 
@@ -651,7 +657,6 @@ machine-readable form (without formatting):
 
     groupstats -m 2010-01:2010-12 -f dump --filetemplate stats
 
-
 =head1 FILES
 
 =over 4
@@ -685,7 +690,7 @@ L<doc/README>
 
 =item -
 
-l>doc/INSTALL>
+L<doc/INSTALL>
 
 =item -
 

bin/hoststats.pl

@@ -1,9 +1,9 @@
 #! /usr/bin/perl
 #
-# cliservstats.pl
+# hoststats.pl
 #
-# This script will get statistical data on client (newsreader) and
+# This script will get statistical data on server (host) usage
-# server (host) usage from a database.
+# from a database.
 #
 # It is part of the NewsStats package.
 #
@@ -31,7 +31,7 @@ Getopt::Long::config ('bundling');
 ### read commandline options
 my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat,
     $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy,
-    $OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile);
+    $OptReportType,$OptSums,$UppBound,$OptConfFile);
 GetOptions ('c|captions!'    => \$OptCaptions,
             'comments!'      => \$OptComments,
             'db=s'           => \$OptDB,
@@ -44,7 +44,6 @@ GetOptions ('c|captions!'    => \$OptCaptions,
             'o|order-by=s'   => \$OptOrderBy,
             'r|report=s'     => \$OptReportType,
             's|sums!'        => \$OptSums,
-            't|type=s'       => \$OptType,
             'u|upper=i'      => \$UppBound,
             'conffile=s'     => \$OptConfFile,
             'h|help'         => \&ShowPOD,
@@ -52,15 +51,6 @@ GetOptions ('c|captions!'    => \$OptCaptions,
 # parse parameters
 # $OptComments defaults to TRUE if --filetemplate is not used
 $OptComments = 1 if (!$OptFileTemplate && !defined($OptComments));
-# parse $OptType
-if ($OptType) {
-  if ($OptType =~ /(host|server)s?/i) {
-    $OptType = 'host';
-  } elsif ($OptType =~ /(newsreader|client)s?/i) {
-    $OptType  = 'client';
-  }
-}
-&Bleat(2, "Please use '--type server' or '-type newsreader'.") if !$OptType;
 # parse $OptReportType
 if ($OptReportType) {
   if ($OptReportType =~ /sums?/i) {
@@ -74,14 +64,8 @@ if ($OptReportType) {
 my %Conf = %{ReadConfig($OptConfFile)};
 
 ### set DBTable
-if ($OptDB) {
-  $Conf{'DBTable'} = $OptDB;
-}
-elsif ($OptType eq 'host') {
 $Conf{'DBTable'} = $Conf{'DBTableHosts'};
-} else {
+$Conf{'DBTable'} = $OptDB if $OptDB;
-  $Conf{'DBTable'} = $Conf{'DBTableClnts'};
-}
   
 ### init database
 my $DBHandle = InitDB(\%Conf,1);
@@ -97,14 +81,14 @@ my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth);
 # with placeholders as well as a list of names to bind to them
 my ($SQLWhereNames,@SQLBindNames);
 if ($OptNames) {
-  ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType);
+  ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,'host');
   # bail out if --names is invalid
   &Bleat(2,"--names option has an invalid format!")
     if !$SQLWhereNames;
 }
 
 ### build SQL WHERE clause
-my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",$OptType);
+my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",'host');
 my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
                                     $ExcludeSums,
                                     &SQLSetBounds('default',$LowBound,$UppBound));
@@ -118,8 +102,8 @@ $OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
                          and $OptNames and $OptNames !~ /[:*%]/);
 # parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
 # if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
-my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType);
+my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, 'host');
-# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy)
+# $GroupBy will contain 'month' or 'host' (parsed result of $OptGroupBy)
 # set it to 'month' or 'key' for OutputData()
 $GroupBy = ($GroupBy eq 'month') ? 'month' : 'key';
 
@@ -128,19 +112,19 @@ my $SQLSelect;
 my $SQLGroupClause = '';
 my $Precision = 0;       # number of digits right of decimal point for output
 if ($OptReportType and $OptReportType ne 'default') {
-  $SQLGroupClause = "GROUP BY $OptType";
+  $SQLGroupClause = "GROUP BY host";
   # change $SQLOrderClause: replace everything before 'postings'
   $SQLOrderClause =~ s/BY.+postings/BY postings/;
-  $SQLSelect = "'All months',$OptType,SUM(postings)";
+  $SQLSelect = "'All months',host,SUM(postings)";
   # change $SQLOrderClause: replace 'postings' with 'SUM(postings)'
   $SQLOrderClause =~ s/postings/SUM(postings)/;
  } else {
-  $SQLSelect = "month,$OptType,postings";
+  $SQLSelect = "month,host,postings";
 };
 
 ### get length of longest name delivered by query
 ### for formatting purposes
-my $Field = ($GroupBy eq 'month') ? $OptType : 'month';
+my $Field = ($GroupBy eq 'month') ? 'host' : 'month';
 my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
                                               $Field,'postings',$SQLWhereClause,
                                               '',@SQLBindNames);
@@ -155,8 +139,8 @@ $DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s',
                                       $SQLOrderClause));
 # execute query
 $DBQuery->execute(@SQLBindNames)
-  or &Bleat(2,sprintf("Can't get %s data for %s from %s.%s: %s\n",
+  or &Bleat(2,sprintf("Can't get host data for %s from %s.%s: %s\n",
-                      $OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
+                      $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
                       $DBI::errstr));
 
 ### output results
@@ -175,7 +159,7 @@ if ($OptCaptions && $OptComments) {
   $LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames)))
     if $OptNames;
   # print boundaries, if set
-  my $CaptionBoundary= '(counting only month fulfilling this condition)';
+  my $CaptionBoundary= '(counting only months fulfilling this condition)';
   $LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n",
          $LowBound ? $LowBound : '',$LowBound ? '=>' : '',
          $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary)
@@ -201,11 +185,11 @@ __END__
 
 =head1 NAME
 
-cliservstats - create reports on host or client usage
+hoststats - create reports on host usage
 
 =head1 SYNOPSIS
 
-B<cliservstats> B<-t> I<host|client> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)|client(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
+B<hoststats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
 
 =head1 REQUIREMENTS
 
@@ -213,9 +197,8 @@ See L<doc/README>.
 
 =head1 DESCRIPTION
 
-This script create reports on newsgroup usage (number of postings from
+This script creates reports on newsgroup usage (number of postings from
-each host or using each client per month) taken from result tables
+each host) taken from result tables created by B<gatherstats.pl>.
-created by B<gatherstats.pl>.
 
 =head2 Features and options
 
@@ -225,9 +208,9 @@ The time period to act on defaults to last month; you can assign another
 time period or a single month (or drop all time constraints) via the
 B<--month> option (see below).
 
-B<cliservstats> will process all hosts or clients by default; you can
+B<hoststats> will process all hosts by default; you can limit
-limit processing to only some hosts or clients by supplying a list of
+processing to only some hosts by supplying a list of those names by
-those names by using the B<--names> option (see below).
+using the B<--names> option (see below).
 
 =head3 Report type
 
@@ -238,33 +221,34 @@ or all postings summed up; for details, see below.
 
 Furthermore you can set an upper and/or lower boundary to exclude some
 results from output via the B<--lower> and B<--upper> options,
-respectively. By default, all hosts/clients with more and/or less
+respectively. By default, all hosts with more and/or less postings
-postings per month will be excluded from the result set (i.e. not
+per month will be excluded from the result set (i.e. not shown and
-shown and not considered forsum reports).
+not considered for sum reports).
 
 =head3 Sorting and formatting the output
 
 By default, all results are grouped by month; you can group results by
-hosts/clients instead via the B<--group-by> option. Within those
+hosts instead via the B<--group-by> option. Within those groups, the
-groups, the list of hosts/clients (or months) is sorted alphabetically
+list of hosts (or months) is sorted alphabetically (or chronologically,
-(or chronologically, respectively) ascending. You can change that order
+respectively) ascending. You can change that order (and sort by number
-(and sort by number of postings) with the B<--order-by> option. For
+of postings) with the B<--order-by> option. For details and exceptions,
-details and exceptions, please see below.
+please see below.
 
 The results will be formatted as a kind of table; you can change the
 output format to a simple list or just a list of names and number of
-postings with the B<--format> option. Captions will be added by means of
+postings with the B<--format> option. Captions will be added by means
-the B<--caption> option; all comments (and captions) can be supressed by
+of the B<--caption> option; all comments (and captions) can be
-using B<--nocomments>.
+supressed by using B<--nocomments>.
 
-Last but not least you can redirect all output to a number of files, e.g.
+Last but not least you can redirect all output to a number of files,
-one for each month, by submitting the B<--filetemplate> option, see below.
+e.g. one for each month, by submitting the B<--filetemplate> option,
+see below.
 
 =head2 Configuration
 
-B<cliservstats> will read its configuration from F<newsstats.conf>
+B<hoststats> will read its configuration from F<newsstats.conf>
-which should be present in etc/ via Config::Auto or from a configuration file
+which should be present in etc/ via Config::Auto or from a configuration
-submitted by the B<--conffile> option.
+file submitted by the B<--conffile> option.
 
 See doc/INSTALL for an overview of possible configuration options.
 
@@ -276,27 +260,22 @@ You can override some configuration options via the B<--db> option.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
-
-=item B<-t>, B<--type> I<host|client>
-
-Create report for hosts (servers) or clients (newsreaders), using
-I<DBTableHosts> or I<DBTableClnts> respectively.
 
 =item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
 
 Set processing period to a single month in YYYY-MM format or to a time
 period between two month in YYYY-MM:YYYY-MM format (two month, separated
 by a colon). By using the keyword I<all> instead, you can set no
-processing period to process the whole database.
+processing period to process the whole database. Defaults to last month.
 
 =item B<-n>, B<--names> I<name(s)>
 
-Limit processing to a certain set of host or client names. I<names(s)>
+Limit processing to a certain set of hostnames. I<names(s)>
 can be a single name (eternal-september.org), a group of names
 (*.inka.de) or a list of either of these, separated by colons, for
 example
@@ -306,15 +285,16 @@ example
 =item B<-s>, B<--sums|--nosums> (sum per month)
 
 Include a "virtual" host named "ALL" for every month in output,
-containing the sum of all detected hosts for that month.
+containing the sum of all detected hosts for that month. False
+by default.
 
 =item B<-r>, B<--report> I<default|sums>
 
 Choose the report type: I<default> or I<sums>
 
-By default, B<cliservstats> will report the number of postings for each
+By default, B<hoststats> will report the number of postings for each
-host/client in each month. But it can also report the total sum of postings
+host in each month. But it can also report the total sum of postings
-per host/client for all months.
+per host for all months.
 
 For report type I<sums>, the B<group-by> option has no meaning and
 will be silently ignored (see below).
@@ -327,18 +307,18 @@ Set the lower boundary. See below.
 
 Set the upper boundary.
 
-By default, all hosts/clients with more postings per month than the
+By default, all hosts with more postings per month than the upper
-upper boundary and/or less postings per month than the lower boundary
+boundary and/or less postings per month than the lower boundary
 will be excluded from further processing. For the default report that
-means each month only hosts/clients with a number of postings between
+means each month only hosts with a number of postings between the
-the boundaries will be displayed. For the sums report, hosts/clients
+boundaries will be displayed. For the sums report, hosts with a number
-with a number of postings exceeding the boundaries in all (!) months
+of postings exceeding the boundaries in all (!) months will not be
-will not be considered.
+considered.
 
 =item B<-g>, B<--group-by> I<month[-desc]|name[-desc]>
 
-By default, all results are grouped by month, sorted chronologically in
+By default, all results are grouped by month, sorted chronologically
-ascending order, like this:
+in ascending order, like this:
 
     # ----- 2012-01:
     arcor-online.net :  9379
@@ -349,8 +329,7 @@ ascending order, like this:
     individual.net   : 16768
     news.albasani.net:  7879
 
-The results can be grouped by host/client instead via
+The results can be grouped by host instead via B<--group-by> I<name>:
-B<--group-by> I<name>:
 
     ----- individual.net
     2012-01: 19525
@@ -362,8 +341,8 @@ B<--group-by> I<name>:
     2012-01: 9063
     2012-02: 7879
 
-By appending I<-desc> to the group-by option parameter, you can reverse
+By appending I<-desc> to the group-by option parameter, you can
-the sort order - e.g. B<--group-by> I<month-desc> will give:
+reverse the sort order - e.g. B<--group-by> I<month-desc> will give:
 
     # ----- 2012-02:
     arcor-online.net :  8606
@@ -379,8 +358,8 @@ will therefore be ignored.
 
 =item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
 
-Within each group (a single month or single host/client, see above),
+Within each group (a single month or single host, see above), the
-the report will be sorted by name (or month) in ascending alphabetical
+report will be sorted by host (or month) in ascending alphabetical
 order by default. You can change the sort order to descending or sort
 by number of postings instead.
 
@@ -426,19 +405,19 @@ False by default.
 
 =item B<--comments|--nocomments>
 
-Add comments (group headers) to I<dump> and I<pretty> output. True by default
+Add comments (group headers) to I<dump> and I<pretty> output. True by 
-as logn as B<--filetemplate> is not set.
+default as long as B<--filetemplate> is not set.
 
-Use I<--nocomments> to suppress anything except host/client names or months and
+Use I<--nocomments> to suppress anything except host names or months
-numbers of postings.
+and numbers of postings.
 
 =item B<--filetemplate> I<filename template>
 
-Save output to file(s) instead of dumping it to STDOUT. B<cliservstats> will
+Save output to file(s) instead of dumping it to STDOUT. B<hoststats>
-create one file for each month (or each host/client, accordant to the
+will create one file for each month (or each host, according to the
 setting of B<--group-by>, see above), with filenames composed by adding
-year and month (or host/client names) to the I<filename template>, for
+year and month (or hostnames) to the I<filename template>, for example
-example with B<--filetemplate> I<stats>:
+with B<--filetemplate> I<stats>:
 
     stats-2012-01
     stats-2012-02
@@ -446,11 +425,11 @@ example with B<--filetemplate> I<stats>:
 
 =item B<--db> I<database table>
 
-Override I<DBTableHosts> or I<DBTableClnts> from F<newsstats.conf>.
+Override I<DBTableHosts> from F<newsstats.conf>.
 
 =item B<--conffile> I<filename>
 
-Load configuration from I<filename> instead of F<newsstats.conf>.
+Read configuration from I<filename> instead of F<newsstats.conf>.
 
 =back
 
@@ -460,31 +439,30 @@ See L<doc/INSTALL>.
 
 =head1 EXAMPLES
 
-Show number of postings per group for lasth month in I<pretty> format:
+Show number of postings per host for lasth month in I<pretty> format:
 
-    cliservstats --type host
+    hoststats
 
 Show that report for January of 2010 and *.inka plus individual.net:
 
-    cliservstats --type host --month 2010-01 --names *.inka:individual.net:
+    hoststats --month 2010-01 --names *.inka:individual.net:
 
-Only show clients with 30 postings or less last month, ordered
+Only show hosts with 30 postings or less last month, ordered
 by number of postings, descending, in I<pretty> format:
 
-    cliservstats --type client --upper 30 --order-by postings-desc
+    hoststats --upper 30 --order-by postings-desc
 
 List number of postings per host for each month of 2010 and redirect
 output to one file for each month, named hosts-2010-01 and so on, in
 machine-readable form (without formatting):
 
-    cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts
+    hoststats -m 2010-01:2010-12 -f dump --filetemplate hosts
-
 
 =head1 FILES
 
 =over 4
 
-=item F<bin/cliservstats.pl>
+=item F<bin/hoststats.pl>
 
 The script itself.
 
@@ -513,7 +491,7 @@ L<doc/README>
 
 =item -
 
-l>doc/INSTALL>
+L<doc/INSTALL>
 
 =item -
 

bin/postingstats.pl

@@ -15,7 +15,8 @@
 #
 # Usage:
 # $~ groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups
-# $~ cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts
+# $~ hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts
+# $~ clientstats.pl --nocomments --sums --versions --format dump | postingstats.pl -t clients
 #
 
 BEGIN {
@@ -53,19 +54,22 @@ if (!$Type) {
   $Type = 'GroupStats';
 } elsif ($Type =~ /(host|server)s?/i) {
   $Type  = 'HostStats';
+} elsif ($Type =~ /(client|reader)s?/i) {
+  $Type  = 'ClientStats';
 };
 my $Timestamp = time;
 
 ##### ----- configuration --------------------------------------------
 my $TLH     = 'de';
 my %Heading = ('GroupStats'  => 'Postingstatistik fuer de.* im Monat '.$Month,
-               'HostStats'  => 'Serverstatistik fuer de.* im Monat '.$Month
+               'HostStats'   => 'Serverstatistik fuer de.* im Monat '.$Month,
+               'ClientStats' => 'Newsreaderstatistik fuer de.* im Monat '.$Month
               );
 my %TH      = ('counter'    => 'Nr.',
                'value'      => 'Anzahl',
                'percentage' => 'Prozent'
               );
-my %LeadIn  = ('GroupStats'  => <<GROUPSIN, 'HostStats' => <<HOSTSIN);
+my %LeadIn  = ('GroupStats'  => <<GROUPSIN, 'HostStats' => <<HOSTSIN, 'ClientStats' => <<CLIENTSIN);
 From: Thomas Hochstein <thh\@thh.name>
 Newsgroups: local.test
 Subject: Postingstatistik fuer de.* im Monat $Month
@@ -88,7 +92,18 @@ Content-Transfer-Encoding: 7bit
 User-Agent: postingstats.pl/$VERSION (NewsStats)
 
 HOSTSIN
-my %LeadOut  = ('GroupStats'  => <<GROUPSOUT, 'HostStats' => <<HOSTSOUT);
+From: Thomas Hochstein <thh\@thh.name>
+Newsgroups: local.test
+Subject: Newsreaderstatistik fuer de.* im Monat $Month
+Message-ID: <destat-clients-$Month.$Timestamp\@mid.news.szaf.org>
+Approved: thh\@thh.name
+Mime-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+User-Agent: postingstats.pl/$VERSION (NewsStats)
+
+CLIENTSIN
+my %LeadOut  = ('GroupStats'  => <<GROUPSOUT, 'HostStats' => <<HOSTSOUT, 'ClientStats' => <<CLIENTSOUT);
 
 Alle Zahlen wurden ermittelt auf einem Newsserver mit redundanter Anbin-
 dung fuer de.*  unter Anwendung ueblicher  Filtermassnahmen. Steuernach-
@@ -110,6 +125,19 @@ wurden, bleiben  erfasst, sofern sie das System  ueberhaupt (und vor der
 Loeschnachricht) erreicht haben.
 HOSTSOUT
 
+Alle Zahlen wurden ermittelt auf einem Newsserver mit redundanter Anbin-
+dung fuer de.*  unter Anwendung ueblicher  Filtermassnahmen. Steuernach-
+richten werden  nicht erfasst; Postings, die  supersedet  oder gecancelt
+wurden, bleiben  erfasst, sofern sie das System  ueberhaupt (und vor der
+Loeschnachricht) erreicht  haben.  Versionsangaben werden  nur gezaehlt,
+wenn Sie ermittelbar sind; daher kann die Summe der Newsreader-Versionen
+kleiner sein als die Postingzahl fuer den Newsreader.  Ausserdem koennen
+an einem Beitrag  mehrere Clients beteiligt  sein,  bspw. der Newsreader
+und ein  lokaler Server  wie der  Hamster.  Daher kann  die Summe  aller
+Newsreader groesser sein  als die Summe  der Postings;  auch ergeben die
+Prozentzahlen dementsprechend in der Summe mehr als 100%.
+CLIENTSOUT
+
 ##### ----- subroutines ----------------------------------------------
 
 sub Percentage {
@@ -124,35 +152,76 @@ sub Divider {
     return ':' . $Symbol x ($MaxLength+TABLEWIDTH) . ":\n";
 }
 
+sub SingleVersion {
+    my ($LastName,$RSubValue,$RValue,$RMaxLength) = @_;
+
+    # get version to add to client name
+    my ($Version) = keys %{$$RSubValue{$LastName}};
+    $Version =~ s/^- //;
+    # add version to client name by creating a new name
+    # and deleting the old one
+    my ($NameVersion) = $LastName . ' ' . $Version;
+    $$RValue{$NameVersion} = $$RValue{$LastName};
+    delete($$RValue{$LastName});
+    $$RMaxLength = length($NameVersion) if length($NameVersion) > $$RMaxLength;
+
+    # delete single version
+    delete($$RSubValue{$LastName});
+}
+
 ##### ----- main loop ------------------------------------------------
 
-my (%Value, $SumName, $SumTotal, $MaxLength);
+my (%Value, %SubValue, $SubCounter, $LastName, $SumName, $SumTotal,
-$MaxLength = 0;
+  $MaxLength);
+
 if ($Type eq 'GroupStats') {
     $SumName    = "$TLH.ALL";
     $TH{'name'} = 'Newsgroup'
 } elsif ($Type eq 'HostStats') {
     $SumName    = 'ALL';
-    $TH{'name'} = 'Server'
+    $TH{'name'} = 'Postingserver'
+} elsif ($Type eq 'ClientStats') {
+    $SumName    = 'ALL';
+    $TH{'name'} = 'Newsreader / Client'
 }
 
-# read from STDIN
+### read from STDIN
+$MaxLength = 0;
 while(<>) {
-    my ($Name, $Value) = split;
+    my ($Name, $Value) = $_ =~ /(.+) (\d+)$/;
     $SumTotal = $Value if $Name eq $SumName;
     next if $Name =~ /ALL$/;
+
+    # handle client versions
+    if ($Type eq 'ClientStats' and $Name =~ /^- /) {
+        $SubValue{$LastName}{$Name} = $Value;
+        $SubCounter++;
+    } else {
+        # clients with just one version
+        &SingleVersion($LastName,\%SubValue,\%Value,\$MaxLength)
+          if ($LastName && $SubCounter == 1);
+
+        # reset version counter and client name
+        $SubCounter = 0;
+        $LastName = $Name;
+        
         $Value{$Name} = $Value;
         $MaxLength = length($Name) if length($Name) > $MaxLength;
     }
+}
+# clients with just one version (last iteration)
+&SingleVersion($LastName,\%SubValue,\%Value,\$MaxLength)
+  if ($LastName && $SubCounter == 1);
 
-# print to STDOUT
+### print to STDOUT
+# calculate padding for $Heading
 my $PaddingLeft = ' ' x int((($MaxLength+TABLEWIDTH-2-length($Heading{$Type}))/2));
 my $PaddingRight = $PaddingLeft;
-$PaddingLeft .= ' ' if (length($Heading{$Type}) + (length($PaddingLeft) * 2) < $MaxLength+TABLEWIDTH);
+$PaddingLeft .= ' ' if (length($Heading{$Type}) + (length($PaddingLeft) * 2) +2 < $MaxLength+TABLEWIDTH);
-my $Counter = 0;
 
 print $LeadIn{$Type};
 
+# print table header
 print &Divider('=',$MaxLength);
 printf(": %s%s%s :\n",$PaddingLeft,$Heading{$Type},$PaddingRight);
 print &Divider('=',$MaxLength);
@@ -163,11 +232,26 @@ printf(": %-3s  : %-6s : %-7s : %-*s :\n",
   $MaxLength,$TH{'name'});
 print &Divider('-',$MaxLength);
 
+# print table
+my $Counter = 0;
 foreach my $Name (sort { $Value{$b} <=> $Value{$a} } keys %Value) {
     $Counter++;
-    printf(": %3u. : %6u : %6.2f%% : %-*s :\n",$Counter,$Value{$Name},&Percentage($SumTotal,$Value{$Name}),$MaxLength,$Name);
+    printf(": %3u. : %6u : %6.2f%% : %-*s :\n",
+      $Counter,$Value{$Name},&Percentage($SumTotal,$Value{$Name}),
+      $MaxLength,$Name);
+    # handle client versions
+    if ($SubValue{$Name}) {
+        foreach my $SubName (sort { $SubValue{$Name}{$b} <=> $SubValue{$Name}{$a} }
+          keys %{$SubValue{$Name}}) {
+            printf(":      : %6u : %6.2f%% : %-*s :\n",
+              $SubValue{$Name}{$SubName},
+              &Percentage($SumTotal,$SubValue{$Name}{$SubName}),
+              $MaxLength,$SubName);
+        }
+    }
 }
 
+# print table footer
 print &Divider('-',$MaxLength);
 printf(":      : %6u : %s : %-*s :\n",$SumTotal,'100.00%',$MaxLength,'');
 print &Divider('=',$MaxLength);
@@ -184,7 +268,7 @@ postingstats - format and post reports
 
 =head1 SYNOPSIS
 
-B<postingstats> B<-t> I<groups|hosts> [B<-Vh> [B<-m> I<YYYY-MM>]
+B<postingstats> [B<-Vh>] [B<-t> I<groups|hosts|clients>] [B<-m> I<YYYY-MM>]
 
 =head1 REQUIREMENTS
 
@@ -193,19 +277,21 @@ See L<doc/README>.
 =head1 DESCRIPTION
 
 This script will re-format reports on newsgroup usage created by
-B<groupstats.pl> or B<cliservstats.pl> and create a message that can
+B<groupstats.pl>, B<hoststats.pl> or B<clientstats.pl> and create a
-be posted to Usenet.
+message that can be posted to Usenet.
 
 =head2 Features and options
 
 B<postingstats> will create a table with entries numbered from most
 to least and percentages calculated from the sum total of all values.
 
-It depends on a sorted list on STDIN in I<dump> format with I<sums>.
+It depends on a sorted list on STDIN in I<dump> format with I<sums>;
+I<versions> from B<clientstas.pl> are optional.
 
 B<postingstats> needs a B<--type> and a B<--month> to create a caption
 and select matching lead-ins and lead-outs. B<--type> is also needed
-to catch the correct sum total from input.
+to catch the correct sum total from input which differs between I<groups>
+on one hand and I<hosts> or I<clients> on the other hand.
 
 It will default to posting statistics (number of postings per group)
 and last month.
@@ -224,12 +310,12 @@ C<----- configuration -----> section.
 =item C<$TLH>
 
 Top level hierarchy the report was created for. Used for display and
-sum total.
+sum total (only for I<groups>).
 
 =item C<%Heading>
 
-Hash with keys for I<GroupStats> and I<HostStats>. Used to display a
+Hash with keys for I<GroupStats>, I<HostStats> and I<ClientStats>.
-heading.
+Used to display a heading.
 
 =item C<%TH>
 
@@ -242,14 +328,14 @@ Output will be truncated otherwise.
 
 =item C<%LeadIn>
 
-Hash with keys for I<GroupStats> and I<HostStats>. Used to create the
+Hash with keys for I<GroupStats>, I<HostStats> and I<ClientStats>.
-headers for our posting. Can contain other text that will be shown
+Used to create the headers for the postings. Can contain other text
-before C<%Heading>.
+that will be shown before C<%Heading>.
 
 =item C<%LeadOut>
 
-Hash with keys for I<GroupStats> and I<HostStats>. Will be shown at the
+Hash with keys for I<GroupStats>, I<HostStats> and I<ClientStats>.
-end of our posting.
+Will be shown at the end of the posting.
 
 =back
 
@@ -259,19 +345,20 @@ end of our posting.
 
 =item B<-V>, B<--version>
 
-Print out version and copyright information and exit.
+Display version and copyright information and exit.
 
 =item B<-h>, B<--help>
 
-Print this man page and exit.
+Display this man page and exit.
 
-=item B<-t>, B<--type> I<groups|hosts>
+=item B<-t>, B<--type> I<groups|hosts|clients>
 
-Set report type to posting statistics or hosts statistics accordingly.
+Set report type to posting statistics, hosts statistics or client
+statistics accordingly.
 
 =item B<-m>, B<--month> I<YYYY-MM>
 
-Set month for display.
+Set month (for display only).
 
 =back
 
@@ -287,11 +374,15 @@ Create a posting from a posting statistics report for last month:
 
 Create a posting from a posting statistics report for 2012-01:
 
-    groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups -m 2012-01
+    groupstats.pl --nocomments --sums --format dump -m 2012-01 | postingstats.pl -t groups -m 2012-01
 
 Create a posting from a host statistics report for last month:
 
-    cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts
+    hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts
+
+Create a posting from a client statistics report for last month:
+
+    clientstats.pl --nocomments --sums --versions --format dump | postingstats.pl -t clients
 
 =head1 FILES
 
@@ -326,7 +417,7 @@ L<doc/README>
 
 =item -
 
-l>doc/INSTALL>
+L<doc/INSTALL>
 
 =item -
 
@@ -334,7 +425,11 @@ groupstats -h
 
 =item -
 
-cliservstats -h
+hoststats -h
+
+=item -
+
+clientstats -h
 
 =back
 

contrib/dopostingstats.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
 # installation path is /srv/newsstats/, please adjust accordingly
-if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then
+
-  /srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y
+# get month
-  /srv/newsstats/bin/cliservstats.pl -t server --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y
+MONTH=$1
-else
+if ! [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then
-  echo 'Input error, please use dopostingstats.sh YYYY-MM'
+  MONTH=$(date -d "$(date +%Y-%m-15) -1 month" '+%Y-%m')
 fi
 
+# post stats
+/srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y
+/srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t server --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y
+/srv/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t client --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y

doc/ChangeLog

@@ -1,3 +1,23 @@
+NewsStats 0.4.0 (2025-06-02)
+  * Reformat $Conf{TLH} for GroupStats only.
+  * Extract TLH check from HostStats to subroutine, fix no-op check.
+  * Extract getting raw headers from HostStats to subroutine.
+  * Improve documentation for config file.
+  * ParseHeader: re-merge continuation lines.
+  * Add ClientStats to gatherstats.
+  * Move cliservstats to hoststats.
+  * Add clientstats (for clients).
+  * Add ClientStats to postingstats.
+  * gatherstats: Don't die on parsing errors.
+  * DBClnts: set version length to 50.
+  * gatherstats: Truncate overlong clients or versions.
+  * gatherstats: Remove whitespace from client and version.
+  * Fix version queries.
+  * Add ClientStats to dopostingstats.
+  * Let dopostingstats default to last month.
+  * Set executable bit for new scripts.
+  * Update documentation.
+
 NewsStats 0.3.0 (2025-05-18)
   * Extract GroupStats (in gatherstats) to subroutine.
   * Add ParseHeader() to library.

doc/INSTALL

@@ -1,11 +1,12 @@
 NewsStats (c) 2010-2013, 2025 Thomas Hochstein <thh@thh.name>
 
-NewsStats is a software package used to gather statistical information
+NewsStats is a software package that can be used to collect
-from a live Usenet feed and for its subsequent examination.
+statistical information from a live Usenet feed and then analyze it
+to create statistical reports.
 
-This script package is free software; you can redistribute it and/or
+This  package is free software; you can redistribute it and/or modify
-modify it under the terms of the GNU Public License as published by
+it under the terms of the GNU Public License as published by the Free
-the Free Software Foundation.
+Software Foundation.
 
 ---------------------------------------------------------------------
 
@@ -16,9 +17,10 @@ INSTALLATION INSTRUCTIONS
    * Download the current version of NewsStats from
      <https://th-h.de/net/software/newsstats/>.
 
-   * Untar it into a directory of your choice:
+   * Untar it into a directory of your choice, i.e. /srv/newsstats:
 
-     # tar -xzf newsstats-nn.tar.gz
+     $ cd /srv
+     $ tar -xzf newsstats-n.n.n.tar.gz
 
      Scripts in this path - at least feedlog.pl - should be executable by the
      news user.
@@ -28,8 +30,8 @@ INSTALLATION INSTRUCTIONS
    * Copy the sample configuration file newsstats.conf.sample to
      newsstats.conf and modify it for your purposes:
 
-     # cp etc/newsstats.conf.sample etc/newsstats.conf
+     $ cp etc/newsstats.conf.sample etc/newsstats.conf
-     # vim etc/newsstats.conf
+     $ vim etc/newsstats.conf
 
    a) Mandatory configuration options
 
@@ -60,22 +62,28 @@ INSTALLATION INSTRUCTIONS
    * DBTableHosts = hosts_de
      Table holding data on postings per server.
 
+   * DBTableClnts = clients_de
+     Table holding data on postings per client.
+
    b) Optional configuration options
 
-   * TLH = de
+   * TLH = de.alt,news.admin
-     Limit examination to that top-level hierarchy.
+     Limit examination to that top-level hierarchy/hierarchies.
+     Comma-separated list.
 
 3) Database (mysql) setup
 
-   * Setup your database server with a username, password and
+   * Setup your database server with an username, a password and
-     database matching the NewsStats configuration (see 2 a).
+     (optionally) a database matching the NewsStats configuration
+     (see 2 a).
 
    * Start the database creation script:
 
-     # bin/dbcreate.pl
+     $ bin/dbcreate.pl
 
-     It will setup the necessary database tables and display some
+     It will create the database (if not already present), create the
-     information on the next steps.
+     necessary database tables and display some information on the
+     next steps.
 
 4) Feed (INN) setup
 
@@ -89,39 +97,39 @@ INSTALLATION INSTRUCTIONS
              :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
 
    * You should only feed that hierarchy (those hierarchies ...) to
-     feedlog.pl you'll want to cover with your statistical
+     feedlog.pl that you want to cover with your statistical analysis.
-     examination. It may be a good idea to setup different feeds (to
+     It may be a good idea to setup different feeds (to different
-     different databases ...) for different hierarchies.
+     databases ...) for different hierarchies.
 
    * Please double check that your path to feedlog.pl is correct and
      feedlog.pl can be executed by the news user
 
    * Check your 'newsfeeds' syntax:
 
-     # ctlinnd checkfile
+     $ ctlinnd checkfile
 
    * Reload 'newsfeeds':
 
-     # ctlinnd reload newsfeeds 'Adding newsstats! feed'
+     $ ctlinnd reload newsfeeds 'Adding newsstats! feed'
 
    * Watch your 'news.notice' and 'errlog' files:
 
-     # tail -f /var/log/news/news.notice
+     $ tail -f /var/log/news/news.notice
      ...
-     # tail -f /var/log/news/errlog
+     $ tail -f /var/log/news/errlog
 
    Everything should be going smoothly now.
 
    * If INN is spewing error messages to 'errlog' or reporting
      continous respawns of feedlog.pl to 'news.notice', stop your feed:
 
-     # ctlinnd drop 'newsstats!'
+     $ ctlinnd drop 'newsstats!'
 
      and investigate. 'errlog' may be helpful here.
 
    * You can restart the feed with
 
-     # ctlinnd begin 'newsstats!'
+     $ ctlinnd begin 'newsstats!'
 
      later.
 

doc/README

@@ -1,21 +1,21 @@
 NewsStats (c) 2010-2013, 2025 Thomas Hochstein <thh@thh.name>
 
 NewsStats is a software package for gathering statistical data live
-from a Usenet feed and subsequent examination.
+from a Usenet feed and subsequent analysis.
 
-This script package is free software; you can redistribute it and/or
+This package is free software; you can redistribute it and/or modify
-modify it under the terms of the GNU Public License as published by
+it under the terms of the GNU Public License as published by the Free
-the Free Software Foundation.
+Software Foundation.
 
 ---------------------------------------------------------------------
 
 What's that?
 
-    There's a multitude of tools for the statistical examination of
+    There's a multitude of tools to create statistics about newsgroup
-    newsgroups: number of postings per month or per person, longest
+    usage: number of postings per month or per person, longest threads,
-    threads, and so on (see <https://th-h.de/net/usenet/stats/>
+    and so on (see <https://th-h.de/net/usenet/stats/> [German language]
-    [German language] for an incomplete list). Most of them use a per-
+    for an incomplete list). Most of them use a per-newsgroup approach
-    newsgroup approach while NewsStats is hierarchy oriented.
+    while NewsStats is hierarchy oriented.
 
     NewsStats will accumulate data from a live INN feed, allowing you
     to process the saved information later on.
@@ -40,7 +40,9 @@ Prerequisites
 
     * Perl 5.8.x with standard modules
       - Cwd
+      - Encode
       - File::Basename
+      - Getopt::Long
       - Sys::Syslog
 
     * Perl modules from CPAN
@@ -50,7 +52,7 @@ Prerequisites
 
     * mysql 5.0.x
 
-    * working installation of INN
+    * a working installation of INN
 
 Installation instructions
 
@@ -67,15 +69,52 @@ Getting Started
     table. See the feedlog.pl man page for more information.
 
     You can process that data via 'gatherstats.pl'; currently the
-    tabulation of postings per group and injection server per month is
+    tabulation of postings per group, injection server and posting
-    supported. Tabulation of clients (newsreaders) is planned. See
+    agent (newsreader) per month is supported. See the gatherstats.pl
-    the gatherstats.pl man page for more information.
+    man page for more information.
+
+    Example:
+
+        bin/gatherstats.pl
+
+    will parse raw data from the last month and save the results in
+    tables for postings per group, server and client, respectively.
 
     Report generation is handled by specialised scripts for each
-    report type. Currently reports on the number of postings per group
+    report type: 'groupstats.pl' for postings per group
-    and month and injection server and month are supported; you can
+    (s), 'hoststats.pl' for postings per injection server
-    use 'groupstats.pl' and 'cliservstats.pl' for that. See the
+    (s) and 'clientstats.pl' for postings per posting agent. See the
-    groupstats.pl and cliservstats.pl man pages for more information.
+    groupstats.pl, hoststats.pl and clientstats.pl man pages for more
+    information.
+
+    Example:
+
+        bin/groupstats.pl  -o postings-desc
+        bin/hoststats.pl   -o postings-desc
+        bin/clientstats.pl -o postings-desc -v
+
+    will show reports for postings per group, per injection server and
+    per client (with detailed client versions) for the last month,
+    using the result tables filled by gatherstats.
+
+    To post those reports to Usenet, change postingstats.pl according
+    to your needs (sender, newsgroups and other headers, translation
+    of table headers and text templates) and display a test posting
+    by piping report data into postingstats.pl:
+
+        bin/groupstats.pl --nocomments -s -f dump | bin/postingstats.pl
+
+    If the result is to your liking, add a pipe to a inews
+    implementation.
+
+    Example:
+
+        bin/groupstats.pl --nocomments -s -f dump | bin/postingstats.pl | contrib/tinews.pl -X
+
+More information
+
+    See the man pages for 'gatherstats' and the report generating
+    scripts.
 
 Reporting Bugs
 
@@ -87,7 +126,7 @@ Reporting Bugs
 
 Development
 
-    This program is maintained using the Git version control system at
+    This package is maintained using the Git version control system at
     <https://code.virtcomm.de/thh/newsstats/>.
 
 Related projects

doc/TODO

@@ -1,12 +1,10 @@
                          NewsStats To-Do List
                          ====================
 
-This is a list of planned bug fixes, improvements and enhancements for
+This is a list of possible bug fixes, improvements and enhancements for
 NewsStats.
 
 * General
-  - Improve Documentation
-    The documentation is rather sparse and could use some improvement.
   - Add a test suite
     There is currently no kind of test suite or regression tests. Something like
     that is badly needed.
@@ -27,8 +25,6 @@ NewsStats.
     for late creation and deletion), optionally including the previously
     mentioned information; and you should be able to get the history of any
     group.
-  - Add other reports
-    NewsStats should include some other kinds of reports (stats on used clients)
   - Add tools for database management
     NewsStats should offer tools e.g. to inject postings into the 'raw' database,
     or to split databases.
@@ -53,23 +49,11 @@ NewsStats.
       Some other tests - working database connection, valid database and table
       names - would be nice.
 
-  + install/install.pl
-    - Read current version from a file dropped and updated by installer
-    - Add / enhance / test error handling
-    - General tests and optimisations
-
-  + feedlog.pl
-    - Add / enhance / test error handling
-    - General tests and optimisations
-
   + gatherstats.pl
     - Use hierarchy information (see GroupInfo above)
-    - Add gathering of other stats (clients, ...)
-    - better modularisation (code reuse for other reports!)
     - Add / enhance / test error handling
-    - General tests and optimisations
 
-  + groupstats.pl
+  + groupstats.pl, hoststats.pl, clientstats.pl
-    - better modularisation (code reuse for other reports!)
+    - better modularisation (code reuse)
     - Add / enhance / test error handling
     - General tests and optimisations

etc/newsstats.conf.sample

@@ -13,7 +13,9 @@ DBDatabase = newsstats
 DBTableRaw   = raw_de
 DBTableGrps  = groups_de
 DBTableHosts = hosts_de
-#DBTableClnts =
+DBTableClnts = clnts_de
 
 ### hierarchy configuration
+# comma-separated list of TLHs to parse
+# newsgroups not starting with one of those patterns are not counted
 TLH = de

lib/NewsStats.pm

@@ -49,9 +49,8 @@ require Exporter;
                  Output      => [qw(OutputData FormatOutput)],
                  SQLHelper   => [qw(SQLHierarchies SQLSortOrder SQLGroupList
                                  SQLSetBounds SQLBuildClause GetMaxLength)]);
-$VERSION = '0.3.0';
+$VERSION = '0.4.0';
 
-use Data::Dumper;
 use File::Basename;
 use Cwd qw(realpath);
 
@@ -280,7 +279,8 @@ sub ParseHeaders {
     } elsif (/^\s/) {
       # continuation lines
       if ($Label) {
-        $Header{lc($Label)} .= "\n$_";
+        s/^\s+/ /;
+        $Header{lc($Label)} .= $_;
       } else {
         warn (sprintf("Non-header line: %s\n",$_));
       }
@@ -439,8 +439,7 @@ sub OutputData {
   my ($Format, $Comments, $GroupBy, $Precision, $ValidKeys, $LeadIn, $FileTempl,
       $DBQuery, $PadField, $PadValue) = @_;
   my %ValidKeys = %{$ValidKeys} if $ValidKeys;
-  my ($FileName, $Handle, $OUT);
+  my ($LastIteration, $FileName, $Handle, $OUT);
-  our $LastIteration;
 
   # define output types
   my %LegalOutput;
@@ -480,7 +479,7 @@ sub OutputData {
       $Handle = $OUT;
     };
     print $Handle &FormatOutput($Format, $Comments, $LeadIn, $Caption,
-                                $Key, $Value, $Precision, $PadField, $PadValue);
+                                $Key, $Value, $Precision, $PadField, $PadValue, $LastIteration);
     $LastIteration = $Caption;
   };
   close $OUT if ($FileTempl);
@@ -500,24 +499,26 @@ sub FormatOutput {
 ###      $PadValue : padding length for value field (optional) for 'pretty'
 ### OUT: $Output: formatted output
   my ($Format, $Comments, $LeadIn, $Caption, $Key, $Value, $Precision, $PadField,
-      $PadValue) = @_;
+      $PadValue, $LastIteration) = @_;
   my ($Output);
-  # keep last caption in mind
-  our ($LastIteration);
   # create one line of output
   if ($Format eq 'dump') {
     # output as dump (key value)
     $Output = sprintf ("# %s:\n",$Caption)
-      if ($Comments and (!defined($LastIteration) or $Caption ne $LastIteration));
+      if ($Caption and $Comments and (!defined($LastIteration) or $Caption ne $LastIteration));
     $Output .= sprintf ("%s %u\n",$Key,$Value);
   } elsif ($Format eq 'list') {
     # output as list (caption key value)
+    if ($Caption) {
       $Output = sprintf ("%s %s %u\n",$Caption,$Key,$Value);
+    } else {
+      $Output = sprintf ("%s %u\n",$Key,$Value);
+    }
   } elsif ($Format eq 'pretty') {
     # output as a table
     if ($Comments and (!defined($LastIteration) or $Caption ne $LastIteration)) {
       $Output = $LeadIn;
-      $Output .= sprintf ("# ----- %s:\n",$Caption);
+      $Output .= sprintf ("# ----- %s:\n",$Caption) if $Caption;
     }
     # increase $PadValue for numbers with decimal point
     $PadValue += $Precision+1 if $Precision;
@@ -582,7 +583,7 @@ sub SQLSortOrder {
 ### IN : $GroupBy: primary sort by 'month' (default) or 'newsgroups'
 ###      $OrderBy: secondary sort by month/newsgroups (default)
 ###                or number of 'postings'
-###      $Type   : newsgroup, host, client
+###      $Type   : newsgroup, host or client+version
 ### OUT: a SQL ORDER BY clause
   my ($GroupBy,$OrderBy,$Type) = @_;
   my ($GroupSort,$OrderSort) = ('','');
@@ -640,9 +641,11 @@ sub SQLGroupList {
 ### OUT: SQL code to become part of a 'WHERE' clause,
 ###      list of names for SQL bindings
   my ($Names,$Type) = @_;
+  my $InvalidCharRegExp;
   # substitute '*' wildcard with SQL wildcard character '%'
   $Names =~ s/\*/%/g;
-  return (undef,undef) if !CheckValidNames($Names);
+  $InvalidCharRegExp = ',;' if $Type eq 'client';
+  return (undef,undef) if !CheckValidNames($Names,$InvalidCharRegExp);
   # just one name/newsgroup?
   return (SQLGroupWildcard($Names,$Type),$Names) if $Names !~ /:/;
   my ($SQL,@WildcardNames,@NoWildcardNames);
@@ -806,9 +809,10 @@ sub CheckValidNames {
 ################################################################################
 ### syntax check of a list
 ### IN : $Names           : list of names, e.g. newsgroups (group.one.*:group.two:group.three.*)
+###      InvalidCharRegExp: regular expression for invalid characters
 ### OUT: boolean
-  my ($Names) = @_;
+  my ($Names,$InvalidCharRegExp) = @_;
-  my $InvalidCharRegExp = ',; ';
+  $InvalidCharRegExp = ',; ' if (!$InvalidCharRegExp);
   return ($Names =~ /[$InvalidCharRegExp]/) ? 0 : 1;
 };