From 988e7b2f13706837cf4bb0a2be869f998b05281c Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 18 May 2025 17:17:22 +0200 Subject: [PATCH 01/29] Bump version. Signed-off-by: Thomas Hochstein --- doc/ChangeLog | 3 +++ lib/NewsStats.pm | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index e6da31e..b53a702 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,6 @@ +NewsStats 0.4.0 (unreleased) + + NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. * Add ParseHeader() to library. diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index d16965b..51e939b 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -49,7 +49,7 @@ require Exporter; Output => [qw(OutputData FormatOutput)], SQLHelper => [qw(SQLHierarchies SQLSortOrder SQLGroupList SQLSetBounds SQLBuildClause GetMaxLength)]); -$VERSION = '0.3.0'; +$VERSION = '0.4.0'; use Data::Dumper; use File::Basename; From 6122d1a49d419a7482fc0cbba869091a276d1f6f Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 18 May 2025 17:20:36 +0200 Subject: [PATCH 02/29] Fix POD. Signed-off-by: Thomas Hochstein --- bin/cliservstats.pl | 2 +- bin/groupstats.pl | 2 +- bin/postingstats.pl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/cliservstats.pl b/bin/cliservstats.pl index a38cba0..3c3c7bb 100644 --- a/bin/cliservstats.pl +++ b/bin/cliservstats.pl @@ -513,7 +513,7 @@ L =item - -l>doc/INSTALL> +L =item - diff --git a/bin/groupstats.pl b/bin/groupstats.pl index cc13550..0c193a8 100755 --- a/bin/groupstats.pl +++ b/bin/groupstats.pl @@ -685,7 +685,7 @@ L =item - -l>doc/INSTALL> +L =item - diff --git a/bin/postingstats.pl b/bin/postingstats.pl index e6fe3db..fed9877 100644 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -326,7 +326,7 @@ L =item - -l>doc/INSTALL> +L =item - From 671ae67be067497899efe57065ff271b4d3a8fef Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 17:11:49 +0200 Subject: [PATCH 03/29] Fix typo. Signed-off-by: Thomas Hochstein --- bin/cliservstats.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cliservstats.pl b/bin/cliservstats.pl index 3c3c7bb..f92eb24 100644 --- a/bin/cliservstats.pl +++ b/bin/cliservstats.pl @@ -60,7 +60,7 @@ if ($OptType) { $OptType = 'client'; } } -&Bleat(2, "Please use '--type server' or '-type newsreader'.") if !$OptType; +&Bleat(2, "Please use '--type server' or '--type client'.") if !$OptType; # parse $OptReportType if ($OptReportType) { if ($OptReportType =~ /sums?/i) { From 3447cdabff371232bb774d95022771563ebfa010 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 17:42:20 +0200 Subject: [PATCH 04/29] Reformat Conf(TLH) for GroupStats only. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 57 +++++++++++++++++++++++----------------------- doc/ChangeLog | 2 +- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 15b7ad4..e900104 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -84,35 +84,6 @@ my ($Period) = &GetTimePeriod($OptMonth); &Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ". "'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time'); -### reformat $Conf{'TLH'} -my $TLH; -if ($Conf{'TLH'}) { - # $Conf{'TLH'} is parsed as an array by Config::Auto; - # make a flat list again, separated by : - if (ref($Conf{'TLH'}) eq 'ARRAY') { - $TLH = join(':',@{$Conf{'TLH'}}); - } else { - $TLH = $Conf{'TLH'}; - } - # strip whitespace - $TLH =~ s/\s//g; - # add trailing dots if none are present yet - # (using negative look-behind assertions) - $TLH =~ s/(? Date: Thu, 29 May 2025 18:03:10 +0200 Subject: [PATCH 05/29] Refactor and fix TLH check. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 49 ++++++++++++++++++++++++++++++++++------------ doc/ChangeLog | 1 + 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index e900104..ca90cc7 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -137,7 +137,7 @@ foreach my $Month (&ListMonth($Period)) { googlegroups.com heirich.name news.neostrada.pl netcologne.de newsdawg.com newscene.com news-service.com octanews.com readnews.com wieslauf.sub.de highway.telekom.at united-newsserver.de xennanews.com xlned.com xsnews.nl news.xs4all.nl); - &HostStats($DBHandle,$DBRaw,$DBHosts,$Month,$OptMID,$OptTest,$OptDebug,@KnownHosts); + &HostStats($DBHandle,$DBRaw,$DBHosts,$Month,$OptTLH,$OptMID,$OptTest,$OptDebug,@KnownHosts); }; }; @@ -153,10 +153,10 @@ sub GroupStats { ### $DBRaw : database table for raw data (to read from) ### $DBGrps : database table for groups data (to write to) ### $Month : current month to do -### $MID : specific Message-ID to fetch (testing purposes) ### $TLH : TLHs to collect ### $Checkgroupsfile : filename template for checkgroups file ### (expanded to $Checkgroupsfile-$Month) +### $MID : specific Message-ID to fetch (testing purposes) ### $Test : test mode ### $Debug : debug mode ### OUT: (nothing) @@ -251,12 +251,13 @@ sub HostStats { ### $DBRaw : database table for raw data (to read from) ### $DBHosts : database table for hosts data (to write to) ### $Month : current month to do +### $TLH : TLHs to collect ### $MID : specific Message-ID to fetch (testing purposes) ### $Test : test mode ### $Debug : debug mode ### @KnownHosts : list of known hosts with subdomains ### OUT: (nothing) - my ($DBHandle,$DBRaw,$DBHosts,$Month,$MID,$Test,$Debug,@KnownHosts) = @_; + my ($DBHandle,$DBRaw,$DBHosts,$Month,$TLH,$MID,$Test,$Debug,@KnownHosts) = @_; my (%Postings,$DBQuery); @@ -281,16 +282,7 @@ sub HostStats { ### parse headers while (my ($Newsgroups,$Headers) = $DBQuery->fetchrow_array) { ### skip postings with wrong TLH - # remove whitespace from contents of Newsgroups: - chomp($Newsgroups); - $Newsgroups =~ s/\s//; - my $GroupCount; - for (split /,/, $Newsgroups) { - # don't count newsgroup/hierarchy in wrong TLH - next if($TLH and !/^$TLH/); - $GroupCount++; - }; - next if !$GroupCount; + next if ($TLH && !CheckTLH($Newsgroups,$TLH)); my $Host; my %Header = ParseHeaders(split(/\n/,$Headers)); @@ -380,6 +372,37 @@ sub HostStats { }; }; +sub CheckTLH { +### ---------------------------------------------------------------------------- +### count newsgroups from legal TLH(s) +### IN : $Newsgroups : comma separated list of newsgroups +### $TLH : (reference to an array of) legal TLH(s) +### OUT: number of newsgroups from legal TLH(s) + my ($Newsgroups,$TLH) = @_; + + my (@TLH,$GroupCount); + + # fill @TLH from $TLH, which can be an array reference or a scalar value + if (ref($TLH) eq 'ARRAY') { + @TLH = @{$TLH}; + } else { + push @TLH, $TLH; + } + + # remove whitespace from contents of Newsgroups: + chomp($Newsgroups); + $Newsgroups =~ s/\s//; + for (split /,/, $Newsgroups) { + my $Newsgroup = $_; + foreach (@TLH) { + # increment $GroupCount if $Newsgroup starts with $TLH + $GroupCount++ if $Newsgroup =~ /^$_/; + } + }; + + return $GroupCount; +} + __END__ ################################ Documentation ################################# diff --git a/doc/ChangeLog b/doc/ChangeLog index 9664638..9f248cc 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,6 @@ NewsStats 0.4.0 (unreleased) * Reformat $Conf{TLH} for GroupStats only. + * Extract TLH check from HostStats to subroutine, fix no-op check. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From f78d4c2158439fea62c73b8aa9568f87842b09f7 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 18:03:44 +0200 Subject: [PATCH 06/29] Refactor getting raw headers. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 46 +++++++++++++++++++++++++++++++--------------- doc/ChangeLog | 1 + 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index ca90cc7..6118a70 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -261,21 +261,7 @@ sub HostStats { my (%Postings,$DBQuery); - if (!$MID) { - # get raw header data from raw table for given month - $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ". - "WHERE day LIKE ? AND NOT disregard", - $DBRaw)); - $DBQuery->execute($Month.'-%') - or &Bleat(2,sprintf("Can't get hosts data for %s from %s: ". - "$DBI::errstr\n",$Month,$DBRaw)); - } else { - $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ". - "WHERE mid = ?", $DBRaw)); - $DBQuery->execute($MID) - or &Bleat(2,sprintf("Can't get hosts data for %s from %s: ". - "$DBI::errstr\n",$MID,$DBRaw)); - } + $DBQuery = GetHeaders($DBHandle,$DBRaw,$Month,$MID); ### ---------------------------------------------- print "----- HostStats -----\n" if $Debug; @@ -372,6 +358,36 @@ sub HostStats { }; }; +sub GetHeaders { +### ---------------------------------------------------------------------------- +### get (newsgroups and) raw headers from database +### IN : $DBHandle : database handle +### $DBRaw : database table for raw data (to read from) +### $Month : current month to do +### $MID : specific Message-ID to fetch (testing purposes) +### OUT: DBI statement handle + my ($DBHandle,$DBRaw,$Month,$MID) = @_; + + my $DBQuery; + + if (!$MID) { + # get raw header data from raw table for given month + $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ". + "WHERE day LIKE ? AND NOT disregard", + $DBRaw)); + $DBQuery->execute($Month.'-%') + or &Bleat(2,sprintf("Can't get header data for %s from %s: ". + "$DBI::errstr\n",$Month,$DBRaw)); + } else { + $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups,headers FROM %s ". + "WHERE mid = ?", $DBRaw)); + $DBQuery->execute($MID) + or &Bleat(2,sprintf("Can't get header data for %s from %s: ". + "$DBI::errstr\n",$MID,$DBRaw)); + } + return $DBQuery; +} + sub CheckTLH { ### ---------------------------------------------------------------------------- ### count newsgroups from legal TLH(s) diff --git a/doc/ChangeLog b/doc/ChangeLog index 9f248cc..3f46f78 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,6 +1,7 @@ NewsStats 0.4.0 (unreleased) * Reformat $Conf{TLH} for GroupStats only. * Extract TLH check from HostStats to subroutine, fix no-op check. + * Extract getting raw headers from HostStats to subroutine. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From c985e29b7e2ccccf9a0d882da5b4e2dd0fe5504d Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 18:35:17 +0200 Subject: [PATCH 07/29] Improve documentation for config file. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 4 +++- doc/ChangeLog | 1 + doc/INSTALL | 5 +++-- etc/newsstats.conf.sample | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 6118a70..b9070af 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -549,10 +549,12 @@ Newsgroups not found in the checkgroups file will be dropped (and logged to STDERR), and newsgroups found there but having no postings will be added with a count of 0 (and logged to STDERR). -=item B<--hierarchy> I (newsgroup hierarchy) +=item B<--hierarchy> I (newsgroup hierarchy/hierarchies) Override I from F. +I can be a single word or a comma-separated list. + =item B<--rawdb> I (raw data table) Override I from F. diff --git a/doc/ChangeLog b/doc/ChangeLog index 3f46f78..a3de323 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -2,6 +2,7 @@ NewsStats 0.4.0 (unreleased) * Reformat $Conf{TLH} for GroupStats only. * Extract TLH check from HostStats to subroutine, fix no-op check. * Extract getting raw headers from HostStats to subroutine. + * Improve documentation for config file. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/doc/INSTALL b/doc/INSTALL index 2ef057e..9a37207 100644 --- a/doc/INSTALL +++ b/doc/INSTALL @@ -62,8 +62,9 @@ INSTALLATION INSTRUCTIONS b) Optional configuration options - * TLH = de - Limit examination to that top-level hierarchy. + * TLH = de.alt,news.admin + Limit examination to that top-level hierarchy/hierarchies. + Comma-separated list. 3) Database (mysql) setup diff --git a/etc/newsstats.conf.sample b/etc/newsstats.conf.sample index a960644..1ea3430 100644 --- a/etc/newsstats.conf.sample +++ b/etc/newsstats.conf.sample @@ -16,4 +16,6 @@ DBTableHosts = hosts_de #DBTableClnts = ### hierarchy configuration +# comma-separated list of TLHs to parse +# newsgroups not starting with one of those patterns are not counted TLH = de From d194ef754f2c8e8e7866850904f6e78f0812b9dc Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 19:04:16 +0200 Subject: [PATCH 08/29] Move lc() to counting. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index b9070af..3316f1f 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -321,11 +321,9 @@ sub HostStats { } } - # lowercase - $Host = lc($Host); - # count host if ($Host) { + $Host = lc($Host); $Postings{$Host}++; $Postings{'ALL'}++; } else { From eea296391cc0221f6e6901e47321c45c0e371c6d Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 19:32:39 +0200 Subject: [PATCH 09/29] ParseHeader will now re-merge continuation lines. Signed-off-by: Thomas Hochstein --- doc/ChangeLog | 1 + lib/NewsStats.pm | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index a3de323..dfa6de7 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -3,6 +3,7 @@ NewsStats 0.4.0 (unreleased) * Extract TLH check from HostStats to subroutine, fix no-op check. * Extract getting raw headers from HostStats to subroutine. * Improve documentation for config file. + * ParseHeader: re-merge continuation lines. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index 51e939b..97e75f7 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -280,7 +280,8 @@ sub ParseHeaders { } elsif (/^\s/) { # continuation lines if ($Label) { - $Header{lc($Label)} .= "\n$_"; + s/^\s+/ /; + $Header{lc($Label)} .= $_; } else { warn (sprintf("Non-header line: %s\n",$_)); } From 3e73346b2071132f3a47ceece155a84f461426bf Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Fri, 30 May 2025 19:48:35 +0200 Subject: [PATCH 10/29] OutputData(): Change handover of LastIteration. Signed-off-by: Thomas Hochstein --- lib/NewsStats.pm | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index 97e75f7..35b59c0 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -434,14 +434,13 @@ sub OutputData { ### $LeadIn : print at start of output ### $FileTempl: file name template (--filetemplate): filetempl-YYYY-MM ### $DBQuery : database query handle with executed query, -### containing $Month, $Key, $Value +### containing $Month, $Key, $Value ### $PadField : padding length for key field (optional) for 'pretty' ### $PadValue : padding length for value field (optional) for 'pretty' my ($Format, $Comments, $GroupBy, $Precision, $ValidKeys, $LeadIn, $FileTempl, $DBQuery, $PadField, $PadValue) = @_; my %ValidKeys = %{$ValidKeys} if $ValidKeys; - my ($FileName, $Handle, $OUT); - our $LastIteration; + my ($LastIteration, $FileName, $Handle, $OUT); # define output types my %LegalOutput; @@ -481,7 +480,7 @@ sub OutputData { $Handle = $OUT; }; print $Handle &FormatOutput($Format, $Comments, $LeadIn, $Caption, - $Key, $Value, $Precision, $PadField, $PadValue); + $Key, $Value, $Precision, $PadField, $PadValue, $LastIteration); $LastIteration = $Caption; }; close $OUT if ($FileTempl); @@ -501,10 +500,8 @@ sub FormatOutput { ### $PadValue : padding length for value field (optional) for 'pretty' ### OUT: $Output: formatted output my ($Format, $Comments, $LeadIn, $Caption, $Key, $Value, $Precision, $PadField, - $PadValue) = @_; + $PadValue, $LastIteration) = @_; my ($Output); - # keep last caption in mind - our ($LastIteration); # create one line of output if ($Format eq 'dump') { # output as dump (key value) @@ -583,7 +580,7 @@ sub SQLSortOrder { ### IN : $GroupBy: primary sort by 'month' (default) or 'newsgroups' ### $OrderBy: secondary sort by month/newsgroups (default) ### or number of 'postings' -### $Type : newsgroup, host, client +### $Type : newsgroup, host or client+version ### OUT: a SQL ORDER BY clause my ($GroupBy,$OrderBy,$Type) = @_; my ($GroupSort,$OrderSort) = ('',''); From a553b374ce30693716d8d4d5f2861baaad97d027 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Thu, 29 May 2025 18:54:39 +0200 Subject: [PATCH 11/29] Add ClientStats to gatherstats. Signed-off-by: Thomas Hochstein --- bin/dbcreate.pl | 19 ++- bin/gatherstats.pl | 351 +++++++++++++++++++++++++++++++++++--- doc/ChangeLog | 1 + etc/newsstats.conf.sample | 2 +- 4 files changed, 352 insertions(+), 21 deletions(-) diff --git a/bin/dbcreate.pl b/bin/dbcreate.pl index ea0fd6c..fee67ab 100755 --- a/bin/dbcreate.pl +++ b/bin/dbcreate.pl @@ -46,7 +46,7 @@ my $DBCreate = < < < < < < < < 1 } @DropAgents; + + $DBQuery = GetHeaders($DBHandle,$DBRaw,$Month,$MID); + + ### ---------------------------------------------- + print "----- ClientStats -----\n" if $Debug; + ### parse headers + while (my ($Newsgroups,$Headers) = $DBQuery->fetchrow_array) { + ### skip postings with wrong TLH + next if ($TLH && !CheckTLH($Newsgroups,$TLH)); + + my (@Clients, $Client, $Version); + my %Header = ParseHeaders(split(/\n/,$Headers)); + + ### X-Mailer + if ($Header{'x-mailer'}) { + # transfer to x-newsreader and parse from there + $Header{'x-newsreader'} = $Header{'x-mailer'}; + } + ### X-Newsreader + if ($Header{'x-newsreader'}) { + $Header{'x-newsreader'} = RemoveComments($Header{'x-newsreader'}); + # remove 'http://' and 'via' (CrossPoint) + $Header{'x-newsreader'} =~ s/https?:\/\///; + $Header{'x-newsreader'} =~ s/ ?via(.+)?$//; + # parse header + # User-Agent style + if ($Header{'x-newsreader'} =~ /^([^\/ ]+\/[^\/ ]+ ?)+$/) { + # transfer to user-agent and parse from there + $Header{'user-agent'} = $Header{'x-newsreader'}; + # "client name version" + } elsif ($Header{'x-newsreader'} =~ / /) { + ($Client, $Version) = ParseXNewsreader($Header{'x-newsreader'}); + } else { + $Client = $Header{'x-newsreader'}; + $Version = ''; + } + if ($Client) { + # special cases + $Client = 'CrossPoint' if $Client =~ /^CrossPoint\//; + $Client = 'Virtual Access' if $Client =~ /^Virtual Access/; + my %UserAgent = (agent => $Client, + version => $Version); + push @Clients, { %UserAgent }; + } else { + $Header{'user-agent'} = $Header{'x-newsreader'}; + } + } + ### User-Agent + if(!@Clients && $Header{'user-agent'}) { + $Header{'user-agent'} = RemoveComments($Header{'user-agent'}); + ### well-formed? + if ($Header{'user-agent'} =~ /^([^\/ ]+\/[^\/ ]+ ?)+$/) { + @Clients = ParseUserAgent($Header{'user-agent'}); + } else { + # snip and add known well-formed agents from the trailing end + while ($Header{'user-agent'} =~ /(((Hamster)|(Hamster-Pg)|(KorrNews)|(OE-Tools)|(Mime-proxy))(\/[^\/ ]+))$/) { + push @Clients, ParseUserAgent($1); + $Header{'user-agent'} =~ s/ [^\/ ]+\/[^\/ ]+$//; + } + ### special cases + # remove 'http://open-news-network.org' + $Header{'user-agent'} =~ s/^https?:\/\/open-news-network.org(\S+)?//; + # Thunderbird + if ($Header{'user-agent'} =~ /((Mozilla[- ])?Thunderbird) ?([0-9.]+)?/) { + $Client = 'Thunderbird'; + $Version = $3; + # XP + } elsif ($Header{'user-agent'} =~ /((TrueXP|FreeXP|XP2(\/Agent)?)) \/(.+)$/) { + $Client = $1; + $Version = $4; + $Client = 'XP2' if $Client eq 'XP2/Agent'; + ### most general case + # client version + # client/version + # client/32 version + # - version may end in one non-numeric character + # - including trailing beta/pre/... + # 1) client: (([^0-9]+)|(\D+\/\d+)) + # 2) version: (\S+\d\D?) + # 3) trailing: (( alpha\d?)|( beta\d?)|( rc\d)| pre| trialware)? + } elsif ($Header{'user-agent'} =~ /^(([^0-9]+)|(\D+\/\d+))[\/ ]((\S+\d\D?)(( alpha\d?)|( beta\d?)|( rc\d)| pre| trialware)?)$/) { + $Client = $1; + $Version = $4; + ### some very special cases + # SeaMonkey/nn + } elsif ($Header{'user-agent'} =~ /SeaMonkey\/([0-9.]+)/) { + $Client = 'Seamonkey'; + $Version = $1; + # Emacs nn/Gnus nn + } elsif ($Header{'user-agent'} =~ /Emacs [0-9.]+\/Gnus ([0-9.]+)/) { + $Client = 'Gnus'; + $Version = $1; + # failed to parse + } else { + $Client = $Header{'user-agent'}; + } + # count client, if found + if ($Client) { + my %UserAgent = (agent => $Client, + version => $Version); + push @Clients, { %UserAgent }; + } else { + &Bleat(2,sprintf("%s FAILED", $Header{'message-id'})) if !@Clients; + } + } + } + + if (@Clients) { + $Postings{'ALL'}{'ALL'}++; + foreach (@Clients) { + # filter agents for User-Agent with multiple agents + next if $#Clients && exists($DropAgent{lc($_->{'agent'})}); + # encode to utf-8, if necessary + $_->{'agent'} = encode('UTF-8', $_->{'agent'}) if $_->{'agent'} =~ /[\x80-\x{ffff}]/; + $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/; + # special cases + # Mozilla + $_->{'agent'} = 'Mozilla' if $_->{'agent'} eq '•Mozilla'; + $_->{'agent'} =~ s/^Mozilla //; + # Forte Agent + $_->{'agent'} = 'Forte Agent' if $_->{'agent'} eq 'ForteAgent'; + if ($_->{'agent'} eq 'Forte Agent') { + $_->{'version'} =~ s/-/\//; + $_->{'version'} = '' if $_->{'version'} eq '32Bit'; + } + # count client ('ALL') and client/version (if version is present) + $Postings{$_->{'agent'}}{'ALL'}++; + $Postings{$_->{'agent'}}{$_->{'version'}}++ if $_->{'version'}; + + printf("%s: %s {%s}\n", $Header{'message-id'}, $_->{'agent'}, + $_->{'version'} ? $Postings{$_->{'agent'}}{$_->{'version'}} : '') + if ($MID or $Debug && $Debug >1); + } + } + }; + + # delete old data for that month + if (!$Test) { + $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s WHERE month = ?", + $DBClients),undef,$Month) + or &Bleat(2,sprintf("Can't delete old client data for %s from %s: ". + "$DBI::errstr\n",$Month,$DBClients)); + }; + + foreach my $Client (sort keys %Postings) { + foreach my $Version (sort keys %{$Postings{$Client}}) { + printf ("%s {%s}: %d\n",$Client,$Version,$Postings{$Client}{$Version}) if $Debug; + + if (!$Test) { + # write to database + $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s ". + "(month,client,version,postings) ". + "VALUES (?, ?, ?, ?)",$DBClients)); + $DBQuery->execute($Month, $Client, $Version, $Postings{$Client}{$Version}) + or &Bleat(2,sprintf("Can't write groups data for %s/%s/%s to %s: ". + "$DBI::errstr\n",$Month,$Client,$Version,$DBClients)); + $DBQuery->finish; + }; + } + }; + +}; + sub GetHeaders { ### ---------------------------------------------------------------------------- ### get (newsgroups and) raw headers from database -### IN : $DBHandle : database handle -### $DBRaw : database table for raw data (to read from) -### $Month : current month to do -### $MID : specific Message-ID to fetch (testing purposes) +### IN : $DBHandle: database handle +### $DBRaw : database table for raw data (to read from) +### $Month : current month to do +### $MID : specific Message-ID to fetch (testing purposes) ### OUT: DBI statement handle my ($DBHandle,$DBRaw,$Month,$MID) = @_; @@ -389,8 +581,8 @@ sub GetHeaders { sub CheckTLH { ### ---------------------------------------------------------------------------- ### count newsgroups from legal TLH(s) -### IN : $Newsgroups : comma separated list of newsgroups -### $TLH : (reference to an array of) legal TLH(s) +### IN : $Newsgroups: comma separated list of newsgroups +### $TLH : (reference to an array of) legal TLH(s) ### OUT: number of newsgroups from legal TLH(s) my ($Newsgroups,$TLH) = @_; @@ -417,6 +609,116 @@ sub CheckTLH { return $GroupCount; } +sub RemoveComments { +### ---------------------------------------------------------------------------- +### remove comments and other junk from header +### IN : $Header: a header +### OUT: the header, with comments and other junk removed + my $Header = shift; + + # decode MIME encoded words + if ($Header =~ /=\?\S+\?[BQ]\?/) { + $Header = decode("MIME-Header",$Header); + } + + # remove nested comments from '(' to first ')' + while ($Header =~ /\([^)]+\)/) { + $Header =~ s/\([^()]+?\)//; + } + + # remove dangling ')' + $Header =~ s/\S+\)//; + + # remove from dangling '(' to end of header + $Header =~ s/\(.+$//; + + # remove from '[' to first ']' + $Header =~ s/\[[^\[\]]+?\]//; + + # remove 'Nr. ... lebt' + $Header =~ s/Nr\. \d+ lebt//; + + # remove nn:nn:nn + $Header =~ s/\d\d:\d\d:\d\d//; + + # remove 'mm/... ' + $Header =~ s/\/mm\/\S+//; + + # remove ' DE' / _DE' + $Header =~ s/[ _]DE//; + + # remove trailing 'eol' or '-shl' + $Header =~ s/(eol)|(-shl)$//; + + # remove from ';' or ',' (CrossPoint) + # or '&' to end of header + $Header =~ s/[;,&].+$//; + + # remove from 'by ' or 'unter Windows' or '@ Windows' + # to end of header + $Header =~ s/((by )|(unter +Windows)|(@ Windows)).+$//; + + # remove superfluous whitespace in header + # and whitespace around header + $Header =~ s/\s+/ /g; + $Header =~ s/^\s+//; + $Header =~ s/\s+$//; + + return $Header; +} + +sub ParseXNewsreader { +### ---------------------------------------------------------------------------- +### parse X-Newsreader header (client and version, if present) +### IN : $XNR: a X-Newsreader header +### OUT: client and version, if present + my $XNR = shift; + + my ($Client, $Version); + + foreach (split(/ /,$XNR)) { + # add to client name if no digit present + if (!/\d[0-9.]/ or /\/\d$/) { + $Client .= $_ . ' ' ; + # otherwise, use as version and terminate parsing + } else { + $Version = $_; + last; + } + } + + # remove trailing whitespace + $Client =~ s/\s+$// if $Client; + + # set $Version + $Version = '' if !$Version; + + return $Client, $Version; +} + + +sub ParseUserAgent { +### ---------------------------------------------------------------------------- +### parse User-Agent header (agent and version) +### IN : $UserAgent: a User-Agent header +### OUT: array of hashes (agent/version) + my $UserAgent = shift; + + my @UserAgents; + + # a well-formed User-Agent header will contain pairs of + # client/version, i.e. 'slrn/0.9.7.3' + foreach (split(/ /,$UserAgent)) { + my %UserAgent; + /^(.+)\/(.+)$/; + $UserAgent{'agent'} = $1; + $UserAgent{'version'} = $2; + push @UserAgents, { %UserAgent }; + } + + return @UserAgents; +} + __END__ ################################ Documentation ################################# @@ -427,7 +729,7 @@ gatherstats - process statistical data from a raw source =head1 SYNOPSIS -B [B<-Vhdt>] [B<-m> I | I] [B<-s> I] [B<-c> I]] [B<--hierarchy> I] [B<--rawdb> I] [B<-groupsdb> I] [B<--clientsdb> I] [B<--hostsdb> I] [B<--conffile> I] +B [B<-Vhdt>] [B<-m> I | I] [B<-s> I] [B<-c> I]] [B<--hierarchy> I] [B<--rawdb> I] [B<-groupsdb> I] [B<--hostsdb> I] [B<--clientsdb> I] [B<--conffile> I] =head1 REQUIREMENTS @@ -474,12 +776,23 @@ override that default through the B<--groupsdb> option. =item B (postings from host per month) B will examine Injection-Info:, X-Trace: and Path: -headers and try to normalize them. Groups not in I will be -ignored. The sum of all detected hosts will also saved for each month. +headers and try to normalize them. The sum of all detected hosts will +also be saved for each month. Groups not in I will be ignored. Data is written to I (see L); you can override that default through the B<--hostsdb> option. +=item B (postings by client per month) + +B will examine User-Agent:, X-Newsreader: and X-Mailer: +headers and try to remove comments and non-standard contents. Clients +and client versions are counted separately. The sum of all detected +clients will also be saved for each month. Groups not in I will +be ignored. + +Data is written to I (see L); you can +override that default through the B<--clientsdb> option. + =back =head2 Configuration @@ -561,14 +874,14 @@ Override I from F. Override I from F. -=item B<--clientsdb> I
(client data table) - -Override I from F. - =item B<--hostsdb> I
(host data table) Override I from F. +=item B<--clientsdb> I
(client data table) + +Override I from F. + =item B<--conffile> I Load configuration from I instead of F. diff --git a/doc/ChangeLog b/doc/ChangeLog index dfa6de7..6792cc6 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -4,6 +4,7 @@ NewsStats 0.4.0 (unreleased) * Extract getting raw headers from HostStats to subroutine. * Improve documentation for config file. * ParseHeader: re-merge continuation lines. + * Add ClientStats to gatherstats. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/etc/newsstats.conf.sample b/etc/newsstats.conf.sample index 1ea3430..a68ee6a 100644 --- a/etc/newsstats.conf.sample +++ b/etc/newsstats.conf.sample @@ -13,7 +13,7 @@ DBDatabase = newsstats DBTableRaw = raw_de DBTableGrps = groups_de DBTableHosts = hosts_de -#DBTableClnts = +DBTableClnts = clnts_de ### hierarchy configuration # comma-separated list of TLHs to parse From 963f07432c98b247bc4c3df1bbef955d120db74a Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Fri, 30 May 2025 16:38:18 +0200 Subject: [PATCH 12/29] Move cliservstats to hoststats. Signed-off-by: Thomas Hochstein --- bin/{cliservstats.pl => hoststats.pl} | 144 +++++++++++--------------- bin/postingstats.pl | 8 +- contrib/dopostingstats.sh | 2 +- doc/ChangeLog | 1 + doc/README | 4 +- 5 files changed, 68 insertions(+), 91 deletions(-) rename bin/{cliservstats.pl => hoststats.pl} (74%) diff --git a/bin/cliservstats.pl b/bin/hoststats.pl similarity index 74% rename from bin/cliservstats.pl rename to bin/hoststats.pl index f92eb24..9e949ff 100644 --- a/bin/cliservstats.pl +++ b/bin/hoststats.pl @@ -1,9 +1,9 @@ #! /usr/bin/perl # -# cliservstats.pl +# hoststats.pl # -# This script will get statistical data on client (newsreader) and -# server (host) usage from a database. +# This script will get statistical data on server (host) usage +# from a database. # # It is part of the NewsStats package. # @@ -31,7 +31,7 @@ Getopt::Long::config ('bundling'); ### read commandline options my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat, $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy, - $OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile); + $OptReportType,$OptSums,$UppBound,$OptConfFile); GetOptions ('c|captions!' => \$OptCaptions, 'comments!' => \$OptComments, 'db=s' => \$OptDB, @@ -44,7 +44,6 @@ GetOptions ('c|captions!' => \$OptCaptions, 'o|order-by=s' => \$OptOrderBy, 'r|report=s' => \$OptReportType, 's|sums!' => \$OptSums, - 't|type=s' => \$OptType, 'u|upper=i' => \$UppBound, 'conffile=s' => \$OptConfFile, 'h|help' => \&ShowPOD, @@ -52,15 +51,6 @@ GetOptions ('c|captions!' => \$OptCaptions, # parse parameters # $OptComments defaults to TRUE if --filetemplate is not used $OptComments = 1 if (!$OptFileTemplate && !defined($OptComments)); -# parse $OptType -if ($OptType) { - if ($OptType =~ /(host|server)s?/i) { - $OptType = 'host'; - } elsif ($OptType =~ /(newsreader|client)s?/i) { - $OptType = 'client'; - } -} -&Bleat(2, "Please use '--type server' or '--type client'.") if !$OptType; # parse $OptReportType if ($OptReportType) { if ($OptReportType =~ /sums?/i) { @@ -74,14 +64,8 @@ if ($OptReportType) { my %Conf = %{ReadConfig($OptConfFile)}; ### set DBTable -if ($OptDB) { - $Conf{'DBTable'} = $OptDB; -} -elsif ($OptType eq 'host') { - $Conf{'DBTable'} = $Conf{'DBTableHosts'}; -} else { - $Conf{'DBTable'} = $Conf{'DBTableClnts'}; -} +$Conf{'DBTable'} = $Conf{'DBTableHosts'}; +$Conf{'DBTable'} = $OptDB if $OptDB; ### init database my $DBHandle = InitDB(\%Conf,1); @@ -97,14 +81,14 @@ my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth); # with placeholders as well as a list of names to bind to them my ($SQLWhereNames,@SQLBindNames); if ($OptNames) { - ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType); + ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,'host'); # bail out if --names is invalid &Bleat(2,"--names option has an invalid format!") if !$SQLWhereNames; } ### build SQL WHERE clause -my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",$OptType); +my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",'host'); my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, $ExcludeSums, &SQLSetBounds('default',$LowBound,$UppBound)); @@ -118,8 +102,8 @@ $OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/ and $OptNames and $OptNames !~ /[:*%]/); # parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause # if $OptGroupBy is still not set, SQLSortOrder() will default to 'month' -my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType); -# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy) +my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, 'host'); +# $GroupBy will contain 'month' or 'host' (parsed result of $OptGroupBy) # set it to 'month' or 'key' for OutputData() $GroupBy = ($GroupBy eq 'month') ? 'month' : 'key'; @@ -128,19 +112,19 @@ my $SQLSelect; my $SQLGroupClause = ''; my $Precision = 0; # number of digits right of decimal point for output if ($OptReportType and $OptReportType ne 'default') { - $SQLGroupClause = "GROUP BY $OptType"; + $SQLGroupClause = "GROUP BY host"; # change $SQLOrderClause: replace everything before 'postings' $SQLOrderClause =~ s/BY.+postings/BY postings/; - $SQLSelect = "'All months',$OptType,SUM(postings)"; + $SQLSelect = "'All months',host,SUM(postings)"; # change $SQLOrderClause: replace 'postings' with 'SUM(postings)' $SQLOrderClause =~ s/postings/SUM(postings)/; } else { - $SQLSelect = "month,$OptType,postings"; + $SQLSelect = "month,host,postings"; }; ### get length of longest name delivered by query ### for formatting purposes -my $Field = ($GroupBy eq 'month') ? $OptType : 'month'; +my $Field = ($GroupBy eq 'month') ? 'host' : 'month'; my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, $Field,'postings',$SQLWhereClause, '',@SQLBindNames); @@ -155,8 +139,8 @@ $DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s', $SQLOrderClause)); # execute query $DBQuery->execute(@SQLBindNames) - or &Bleat(2,sprintf("Can't get %s data for %s from %s.%s: %s\n", - $OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, + or &Bleat(2,sprintf("Can't get host data for %s from %s.%s: %s\n", + $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, $DBI::errstr)); ### output results @@ -175,7 +159,7 @@ if ($OptCaptions && $OptComments) { $LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames))) if $OptNames; # print boundaries, if set - my $CaptionBoundary= '(counting only month fulfilling this condition)'; + my $CaptionBoundary= '(counting only months fulfilling this condition)'; $LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n", $LowBound ? $LowBound : '',$LowBound ? '=>' : '', $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary) @@ -201,11 +185,11 @@ __END__ =head1 NAME -cliservstats - create reports on host or client usage +hoststats - create reports on host usage =head1 SYNOPSIS -B B<-t> I [B<-Vhcs> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] +B [B<-Vhcs> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] =head1 REQUIREMENTS @@ -214,8 +198,7 @@ See L. =head1 DESCRIPTION This script create reports on newsgroup usage (number of postings from -each host or using each client per month) taken from result tables -created by B. +each host) taken from result tables created by B. =head2 Features and options @@ -225,9 +208,9 @@ The time period to act on defaults to last month; you can assign another time period or a single month (or drop all time constraints) via the B<--month> option (see below). -B will process all hosts or clients by default; you can -limit processing to only some hosts or clients by supplying a list of -those names by using the B<--names> option (see below). +B will process all hosts by default; you can limit +processing to only some hosts by supplying a list of those names by +using the B<--names> option (see below). =head3 Report type @@ -238,18 +221,18 @@ or all postings summed up; for details, see below. Furthermore you can set an upper and/or lower boundary to exclude some results from output via the B<--lower> and B<--upper> options, -respectively. By default, all hosts/clients with more and/or less -postings per month will be excluded from the result set (i.e. not -shown and not considered forsum reports). +respectively. By default, all hosts with more and/or less postings +per month will be excluded from the result set (i.e. not shown and +not considered for sum reports). =head3 Sorting and formatting the output By default, all results are grouped by month; you can group results by -hosts/clients instead via the B<--group-by> option. Within those -groups, the list of hosts/clients (or months) is sorted alphabetically -(or chronologically, respectively) ascending. You can change that order -(and sort by number of postings) with the B<--order-by> option. For -details and exceptions, please see below. +hosts instead via the B<--group-by> option. Within those groups, the +list of hosts (or months) is sorted alphabetically (or chronologically, +respectively) ascending. You can change that order (and sort by number +of postings) with the B<--order-by> option. For details and exceptions, +please see below. The results will be formatted as a kind of table; you can change the output format to a simple list or just a list of names and number of @@ -262,7 +245,7 @@ one for each month, by submitting the B<--filetemplate> option, see below. =head2 Configuration -B will read its configuration from F +B will read its configuration from F which should be present in etc/ via Config::Auto or from a configuration file submitted by the B<--conffile> option. @@ -282,11 +265,6 @@ Print out version and copyright information and exit. Print this man page and exit. -=item B<-t>, B<--type> I - -Create report for hosts (servers) or clients (newsreaders), using -I or I respectively. - =item B<-m>, B<--month> I Set processing period to a single month in YYYY-MM format or to a time @@ -296,7 +274,7 @@ processing period to process the whole database. =item B<-n>, B<--names> I -Limit processing to a certain set of host or client names. I +Limit processing to a certain set of hostnames. I can be a single name (eternal-september.org), a group of names (*.inka.de) or a list of either of these, separated by colons, for example @@ -312,9 +290,9 @@ containing the sum of all detected hosts for that month. Choose the report type: I or I -By default, B will report the number of postings for each -host/client in each month. But it can also report the total sum of postings -per host/client for all months. +By default, B will report the number of postings for each +host in each month. But it can also report the total sum of postings +per host for all months. For report type I, the B option has no meaning and will be silently ignored (see below). @@ -327,13 +305,13 @@ Set the lower boundary. See below. Set the upper boundary. -By default, all hosts/clients with more postings per month than the -upper boundary and/or less postings per month than the lower boundary +By default, all hosts with more postings per month than the upper +boundary and/or less postings per month than the lower boundary will be excluded from further processing. For the default report that -means each month only hosts/clients with a number of postings between -the boundaries will be displayed. For the sums report, hosts/clients -with a number of postings exceeding the boundaries in all (!) months -will not be considered. +means each month only hosts with a number of postings between the +boundaries will be displayed. For the sums report, hosts with a number +of postings exceeding the boundaries in all (!) months will not be +considered. =item B<-g>, B<--group-by> I @@ -349,8 +327,7 @@ ascending order, like this: individual.net : 16768 news.albasani.net: 7879 -The results can be grouped by host/client instead via -B<--group-by> I: +The results can be grouped by host instead via B<--group-by> I: ----- individual.net 2012-01: 19525 @@ -379,8 +356,8 @@ will therefore be ignored. =item B<-o>, B<--order-by> I -Within each group (a single month or single host/client, see above), -the report will be sorted by name (or month) in ascending alphabetical +Within each group (a single month or single host, see above), the +report will be sorted by host (or month) in ascending alphabetical order by default. You can change the sort order to descending or sort by number of postings instead. @@ -426,19 +403,19 @@ False by default. =item B<--comments|--nocomments> -Add comments (group headers) to I and I output. True by default -as logn as B<--filetemplate> is not set. +Add comments (group headers) to I and I output. True by +default as long as B<--filetemplate> is not set. -Use I<--nocomments> to suppress anything except host/client names or months and -numbers of postings. +Use I<--nocomments> to suppress anything except host names or months +and numbers of postings. =item B<--filetemplate> I -Save output to file(s) instead of dumping it to STDOUT. B will -create one file for each month (or each host/client, accordant to the +Save output to file(s) instead of dumping it to STDOUT. B +will create one file for each month (or each host, according to the setting of B<--group-by>, see above), with filenames composed by adding -year and month (or host/client names) to the I, for -example with B<--filetemplate> I: +year and month (or hostnames) to the I, for example +with B<--filetemplate> I: stats-2012-01 stats-2012-02 @@ -446,7 +423,7 @@ example with B<--filetemplate> I: =item B<--db> I -Override I or I from F. +Override I from F. =item B<--conffile> I @@ -462,29 +439,28 @@ See L. Show number of postings per group for lasth month in I format: - cliservstats --type host + hoststats Show that report for January of 2010 and *.inka plus individual.net: - cliservstats --type host --month 2010-01 --names *.inka:individual.net: + hoststats --month 2010-01 --names *.inka:individual.net: -Only show clients with 30 postings or less last month, ordered +Only show hosts with 30 postings or less last month, ordered by number of postings, descending, in I format: - cliservstats --type client --upper 30 --order-by postings-desc + hoststats --upper 30 --order-by postings-desc List number of postings per host for each month of 2010 and redirect output to one file for each month, named hosts-2010-01 and so on, in machine-readable form (without formatting): - cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts - + hoststats -m 2010-01:2010-12 -f dump --filetemplate hosts =head1 FILES =over 4 -=item F +=item F The script itself. diff --git a/bin/postingstats.pl b/bin/postingstats.pl index fed9877..f74d89e 100644 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -15,7 +15,7 @@ # # Usage: # $~ groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups -# $~ cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts +# $~ hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts # BEGIN { @@ -193,7 +193,7 @@ See L. =head1 DESCRIPTION This script will re-format reports on newsgroup usage created by -B or B and create a message that can +B or B and create a message that can be posted to Usenet. =head2 Features and options @@ -291,7 +291,7 @@ Create a posting from a posting statistics report for 2012-01: Create a posting from a host statistics report for last month: - cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts + hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts =head1 FILES @@ -334,7 +334,7 @@ groupstats -h =item - -cliservstats -h +hoststats -h =back diff --git a/contrib/dopostingstats.sh b/contrib/dopostingstats.sh index 30ef8f1..13147b2 100644 --- a/contrib/dopostingstats.sh +++ b/contrib/dopostingstats.sh @@ -2,7 +2,7 @@ # installation path is /srv/newsstats/, please adjust accordingly if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then /srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y - /srv/newsstats/bin/cliservstats.pl -t server --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y + /srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y else echo 'Input error, please use dopostingstats.sh YYYY-MM' fi diff --git a/doc/ChangeLog b/doc/ChangeLog index 6792cc6..4a54296 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -5,6 +5,7 @@ NewsStats 0.4.0 (unreleased) * Improve documentation for config file. * ParseHeader: re-merge continuation lines. * Add ClientStats to gatherstats. + * Move cliservstats to hoststats. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/doc/README b/doc/README index 222cdba..f6b1f44 100644 --- a/doc/README +++ b/doc/README @@ -74,8 +74,8 @@ Getting Started Report generation is handled by specialised scripts for each report type. Currently reports on the number of postings per group and month and injection server and month are supported; you can - use 'groupstats.pl' and 'cliservstats.pl' for that. See the - groupstats.pl and cliservstats.pl man pages for more information. + use 'groupstats.pl' and 'hoststats.pl' for that. See the + groupstats.pl and hoststats.pl man pages for more information. Reporting Bugs From 66a175c7f8d8a35b7c015b68c897eecec84244e7 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Fri, 30 May 2025 19:48:02 +0200 Subject: [PATCH 13/29] Add clientstats (for clients). Signed-off-by: Thomas Hochstein --- bin/clientstats.pl | 598 +++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeLog | 1 + lib/NewsStats.pm | 10 +- 3 files changed, 606 insertions(+), 3 deletions(-) create mode 100644 bin/clientstats.pl diff --git a/bin/clientstats.pl b/bin/clientstats.pl new file mode 100644 index 0000000..0f7ae20 --- /dev/null +++ b/bin/clientstats.pl @@ -0,0 +1,598 @@ +#! /usr/bin/perl +# +# clientstats.pl +# +# This script will get statistical data on newsreader (client) usage +# from a database. +# +# It is part of the NewsStats package. +# +# Copyright (c) 2025 Thomas Hochstein +# +# It can be redistributed and/or modified under the same terms under +# which Perl itself is published. + +BEGIN { + use File::Basename; + # we're in .../bin, so our module is in ../lib + push(@INC, dirname($0).'/../lib'); +} +use strict; +use warnings; + +use NewsStats qw(:DEFAULT :TimePeriods :Output :SQLHelper ReadGroupList); + +use DBI; +use Getopt::Long qw(GetOptions); +Getopt::Long::config ('bundling'); + +################################# Main program ################################# + +### read commandline options +my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat, + $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy, + $OptReportType,$OptSums,$UppBound,$OptVersions,$OptConfFile); +GetOptions ('c|captions!' => \$OptCaptions, + 'comments!' => \$OptComments, + 'db=s' => \$OptDB, + 'filetemplate=s' => \$OptFileTemplate, + 'f|format=s' => \$OptFormat, + 'g|group-by=s' => \$OptGroupBy, + 'l|lower=i' => \$LowBound, + 'm|month=s' => \$OptMonth, + 'n|names=s' => \$OptNames, + 'o|order-by=s' => \$OptOrderBy, + 'r|report=s' => \$OptReportType, + 's|sums!' => \$OptSums, + 'u|upper=i' => \$UppBound, + 'v|versions!' => \$OptVersions, + 'conffile=s' => \$OptConfFile, + 'h|help' => \&ShowPOD, + 'V|version' => \&ShowVersion) or exit 1; +# parse parameters +# $OptComments defaults to TRUE if --filetemplate is not used +$OptComments = 1 if (!$OptFileTemplate && !defined($OptComments)); +# parse $OptReportType +if ($OptReportType) { + if ($OptReportType =~ /sums?/i) { + $OptReportType = 'sum'; + } else { + $OptReportType = 'default'; + } +} + +### read configuration +my %Conf = %{ReadConfig($OptConfFile)}; + +### set DBTable +$Conf{'DBTable'} = $Conf{'DBTableClnts'}; +$Conf{'DBTable'} = $OptDB if $OptDB; + +### init database +my $DBHandle = InitDB(\%Conf,1); + +### get time period and names, prepare SQL 'WHERE' clause +# get time period +# and set caption for output and expression for SQL 'WHERE' clause +my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth); +# bail out if --month is invalid +&Bleat(2,"--month option has an invalid format - ". + "please use 'YYYY-MM', 'YYYY-MM:YYYY-MM' or 'ALL'!") if !$CaptionPeriod; +# get list of clients and set expression for SQL 'WHERE' clause +# with placeholders as well as a list of names to bind to them +my ($SQLWhereNames,@SQLBindNames); +if ($OptNames) { + ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,'client'); + # bail out if --names is invalid + &Bleat(2,"--names option has an invalid format!") + if !$SQLWhereNames; +} + +### build SQL WHERE clause +my $ExcludeSums = $OptSums ? '' : "client != 'ALL'"; +my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, + $ExcludeSums,"version = 'ALL'", + &SQLSetBounds('default',$LowBound,$UppBound)); + +### get sort order and build SQL 'ORDER BY' clause +# force to 'month' for $OptReportType 'sum' +$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default'); +# default to 'name' if $OptGroupBy is not set and +# just one name is requested, but more than one month +$OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/ + and $OptNames and $OptNames !~ /[:*%]/); +# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause +# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month' +my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, 'client, version'); +# $GroupBy will contain 'month' or 'client, version' (parsed result of $OptGroupBy) +# set it to 'month' or 'key' for OutputData() +$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key'; + +### get report type and build SQL 'SELECT' query +my $SQLSelect; +my $SQLGroupClause = ''; + +if ($OptReportType and $OptReportType ne 'default') { + $SQLGroupClause = "GROUP BY client, version"; + # change $SQLOrderClause: replace everything before 'postings' + $SQLOrderClause =~ s/BY.+postings/BY postings/; + $SQLSelect = "'All months',LEFT(client,40),SUM(postings)"; + # change $SQLOrderClause: replace 'postings' with 'SUM(postings)' + $SQLOrderClause =~ s/postings/SUM(postings)/; + } else { + $SQLSelect = "month,LEFT(client,40),postings"; +}; + +### get length of longest name delivered by query +### for formatting purposes +my $Field = ($GroupBy eq 'month') ? 'LEFT(client,40)' : 'month'; +my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, + $Field,'postings',$SQLWhereClause, + '',@SQLBindNames); + +### build and execute SQL query +my ($DBQuery); +# prepare query +$DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s', + $SQLSelect, + $Conf{'DBDatabase'},$Conf{'DBTable'}, + $SQLWhereClause,$SQLGroupClause, + $SQLOrderClause)); +# execute query +$DBQuery->execute(@SQLBindNames) + or &Bleat(2,sprintf("Can't get client data for %s from %s.%s: %s\n", + $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, + $DBI::errstr)); + +### output results +# set default to 'pretty' +$OptFormat = 'pretty' if !$OptFormat; +# print captions if --caption is set +my $LeadIn; +if ($OptCaptions && $OptComments) { + # print time period with report type + my $CaptionReportType = '(number of postings for each month)'; + if ($OptReportType and $OptReportType ne 'default') { + $CaptionReportType = '(number of all postings for that time period)'; + } + $LeadIn .= sprintf("# ----- Report for %s %s\n",$CaptionPeriod,$CaptionReportType); + # print name list if --names is set + $LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames))) + if $OptNames; + # print boundaries, if set + my $CaptionBoundary= '(counting only months fulfilling this condition)'; + $LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n", + $LowBound ? $LowBound : '',$LowBound ? '=>' : '', + $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary) + if ($LowBound or $UppBound); + # print primary and secondary sort order + $LeadIn .= sprintf("# ----- Grouped by %s (%s), sorted %s%s\n", + ($GroupBy eq 'month') ? 'Months' : 'Names', + ($OptGroupBy and $OptGroupBy =~ /-?desc$/i) ? 'descending' : 'ascending', + ($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '', + ($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending'); +} + +# output data +# (changed code copy from NewsStats::OutputData) +my ($LastIteration, $FileName, $Handle, $OUT); + +# define output types +my %LegalOutput; +@LegalOutput{('dump','list','pretty')} = (); +# bail out if format is unknown +&Bleat(2,"Unknown output type '$OptFormat'!") if !exists($LegalOutput{$OptFormat}); + +while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { + # save client for later use + my $Client = $Key; + # care for correct sorting order and abstract from month and keys: + # $Caption will be $Month or $Key, according to sorting order, + # and $Key will be $Key or $Month, respectively + my $Caption; + if ($GroupBy eq 'key') { + $Caption = $Key; + $Key = $Month; + } else { + $Caption = $Month; + } + # set output file handle + if (!$OptFileTemplate) { + $Handle = *STDOUT{IO}; # set $Handle to a reference to STDOUT + } elsif (!defined($LastIteration) or $LastIteration ne $Caption) { + close $OUT if ($LastIteration); + # safeguards for filename creation: + # replace potential problem characters with '_' + $FileName = sprintf('%s-%s',$OptFileTemplate,$Caption); + $FileName =~ s/[^a-zA-Z0-9_-]+/_/g; + open ($OUT,">$FileName") + or &Bleat(2,sprintf("Cannot open output file '%s': $!", + $FileName)); + $Handle = $OUT; + }; + print $Handle &FormatOutput($OptFormat, $OptComments, $LeadIn, $Caption, + $Key, $Value, 0, $MaxLength, $MaxValLength, $LastIteration); + # output client versions + if ($OptVersions) { + ### get client versions + # $SQLWhereClause without 'ALL' version + $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, + $ExcludeSums,"version != 'ALL'","client = '$Client'", + &SQLSetBounds('default',$LowBound,$UppBound)); + + # save length of longest client + my $ClientMaxLenght = $MaxLength; + my $ClientMaxValLenght = $MaxValLength; + # get length of longest version delivered by query + # for formatting purposes + my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, + 'version','postings',$SQLWhereClause, + '',@SQLBindNames); + if ($MaxLength) { + # add lenght of '- ' + $MaxLength += 2; + # set to length of longest client, if longer + $MaxLength = $ClientMaxLenght if $ClientMaxLenght > $MaxLength; + $MaxValLength = $ClientMaxValLenght if $ClientMaxValLenght > $MaxValLength; + } + + # prepare query + my $DBVersQuery = $DBHandle->prepare(sprintf('SELECT version,postings FROM %s.%s %s %s %s', + $Conf{'DBDatabase'},$Conf{'DBTable'}, + $SQLWhereClause,$SQLGroupClause, + $SQLOrderClause)); + # execute query + $DBVersQuery->execute(@SQLBindNames) + or &Bleat(2,sprintf("Can't get version data for %s from %s.%s: %s\n", + $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, + $DBI::errstr)); + # output versions + while (my ($Version, $Postings) = $DBVersQuery->fetchrow_array) { + $Version = '- ' . $Version; + print $Handle &FormatOutput($OptFormat, $OptComments, $LeadIn, '', + $Version, $Postings, 0, $MaxLength, $MaxValLength, + ''); + } + } + $LastIteration = $Caption; +}; +close $OUT if ($OptFileTemplate); + +### close handles +$DBHandle->disconnect; + +__END__ + +################################ Documentation ################################# + +=head1 NAME + +clientstats - create reports on client usage + +=head1 SYNOPSIS + +B [B<-Vhcs> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] + +=head1 REQUIREMENTS + +See L. + +=head1 DESCRIPTION + +This script create reports on newsgroup usage (number of postings +using each client per month) taken from result tables created by +B. + +=head2 Features and options + +=head3 Time period and names + +The time period to act on defaults to last month; you can assign another +time period or a single month (or drop all time constraints) via the +B<--month> option (see below). + +B will process all clients by default; you can limit +processing to only some clients by supplying a list of those names by +using the B<--names> option (see below). + +=head3 Report type + +You can choose between different B<--report> types: postings per month +or all postings summed up; for details, see below. + +=head3 Upper and lower boundaries + +Furthermore you can set an upper and/or lower boundary to exclude some +results from output via the B<--lower> and B<--upper> options, +respectively. By default, all clients with more and/or less postings +per month will be excluded from the result set (i.e. not shown and +not considered for sum reports). + +=head3 Sorting and formatting the output + +By default, all results are grouped by month; you can group results by +clients instead via the B<--group-by> option. Within those groups, +the list of clients (or months) is sorted alphabetically +(or chronologically, respectively) ascending. You can change that order +(and sort by number of postings) with the B<--order-by> option. For +details and exceptions, please see below. + +The results will be formatted as a kind of table; you can change the +output format to a simple list or just a list of names and number of +postings with the B<--format> option. Captions will be added by means of +the B<--caption> option; all comments (and captions) can be supressed by +using B<--nocomments>. + +Last but not least you can redirect all output to a number of files, e.g. +one for each month, by submitting the B<--filetemplate> option, see below. + +=head2 Configuration + +B will read its configuration from F +which should be present in etc/ via Config::Auto or from a configuration file +submitted by the B<--conffile> option. + +See doc/INSTALL for an overview of possible configuration options. + +You can override some configuration options via the B<--db> option. + +=head1 OPTIONS + +=over 3 + +=item B<-V>, B<--version> + +Print out version and copyright information and exit. + +=item B<-h>, B<--help> + +Print this man page and exit. + +=item B<-m>, B<--month> I + +Set processing period to a single month in YYYY-MM format or to a time +period between two month in YYYY-MM:YYYY-MM format (two month, separated +by a colon). By using the keyword I instead, you can set no +processing period to process the whole database. + +=item B<-n>, B<--names> I + +Limit processing to a certain set of client names. I +can be a single name (eternal-september.org), a group of names +(*.inka.de) or a list of either of these, separated by colons, for +example + + eternal-september.org:solani.org:*.inka.de + +=item B<-s>, B<--sums|--nosums> (sum per month) + +Include "virtual" clients named "ALL" for every month in output, +containing the sum of all detected clients for that month. + +=item B<-r>, B<--report> I + +Choose the report type: I or I + +By default, B will report the number of postings for each +client in each month. But it can also report the total sum of postings +per client for all months. + +For report type I, the B option has no meaning and +will be silently ignored (see below). + +=item B<-l>, B<--lower> I + +Set the lower boundary. See below. + +=item B<-l>, B<--upper> I + +Set the upper boundary. + +By default, all clients with more postings per month than the +upper boundary and/or less postings per month than the lower boundary +will be excluded from further processing. For the default report that +means each month only /clients with a number of postings between +the boundaries will be displayed. For the sums report, /clients +with a number of postings exceeding the boundaries in all (!) months +will not be considered. + +=item B<-g>, B<--group-by> I + +By default, all results are grouped by month, sorted chronologically in +ascending order, like this: + + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + news.albasani.net: 9063 + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + news.albasani.net: 7879 + +The results can be grouped by client instead via +B<--group-by> I: + + ----- individual.net + 2012-01: 19525 + 2012-02: 16768 + ----- arcor-online.net + 2012-01: 9379 + 2012-02: 8606 + ----- news.albasani.net + 2012-01: 9063 + 2012-02: 7879 + +By appending I<-desc> to the group-by option parameter, you can reverse +the sort order - e.g. B<--group-by> I will give: + + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + news.albasani.net: 7879 + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + news.albasani.net: 9063 + +Sums reports (see above) will always be grouped by months; this option +will therefore be ignored. + +=item B<-o>, B<--order-by> I + +Within each group (a single month or single client, see above), +the report will be sorted by name (or month) in ascending alphabetical +order by default. You can change the sort order to descending or sort +by number of postings instead. + +=item B<-f>, B<--format> I + +Select the output format, I being the default: + + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + +I format looks like this: + + 2012-01 arcor-online.net 9379 + 2012-01 individual.net 19525 + 2012-02 arcor-online.net 8606 + 2012-02 individual.net 16768 + +And I format looks like this: + + # 2012-01: + arcor-online.net 9379 + individual.net 19525 + # 2012-02: + arcor-online.net 8606 + individual.net 16768 + +You can remove the comments by using B<--nocomments>, see below. + +=item B<-c>, B<--captions|--nocaptions> + +Add captions to output, like this: + + ----- Report for 2012-01 to 2012-02 (number of postings for each month) + ----- Names: individual.net + ----- Threshold: 8000 => x (counting only month fulfilling this condition) + ----- Grouped by Month (ascending), sorted by number of postings descending + +False by default. + +=item B<--comments|--nocomments> + +Add comments (group headers) to I and I output. True by default +as logn as B<--filetemplate> is not set. + +Use I<--nocomments> to suppress anything except client names or months and +numbers of postings. + +=item B<--filetemplate> I + +Save output to file(s) instead of dumping it to STDOUT. B +will create one file for each month (or each client, according to the +setting of B<--group-by>, see above), with filenames composed by adding +year and month (or client names) to the I, for +example with B<--filetemplate> I: + + stats-2012-01 + stats-2012-02 + ... and so on + +=item B<--db> I + +Override I or I from F. + +=item B<--conffile> I + +Load configuration from I instead of F. + +=back + +=head1 INSTALLATION + +See L. + +=head1 EXAMPLES + +Show number of postings per group for lasth month in I format: + + clientstats + +Show that report for January of 2010 and *.inka plus individual.net: + + clientstats --month 2010-01 --names *.inka:individual.net: + +Only show clients with 30 postings or less last month, ordered +by number of postings, descending, in I format: + + clientstats --upper 30 --order-by postings-desc + +List number of postings per host for each month of 2010 and redirect +output to one file for each month, named hosts-2010-01 and so on, in +machine-readable form (without formatting): + + clientstats -m 2010-01:2010-12 -f dump --filetemplate hosts + + +=head1 FILES + +=over 4 + +=item F + +The script itself. + +=item F + +Library functions for the NewsStats package. + +=item F + +Runtime configuration file. + +=back + +=head1 BUGS + +Please report any bugs or feature requests to the author or use the +bug tracker at L! + +=head1 SEE ALSO + +=over 2 + +=item - + +L + +=item - + +L + +=item - + +gatherstats -h + +=back + +This script is part of the B package. + +=head1 AUTHOR + +Thomas Hochstein + +=head1 COPYRIGHT AND LICENSE + +Copyright (c) 2025 Thomas Hochstein + +This program is free software; you may redistribute it and/or modify it +under the same terms as Perl itself. + +=cut diff --git a/doc/ChangeLog b/doc/ChangeLog index 4a54296..309ff60 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -6,6 +6,7 @@ NewsStats 0.4.0 (unreleased) * ParseHeader: re-merge continuation lines. * Add ClientStats to gatherstats. * Move cliservstats to hoststats. + * Add clientstats (for clients). NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index 35b59c0..8a5dfde 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -506,16 +506,20 @@ sub FormatOutput { if ($Format eq 'dump') { # output as dump (key value) $Output = sprintf ("# %s:\n",$Caption) - if ($Comments and (!defined($LastIteration) or $Caption ne $LastIteration)); + if ($Caption and $Comments and (!defined($LastIteration) or $Caption ne $LastIteration)); $Output .= sprintf ("%s %u\n",$Key,$Value); } elsif ($Format eq 'list') { # output as list (caption key value) - $Output = sprintf ("%s %s %u\n",$Caption,$Key,$Value); + if ($Caption) { + $Output = sprintf ("%s %s %u\n",$Caption,$Key,$Value); + } else { + $Output = sprintf ("%s %u\n",$Key,$Value); + } } elsif ($Format eq 'pretty') { # output as a table if ($Comments and (!defined($LastIteration) or $Caption ne $LastIteration)) { $Output = $LeadIn; - $Output .= sprintf ("# ----- %s:\n",$Caption); + $Output .= sprintf ("# ----- %s:\n",$Caption) if $Caption; } # increase $PadValue for numbers with decimal point $PadValue += $Precision+1 if $Precision; From 09a91126796dcecf49cf769cd2848111d25f7c72 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Fri, 30 May 2025 23:29:40 +0200 Subject: [PATCH 14/29] Fix CheckValidNames(). - Make RegExp configurable. - Change default for clients (client names have spaces). Signed-off-by: Thomas Hochstein --- lib/NewsStats.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index 8a5dfde..f50ba94 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -642,9 +642,11 @@ sub SQLGroupList { ### OUT: SQL code to become part of a 'WHERE' clause, ### list of names for SQL bindings my ($Names,$Type) = @_; + my $InvalidCharRegExp; # substitute '*' wildcard with SQL wildcard character '%' $Names =~ s/\*/%/g; - return (undef,undef) if !CheckValidNames($Names); + $InvalidCharRegExp = ',;' if $Type eq 'client'; + return (undef,undef) if !CheckValidNames($Names,$InvalidCharRegExp); # just one name/newsgroup? return (SQLGroupWildcard($Names,$Type),$Names) if $Names !~ /:/; my ($SQL,@WildcardNames,@NoWildcardNames); @@ -807,10 +809,11 @@ sub SQLBuildClause { sub CheckValidNames { ################################################################################ ### syntax check of a list -### IN : $Names: list of names, e.g. newsgroups (group.one.*:group.two:group.three.*) +### IN : $Names : list of names, e.g. newsgroups (group.one.*:group.two:group.three.*) +### InvalidCharRegExp: regular expression for invalid characters ### OUT: boolean - my ($Names) = @_; - my $InvalidCharRegExp = ',; '; + my ($Names,$InvalidCharRegExp) = @_; + $InvalidCharRegExp = ',; ' if (!$InvalidCharRegExp); return ($Names =~ /[$InvalidCharRegExp]/) ? 0 : 1; }; From 39e845d5529ced795b07184ebb34a96d7562182e Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Fri, 30 May 2025 22:43:39 +0200 Subject: [PATCH 15/29] Add ClientStats to postingstats. Signed-off-by: Thomas Hochstein --- bin/postingstats.pl | 151 +++++++++++++++++++++++++++++++++++--------- doc/ChangeLog | 1 + 2 files changed, 123 insertions(+), 29 deletions(-) diff --git a/bin/postingstats.pl b/bin/postingstats.pl index f74d89e..a9d1e9a 100644 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -16,6 +16,7 @@ # Usage: # $~ groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups # $~ hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts +# $~ clientstats.pl --nocomments --sums --versions --format dump | postingstats.pl -t clients # BEGIN { @@ -53,19 +54,22 @@ if (!$Type) { $Type = 'GroupStats'; } elsif ($Type =~ /(host|server)s?/i) { $Type = 'HostStats'; +} elsif ($Type =~ /(client|reader)s?/i) { + $Type = 'ClientStats'; }; my $Timestamp = time; ##### ----- configuration -------------------------------------------- my $TLH = 'de'; -my %Heading = ('GroupStats' => 'Postingstatistik fuer de.* im Monat '.$Month, - 'HostStats' => 'Serverstatistik fuer de.* im Monat '.$Month +my %Heading = ('GroupStats' => 'Postingstatistik fuer de.* im Monat '.$Month, + 'HostStats' => 'Serverstatistik fuer de.* im Monat '.$Month, + 'ClientStats' => 'Newsreaderstatistik fuer de.* im Monat '.$Month ); my %TH = ('counter' => 'Nr.', 'value' => 'Anzahl', 'percentage' => 'Prozent' ); -my %LeadIn = ('GroupStats' => < < < < < Newsgroups: local.test Subject: Postingstatistik fuer de.* im Monat $Month @@ -88,7 +92,18 @@ Content-Transfer-Encoding: 7bit User-Agent: postingstats.pl/$VERSION (NewsStats) HOSTSIN -my %LeadOut = ('GroupStats' => < < +Newsgroups: local.test +Subject: Newsreaderstatistik fuer de.* im Monat $Month +Message-ID: +Approved: thh\@thh.name +Mime-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit +User-Agent: postingstats.pl/$VERSION (NewsStats) + +CLIENTSIN +my %LeadOut = ('GroupStats' => < < < $$RMaxLength; + + # delete single version + delete($$RSubValue{$LastName}); +} + ##### ----- main loop ------------------------------------------------ -my (%Value, $SumName, $SumTotal, $MaxLength); -$MaxLength = 0; +my (%Value, %SubValue, $SubCounter, $LastName, $SumName, $SumTotal, + $MaxLength); + if ($Type eq 'GroupStats') { $SumName = "$TLH.ALL"; $TH{'name'} = 'Newsgroup' } elsif ($Type eq 'HostStats') { $SumName = 'ALL'; - $TH{'name'} = 'Server' + $TH{'name'} = 'Postingserver' +} elsif ($Type eq 'ClientStats') { + $SumName = 'ALL'; + $TH{'name'} = 'Newsreader / Client' } -# read from STDIN +### read from STDIN +$MaxLength = 0; while(<>) { - my ($Name, $Value) = split; + my ($Name, $Value) = $_ =~ /(.+) (\d+)$/; $SumTotal = $Value if $Name eq $SumName; next if $Name =~ /ALL$/; - $Value{$Name} = $Value; - $MaxLength = length($Name) if length($Name) > $MaxLength; -} -# print to STDOUT + # handle client versions + if ($Type eq 'ClientStats' and $Name =~ /^- /) { + $SubValue{$LastName}{$Name} = $Value; + $SubCounter++; + } else { + # clients with just one version + &SingleVersion($LastName,\%SubValue,\%Value,\$MaxLength) + if ($LastName && $SubCounter == 1); + + # reset version counter and client name + $SubCounter = 0; + $LastName = $Name; + + $Value{$Name} = $Value; + $MaxLength = length($Name) if length($Name) > $MaxLength; + } +} +# clients with just one version (last iteration) +&SingleVersion($LastName,\%SubValue,\%Value,\$MaxLength) + if ($LastName && $SubCounter == 1); + +### print to STDOUT +# calculate padding for $Heading my $PaddingLeft = ' ' x int((($MaxLength+TABLEWIDTH-2-length($Heading{$Type}))/2)); my $PaddingRight = $PaddingLeft; -$PaddingLeft .= ' ' if (length($Heading{$Type}) + (length($PaddingLeft) * 2) < $MaxLength+TABLEWIDTH); -my $Counter = 0; +$PaddingLeft .= ' ' if (length($Heading{$Type}) + (length($PaddingLeft) * 2) +2 < $MaxLength+TABLEWIDTH); print $LeadIn{$Type}; +# print table header print &Divider('=',$MaxLength); printf(": %s%s%s :\n",$PaddingLeft,$Heading{$Type},$PaddingRight); print &Divider('=',$MaxLength); @@ -163,11 +232,26 @@ printf(": %-3s : %-6s : %-7s : %-*s :\n", $MaxLength,$TH{'name'}); print &Divider('-',$MaxLength); -foreach my $Name (sort { $Value{$b} <=> $Value {$a}} keys %Value) { +# print table +my $Counter = 0; +foreach my $Name (sort { $Value{$b} <=> $Value{$a} } keys %Value) { $Counter++; - printf(": %3u. : %6u : %6.2f%% : %-*s :\n",$Counter,$Value{$Name},&Percentage($SumTotal,$Value{$Name}),$MaxLength,$Name); + printf(": %3u. : %6u : %6.2f%% : %-*s :\n", + $Counter,$Value{$Name},&Percentage($SumTotal,$Value{$Name}), + $MaxLength,$Name); + # handle client versions + if ($SubValue{$Name}) { + foreach my $SubName (sort { $SubValue{$Name}{$b} <=> $SubValue{$Name}{$a} } + keys %{$SubValue{$Name}}) { + printf(": : %6u : %6.2f%% : %-*s :\n", + $SubValue{$Name}{$SubName}, + &Percentage($SumTotal,$SubValue{$Name}{$SubName}), + $MaxLength,$SubName); + } + } } +# print table footer print &Divider('-',$MaxLength); printf(": : %6u : %s : %-*s :\n",$SumTotal,'100.00%',$MaxLength,''); print &Divider('=',$MaxLength); @@ -184,7 +268,7 @@ postingstats - format and post reports =head1 SYNOPSIS -B B<-t> I [B<-Vh> [B<-m> I] +B B<-t> I [B<-Vh> [B<-m> I] =head1 REQUIREMENTS @@ -193,8 +277,8 @@ See L. =head1 DESCRIPTION This script will re-format reports on newsgroup usage created by -B or B and create a message that can -be posted to Usenet. +B, B or B and create a +message that can be posted to Usenet. =head2 Features and options @@ -228,8 +312,8 @@ sum total. =item C<%Heading> -Hash with keys for I and I. Used to display a -heading. +Hash with keys for I, I and I. +Used to display a heading. =item C<%TH> @@ -242,14 +326,14 @@ Output will be truncated otherwise. =item C<%LeadIn> -Hash with keys for I and I. Used to create the -headers for our posting. Can contain other text that will be shown -before C<%Heading>. +Hash with keys for I, I and I. +Used to create the headers for our posting. Can contain other text +that will be shown before C<%Heading>. =item C<%LeadOut> -Hash with keys for I and I. Will be shown at the -end of our posting. +Hash with keys for I, I and I. +Will be shown at the end of our posting. =back @@ -265,9 +349,10 @@ Print out version and copyright information and exit. Print this man page and exit. -=item B<-t>, B<--type> I +=item B<-t>, B<--type> I -Set report type to posting statistics or hosts statistics accordingly. +Set report type to posting statistics, hosts statistics or client +statistics accordingly. =item B<-m>, B<--month> I @@ -293,6 +378,10 @@ Create a posting from a host statistics report for last month: hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts +Create a posting from a client statistics report for last month: + + clientstats.pl --nocomments --sums --versions --format dump | postingstats.pl -t clients + =head1 FILES =over 4 @@ -336,6 +425,10 @@ groupstats -h hoststats -h +=item - + +clientstats -h + =back This script is part of the B package. diff --git a/doc/ChangeLog b/doc/ChangeLog index 309ff60..f230009 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -7,6 +7,7 @@ NewsStats 0.4.0 (unreleased) * Add ClientStats to gatherstats. * Move cliservstats to hoststats. * Add clientstats (for clients). + * Add ClientStats to postingstats. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 06bcdfb2bed6f65d39018f52b868c34c3bbe41f4 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:01:57 +0200 Subject: [PATCH 16/29] gatherstats: Don't die on parsing errors. Just warn if host or client can't be identified. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 4 ++-- doc/ChangeLog | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index b4c844e..9b02bfa 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -339,7 +339,7 @@ sub HostStats { $Postings{$Host}++; $Postings{'ALL'}++; } else { - &Bleat(2,sprintf("%s FAILED", $Header{'message-id'})) if !$Host; + &Bleat(1,sprintf("%s FAILED", $Header{'message-id'})) if !$Host; } printf("%s: %s\n", $Header{'message-id'}, $Host) if ($MID or $Debug && $Debug >1); @@ -487,7 +487,7 @@ sub ClientStats { version => $Version); push @Clients, { %UserAgent }; } else { - &Bleat(2,sprintf("%s FAILED", $Header{'message-id'})) if !@Clients; + &Bleat(1,sprintf("%s FAILED", $Header{'message-id'})) if !@Clients; } } } diff --git a/doc/ChangeLog b/doc/ChangeLog index f230009..e102c34 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -8,6 +8,7 @@ NewsStats 0.4.0 (unreleased) * Move cliservstats to hoststats. * Add clientstats (for clients). * Add ClientStats to postingstats. + * gatherstats: Don't die on parsing errors. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 462f28505dcfd604fb1b0021b1376e6f940f5184 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:33:46 +0200 Subject: [PATCH 17/29] DBClnts: set version length to 50. Signed-off-by: Thomas Hochstein --- bin/dbcreate.pl | 2 +- doc/ChangeLog | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/dbcreate.pl b/bin/dbcreate.pl index fee67ab..e3ad8c6 100755 --- a/bin/dbcreate.pl +++ b/bin/dbcreate.pl @@ -110,7 +110,7 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableClnts'}` ( `id` bigint(20) unsigned NOT NULL auto_increment, `month` varchar(7) character set ascii NOT NULL, `client` varchar(150) NOT NULL, - `version` varchar(20) NOT NULL, + `version` varchar(50) NOT NULL, `postings` int(11) NOT NULL, `revision` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (`id`), diff --git a/doc/ChangeLog b/doc/ChangeLog index e102c34..c829e19 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -9,6 +9,7 @@ NewsStats 0.4.0 (unreleased) * Add clientstats (for clients). * Add ClientStats to postingstats. * gatherstats: Don't die on parsing errors. + * DBClnts: set version length to to 50. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From ed3fb3cda0f2d5cd805be92ce4c8521532504830 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 01:06:15 +0200 Subject: [PATCH 18/29] Truncate overlong clients or versions. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 3 +++ doc/ChangeLog | 1 + 2 files changed, 4 insertions(+) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 9b02bfa..9cb0a9b 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -500,6 +500,9 @@ sub ClientStats { # encode to utf-8, if necessary $_->{'agent'} = encode('UTF-8', $_->{'agent'}) if $_->{'agent'} =~ /[\x80-\x{ffff}]/; $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/; + # truncate overlong clients or versions + $_->{'agent'} = substr($_->{'agent'}, 0, 150) if length($_->{'agent'}) > 150; + $_->{'version'} = substr($_->{'version'}, 0, 50) if $_->{'version'} and length($_->{'version'}) > 50; # special cases # Mozilla $_->{'agent'} = 'Mozilla' if $_->{'agent'} eq '•Mozilla'; diff --git a/doc/ChangeLog b/doc/ChangeLog index c829e19..aa498ca 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -10,6 +10,7 @@ NewsStats 0.4.0 (unreleased) * Add ClientStats to postingstats. * gatherstats: Don't die on parsing errors. * DBClnts: set version length to to 50. + * gatherstats: Truncate overlong clients or versions. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 0102b72971a5889173d67906aa799ea49cc8bcc6 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:48:23 +0200 Subject: [PATCH 19/29] Remove whitespace from client and version. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 6 ++++-- doc/ChangeLog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 9cb0a9b..55c2dad 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -497,6 +497,9 @@ sub ClientStats { foreach (@Clients) { # filter agents for User-Agent with multiple agents next if $#Clients && exists($DropAgent{lc($_->{'agent'})}); + # remove whitespace + $_->{'agent'} =~ s/^\s+|\s+$//g; + $_->{'version'} =~ s/^\s+|\s+$//g if $_->{'version'}; # encode to utf-8, if necessary $_->{'agent'} = encode('UTF-8', $_->{'agent'}) if $_->{'agent'} =~ /[\x80-\x{ffff}]/; $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/; @@ -664,8 +667,7 @@ sub RemoveComments { # remove superfluous whitespace in header # and whitespace around header $Header =~ s/\s+/ /g; - $Header =~ s/^\s+//; - $Header =~ s/\s+$//; + $Header =~ s/^\s+|\s+$//g; return $Header; } diff --git a/doc/ChangeLog b/doc/ChangeLog index aa498ca..6bf2a46 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -11,6 +11,7 @@ NewsStats 0.4.0 (unreleased) * gatherstats: Don't die on parsing errors. * DBClnts: set version length to to 50. * gatherstats: Truncate overlong clients or versions. + * gatherstats: Remove whitespace from client and version. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 07e45437174ef6551a6657df0f257186a061d6a5 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:58:23 +0200 Subject: [PATCH 20/29] Fix typos. Signed-off-by: Thomas Hochstein --- bin/postingstats.pl | 2 +- doc/ChangeLog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/postingstats.pl b/bin/postingstats.pl index a9d1e9a..e517a45 100644 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -134,7 +134,7 @@ wenn Sie ermittelbar sind; daher kann die Summe der Newsreader-Versionen kleiner sein als die Postingzahl fuer den Newsreader. Ausserdem koennen an einem Beitrag mehrere Clients beteiligt sein, bspw. der Newsreader und ein lokaler Server wie der Hamster. Daher kann die Summe aller -Newsreader groesser sein als die Summer der Postings; auch ergeben die +Newsreader groesser sein als die Summe der Postings; auch ergeben die Prozentzahlen dementsprechend in der Summe mehr als 100%. CLIENTSOUT diff --git a/doc/ChangeLog b/doc/ChangeLog index 6bf2a46..a6e7ed9 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -9,7 +9,7 @@ NewsStats 0.4.0 (unreleased) * Add clientstats (for clients). * Add ClientStats to postingstats. * gatherstats: Don't die on parsing errors. - * DBClnts: set version length to to 50. + * DBClnts: set version length to 50. * gatherstats: Truncate overlong clients or versions. * gatherstats: Remove whitespace from client and version. From d02ae5e2ff3f013febc4d84d22651610eb7a88b2 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 09:38:41 +0200 Subject: [PATCH 21/29] Fix version queries. Add month to WHERE clause, use bind values. Signed-off-by: Thomas Hochstein --- bin/clientstats.pl | 12 ++++++++---- doc/ChangeLog | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/clientstats.pl b/bin/clientstats.pl index 0f7ae20..b94fa16 100644 --- a/bin/clientstats.pl +++ b/bin/clientstats.pl @@ -215,10 +215,14 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { # output client versions if ($OptVersions) { ### get client versions - # $SQLWhereClause without 'ALL' version + # $SQLWhereClause without 'ALL' version, with client and month set $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, - $ExcludeSums,"version != 'ALL'","client = '$Client'", + $ExcludeSums,"version != 'ALL'", + 'client = ?','month = ?', &SQLSetBounds('default',$LowBound,$UppBound)); + # push client and month to @SQLVersBindNames + my @SQLVersBindNames = @SQLBindNames; + push (@SQLVersBindNames, ($Client, $Month)); # save length of longest client my $ClientMaxLenght = $MaxLength; @@ -227,7 +231,7 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { # for formatting purposes my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, 'version','postings',$SQLWhereClause, - '',@SQLBindNames); + '',@SQLVersBindNames); if ($MaxLength) { # add lenght of '- ' $MaxLength += 2; @@ -242,7 +246,7 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { $SQLWhereClause,$SQLGroupClause, $SQLOrderClause)); # execute query - $DBVersQuery->execute(@SQLBindNames) + $DBVersQuery->execute(@SQLVersBindNames) or &Bleat(2,sprintf("Can't get version data for %s from %s.%s: %s\n", $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, $DBI::errstr)); diff --git a/doc/ChangeLog b/doc/ChangeLog index a6e7ed9..ef17bfc 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -12,6 +12,7 @@ NewsStats 0.4.0 (unreleased) * DBClnts: set version length to 50. * gatherstats: Truncate overlong clients or versions. * gatherstats: Remove whitespace from client and version. + * Fix version queries. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 8afeb09cc2a6b8b32d4fb01574ee40e961f0defe Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 18:33:33 +0200 Subject: [PATCH 22/29] Add clientstats to dopostingstats. Signed-off-by: Thomas Hochstein --- contrib/dopostingstats.sh | 1 + doc/ChangeLog | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/dopostingstats.sh b/contrib/dopostingstats.sh index 13147b2..c0e5e8c 100644 --- a/contrib/dopostingstats.sh +++ b/contrib/dopostingstats.sh @@ -3,6 +3,7 @@ if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then /srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y /srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y + /srv/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t client --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y else echo 'Input error, please use dopostingstats.sh YYYY-MM' fi diff --git a/doc/ChangeLog b/doc/ChangeLog index ef17bfc..1a442c5 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -13,6 +13,7 @@ NewsStats 0.4.0 (unreleased) * gatherstats: Truncate overlong clients or versions. * gatherstats: Remove whitespace from client and version. * Fix version queries. + * Add ClientStats to dopostingstats. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 18db200aea331ad9a7bbf9f81b05bb6b795c6665 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 18:34:41 +0200 Subject: [PATCH 23/29] Let dopostingstats default to last month. Signed-off-by: Thomas Hochstein --- contrib/dopostingstats.sh | 15 +++++++++------ doc/ChangeLog | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/dopostingstats.sh b/contrib/dopostingstats.sh index c0e5e8c..61ffd53 100644 --- a/contrib/dopostingstats.sh +++ b/contrib/dopostingstats.sh @@ -1,10 +1,13 @@ #!/bin/bash # installation path is /srv/newsstats/, please adjust accordingly -if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then - /srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y - /srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y - /srv/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t client --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y -else - echo 'Input error, please use dopostingstats.sh YYYY-MM' + +# get month +MONTH=$1 +if ! [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then + MONTH=$(date -d "$(date +%Y-%m-15) -1 month" '+%Y-%m') fi +# post stats +/srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y +/srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t server --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y +/srv/newsstats/bin/clientstats.pl --nocomments --sums --versions --format dump --month $MONTH | /srv/newsstats/bin/postingstats.pl -t client --month $MONTH | /srv/newsstats/contrib/tinews.pl -X -Y diff --git a/doc/ChangeLog b/doc/ChangeLog index 1a442c5..323e178 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -14,6 +14,7 @@ NewsStats 0.4.0 (unreleased) * gatherstats: Remove whitespace from client and version. * Fix version queries. * Add ClientStats to dopostingstats. + * Let dopostingstats default to last month. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From cff76a3c65cc311610dacaf33e103a1e3c8fd64c Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 21:47:59 +0200 Subject: [PATCH 24/29] Set executable bit for new scripts. Signed-off-by: Thomas Hochstein --- bin/clientstats.pl | 0 bin/hoststats.pl | 0 bin/postingstats.pl | 0 contrib/dopostingstats.sh | 0 contrib/tinews.pl | 0 contrib/yearstats.sh | 0 doc/ChangeLog | 1 + 7 files changed, 1 insertion(+) mode change 100644 => 100755 bin/clientstats.pl mode change 100644 => 100755 bin/hoststats.pl mode change 100644 => 100755 bin/postingstats.pl mode change 100644 => 100755 contrib/dopostingstats.sh mode change 100644 => 100755 contrib/tinews.pl mode change 100644 => 100755 contrib/yearstats.sh diff --git a/bin/clientstats.pl b/bin/clientstats.pl old mode 100644 new mode 100755 diff --git a/bin/hoststats.pl b/bin/hoststats.pl old mode 100644 new mode 100755 diff --git a/bin/postingstats.pl b/bin/postingstats.pl old mode 100644 new mode 100755 diff --git a/contrib/dopostingstats.sh b/contrib/dopostingstats.sh old mode 100644 new mode 100755 diff --git a/contrib/tinews.pl b/contrib/tinews.pl old mode 100644 new mode 100755 diff --git a/contrib/yearstats.sh b/contrib/yearstats.sh old mode 100644 new mode 100755 diff --git a/doc/ChangeLog b/doc/ChangeLog index 323e178..857b23e 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -15,6 +15,7 @@ NewsStats 0.4.0 (unreleased) * Fix version queries. * Add ClientStats to dopostingstats. * Let dopostingstats default to last month. + * Set executable bit for new scripts. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 1a5b9dbcb1e73bbfd0ebdc4ea1626745137ed34b Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 1 Jun 2025 10:53:52 +0200 Subject: [PATCH 25/29] Remove debugging code. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 1 - lib/NewsStats.pm | 1 - 2 files changed, 2 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 55c2dad..302e587 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -23,7 +23,6 @@ use warnings; use NewsStats qw(:DEFAULT :TimePeriods ListNewsgroups ParseHierarchies ReadGroupList ParseHeaders); use DBI; -use Data::Dumper; use Encode qw(decode encode); use Getopt::Long qw(GetOptions); Getopt::Long::config ('bundling'); diff --git a/lib/NewsStats.pm b/lib/NewsStats.pm index f50ba94..99bc185 100644 --- a/lib/NewsStats.pm +++ b/lib/NewsStats.pm @@ -51,7 +51,6 @@ require Exporter; SQLSetBounds SQLBuildClause GetMaxLength)]); $VERSION = '0.4.0'; -use Data::Dumper; use File::Basename; use Cwd qw(realpath); From 0b87e81b08f67c963dcc763ade8d92a194a8c22a Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 1 Jun 2025 11:00:01 +0200 Subject: [PATCH 26/29] Add parsing exemptions. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 302e587..1b9292c 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -653,7 +653,8 @@ sub RemoveComments { $Header =~ s/[ _]DE//; # remove trailing 'eol' or '-shl' - $Header =~ s/(eol)|(-shl)$//; + # or ml-inews[-sig] + $Header =~ s/(eol)|(-shl)|(ml-inews(-sig)?)$//; # remove from ';' or ',' (CrossPoint) # or '&' to end of header From 66890b68d8811f77af3901fb57bb848c9f187e14 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 1 Jun 2025 16:39:25 +0200 Subject: [PATCH 27/29] Update documentation. - Fix clientstats doc (copied from hoststats). - Add some more examples ro README. Signed-off-by: Thomas Hochstein --- bin/clientstats.pl | 178 ++++++++++++++++++++++++++------------------ bin/dbcreate.pl | 19 ++--- bin/feedlog.pl | 13 ++-- bin/gatherstats.pl | 22 +++--- bin/groupstats.pl | 147 ++++++++++++++++++------------------ bin/hoststats.pl | 38 +++++----- bin/postingstats.pl | 22 +++--- doc/ChangeLog | 1 + doc/INSTALL | 55 ++++++++------ doc/README | 75 ++++++++++++++----- doc/TODO | 22 +----- 11 files changed, 329 insertions(+), 263 deletions(-) diff --git a/bin/clientstats.pl b/bin/clientstats.pl index b94fa16..71923c3 100755 --- a/bin/clientstats.pl +++ b/bin/clientstats.pl @@ -275,7 +275,7 @@ clientstats - create reports on client usage =head1 SYNOPSIS -B [B<-Vhcs> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] +B [B<-Vhcsv> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] =head1 REQUIREMENTS @@ -323,18 +323,19 @@ details and exceptions, please see below. The results will be formatted as a kind of table; you can change the output format to a simple list or just a list of names and number of -postings with the B<--format> option. Captions will be added by means of -the B<--caption> option; all comments (and captions) can be supressed by -using B<--nocomments>. +postings with the B<--format> option. Captions will be added by means +of the B<--caption> option; all comments (and captions) can be +supressed by using B<--nocomments>. -Last but not least you can redirect all output to a number of files, e.g. -one for each month, by submitting the B<--filetemplate> option, see below. +Last but not least you can redirect all output to a number of files, +e.g. one for each month, by submitting the B<--filetemplate> option, +see below. =head2 Configuration B will read its configuration from F -which should be present in etc/ via Config::Auto or from a configuration file -submitted by the B<--conffile> option. +which should be present in etc/ via Config::Auto or from a configuration +file submitted by the B<--conffile> option. See doc/INSTALL for an overview of possible configuration options. @@ -346,32 +347,47 @@ You can override some configuration options via the B<--db> option. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-m>, B<--month> I Set processing period to a single month in YYYY-MM format or to a time period between two month in YYYY-MM:YYYY-MM format (two month, separated by a colon). By using the keyword I instead, you can set no -processing period to process the whole database. +processing period to process the whole database. Defaults to last month. =item B<-n>, B<--names> I Limit processing to a certain set of client names. I -can be a single name (eternal-september.org), a group of names -(*.inka.de) or a list of either of these, separated by colons, for -example +can be a single name (Thunderbird), a group of names (Ice*) or a list +of either of these, separated by colons, for example - eternal-september.org:solani.org:*.inka.de + Forte Agent:Thunderbird:Ice* + +Spaces or special characters like "*" need to be quoted from the shell, +like + + -n 'Forte Agent:Thunderbird:Ice*' + +There is no way to limit processing to a specific version, but you can +alway grep through the output. =item B<-s>, B<--sums|--nosums> (sum per month) Include "virtual" clients named "ALL" for every month in output, -containing the sum of all detected clients for that month. +containing the sum of all detected clients for that month. False +by default. + +=item B<-v>, B<--versions|--noversions> (client versions) + +Include a list of all observed versions of each client in output. +Version information will be displayed with indents ('-') below each +client, sorted in the same way (by postings or alphanumeric). False +by default. =item B<-r>, B<--report> I @@ -379,7 +395,7 @@ Choose the report type: I or I By default, B will report the number of postings for each client in each month. But it can also report the total sum of postings -per client for all months. +per client for all months. Sums of B<--versions> can be included. For report type I, the B option has no meaning and will be silently ignored (see below). @@ -395,10 +411,10 @@ Set the upper boundary. By default, all clients with more postings per month than the upper boundary and/or less postings per month than the lower boundary will be excluded from further processing. For the default report that -means each month only /clients with a number of postings between -the boundaries will be displayed. For the sums report, /clients -with a number of postings exceeding the boundaries in all (!) months -will not be considered. +means each month only clients with a number of postings between the +boundaries will be displayed. For the sums report, clients with a +number of postings exceeding the boundaries in all (!) months will +not be considered. =item B<-g>, B<--group-by> I @@ -406,38 +422,38 @@ By default, all results are grouped by month, sorted chronologically in ascending order, like this: # ----- 2012-01: - arcor-online.net : 9379 - individual.net : 19525 - news.albasani.net: 9063 + 40tude_Dialog: 5873 + Forte Agent : 7735 + Thunderbird : 20925 # ----- 2012-02: - arcor-online.net : 8606 - individual.net : 16768 - news.albasani.net: 7879 + 40tude_Dialog: 4142 + Forte Agent : 5895 + Thunderbird : 19091 The results can be grouped by client instead via B<--group-by> I: - ----- individual.net - 2012-01: 19525 - 2012-02: 16768 - ----- arcor-online.net - 2012-01: 9379 - 2012-02: 8606 - ----- news.albasani.net - 2012-01: 9063 - 2012-02: 7879 + # ----- 40tude_Dialog: + 2012-01: 5873 + 2012-02: 4142 + # ----- Forte Agent: + 2012-01: 7735 + 2012-02: 5895 + # ----- Thunderbird: + 2012-01: 20925 + 2012-02: 19091 By appending I<-desc> to the group-by option parameter, you can reverse the sort order - e.g. B<--group-by> I will give: # ----- 2012-02: - arcor-online.net : 8606 - individual.net : 16768 - news.albasani.net: 7879 + 40tude_Dialog: 4142 + Forte Agent : 5895 + Thunderbird : 19091 # ----- 2012-01: - arcor-online.net : 9379 - individual.net : 19525 - news.albasani.net: 9063 + 40tude_Dialog: 5873 + Forte Agent : 7735 + Thunderbird : 20925 Sums reports (see above) will always be grouped by months; this option will therefore be ignored. @@ -449,41 +465,57 @@ the report will be sorted by name (or month) in ascending alphabetical order by default. You can change the sort order to descending or sort by number of postings instead. -=item B<-f>, B<--format> I - -Select the output format, I being the default: +By default, output is sorted alphabetically: # ----- 2012-01: - arcor-online.net : 9379 - individual.net : 19525 + 40tude_Dialog: 5873 + Forte Agent : 7735 + Thunderbird : 20925 + +Using B<--order-by> I, it will be sorted from most +to least postings: + + # ----- 2012-01: + Thunderbird : 20925 + Forte Agent : 7735 + 40tude_Dialog: 5873 + +=item B<-f>, B<--format> I + +Select the output format, I (a kind of table) being the default: + + # ----- 2012-01: + 40tude_Dialog: 5873 + Forte Agent : 7735 # ----- 2012-02: - arcor-online.net : 8606 - individual.net : 16768 + 40tude_Dialog: 4142 + Forte Agent : 5895 -I format looks like this: +I format looks like this (each client preceded by month): - 2012-01 arcor-online.net 9379 - 2012-01 individual.net 19525 - 2012-02 arcor-online.net 8606 - 2012-02 individual.net 16768 + 2012-01 40tude_Dialog 5873 + 2012-01 Forte Agent 7735 + 2012-02 40tude_Dialog 4142 + 2012-02 Forte Agent 5895 And I format looks like this: # 2012-01: - arcor-online.net 9379 - individual.net 19525 + 40tude_Dialog 5873 + Forte Agent 7735 # 2012-02: - arcor-online.net 8606 - individual.net 16768 + 40tude_Dialog 4142 + Forte Agent 5895 -You can remove the comments by using B<--nocomments>, see below. +You can remove the comments (lines after '#') by using B<--nocomments>, +see below. =item B<-c>, B<--captions|--nocaptions> Add captions to output, like this: ----- Report for 2012-01 to 2012-02 (number of postings for each month) - ----- Names: individual.net + ----- Names: Thunderbird ----- Threshold: 8000 => x (counting only month fulfilling this condition) ----- Grouped by Month (ascending), sorted by number of postings descending @@ -491,11 +523,11 @@ False by default. =item B<--comments|--nocomments> -Add comments (group headers) to I and I output. True by default -as logn as B<--filetemplate> is not set. +Add comments (group headers) to I and I output. True by +default as long as B<--filetemplate> is not set. -Use I<--nocomments> to suppress anything except client names or months and -numbers of postings. +Use I<--nocomments> to suppress anything except client names or months +and numbers of postings. =item B<--filetemplate> I @@ -515,7 +547,7 @@ Override I or I from F. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back @@ -525,26 +557,26 @@ See L. =head1 EXAMPLES -Show number of postings per group for lasth month in I format: +Show number of postings per client for lasth month in I format: clientstats -Show that report for January of 2010 and *.inka plus individual.net: +Show that report for January of 2010 and Thunderbird plus Ice*: - clientstats --month 2010-01 --names *.inka:individual.net: + clientstats --month 2010-01 --names 'Thunderbird:Ice*' -Only show clients with 30 postings or less last month, ordered -by number of postings, descending, in I format: +Only show clients with at least 30 postings last month and the versions +of those clients, ordered each by number of postings, descending, +in I format: - clientstats --upper 30 --order-by postings-desc + clientstats --lower 30 --versions --order-by postings-desc -List number of postings per host for each month of 2010 and redirect +List number of postings per client for each month of 2010 and redirect output to one file for each month, named hosts-2010-01 and so on, in machine-readable form (without formatting): clientstats -m 2010-01:2010-12 -f dump --filetemplate hosts - =head1 FILES =over 4 diff --git a/bin/dbcreate.pl b/bin/dbcreate.pl index e3ad8c6..6bec42d 100755 --- a/bin/dbcreate.pl +++ b/bin/dbcreate.pl @@ -168,7 +168,7 @@ INSTALL my $Upgrade =''; if ($OptUpdate) { - $Upgrade = < 0.02 - # &DoMySQL('...;'); - # print "v0.02: Database upgrades ...\n"; - # &PrintInstructions('0.02',<<" INSTRUCTIONS"); - # INSTRUCTIONS - }; - }; + # TBD # Display general upgrade instructions print $Upgrade; }; @@ -307,11 +298,11 @@ See L for an overview of possible configuration options. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-u>, B<--update> I @@ -319,7 +310,7 @@ Don't do a fresh install, but update from I. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back diff --git a/bin/feedlog.pl b/bin/feedlog.pl index 6359693..9b1fd96 100755 --- a/bin/feedlog.pl +++ b/bin/feedlog.pl @@ -167,8 +167,9 @@ time. All reporting is done to I via I facility. If B fails to initiate a database connection at startup, it will log to -I with I priority and go in an endless loop, as -terminating would only result in a rapid respawn. +I with I priority and go in an endless loop, trying again +to connect every 5 seconds, as terminating would only result in a rapid +respawn. =head2 Configuration @@ -184,15 +185,15 @@ See L for an overview of possible configuration options. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-d>, B<--debug> -Output debugging information to STDERR while parsing STDIN. You'll +Print debugging information to STDERR while parsing STDIN. You'll find that information most probably in your B F file. =item B<-q>, B<--quiet> @@ -201,7 +202,7 @@ Suppress logging to syslog. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 1b9292c..aee6f28 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -803,12 +803,12 @@ override that default through the B<--clientsdb> option. =head2 Configuration B will read its configuration from F -which should be present in etc/ via Config::Auto or from a configuration file -submitted by the B<--conffile> option. +which should be present in etc/ via Config::Auto or from a configuration +file submitted by the B<--conffile> option. See L for an overview of possible configuration options. -You can override configuration options via the B<--hierarchy>, +You can override configuration options by using the B<--hierarchy>, B<--rawdb>, B<--groupsdb>, B<--clientsdb> and B<--hostsdb> options, respectively. @@ -818,15 +818,15 @@ respectively. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-d>, B<--debug> -Output debugging information to STDOUT while processing (number of +Print debugging information to STDOUT while processing (number of postings per group). =item B<-t>, B<--test> @@ -838,15 +838,17 @@ conjunction with B<--test> ... everything else seems a bit pointless. Set processing period to a single month in YYYY-MM format or to a time period between two month in YYYY-MM:YYYY-MM format (two month, separated -by a colon). +by a colon). Defaults to last month. =item B<-s>, B<--stats> I -Set processing type to one of I, I or I. Defaults -to all. +Set processing type to one of I, I, I or I. +Defaults to I. =item B<-c>, B<--checkgroups> I +Relevant only for newsgroup stats (I). + Check each group against a list of valid newsgroups read from a file, one group on each line and ignoring everything after the first whitespace (so you can use a file in checkgroups format or (part of) @@ -889,7 +891,7 @@ Override I from F. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back diff --git a/bin/groupstats.pl b/bin/groupstats.pl index 0c193a8..4da8d41 100755 --- a/bin/groupstats.pl +++ b/bin/groupstats.pl @@ -283,7 +283,7 @@ See L. =head1 DESCRIPTION -This script create reports on newsgroup usage (number of postings per +This script creates reports on newsgroup usage (number of postings per group per month) taken from result tables created by B. @@ -291,16 +291,16 @@ B. =head3 Time period and newsgroups -The time period to act on defaults to last month; you can assign another -time period or a single month (or drop all time constraints) via the -B<--month> option (see below). +The time period to act on defaults to last month; you can assign +another time period or a single month (or drop all time constraints) +via the B<--month> option (see below). B will process all newsgroups by default; you can limit -processing to only some newsgroups by supplying a list of those groups via -B<--newsgroups> option (see below). You can include hierarchy levels in -the output by adding the B<--sums> switch (see below). Optionally -newsgroups not present in a checkgroups file can be excluded from output, -sse B<--checkgroups> below. +processing to only some newsgroups by supplying a list of those groups +via B<--newsgroups> option (see below). You can include hierarchy +levels in the output by adding the B<--sums> switch (see below). +Optionally newsgroups not present in a checkgroups file can be excluded +from output, sse B<--checkgroups> below. =head3 Report type @@ -321,26 +321,27 @@ below. =head3 Sorting and formatting the output By default, all results are grouped by month; you can group results by -newsgroup instead via the B<--groupy-by> option. Within those groups, the -list of newsgroups (or months) is sorted alphabetically (or -chronologically, respectively) ascending. You can change that order (and -sort by number of postings) with the B<--order-by> option. For details and -exceptions, please see below. +newsgroup instead via the B<--groupy-by> option. Within those groups, +the list of newsgroups (or months) is sorted alphabetically (or +chronologically, respectively) ascending. You can change that order +(and sort by number of postings) with the B<--order-by> option. For +details and exceptions, please see below. The results will be formatted as a kind of table; you can change the -output format to a simple list or just a list of newsgroups and number of -postings with the B<--format> option. Captions will be added by means of -the B<--caption> option; all comments (and captions) can be supressed by -using B<--nocomments>. +output format to a simple list or just a list of newsgroups and number +of postings with the B<--format> option. Captions will be added by means +of the B<--caption> option; all comments (and captions) can be supressed +by using B<--nocomments>. -Last but not least you can redirect all output to a number of files, e.g. -one for each month, by submitting the B<--filetemplate> option, see below. +Last but not least you can redirect all output to a number of files, +e.g. one for each month, by submitting the B<--filetemplate> option, +see below. =head2 Configuration B will read its configuration from F -which should be present in etc/ via Config::Auto or from a configuration file -submitted by the B<--conffile> option. +which should be present in etc/ via Config::Auto or from a configuration +file submitted by the B<--conffile> option. See doc/INSTALL for an overview of possible configuration options. @@ -352,18 +353,18 @@ You can override some configuration options via the B<--groupsdb> option. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-m>, B<--month> I Set processing period to a single month in YYYY-MM format or to a time period between two month in YYYY-MM:YYYY-MM format (two month, separated by a colon). By using the keyword I instead, you can set no -processing period to process the whole database. +processing period to process the whole database. Defaults to last month. =item B<-n>, B<--newsgroups> I @@ -388,17 +389,20 @@ See the B man page for details. This option does not work together with the B<--checkgroups> option as all "virtual" groups will not be present in the checkgroups file. +False by default. + =item B<--checkgroups> I -Restrict output to those newgroups present in a file in checkgroups format -(one newgroup name per line; everything after the first whitespace on each -line is ignored). All other newsgroups will be removed from output. +Restrict output to those newgroups present in a file in checkgroups +format (one newgroup name per line; everything after the first +whitespace on each line is ignored). All other newsgroups will be +removed from output. -Contrary to B, I is not a template, but refers to -a single file in checkgroups format. +Contrary to B, I is not a template, but refers +to a single file in checkgroups format. -The B<--sums> option will not work together with this option as "virtual" -groups will not be present in the checkgroups file. +The B<--sums> option will not work together with this option as +"virtual" groups will not be present in the checkgroups file. =item B<-r>, B<--report> I @@ -406,8 +410,8 @@ Choose the report type: I, I or I By default, B will report the number of postings for each newsgroup in each month. But it can also report the average number of -postings per group for all months or the total sum of postings per group -for all months. +postings per group for all months or the total sum of postings per +group for all months. For report types I and I, the B option has no meaning and will be silently ignored (see below). @@ -426,12 +430,13 @@ Set the boundary type to one of I, I, I or I. By default, all newsgroups with more postings per month than the upper -boundary and/or less postings per month than the lower boundary will be -excluded from further processing. For the default report that means each -month only newsgroups with a number of postings between the boundaries -will be displayed. For the other report types, newsgroups with a number of -postings exceeding the boundaries in all (!) months will not be -considered. +boundary and/or less postings per month than the lower boundary will +be +excluded from further processing. For the default report that means +each month only newsgroups with a number of postings between the +boundaries will be displayed. For the other report types, newsgroups +with a number of postings exceeding the boundaries in all (!) months +will not be considered. For example, lets take a list of newsgroups like this: @@ -461,22 +466,23 @@ month. If you want to list all newsgroups with more than 25 postings I, you'll have to set the boundary type to I, see below. A boundary type of I will show only those newsgroups - at all - -that satisfy the boundaries in each and every single month. With the above -list of newsgroups and +that satisfy the boundaries in each and every single month. With the +above list of newsgroups and C, you'll get this result: ----- All months: de.comp.datenbanken.ms-access 293 -de.comp.datenbanken.mysql has not been considered because it had less than -25 postings in 2012-02 (only). +de.comp.datenbanken.mysql has not been considered because it had less +than 25 postings in 2012-02 (only). -You can use that to get a list of newsgroups that have more (or less) then -x postings in every month during the whole reporting period. +You can use that to get a list of newsgroups that have more (or less) +then x postings in every month during the whole reporting period. -A boundary type of I will show only those newsgroups - at all -that -satisfy the boundaries on average. With the above list of newsgroups and +A boundary type of I will show only those newsgroups - at +all - that satisfy the boundaries on average. With the above list of +newsgroups and C, you'll get this result: @@ -491,8 +497,8 @@ The average number of postings in the three groups is: de.comp.datenbanken.mysql 48.33 Last but not least, a boundary type of I will show only those -newsgroups - at all - that satisfy the boundaries with the total sum of -all postings during the reporting period. With the above list of +newsgroups - at all - that satisfy the boundaries with the total sum +of all postings during the reporting period. With the above list of newsgroups and C, you'll finally get this result: @@ -505,8 +511,8 @@ you'll finally get this result: =item B<-g>, B<--group-by> I -By default, all results are grouped by month, sorted chronologically in -ascending order, like this: +By default, all results are grouped by month, sorted chronologically +in ascending order, like this: ----- 2012-01: de.comp.datenbanken.ms-access 84 @@ -525,8 +531,8 @@ B<--group-by> I: 2012-01 88 2012-02 21 -By appending I<-desc> to the group-by option parameter, you can reverse -the sort order - e.g. B<--group-by> I will give: +By appending I<-desc> to the group-by option parameter, you can +reverse the sort order - e.g. B<--group-by> I will give: ----- 2012-02: de.comp.datenbanken.ms-access 126 @@ -541,9 +547,9 @@ this option will therefore be ignored. =item B<-o>, B<--order-by> I Within each group (a single month or single newsgroup, see above), the -report will be sorted by newsgroup names in ascending alphabetical order -by default. You can change the sort order to descending or sort by number -of postings instead. +report will be sorted by newsgroup names in ascending alphabetical +order by default. You can change the sort order to descending or sort +by number of postings instead. =item B<-f>, B<--format> I @@ -587,19 +593,19 @@ False by default. =item B<--comments|--nocomments> -Add comments (group headers) to I and I output. True by default -as logn as B<--filetemplate> is not set. +Add comments (group headers) to I and I output. True by +default as long as B<--filetemplate> is not set. -Use I<--nocomments> to suppress anything except newsgroup names/months and -numbers of postings. +Use I<--nocomments> to suppress anything except newsgroup names/months +and numbers of postings. =item B<--filetemplate> I -Save output to file(s) instead of dumping it to STDOUT. B will -create one file for each month (or each newsgroup, accordant to the -setting of B<--group-by>, see above), with filenames composed by adding -year and month (or newsgroup names) to the I, for -example with B<--filetemplate> I: +Save output to file(s) instead of dumping it to STDOUT. B +will create one file for each month (or each newsgroup, according to +the setting of B<--group-by>, see above), with filenames composed by +adding year and month (or newsgroup names) to the I, +for example with B<--filetemplate> I: stats-2012-01 stats-2012-02 @@ -611,7 +617,7 @@ Override I from F. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back @@ -635,9 +641,9 @@ by number of postings, descending, in I format: groupstats --upper 30 --order-by postings-desc -Show the total of all postings for the year of 2010 for all groups that -had 30 postings or less in every single month in that year, ordered by -number of postings in descending order: +Show the total of all postings for the year of 2010 for all groups +that had 30 postings or less in every single month in that year, +ordered by number of postings in descending order: groupstats -m 2010-01:2010-12 -u 30 -b level -r sums -o postings-desc @@ -651,7 +657,6 @@ machine-readable form (without formatting): groupstats -m 2010-01:2010-12 -f dump --filetemplate stats - =head1 FILES =over 4 diff --git a/bin/hoststats.pl b/bin/hoststats.pl index 9e949ff..7c38d0d 100755 --- a/bin/hoststats.pl +++ b/bin/hoststats.pl @@ -197,7 +197,7 @@ See L. =head1 DESCRIPTION -This script create reports on newsgroup usage (number of postings from +This script creates reports on newsgroup usage (number of postings from each host) taken from result tables created by B. =head2 Features and options @@ -236,18 +236,19 @@ please see below. The results will be formatted as a kind of table; you can change the output format to a simple list or just a list of names and number of -postings with the B<--format> option. Captions will be added by means of -the B<--caption> option; all comments (and captions) can be supressed by -using B<--nocomments>. +postings with the B<--format> option. Captions will be added by means +of the B<--caption> option; all comments (and captions) can be +supressed by using B<--nocomments>. -Last but not least you can redirect all output to a number of files, e.g. -one for each month, by submitting the B<--filetemplate> option, see below. +Last but not least you can redirect all output to a number of files, +e.g. one for each month, by submitting the B<--filetemplate> option, +see below. =head2 Configuration B will read its configuration from F -which should be present in etc/ via Config::Auto or from a configuration file -submitted by the B<--conffile> option. +which should be present in etc/ via Config::Auto or from a configuration +file submitted by the B<--conffile> option. See doc/INSTALL for an overview of possible configuration options. @@ -259,18 +260,18 @@ You can override some configuration options via the B<--db> option. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-m>, B<--month> I Set processing period to a single month in YYYY-MM format or to a time period between two month in YYYY-MM:YYYY-MM format (two month, separated by a colon). By using the keyword I instead, you can set no -processing period to process the whole database. +processing period to process the whole database. Defaults to last month. =item B<-n>, B<--names> I @@ -284,7 +285,8 @@ example =item B<-s>, B<--sums|--nosums> (sum per month) Include a "virtual" host named "ALL" for every month in output, -containing the sum of all detected hosts for that month. +containing the sum of all detected hosts for that month. False +by default. =item B<-r>, B<--report> I @@ -315,8 +317,8 @@ considered. =item B<-g>, B<--group-by> I -By default, all results are grouped by month, sorted chronologically in -ascending order, like this: +By default, all results are grouped by month, sorted chronologically +in ascending order, like this: # ----- 2012-01: arcor-online.net : 9379 @@ -339,8 +341,8 @@ The results can be grouped by host instead via B<--group-by> I: 2012-01: 9063 2012-02: 7879 -By appending I<-desc> to the group-by option parameter, you can reverse -the sort order - e.g. B<--group-by> I will give: +By appending I<-desc> to the group-by option parameter, you can +reverse the sort order - e.g. B<--group-by> I will give: # ----- 2012-02: arcor-online.net : 8606 @@ -427,7 +429,7 @@ Override I from F. =item B<--conffile> I -Load configuration from I instead of F. +Read configuration from I instead of F. =back @@ -437,7 +439,7 @@ See L. =head1 EXAMPLES -Show number of postings per group for lasth month in I format: +Show number of postings per host for lasth month in I format: hoststats diff --git a/bin/postingstats.pl b/bin/postingstats.pl index e517a45..0b362c3 100755 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -268,7 +268,7 @@ postingstats - format and post reports =head1 SYNOPSIS -B B<-t> I [B<-Vh> [B<-m> I] +B [B<-Vh>] [B<-t> I] [B<-m> I] =head1 REQUIREMENTS @@ -285,11 +285,13 @@ message that can be posted to Usenet. B will create a table with entries numbered from most to least and percentages calculated from the sum total of all values. -It depends on a sorted list on STDIN in I format with I. +It depends on a sorted list on STDIN in I format with I; +I from B are optional. B needs a B<--type> and a B<--month> to create a caption and select matching lead-ins and lead-outs. B<--type> is also needed -to catch the correct sum total from input. +to catch the correct sum total from input which differs between I +on one hand and I or I on the other hand. It will default to posting statistics (number of postings per group) and last month. @@ -308,7 +310,7 @@ C<----- configuration -----> section. =item C<$TLH> Top level hierarchy the report was created for. Used for display and -sum total. +sum total (only for I). =item C<%Heading> @@ -327,13 +329,13 @@ Output will be truncated otherwise. =item C<%LeadIn> Hash with keys for I, I and I. -Used to create the headers for our posting. Can contain other text +Used to create the headers for the postings. Can contain other text that will be shown before C<%Heading>. =item C<%LeadOut> Hash with keys for I, I and I. -Will be shown at the end of our posting. +Will be shown at the end of the posting. =back @@ -343,11 +345,11 @@ Will be shown at the end of our posting. =item B<-V>, B<--version> -Print out version and copyright information and exit. +Display version and copyright information and exit. =item B<-h>, B<--help> -Print this man page and exit. +Display this man page and exit. =item B<-t>, B<--type> I @@ -356,7 +358,7 @@ statistics accordingly. =item B<-m>, B<--month> I -Set month for display. +Set month (for display only). =back @@ -372,7 +374,7 @@ Create a posting from a posting statistics report for last month: Create a posting from a posting statistics report for 2012-01: - groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups -m 2012-01 + groupstats.pl --nocomments --sums --format dump -m 2012-01 | postingstats.pl -t groups -m 2012-01 Create a posting from a host statistics report for last month: diff --git a/doc/ChangeLog b/doc/ChangeLog index 857b23e..d327650 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -16,6 +16,7 @@ NewsStats 0.4.0 (unreleased) * Add ClientStats to dopostingstats. * Let dopostingstats default to last month. * Set executable bit for new scripts. + * Update documentation. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. diff --git a/doc/INSTALL b/doc/INSTALL index 9a37207..3588c21 100644 --- a/doc/INSTALL +++ b/doc/INSTALL @@ -1,11 +1,12 @@ NewsStats (c) 2010-2013, 2025 Thomas Hochstein -NewsStats is a software package used to gather statistical information -from a live Usenet feed and for its subsequent examination. +NewsStats is a software package that can be used to collect +statistical information from a live Usenet feed and then analyze it +to create statistical reports. -This script package is free software; you can redistribute it and/or -modify it under the terms of the GNU Public License as published by -the Free Software Foundation. +This package is free software; you can redistribute it and/or modify +it under the terms of the GNU Public License as published by the Free +Software Foundation. --------------------------------------------------------------------- @@ -16,9 +17,10 @@ INSTALLATION INSTRUCTIONS * Download the current version of NewsStats from . - * Untar it into a directory of your choice: + * Untar it into a directory of your choice, i.e. /srv/newsstats: - # tar -xzf newsstats-nn.tar.gz + $ cd /srv + $ tar -xzf newsstats-n.n.n.tar.gz Scripts in this path - at least feedlog.pl - should be executable by the news user. @@ -28,8 +30,8 @@ INSTALLATION INSTRUCTIONS * Copy the sample configuration file newsstats.conf.sample to newsstats.conf and modify it for your purposes: - # cp etc/newsstats.conf.sample etc/newsstats.conf - # vim etc/newsstats.conf + $ cp etc/newsstats.conf.sample etc/newsstats.conf + $ vim etc/newsstats.conf a) Mandatory configuration options @@ -60,6 +62,9 @@ INSTALLATION INSTRUCTIONS * DBTableHosts = hosts_de Table holding data on postings per server. + * DBTableClnts = clients_de + Table holding data on postings per client. + b) Optional configuration options * TLH = de.alt,news.admin @@ -68,19 +73,21 @@ INSTALLATION INSTRUCTIONS 3) Database (mysql) setup - * Setup your database server with a username, password and - database matching the NewsStats configuration (see 2 a). + * Setup your database server with an username, a password and + (optionally) a database matching the NewsStats configuration + (see 2 a). * Start the database creation script: - # bin/dbcreate.pl + $ bin/dbcreate.pl - It will setup the necessary database tables and display some - information on the next steps. + It will create the database (if not already present), create the + necessary database tables and display some information on the + next steps. 4) Feed (INN) setup - You have to setup an INN feed to feedlog.pl. + You have to set up an INN feed to feedlog.pl. * Edit your 'newsfeeds' file and insert something like @@ -90,39 +97,39 @@ INSTALLATION INSTRUCTIONS :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl * You should only feed that hierarchy (those hierarchies ...) to - feedlog.pl you'll want to cover with your statistical - examination. It may be a good idea to setup different feeds (to - different databases ...) for different hierarchies. + feedlog.pl that you want to cover with your statistical analysis. + It may be a good idea to setup different feeds (to different + databases ...) for different hierarchies. * Please double check that your path to feedlog.pl is correct and feedlog.pl can be executed by the news user * Check your 'newsfeeds' syntax: - # ctlinnd checkfile + $ ctlinnd checkfile * Reload 'newsfeeds': - # ctlinnd reload newsfeeds 'Adding newsstats! feed' + $ ctlinnd reload newsfeeds 'Adding newsstats! feed' * Watch your 'news.notice' and 'errlog' files: - # tail -f /var/log/news/news.notice + $ tail -f /var/log/news/news.notice ... - # tail -f /var/log/news/errlog + $ tail -f /var/log/news/errlog Everything should be going smoothly now. * If INN is spewing error messages to 'errlog' or reporting continous respawns of feedlog.pl to 'news.notice', stop your feed: - # ctlinnd drop 'newsstats!' + $ ctlinnd drop 'newsstats!' and investigate. 'errlog' may be helpful here. * You can restart the feed with - # ctlinnd begin 'newsstats!' + $ ctlinnd begin 'newsstats!' later. diff --git a/doc/README b/doc/README index f6b1f44..de7e56c 100644 --- a/doc/README +++ b/doc/README @@ -1,21 +1,21 @@ NewsStats (c) 2010-2013, 2025 Thomas Hochstein NewsStats is a software package for gathering statistical data live -from a Usenet feed and subsequent examination. +from a Usenet feed and subsequent analysis. -This script package is free software; you can redistribute it and/or -modify it under the terms of the GNU Public License as published by -the Free Software Foundation. +This package is free software; you can redistribute it and/or modify +it under the terms of the GNU Public License as published by the Free +Software Foundation. --------------------------------------------------------------------- What's that? - There's a multitude of tools for the statistical examination of - newsgroups: number of postings per month or per person, longest - threads, and so on (see - [German language] for an incomplete list). Most of them use a per- - newsgroup approach while NewsStats is hierarchy oriented. + There's a multitude of tools to create statistics about newsgroup + usage: number of postings per month or per person, longest threads, + and so on (see [German language] + for an incomplete list). Most of them use a per-newsgroup approach + while NewsStats is hierarchy oriented. NewsStats will accumulate data from a live INN feed, allowing you to process the saved information later on. @@ -40,7 +40,9 @@ Prerequisites * Perl 5.8.x with standard modules - Cwd + - Encode - File::Basename + - Getopt::Long - Sys::Syslog * Perl modules from CPAN @@ -50,7 +52,7 @@ Prerequisites * mysql 5.0.x - * working installation of INN + * a working installation of INN Installation instructions @@ -67,15 +69,52 @@ Getting Started table. See the feedlog.pl man page for more information. You can process that data via 'gatherstats.pl'; currently the - tabulation of postings per group and injection server per month is - supported. Tabulation of clients (newsreaders) is planned. See - the gatherstats.pl man page for more information. + tabulation of postings per group, injection server and posting + agent (newsreader) per month is supported. See the gatherstats.pl + man page for more information. + + Example: + + bin/gatherstats.pl + + will parse raw data from the last month and save the results in + tables for postings per group, server and client, respectively. Report generation is handled by specialised scripts for each - report type. Currently reports on the number of postings per group - and month and injection server and month are supported; you can - use 'groupstats.pl' and 'hoststats.pl' for that. See the - groupstats.pl and hoststats.pl man pages for more information. + report type: 'groupstats.pl' for postings per group + (s), 'hoststats.pl' for postings per injection server + (s) and 'clientstats.pl' for postings per posting agent. See the + groupstats.pl, hoststats.pl and clientstats.pl man pages for more + information. + + Example: + + bin/groupstats.pl -o postings-desc + bin/hoststats.pl -o postings-desc + bin/clientstats.pl -o postings-desc -v + + will show reports for postings per group, per injection server and + per client (with detailed client versions) for the last month, + using the result tables filled by gatherstats. + + To post those reports to Usenet, change postingstats.pl according + to your needs (sender, newsgroups and other headers, translation + of table headers and text templates) and display a test posting + by piping report data into postingstats.pl: + + bin/groupstats.pl --nocomments -s -f dump | bin/postingstats.pl + + If the result is to your liking, add a pipe to a inews + implementation. + + Example: + + bin/groupstats.pl --nocomments -s -f dump | bin/postingstats.pl | contrib/tinews.pl -X + +More information + + See the man pages for 'gatherstats' and the report generating + scripts. Reporting Bugs @@ -87,7 +126,7 @@ Reporting Bugs Development - This program is maintained using the Git version control system at + This package is maintained using the Git version control system at . Related projects diff --git a/doc/TODO b/doc/TODO index a376c53..9f5e0a1 100644 --- a/doc/TODO +++ b/doc/TODO @@ -1,12 +1,10 @@ NewsStats To-Do List ==================== -This is a list of planned bug fixes, improvements and enhancements for +This is a list of possible bug fixes, improvements and enhancements for NewsStats. * General - - Improve Documentation - The documentation is rather sparse and could use some improvement. - Add a test suite There is currently no kind of test suite or regression tests. Something like that is badly needed. @@ -27,8 +25,6 @@ NewsStats. for late creation and deletion), optionally including the previously mentioned information; and you should be able to get the history of any group. - - Add other reports - NewsStats should include some other kinds of reports (stats on used clients) - Add tools for database management NewsStats should offer tools e.g. to inject postings into the 'raw' database, or to split databases. @@ -53,23 +49,11 @@ NewsStats. Some other tests - working database connection, valid database and table names - would be nice. - + install/install.pl - - Read current version from a file dropped and updated by installer - - Add / enhance / test error handling - - General tests and optimisations - - + feedlog.pl - - Add / enhance / test error handling - - General tests and optimisations - + gatherstats.pl - Use hierarchy information (see GroupInfo above) - - Add gathering of other stats (clients, ...) - - better modularisation (code reuse for other reports!) - Add / enhance / test error handling - - General tests and optimisations - + groupstats.pl - - better modularisation (code reuse for other reports!) + + groupstats.pl, hoststats.pl, clientstats.pl + - better modularisation (code reuse) - Add / enhance / test error handling - General tests and optimisations From e176b0d5e85d098dbdadc992313e8a28e31f587e Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sun, 1 Jun 2025 16:43:45 +0200 Subject: [PATCH 28/29] Update README.md, add links to other documentation. Signed-off-by: Thomas Hochstein --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 58cf821..a358ac1 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,14 @@ ## Description -**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in a database which in turn can be used to generate various reports. +**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in databases which in turn can be used to generate various reports (postings per group, injection server or posting agent, per month). -The software package is still under development. - -It is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy. +This software is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy. ## More information -Please see the [distribution page](https://th-h.de/net/software/newsstats/) (in German). \ No newline at end of file +Please see the [distribution page](https://th-h.de/net/software/newsstats/) (in German). + +* General overview and examples: [README](doc/README) +* Installation instructions: [INSTALL](doc/INSTALL) +* Changelog: [ChangeLog](doc/ChangeLog) \ No newline at end of file From 20434ab1dc938be46a5a9d1ef426ca42f4fb22b2 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Mon, 2 Jun 2025 16:38:30 +0200 Subject: [PATCH 29/29] Release 0.4.0 Signed-off-by: Thomas Hochstein --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index d327650..417dd70 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,4 @@ -NewsStats 0.4.0 (unreleased) +NewsStats 0.4.0 (2025-06-02) * Reformat $Conf{TLH} for GroupStats only. * Extract TLH check from HostStats to subroutine, fix no-op check. * Extract getting raw headers from HostStats to subroutine.