thh/newsstats
1
0
Fork 0
Code Issues 3 Pull requests Packages Projects Releases Activity

Move cliservstats to hoststats.

Browse source 
Signed-off-by: Thomas Hochstein <thh@thh.name>
This commit is contained in:
Thomas Hochstein 2025-05-30 16:38:18 +02:00
parent a553b374ce
commit 963f07432c
5 changed files with 68 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
bin/cliservstats.pl → bin/hoststats.pl
View file

@ -1,9 +1,9 @@
#! /usr/bin/perl #! /usr/bin/perl
# #
# cliservstats.pl # hoststats.pl
# #
# This script will get statistical data on client (newsreader) and # This script will get statistical data on server (host) usage
# server (host) usage from a database. # from a database.
# #
# It is part of the NewsStats package. # It is part of the NewsStats package.
# #
@ -31,7 +31,7 @@ Getopt::Long::config ('bundling');
### read commandline options ### read commandline options
my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat, my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat,
$OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy, $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy,
$OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile); $OptReportType,$OptSums,$UppBound,$OptConfFile);
GetOptions ('c|captions!' => \$OptCaptions, GetOptions ('c|captions!' => \$OptCaptions,
'comments!' => \$OptComments, 'comments!' => \$OptComments,
'db=s' => \$OptDB, 'db=s' => \$OptDB,
@ -44,7 +44,6 @@ GetOptions ('c|captions!' => \$OptCaptions,
'o|order-by=s' => \$OptOrderBy, 'o|order-by=s' => \$OptOrderBy,
'r|report=s' => \$OptReportType, 'r|report=s' => \$OptReportType,
's|sums!' => \$OptSums, 's|sums!' => \$OptSums,
't|type=s' => \$OptType,
'u|upper=i' => \$UppBound, 'u|upper=i' => \$UppBound,
'conffile=s' => \$OptConfFile, 'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD, 'h|help' => \&ShowPOD,
@ -52,15 +51,6 @@ GetOptions ('c|captions!' => \$OptCaptions,
# parse parameters # parse parameters
# $OptComments defaults to TRUE if --filetemplate is not used # $OptComments defaults to TRUE if --filetemplate is not used
$OptComments = 1 if (!$OptFileTemplate && !defined($OptComments)); $OptComments = 1 if (!$OptFileTemplate && !defined($OptComments));
# parse $OptType
if ($OptType) {
if ($OptType =~ /(host|server)s?/i) {
$OptType = 'host';
} elsif ($OptType =~ /(newsreader|client)s?/i) {
$OptType = 'client';
}
}
&Bleat(2, "Please use '--type server' or '--type client'.") if !$OptType;
# parse $OptReportType # parse $OptReportType
if ($OptReportType) { if ($OptReportType) {
if ($OptReportType =~ /sums?/i) { if ($OptReportType =~ /sums?/i) {
@ -74,14 +64,8 @@ if ($OptReportType) {
my %Conf = %{ReadConfig($OptConfFile)}; my %Conf = %{ReadConfig($OptConfFile)};
### set DBTable ### set DBTable
if ($OptDB) { $Conf{'DBTable'} = $Conf{'DBTableHosts'};
$Conf{'DBTable'} = $OptDB; $Conf{'DBTable'} = $OptDB if $OptDB;
}
elsif ($OptType eq 'host') {
$Conf{'DBTable'} = $Conf{'DBTableHosts'};
} else {
$Conf{'DBTable'} = $Conf{'DBTableClnts'};
}
### init database ### init database
my $DBHandle = InitDB(\%Conf,1); my $DBHandle = InitDB(\%Conf,1);
@ -97,14 +81,14 @@ my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth);
# with placeholders as well as a list of names to bind to them # with placeholders as well as a list of names to bind to them
my ($SQLWhereNames,@SQLBindNames); my ($SQLWhereNames,@SQLBindNames);
if ($OptNames) { if ($OptNames) {
($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType); ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,'host');
# bail out if --names is invalid # bail out if --names is invalid
&Bleat(2,"--names option has an invalid format!") &Bleat(2,"--names option has an invalid format!")
if !$SQLWhereNames; if !$SQLWhereNames;
} }
### build SQL WHERE clause ### build SQL WHERE clause
my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",$OptType); my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",'host');
my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
$ExcludeSums, $ExcludeSums,
&SQLSetBounds('default',$LowBound,$UppBound)); &SQLSetBounds('default',$LowBound,$UppBound));
@ -118,8 +102,8 @@ $OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
and $OptNames and $OptNames !~ /[:*%]/); and $OptNames and $OptNames !~ /[:*%]/);
# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause # parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month' # if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType); my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, 'host');
# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy) # $GroupBy will contain 'month' or 'host' (parsed result of $OptGroupBy)
# set it to 'month' or 'key' for OutputData() # set it to 'month' or 'key' for OutputData()
$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key'; $GroupBy = ($GroupBy eq 'month') ? 'month' : 'key';
@ -128,19 +112,19 @@ my $SQLSelect;
my $SQLGroupClause = ''; my $SQLGroupClause = '';
my $Precision = 0; # number of digits right of decimal point for output my $Precision = 0; # number of digits right of decimal point for output
if ($OptReportType and $OptReportType ne 'default') { if ($OptReportType and $OptReportType ne 'default') {
$SQLGroupClause = "GROUP BY $OptType"; $SQLGroupClause = "GROUP BY host";
# change $SQLOrderClause: replace everything before 'postings' # change $SQLOrderClause: replace everything before 'postings'
$SQLOrderClause =~ s/BY.+postings/BY postings/; $SQLOrderClause =~ s/BY.+postings/BY postings/;
$SQLSelect = "'All months',$OptType,SUM(postings)"; $SQLSelect = "'All months',host,SUM(postings)";
# change $SQLOrderClause: replace 'postings' with 'SUM(postings)' # change $SQLOrderClause: replace 'postings' with 'SUM(postings)'
$SQLOrderClause =~ s/postings/SUM(postings)/; $SQLOrderClause =~ s/postings/SUM(postings)/;
} else { } else {
$SQLSelect = "month,$OptType,postings"; $SQLSelect = "month,host,postings";
}; };
### get length of longest name delivered by query ### get length of longest name delivered by query
### for formatting purposes ### for formatting purposes
my $Field = ($GroupBy eq 'month') ? $OptType : 'month'; my $Field = ($GroupBy eq 'month') ? 'host' : 'month';
my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
$Field,'postings',$SQLWhereClause, $Field,'postings',$SQLWhereClause,
'',@SQLBindNames); '',@SQLBindNames);
@ -155,8 +139,8 @@ $DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s',
$SQLOrderClause)); $SQLOrderClause));
# execute query # execute query
$DBQuery->execute(@SQLBindNames) $DBQuery->execute(@SQLBindNames)
or &Bleat(2,sprintf("Can't get %s data for %s from %s.%s: %s\n", or &Bleat(2,sprintf("Can't get host data for %s from %s.%s: %s\n",
$OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
$DBI::errstr)); $DBI::errstr));
### output results ### output results
@ -175,7 +159,7 @@ if ($OptCaptions && $OptComments) {
$LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames))) $LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames)))
if $OptNames; if $OptNames;
# print boundaries, if set # print boundaries, if set
my $CaptionBoundary= '(counting only month fulfilling this condition)'; my $CaptionBoundary= '(counting only months fulfilling this condition)';
$LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n", $LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n",
$LowBound ? $LowBound : '',$LowBound ? '=>' : '', $LowBound ? $LowBound : '',$LowBound ? '=>' : '',
$UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary) $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary)
@ -201,11 +185,11 @@ __END__
=head1 NAME =head1 NAME
cliservstats - create reports on host or client usage hoststats - create reports on host usage
=head1 SYNOPSIS =head1 SYNOPSIS
B<cliservstats> B<-t> I<host|client> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)|client(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>] B<hoststats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
=head1 REQUIREMENTS =head1 REQUIREMENTS
@ -214,8 +198,7 @@ See L<doc/README>.
=head1 DESCRIPTION =head1 DESCRIPTION
This script create reports on newsgroup usage (number of postings from This script create reports on newsgroup usage (number of postings from
each host or using each client per month) taken from result tables each host) taken from result tables created by B<gatherstats.pl>.
created by B<gatherstats.pl>.
=head2 Features and options =head2 Features and options
@ -225,9 +208,9 @@ The time period to act on defaults to last month; you can assign another
time period or a single month (or drop all time constraints) via the time period or a single month (or drop all time constraints) via the
B<--month> option (see below). B<--month> option (see below).
B<cliservstats> will process all hosts or clients by default; you can B<hoststats> will process all hosts by default; you can limit
limit processing to only some hosts or clients by supplying a list of processing to only some hosts by supplying a list of those names by
those names by using the B<--names> option (see below). using the B<--names> option (see below).
=head3 Report type =head3 Report type
@ -238,18 +221,18 @@ or all postings summed up; for details, see below.
Furthermore you can set an upper and/or lower boundary to exclude some Furthermore you can set an upper and/or lower boundary to exclude some
results from output via the B<--lower> and B<--upper> options, results from output via the B<--lower> and B<--upper> options,
respectively. By default, all hosts/clients with more and/or less respectively. By default, all hosts with more and/or less postings
postings per month will be excluded from the result set (i.e. not per month will be excluded from the result set (i.e. not shown and
shown and not considered forsum reports). not considered for sum reports).
=head3 Sorting and formatting the output =head3 Sorting and formatting the output
By default, all results are grouped by month; you can group results by By default, all results are grouped by month; you can group results by
hosts/clients instead via the B<--group-by> option. Within those hosts instead via the B<--group-by> option. Within those groups, the
groups, the list of hosts/clients (or months) is sorted alphabetically list of hosts (or months) is sorted alphabetically (or chronologically,
(or chronologically, respectively) ascending. You can change that order respectively) ascending. You can change that order (and sort by number
(and sort by number of postings) with the B<--order-by> option. For of postings) with the B<--order-by> option. For details and exceptions,
details and exceptions, please see below. please see below.
The results will be formatted as a kind of table; you can change the The results will be formatted as a kind of table; you can change the
output format to a simple list or just a list of names and number of output format to a simple list or just a list of names and number of
@ -262,7 +245,7 @@ one for each month, by submitting the B<--filetemplate> option, see below.
=head2 Configuration =head2 Configuration
B<cliservstats> will read its configuration from F<newsstats.conf> B<hoststats> will read its configuration from F<newsstats.conf>
which should be present in etc/ via Config::Auto or from a configuration file which should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option. submitted by the B<--conffile> option.
@ -282,11 +265,6 @@ Print out version and copyright information and exit.
Print this man page and exit. Print this man page and exit.
=item B<-t>, B<--type> I<host|client>
Create report for hosts (servers) or clients (newsreaders), using
I<DBTableHosts> or I<DBTableClnts> respectively.
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all> =item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
Set processing period to a single month in YYYY-MM format or to a time Set processing period to a single month in YYYY-MM format or to a time
@ -296,7 +274,7 @@ processing period to process the whole database.
=item B<-n>, B<--names> I<name(s)> =item B<-n>, B<--names> I<name(s)>
Limit processing to a certain set of host or client names. I<names(s)> Limit processing to a certain set of hostnames. I<names(s)>
can be a single name (eternal-september.org), a group of names can be a single name (eternal-september.org), a group of names
(*.inka.de) or a list of either of these, separated by colons, for (*.inka.de) or a list of either of these, separated by colons, for
example example
@ -312,9 +290,9 @@ containing the sum of all detected hosts for that month.
Choose the report type: I<default> or I<sums> Choose the report type: I<default> or I<sums>
By default, B<cliservstats> will report the number of postings for each By default, B<hoststats> will report the number of postings for each
host/client in each month. But it can also report the total sum of postings host in each month. But it can also report the total sum of postings
per host/client for all months. per host for all months.
For report type I<sums>, the B<group-by> option has no meaning and For report type I<sums>, the B<group-by> option has no meaning and
will be silently ignored (see below). will be silently ignored (see below).
@ -327,13 +305,13 @@ Set the lower boundary. See below.
Set the upper boundary. Set the upper boundary.
By default, all hosts/clients with more postings per month than the By default, all hosts with more postings per month than the upper
upper boundary and/or less postings per month than the lower boundary boundary and/or less postings per month than the lower boundary
will be excluded from further processing. For the default report that will be excluded from further processing. For the default report that
means each month only hosts/clients with a number of postings between means each month only hosts with a number of postings between the
the boundaries will be displayed. For the sums report, hosts/clients boundaries will be displayed. For the sums report, hosts with a number
with a number of postings exceeding the boundaries in all (!) months of postings exceeding the boundaries in all (!) months will not be
will not be considered. considered.
=item B<-g>, B<--group-by> I<month[-desc]|name[-desc]> =item B<-g>, B<--group-by> I<month[-desc]|name[-desc]>
@ -349,8 +327,7 @@ ascending order, like this:
individual.net : 16768 individual.net : 16768
news.albasani.net: 7879 news.albasani.net: 7879
The results can be grouped by host/client instead via The results can be grouped by host instead via B<--group-by> I<name>:
B<--group-by> I<name>:
----- individual.net ----- individual.net
2012-01: 19525 2012-01: 19525
@ -379,8 +356,8 @@ will therefore be ignored.
=item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]> =item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
Within each group (a single month or single host/client, see above), Within each group (a single month or single host, see above), the
the report will be sorted by name (or month) in ascending alphabetical report will be sorted by host (or month) in ascending alphabetical
order by default. You can change the sort order to descending or sort order by default. You can change the sort order to descending or sort
by number of postings instead. by number of postings instead.
@ -426,19 +403,19 @@ False by default.
=item B<--comments|--nocomments> =item B<--comments|--nocomments>
Add comments (group headers) to I<dump> and I<pretty> output. True by default Add comments (group headers) to I<dump> and I<pretty> output. True by
as logn as B<--filetemplate> is not set. default as long as B<--filetemplate> is not set.
Use I<--nocomments> to suppress anything except host/client names or months and Use I<--nocomments> to suppress anything except host names or months
numbers of postings. and numbers of postings.
=item B<--filetemplate> I<filename template> =item B<--filetemplate> I<filename template>
Save output to file(s) instead of dumping it to STDOUT. B<cliservstats> will Save output to file(s) instead of dumping it to STDOUT. B<hoststats>
create one file for each month (or each host/client, accordant to the will create one file for each month (or each host, according to the
setting of B<--group-by>, see above), with filenames composed by adding setting of B<--group-by>, see above), with filenames composed by adding
year and month (or host/client names) to the I<filename template>, for year and month (or hostnames) to the I<filename template>, for example
example with B<--filetemplate> I<stats>: with B<--filetemplate> I<stats>:
stats-2012-01 stats-2012-01
stats-2012-02 stats-2012-02
@ -446,7 +423,7 @@ example with B<--filetemplate> I<stats>:
=item B<--db> I<database table> =item B<--db> I<database table>
Override I<DBTableHosts> or I<DBTableClnts> from F<newsstats.conf>. Override I<DBTableHosts> from F<newsstats.conf>.
=item B<--conffile> I<filename> =item B<--conffile> I<filename>
@ -462,29 +439,28 @@ See L<doc/INSTALL>.
Show number of postings per group for lasth month in I<pretty> format: Show number of postings per group for lasth month in I<pretty> format:
cliservstats --type host hoststats
Show that report for January of 2010 and *.inka plus individual.net: Show that report for January of 2010 and *.inka plus individual.net:
cliservstats --type host --month 2010-01 --names *.inka:individual.net: hoststats --month 2010-01 --names *.inka:individual.net:
Only show clients with 30 postings or less last month, ordered Only show hosts with 30 postings or less last month, ordered
by number of postings, descending, in I<pretty> format: by number of postings, descending, in I<pretty> format:
cliservstats --type client --upper 30 --order-by postings-desc hoststats --upper 30 --order-by postings-desc
List number of postings per host for each month of 2010 and redirect List number of postings per host for each month of 2010 and redirect
output to one file for each month, named hosts-2010-01 and so on, in output to one file for each month, named hosts-2010-01 and so on, in
machine-readable form (without formatting): machine-readable form (without formatting):
cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts hoststats -m 2010-01:2010-12 -f dump --filetemplate hosts
=head1 FILES =head1 FILES
=over 4 =over 4
=item F<bin/cliservstats.pl> =item F<bin/hoststats.pl>
The script itself. The script itself.

8
bin/postingstats.pl
View file

@ -15,7 +15,7 @@
# #
# Usage: # Usage:
# $~ groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups # $~ groupstats.pl --nocomments --sums --format dump | postingstats.pl -t groups
# $~ cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts # $~ hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts
# #
BEGIN { BEGIN {
@ -193,7 +193,7 @@ See L<doc/README>.
=head1 DESCRIPTION =head1 DESCRIPTION
This script will re-format reports on newsgroup usage created by This script will re-format reports on newsgroup usage created by
B<groupstats.pl> or B<cliservstats.pl> and create a message that can B<groupstats.pl> or B<hoststats.pl> and create a message that can
be posted to Usenet. be posted to Usenet.
=head2 Features and options =head2 Features and options
@ -291,7 +291,7 @@ Create a posting from a posting statistics report for 2012-01:
Create a posting from a host statistics report for last month: Create a posting from a host statistics report for last month:
cliservstats.pl -t server --nocomments --sums --format dump | postingstats.pl -t hosts hoststats.pl --nocomments --sums --format dump | postingstats.pl -t hosts
=head1 FILES =head1 FILES
@ -334,7 +334,7 @@ groupstats -h
=item - =item -
cliservstats -h hoststats -h
=back =back

2
contrib/dopostingstats.sh
View file

@ -2,7 +2,7 @@
# installation path is /srv/newsstats/, please adjust accordingly # installation path is /srv/newsstats/, please adjust accordingly
if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then if [[ $1 =~ [0-9]{4}-[0-9]{2} ]]; then
/srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y /srv/newsstats/bin/groupstats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y
/srv/newsstats/bin/cliservstats.pl -t server --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y /srv/newsstats/bin/hoststats.pl --nocomments --sums --format dump --month $1 | /srv/newsstats/bin/postingstats.pl -t server --month $1 | /srv/newsstats/contrib/tinews.pl -X -Y
else else
echo 'Input error, please use dopostingstats.sh YYYY-MM' echo 'Input error, please use dopostingstats.sh YYYY-MM'
fi fi

1
doc/ChangeLog
View file

@ -5,6 +5,7 @@ NewsStats 0.4.0 (unreleased)
* Improve documentation for config file. * Improve documentation for config file.
* ParseHeader: re-merge continuation lines. * ParseHeader: re-merge continuation lines.
* Add ClientStats to gatherstats. * Add ClientStats to gatherstats.
* Move cliservstats to hoststats.
NewsStats 0.3.0 (2025-05-18) NewsStats 0.3.0 (2025-05-18)
* Extract GroupStats (in gatherstats) to subroutine. * Extract GroupStats (in gatherstats) to subroutine.

4
doc/README
View file

@ -74,8 +74,8 @@ Getting Started
Report generation is handled by specialised scripts for each Report generation is handled by specialised scripts for each
report type. Currently reports on the number of postings per group report type. Currently reports on the number of postings per group
and month and injection server and month are supported; you can and month and injection server and month are supported; you can
use 'groupstats.pl' and 'cliservstats.pl' for that. See the use 'groupstats.pl' and 'hoststats.pl' for that. See the
groupstats.pl and cliservstats.pl man pages for more information. groupstats.pl and hoststats.pl man pages for more information.
Reporting Bugs Reporting Bugs