Thomas Hochstein
28157570f1
--comments defaulted to true, but --nocomments was enforced if --filetemplate was set. Remove enforcement, but default to --nocomments if --filetemplate is set. Default behaviour is unchanged, but it's now possible to have comments in files. Change handling of captions accordingly (must be sent to output handle now). Update POD. Signed-off-by: Thomas Hochstein <thh@thh.name>
538 lines
17 KiB
Perl
538 lines
17 KiB
Perl
#! /usr/bin/perl
|
|
#
|
|
# cliservstats.pl
|
|
#
|
|
# This script will get statistical data on client (newsreader) and
|
|
# server (host) usage from a database.
|
|
#
|
|
# It is part of the NewsStats package.
|
|
#
|
|
# Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
|
|
#
|
|
# It can be redistributed and/or modified under the same terms under
|
|
# which Perl itself is published.
|
|
|
|
BEGIN {
|
|
use File::Basename;
|
|
# we're in .../bin, so our module is in ../lib
|
|
push(@INC, dirname($0).'/../lib');
|
|
}
|
|
use strict;
|
|
use warnings;
|
|
|
|
use NewsStats qw(:DEFAULT :TimePeriods :Output :SQLHelper ReadGroupList);
|
|
|
|
use DBI;
|
|
use Getopt::Long qw(GetOptions);
|
|
Getopt::Long::config ('bundling');
|
|
|
|
################################# Main program #################################
|
|
|
|
### read commandline options
|
|
my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat,
|
|
$OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy,
|
|
$OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile);
|
|
GetOptions ('c|captions!' => \$OptCaptions,
|
|
'comments!' => \$OptComments,
|
|
'db=s' => \$OptDB,
|
|
'filetemplate=s' => \$OptFileTemplate,
|
|
'f|format=s' => \$OptFormat,
|
|
'g|group-by=s' => \$OptGroupBy,
|
|
'l|lower=i' => \$LowBound,
|
|
'm|month=s' => \$OptMonth,
|
|
'n|names=s' => \$OptNames,
|
|
'o|order-by=s' => \$OptOrderBy,
|
|
'r|report=s' => \$OptReportType,
|
|
's|sums!' => \$OptSums,
|
|
't|type=s' => \$OptType,
|
|
'u|upper=i' => \$UppBound,
|
|
'conffile=s' => \$OptConfFile,
|
|
'h|help' => \&ShowPOD,
|
|
'V|version' => \&ShowVersion) or exit 1;
|
|
# parse parameters
|
|
# $OptComments defaults to TRUE if --filetemplate is not used
|
|
$OptComments = 1 if (!$OptFileTemplate && !defined($OptComments));
|
|
# parse $OptType
|
|
if ($OptType) {
|
|
if ($OptType =~ /(host|server)s?/i) {
|
|
$OptType = 'host';
|
|
} elsif ($OptType =~ /(newsreader|client)s?/i) {
|
|
$OptType = 'client';
|
|
}
|
|
}
|
|
&Bleat(2, "Please use '--type server' or '-type newsreader'.") if !$OptType;
|
|
# parse $OptReportType
|
|
if ($OptReportType) {
|
|
if ($OptReportType =~ /sums?/i) {
|
|
$OptReportType = 'sum';
|
|
} else {
|
|
$OptReportType = 'default';
|
|
}
|
|
}
|
|
|
|
### read configuration
|
|
my %Conf = %{ReadConfig($OptConfFile)};
|
|
|
|
### set DBTable
|
|
if ($OptDB) {
|
|
$Conf{'DBTable'} = $OptDB;
|
|
}
|
|
elsif ($OptType eq 'host') {
|
|
$Conf{'DBTable'} = $Conf{'DBTableHosts'};
|
|
} else {
|
|
$Conf{'DBTable'} = $Conf{'DBTableClnts'};
|
|
}
|
|
|
|
### init database
|
|
my $DBHandle = InitDB(\%Conf,1);
|
|
|
|
### get time period and newsgroups, prepare SQL 'WHERE' clause
|
|
# get time period
|
|
# and set caption for output and expression for SQL 'WHERE' clause
|
|
my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth);
|
|
# bail out if --month is invalid
|
|
&Bleat(2,"--month option has an invalid format - ".
|
|
"please use 'YYYY-MM', 'YYYY-MM:YYYY-MM' or 'ALL'!") if !$CaptionPeriod;
|
|
# get list of hosts and set expression for SQL 'WHERE' clause
|
|
# with placeholders as well as a list of newsgroup to bind to them
|
|
my ($SQLWhereNames,@SQLBindNames);
|
|
if ($OptNames) {
|
|
($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType);
|
|
# bail out if --names is invalid
|
|
&Bleat(2,"--names option has an invalid format!")
|
|
if !$SQLWhereNames;
|
|
}
|
|
|
|
### build SQL WHERE clause
|
|
my $ExcludeSums = $OptSums ? '' : sprintf("%s != 'ALL'",$OptType);
|
|
my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
|
|
$ExcludeSums,
|
|
&SQLSetBounds('default',$LowBound,$UppBound));
|
|
|
|
### get sort order and build SQL 'ORDER BY' clause
|
|
# force to 'month' for $OptReportType 'sum'
|
|
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
|
|
# default to 'name' if $OptGroupBy is not set and
|
|
# just one name is requested, but more than one month
|
|
$OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
|
|
and $OptNames and $OptNames !~ /[:*%]/);
|
|
# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
|
|
# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
|
|
my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType);
|
|
# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy)
|
|
# set it to 'month' or 'key' for OutputData()
|
|
$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key';
|
|
|
|
### get report type and build SQL 'SELECT' query
|
|
my $SQLSelect;
|
|
my $SQLGroupClause = '';
|
|
my $Precision = 0; # number of digits right of decimal point for output
|
|
if ($OptReportType and $OptReportType ne 'default') {
|
|
$SQLGroupClause = "GROUP BY $OptType";
|
|
# change $SQLOrderClause: replace everything before 'postings'
|
|
$SQLOrderClause =~ s/BY.+postings/BY postings/;
|
|
$SQLSelect = "'All months',$OptType,SUM(postings)";
|
|
# change $SQLOrderClause: replace 'postings' with 'SUM(postings)'
|
|
$SQLOrderClause =~ s/postings/SUM(postings)/;
|
|
} else {
|
|
$SQLSelect = "month,$OptType,postings";
|
|
};
|
|
|
|
### get length of longest newsgroup name delivered by query
|
|
### for formatting purposes
|
|
my $Field = ($GroupBy eq 'month') ? $OptType : 'month';
|
|
my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
|
|
$Field,'postings',$SQLWhereClause,
|
|
@SQLBindNames);
|
|
|
|
### build and execute SQL query
|
|
my ($DBQuery);
|
|
# prepare query
|
|
$DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s',
|
|
$SQLSelect,
|
|
$Conf{'DBDatabase'},$Conf{'DBTable'},
|
|
$SQLWhereClause,$SQLGroupClause,
|
|
$SQLOrderClause));
|
|
# execute query
|
|
$DBQuery->execute(@SQLBindNames)
|
|
or &Bleat(2,sprintf("Can't get %ss data for %s from %s.%s: %s\n",
|
|
$OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
|
|
$DBI::errstr));
|
|
|
|
### output results
|
|
# set default to 'pretty'
|
|
$OptFormat = 'pretty' if !$OptFormat;
|
|
# print captions if --caption is set
|
|
my $LeadIn;
|
|
if ($OptCaptions && $OptComments) {
|
|
# print time period with report type
|
|
my $CaptionReportType = '(number of postings for each month)';
|
|
if ($OptReportType and $OptReportType ne 'default') {
|
|
$CaptionReportType = '(number of all postings for that time period)';
|
|
}
|
|
$LeadIn .= sprintf("# ----- Report for %s %s\n",$CaptionPeriod,$CaptionReportType);
|
|
# print name list if --names is set
|
|
$LeadIn .= sprintf("# ----- Names: %s\n",join(',',split(/:/,$OptNames)))
|
|
if $OptNames;
|
|
# print boundaries, if set
|
|
my $CaptionBoundary= '(counting only month fulfilling this condition)';
|
|
$LeadIn .= sprintf("# ----- Threshold: %s %s x %s %s %s\n",
|
|
$LowBound ? $LowBound : '',$LowBound ? '=>' : '',
|
|
$UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary)
|
|
if ($LowBound or $UppBound);
|
|
# print primary and secondary sort order
|
|
$LeadIn .= sprintf("# ----- Grouped by %s (%s), sorted %s%s\n",
|
|
($GroupBy eq 'month') ? 'Months' : 'Names',
|
|
($OptGroupBy and $OptGroupBy =~ /-?desc$/i) ? 'descending' : 'ascending',
|
|
($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
|
|
($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
|
|
}
|
|
|
|
# output data
|
|
&OutputData($OptFormat,$OptComments,$GroupBy,$Precision,'',$LeadIn,
|
|
$OptFileTemplate,$DBQuery,$MaxLength,$MaxValLength);
|
|
|
|
### close handles
|
|
$DBHandle->disconnect;
|
|
|
|
__END__
|
|
|
|
################################ Documentation #################################
|
|
|
|
=head1 NAME
|
|
|
|
cliservstats - create reports on host or client usage
|
|
|
|
=head1 SYNOPSIS
|
|
|
|
B<cliservstats> B<-t> I<host|client> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)|client(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
|
|
|
|
=head1 REQUIREMENTS
|
|
|
|
See L<doc/README>.
|
|
|
|
=head1 DESCRIPTION
|
|
|
|
This script create reports on newsgroup usage (number of postings from
|
|
each host or using each client per month) taken from result tables
|
|
created by B<gatherstats.pl>.
|
|
|
|
=head2 Features and options
|
|
|
|
=head3 Time period and names
|
|
|
|
The time period to act on defaults to last month; you can assign another
|
|
time period or a single month (or drop all time constraints) via the
|
|
B<--month> option (see below).
|
|
|
|
B<cliservstats> will process all hosts or clients by default; you can
|
|
limit processing to only some hosts or clients by supplying a list of
|
|
those names by using the B<--names> option (see below).
|
|
|
|
=head3 Report type
|
|
|
|
You can choose between different B<--report> types: postings per month
|
|
or all postings summed up; for details, see below.
|
|
|
|
=head3 Upper and lower boundaries
|
|
|
|
Furthermore you can set an upper and/or lower boundary to exclude some
|
|
results from output via the B<--lower> and B<--upper> options,
|
|
respectively. By default, all hosts/clients with more and/or less
|
|
postings per month will be excluded from the result set (i.e. not
|
|
shown and not considered forsum reports).
|
|
|
|
=head3 Sorting and formatting the output
|
|
|
|
By default, all results are grouped by month; you can group results by
|
|
hosts/clients instead via the B<--group-by> option. Within those
|
|
groups, the list of hosts/clients (or months) is sorted alphabetically
|
|
(or chronologically, respectively) ascending. You can change that order
|
|
(and sort by number of postings) with the B<--order-by> option. For
|
|
details and exceptions, please see below.
|
|
|
|
The results will be formatted as a kind of table; you can change the
|
|
output format to a simple list or just a list of names and number of
|
|
postings with the B<--format> option. Captions will be added by means of
|
|
the B<--caption> option; all comments (and captions) can be supressed by
|
|
using B<--nocomments>.
|
|
|
|
Last but not least you can redirect all output to a number of files, e.g.
|
|
one for each month, by submitting the B<--filetemplate> option, see below.
|
|
|
|
=head2 Configuration
|
|
|
|
B<cliservstats> will read its configuration from F<newsstats.conf>
|
|
which should be present in etc/ via Config::Auto or from a configuration file
|
|
submitted by the B<--conffile> option.
|
|
|
|
See doc/INSTALL for an overview of possible configuration options.
|
|
|
|
You can override some configuration options via the B<--db> option.
|
|
|
|
=head1 OPTIONS
|
|
|
|
=over 3
|
|
|
|
=item B<-V>, B<--version>
|
|
|
|
Print out version and copyright information and exit.
|
|
|
|
=item B<-h>, B<--help>
|
|
|
|
Print this man page and exit.
|
|
|
|
=item B<-t>, B<--type> I<host|client>
|
|
|
|
Create report for hosts (servers) or clients (newsreaders), using
|
|
I<DBTableHosts> or I<DBTableClnts> respectively.
|
|
|
|
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
|
|
|
|
Set processing period to a single month in YYYY-MM format or to a time
|
|
period between two month in YYYY-MM:YYYY-MM format (two month, separated
|
|
by a colon). By using the keyword I<all> instead, you can set no
|
|
processing period to process the whole database.
|
|
|
|
=item B<-n>, B<--names> I<name(s)>
|
|
|
|
Limit processing to a certain set of host or client names. I<names(s)>
|
|
can be a single name (eternal-september.org), a group of names
|
|
(*.inka.de) or a list of either of these, separated by colons, for
|
|
example
|
|
|
|
eternal-september.org:solani.org:*.inka.de
|
|
|
|
=item B<-s>, B<--sums|--nosums> (sum per month)
|
|
|
|
Include a "virtual" host named "ALL" for every month in output,
|
|
containing the sum of all detected hosts for that month.
|
|
|
|
=item B<-r>, B<--report> I<default|sums>
|
|
|
|
Choose the report type: I<default> or I<sums>
|
|
|
|
By default, B<cliservstats> will report the number of postings for each
|
|
host/client in each month. But it can also report the total sum of postings
|
|
per host/client for all months.
|
|
|
|
For report type I<sums>, the B<group-by> option has no meaning and
|
|
will be silently ignored (see below).
|
|
|
|
=item B<-l>, B<--lower> I<lower boundary>
|
|
|
|
Set the lower boundary. See below.
|
|
|
|
=item B<-l>, B<--upper> I<upper boundary>
|
|
|
|
Set the upper boundary.
|
|
|
|
By default, all hosts/clients with more postings per month than the
|
|
upper boundary and/or less postings per month than the lower boundary
|
|
will be excluded from further processing. For the default report that
|
|
means each month only hosts/clients with a number of postings between
|
|
the boundaries will be displayed. For the sums report, hosts/clients
|
|
with a number of postings exceeding the boundaries in all (!) months
|
|
will not be considered.
|
|
|
|
=item B<-g>, B<--group-by> I<month[-desc]|name[-desc]>
|
|
|
|
By default, all results are grouped by month, sorted chronologically in
|
|
ascending order, like this:
|
|
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
news.albasani.net: 9063
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
news.albasani.net: 7879
|
|
|
|
The results can be grouped by host/client instead via
|
|
B<--group-by> I<name>:
|
|
|
|
----- individual.net
|
|
2012-01: 19525
|
|
2012-02: 16768
|
|
----- arcor-online.net
|
|
2012-01: 9379
|
|
2012-02: 8606
|
|
----- news.albasani.net
|
|
2012-01: 9063
|
|
2012-02: 7879
|
|
|
|
By appending I<-desc> to the group-by option parameter, you can reverse
|
|
the sort order - e.g. B<--group-by> I<month-desc> will give:
|
|
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
news.albasani.net: 7879
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
news.albasani.net: 9063
|
|
|
|
Sums reports (see above) will always be grouped by months; this option
|
|
will therefore be ignored.
|
|
|
|
=item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
|
|
|
|
Within each group (a single month or single host/client, see above),
|
|
the report will be sorted by name (or month) in ascending alphabetical
|
|
order by default. You can change the sort order to descending or sort
|
|
by number of postings instead.
|
|
|
|
=item B<-f>, B<--format> I<pretty|list|dump>
|
|
|
|
Select the output format, I<pretty> being the default:
|
|
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
|
|
I<list> format looks like this:
|
|
|
|
2012-01 arcor-online.net 9379
|
|
2012-01 individual.net 19525
|
|
2012-02 arcor-online.net 8606
|
|
2012-02 individual.net 16768
|
|
|
|
And I<dump> format looks like this:
|
|
|
|
# 2012-01:
|
|
arcor-online.net 9379
|
|
individual.net 19525
|
|
# 2012-02:
|
|
arcor-online.net 8606
|
|
individual.net 16768
|
|
|
|
You can remove the comments by using B<--nocomments>, see below.
|
|
|
|
=item B<-c>, B<--captions|--nocaptions>
|
|
|
|
Add captions to output, like this:
|
|
|
|
----- Report for 2012-01 to 2012-02 (number of postings for each month)
|
|
----- Names: individual.net
|
|
----- Threshold: 8000 => x (counting only month fulfilling this condition)
|
|
----- Grouped by Month (ascending), sorted by number of postings descending
|
|
|
|
False by default.
|
|
|
|
=item B<--comments|--nocomments>
|
|
|
|
Add comments (group headers) to I<dump> and I<pretty> output. True by default
|
|
as logn as B<--filetemplate> is not set.
|
|
|
|
Use I<--nocomments> to suppress anything except host/client names or months and
|
|
numbers of postings.
|
|
|
|
=item B<--filetemplate> I<filename template>
|
|
|
|
Save output to file(s) instead of dumping it to STDOUT. B<cliservstats> will
|
|
create one file for each month (or each host/client, accordant to the
|
|
setting of B<--group-by>, see above), with filenames composed by adding
|
|
year and month (or host/client names) to the I<filename template>, for
|
|
example with B<--filetemplate> I<stats>:
|
|
|
|
stats-2012-01
|
|
stats-2012-02
|
|
... and so on
|
|
|
|
=item B<--db> I<database table>
|
|
|
|
Override I<DBTableHosts> or I<DBTableClnts> from F<newsstats.conf>.
|
|
|
|
=item B<--conffile> I<filename>
|
|
|
|
Load configuration from I<filename> instead of F<newsstats.conf>.
|
|
|
|
=back
|
|
|
|
=head1 INSTALLATION
|
|
|
|
See L<doc/INSTALL>.
|
|
|
|
=head1 EXAMPLES
|
|
|
|
Show number of postings per group for lasth month in I<pretty> format:
|
|
|
|
cliservstats --type host
|
|
|
|
Show that report for January of 2010 and *.inka plus individual.net:
|
|
|
|
cliservstats --type host --month 2010-01 --names *.inka:individual.net:
|
|
|
|
Only show clients with 30 postings or less last month, ordered
|
|
by number of postings, descending, in I<pretty> format:
|
|
|
|
cliservstats --type client --upper 30 --order-by postings-desc
|
|
|
|
List number of postings per host for each month of 2010 and redirect
|
|
output to one file for each month, named hosts-2010-01 and so on, in
|
|
machine-readable form (without formatting):
|
|
|
|
cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts
|
|
|
|
|
|
=head1 FILES
|
|
|
|
=over 4
|
|
|
|
=item F<bin/cliservstats.pl>
|
|
|
|
The script itself.
|
|
|
|
=item F<lib/NewsStats.pm>
|
|
|
|
Library functions for the NewsStats package.
|
|
|
|
=item F<etc/newsstats.conf>
|
|
|
|
Runtime configuration file.
|
|
|
|
=back
|
|
|
|
=head1 BUGS
|
|
|
|
Please report any bugs or feature requests to the author or use the
|
|
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
|
|
|
=head1 SEE ALSO
|
|
|
|
=over 2
|
|
|
|
=item -
|
|
|
|
L<doc/README>
|
|
|
|
=item -
|
|
|
|
l>doc/INSTALL>
|
|
|
|
=item -
|
|
|
|
gatherstats -h
|
|
|
|
=back
|
|
|
|
This script is part of the B<NewsStats> package.
|
|
|
|
=head1 AUTHOR
|
|
|
|
Thomas Hochstein <thh@thh.name>
|
|
|
|
=head1 COPYRIGHT AND LICENSE
|
|
|
|
Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
|
|
|
|
This program is free software; you may redistribute it and/or modify it
|
|
under the same terms as Perl itself.
|
|
|
|
=cut
|