535 lines
17 KiB
Perl
535 lines
17 KiB
Perl
#! /usr/bin/perl
|
|
#
|
|
# cliservstats.pl
|
|
#
|
|
# This script will get statistical data on client (newsreader) and
|
|
# server (host) usage from a database.
|
|
#
|
|
# It is part of the NewsStats package.
|
|
#
|
|
# Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
|
|
#
|
|
# It can be redistributed and/or modified under the same terms under
|
|
# which Perl itself is published.
|
|
|
|
BEGIN {
|
|
use File::Basename;
|
|
# we're in .../bin, so our module is in ../lib
|
|
push(@INC, dirname($0).'/../lib');
|
|
}
|
|
use strict;
|
|
use warnings;
|
|
|
|
use NewsStats qw(:DEFAULT :TimePeriods :Output :SQLHelper ReadGroupList);
|
|
|
|
use DBI;
|
|
use Getopt::Long qw(GetOptions);
|
|
Getopt::Long::config ('bundling');
|
|
|
|
################################# Main program #################################
|
|
|
|
### read commandline options
|
|
my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat,
|
|
$OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy,
|
|
$OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile);
|
|
GetOptions ('c|captions!' => \$OptCaptions,
|
|
'comments!' => \$OptComments,
|
|
'db=s' => \$OptDB,
|
|
'filetemplate=s' => \$OptFileTemplate,
|
|
'f|format=s' => \$OptFormat,
|
|
'g|group-by=s' => \$OptGroupBy,
|
|
'l|lower=i' => \$LowBound,
|
|
'm|month=s' => \$OptMonth,
|
|
'n|names=s' => \$OptNames,
|
|
'o|order-by=s' => \$OptOrderBy,
|
|
'r|report=s' => \$OptReportType,
|
|
's|sums!' => \$OptSums,
|
|
't|type=s' => \$OptType,
|
|
'u|upper=i' => \$UppBound,
|
|
'conffile=s' => \$OptConfFile,
|
|
'h|help' => \&ShowPOD,
|
|
'V|version' => \&ShowVersion) or exit 1;
|
|
# parse parameters
|
|
# TODO: $OptSums is currently a no-op
|
|
# $OptComments defaults to TRUE
|
|
$OptComments = 1 if (!defined($OptComments));
|
|
# force --nocomments when --filetemplate is used
|
|
$OptComments = 0 if ($OptFileTemplate);
|
|
# parse $OptType
|
|
if ($OptType) {
|
|
if ($OptType =~ /(host|server)s?/i) {
|
|
$OptType = 'host';
|
|
} elsif ($OptType =~ /(newsreader|client)s?/i) {
|
|
$OptType = 'client';
|
|
}
|
|
}
|
|
&Bleat(2, "Please use '--type server' or '-type newsreader'.") if !$OptType;
|
|
# parse $OptReportType
|
|
if ($OptReportType) {
|
|
if ($OptReportType =~ /sums?/i) {
|
|
$OptReportType = 'sum';
|
|
} else {
|
|
$OptReportType = 'default';
|
|
}
|
|
}
|
|
|
|
### read configuration
|
|
my %Conf = %{ReadConfig($OptConfFile)};
|
|
|
|
### set DBTable
|
|
if ($OptDB) {
|
|
$Conf{'DBTable'} = $OptDB;
|
|
}
|
|
elsif ($OptType eq 'host') {
|
|
$Conf{'DBTable'} = $Conf{'DBTableHosts'};
|
|
} else {
|
|
$Conf{'DBTable'} = $Conf{'DBTableClnts'};
|
|
}
|
|
|
|
### init database
|
|
my $DBHandle = InitDB(\%Conf,1);
|
|
|
|
### get time period and newsgroups, prepare SQL 'WHERE' clause
|
|
# get time period
|
|
# and set caption for output and expression for SQL 'WHERE' clause
|
|
my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth);
|
|
# bail out if --month is invalid
|
|
&Bleat(2,"--month option has an invalid format - ".
|
|
"please use 'YYYY-MM', 'YYYY-MM:YYYY-MM' or 'ALL'!") if !$CaptionPeriod;
|
|
# get list of hosts and set expression for SQL 'WHERE' clause
|
|
# with placeholders as well as a list of newsgroup to bind to them
|
|
my ($SQLWhereNames,@SQLBindNames);
|
|
if ($OptNames) {
|
|
($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType);
|
|
# bail out if --names is invalid
|
|
&Bleat(2,"--names option has an invalid format!")
|
|
if !$SQLWhereNames;
|
|
}
|
|
|
|
### build SQL WHERE clause
|
|
my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames,
|
|
&SQLSetBounds('default',$LowBound,$UppBound));
|
|
|
|
### get sort order and build SQL 'ORDER BY' clause
|
|
# force to 'month' for $OptReportType 'sum'
|
|
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
|
|
# default to 'name' if $OptGroupBy is not set and
|
|
# just one name is requested, but more than one month
|
|
$OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
|
|
and $OptNames and $OptNames !~ /[:*%]/);
|
|
# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
|
|
# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
|
|
my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType);
|
|
# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy)
|
|
# set it to 'month' or 'key' for OutputData()
|
|
$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key';
|
|
|
|
### get report type and build SQL 'SELECT' query
|
|
my $SQLSelect;
|
|
my $SQLGroupClause = '';
|
|
my $Precision = 0; # number of digits right of decimal point for output
|
|
if ($OptReportType and $OptReportType ne 'default') {
|
|
$SQLGroupClause = "GROUP BY $OptType";
|
|
# change $SQLOrderClause: replace everything before 'postings'
|
|
$SQLOrderClause =~ s/BY.+postings/BY postings/;
|
|
$SQLSelect = "'All months',$OptType,SUM(postings)";
|
|
# change $SQLOrderClause: replace 'postings' with 'SUM(postings)'
|
|
$SQLOrderClause =~ s/postings/SUM(postings)/;
|
|
} else {
|
|
$SQLSelect = "month,$OptType,postings";
|
|
};
|
|
|
|
### get length of longest newsgroup name delivered by query
|
|
### for formatting purposes
|
|
my $Field = ($GroupBy eq 'month') ? $OptType : 'month';
|
|
my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'},
|
|
$Field,'postings',$SQLWhereClause,
|
|
@SQLBindNames);
|
|
|
|
### build and execute SQL query
|
|
my ($DBQuery);
|
|
# prepare query
|
|
$DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s',
|
|
$SQLSelect,
|
|
$Conf{'DBDatabase'},$Conf{'DBTable'},
|
|
$SQLWhereClause,$SQLGroupClause,
|
|
$SQLOrderClause));
|
|
# execute query
|
|
$DBQuery->execute(@SQLBindNames)
|
|
or &Bleat(2,sprintf("Can't get %ss data for %s from %s.%s: %s\n",
|
|
$OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'},
|
|
$DBI::errstr));
|
|
|
|
### output results
|
|
# set default to 'pretty'
|
|
$OptFormat = 'pretty' if !$OptFormat;
|
|
# print captions if --caption is set
|
|
if ($OptCaptions && $OptComments) {
|
|
# print time period with report type
|
|
my $CaptionReportType = '(number of postings for each month)';
|
|
if ($OptReportType and $OptReportType ne 'default') {
|
|
$CaptionReportType = '(number of all postings for that time period)';
|
|
}
|
|
printf("# ----- Report for %s %s\n",$CaptionPeriod,$CaptionReportType);
|
|
# print name list if --names is set
|
|
printf("# ----- Names: %s\n",join(',',split(/:/,$OptNames)))
|
|
if $OptNames;
|
|
# print boundaries, if set
|
|
my $CaptionBoundary= '(counting only month fulfilling this condition)';
|
|
printf("# ----- Threshold: %s %s x %s %s %s\n",
|
|
$LowBound ? $LowBound : '',$LowBound ? '=>' : '',
|
|
$UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary)
|
|
if ($LowBound or $UppBound);
|
|
# print primary and secondary sort order
|
|
printf("# ----- Grouped by %s (%s), sorted %s%s\n",
|
|
($GroupBy eq 'month') ? 'Months' : 'Names',
|
|
($OptGroupBy and $OptGroupBy =~ /-?desc$/i) ? 'descending' : 'ascending',
|
|
($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
|
|
($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
|
|
}
|
|
|
|
# output data
|
|
&OutputData($OptFormat,$OptComments,$GroupBy,$Precision,'',
|
|
$OptFileTemplate,$DBQuery,$MaxLength,$MaxValLength);
|
|
|
|
### close handles
|
|
$DBHandle->disconnect;
|
|
|
|
__END__
|
|
|
|
################################ Documentation #################################
|
|
|
|
=head1 NAME
|
|
|
|
cliservstats - create reports on host or client usage
|
|
|
|
=head1 SYNOPSIS
|
|
|
|
B<cliservstats> B<-t> I<host|client> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<server(s)|client(s)>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--db> I<database table>] [B<--conffile> I<filename>]
|
|
|
|
=head1 REQUIREMENTS
|
|
|
|
See L<doc/README>.
|
|
|
|
=head1 DESCRIPTION
|
|
|
|
This script create reports on newsgroup usage (number of postings from
|
|
each host or using each client per month) taken from result tables
|
|
created by B<gatherstats.pl>.
|
|
|
|
=head2 Features and options
|
|
|
|
=head3 Time period and names
|
|
|
|
The time period to act on defaults to last month; you can assign another
|
|
time period or a single month (or drop all time constraints) via the
|
|
B<--month> option (see below).
|
|
|
|
B<cliservstats> will process all hosts or clients by default; you can
|
|
limit processing to only some hosts or clients by supplying a list of
|
|
those names by using the B<--names> option (see below).
|
|
|
|
=head3 Report type
|
|
|
|
You can choose between different B<--report> types: postings per month
|
|
or all postings summed up; for details, see below.
|
|
|
|
=head3 Upper and lower boundaries
|
|
|
|
Furthermore you can set an upper and/or lower boundary to exclude some
|
|
results from output via the B<--lower> and B<--upper> options,
|
|
respectively. By default, all hosts/clients with more and/or less
|
|
postings per month will be excluded from the result set (i.e. not
|
|
shown and not considered forsum reports).
|
|
|
|
=head3 Sorting and formatting the output
|
|
|
|
By default, all results are grouped by month; you can group results by
|
|
hosts/clients instead via the B<--group-by> option. Within those
|
|
groups, the list of hosts/clients (or months) is sorted alphabetically
|
|
(or chronologically, respectively) ascending. You can change that order
|
|
(and sort by number of postings) with the B<--order-by> option. For
|
|
details and exceptions, please see below.
|
|
|
|
The results will be formatted as a kind of table; you can change the
|
|
output format to a simple list or just a list of names and number of
|
|
postings with the B<--format> option. Captions will be added by means of
|
|
the B<--caption> option; all comments (and captions) can be supressed by
|
|
using B<--nocomments>.
|
|
|
|
Last but not least you can redirect all output to a number of files, e.g.
|
|
one for each month, by submitting the B<--filetemplate> option, see below.
|
|
Captions and comments are automatically disabled in this case.
|
|
|
|
=head2 Configuration
|
|
|
|
B<cliservstats> will read its configuration from F<newsstats.conf>
|
|
which should be present in etc/ via Config::Auto or from a configuration file
|
|
submitted by the B<--conffile> option.
|
|
|
|
See doc/INSTALL for an overview of possible configuration options.
|
|
|
|
You can override some configuration options via the B<--db> option.
|
|
|
|
=head1 OPTIONS
|
|
|
|
=over 3
|
|
|
|
=item B<-V>, B<--version>
|
|
|
|
Print out version and copyright information and exit.
|
|
|
|
=item B<-h>, B<--help>
|
|
|
|
Print this man page and exit.
|
|
|
|
=item B<-t>, B<--type> I<host|client>
|
|
|
|
Create report for hosts (servers) or clients (newsreaders), using
|
|
I<DBTableHosts> or I<DBTableClnts> respectively.
|
|
|
|
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
|
|
|
|
Set processing period to a single month in YYYY-MM format or to a time
|
|
period between two month in YYYY-MM:YYYY-MM format (two month, separated
|
|
by a colon). By using the keyword I<all> instead, you can set no
|
|
processing period to process the whole database.
|
|
|
|
=item B<-n>, B<--names> I<name(s)>
|
|
|
|
Limit processing to a certain set of host or client names. I<names(s)>
|
|
can be a single name (eternal-september.org), a group of names
|
|
(*.inka.de) or a list of either of these, separated by colons, for
|
|
example
|
|
|
|
eternal-september.org:solani.org:*.inka.de
|
|
|
|
=item B<-r>, B<--report> I<default|sums>
|
|
|
|
Choose the report type: I<default> or I<sums>
|
|
|
|
By default, B<cliservstats> will report the number of postings for each
|
|
host/client in each month. But it can also report the total sum of postings
|
|
per host/client for all months.
|
|
|
|
For report type I<sums>, the B<group-by> option has no meaning and
|
|
will be silently ignored (see below).
|
|
|
|
=item B<-l>, B<--lower> I<lower boundary>
|
|
|
|
Set the lower boundary. See below.
|
|
|
|
=item B<-l>, B<--upper> I<upper boundary>
|
|
|
|
Set the upper boundary.
|
|
|
|
By default, all hosts/clients with more postings per month than the
|
|
upper boundary and/or less postings per month than the lower boundary
|
|
will be excluded from further processing. For the default report that
|
|
means each month only hosts/clients with a number of postings between
|
|
the boundaries will be displayed. For the sums report, hosts/clients
|
|
with a number of postings exceeding the boundaries in all (!) months
|
|
will not be considered.
|
|
|
|
=item B<-g>, B<--group-by> I<month[-desc]|name[-desc]>
|
|
|
|
By default, all results are grouped by month, sorted chronologically in
|
|
ascending order, like this:
|
|
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
news.albasani.net: 9063
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
news.albasani.net: 7879
|
|
|
|
The results can be grouped by host/client instead via
|
|
B<--group-by> I<name>:
|
|
|
|
----- individual.net
|
|
2012-01: 19525
|
|
2012-02: 16768
|
|
----- arcor-online.net
|
|
2012-01: 9379
|
|
2012-02: 8606
|
|
----- news.albasani.net
|
|
2012-01: 9063
|
|
2012-02: 7879
|
|
|
|
By appending I<-desc> to the group-by option parameter, you can reverse
|
|
the sort order - e.g. B<--group-by> I<month-desc> will give:
|
|
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
news.albasani.net: 7879
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
news.albasani.net: 9063
|
|
|
|
Sums reports (see above) will always be grouped by months; this option
|
|
will therefore be ignored.
|
|
|
|
=item B<-o>, B<--order-by> I<default[-desc]|postings[-desc]>
|
|
|
|
Within each group (a single month or single host/client, see above),
|
|
the report will be sorted by name (or month) in ascending alphabetical
|
|
order by default. You can change the sort order to descending or sort
|
|
by number of postings instead.
|
|
|
|
=item B<-f>, B<--format> I<pretty|list|dump>
|
|
|
|
Select the output format, I<pretty> being the default:
|
|
|
|
# ----- 2012-01:
|
|
arcor-online.net : 9379
|
|
individual.net : 19525
|
|
# ----- 2012-02:
|
|
arcor-online.net : 8606
|
|
individual.net : 16768
|
|
|
|
I<list> format looks like this:
|
|
|
|
2012-01 arcor-online.net 9379
|
|
2012-01 individual.net 19525
|
|
2012-02 arcor-online.net 8606
|
|
2012-02 individual.net 16768
|
|
|
|
And I<dump> format looks like this:
|
|
|
|
# 2012-01:
|
|
arcor-online.net 9379
|
|
individual.net 19525
|
|
# 2012-02:
|
|
arcor-online.net 8606
|
|
individual.net 16768
|
|
|
|
You can remove the comments by using B<--nocomments>, see below.
|
|
|
|
=item B<-c>, B<--captions|--nocaptions>
|
|
|
|
Add captions to output, like this:
|
|
|
|
----- Report for 2012-01 to 2012-02 (number of postings for each month)
|
|
----- Names: individual.net
|
|
----- Threshold: 8000 => x (counting only month fulfilling this condition)
|
|
----- Grouped by Month (ascending), sorted by number of postings descending
|
|
|
|
False by default.
|
|
|
|
=item B<--comments|--nocomments>
|
|
|
|
Add comments (group headers) to I<dump> and I<pretty> output. True by default.
|
|
|
|
Use I<--nocomments> to suppress anything except newsgroup names/months and
|
|
numbers of postings. This is enforced when using B<--filetemplate>, see below.
|
|
|
|
=item B<--filetemplate> I<filename template>
|
|
|
|
Save output to file(s) instead of dumping it to STDOUT. B<cliservstats> will
|
|
create one file for each month (or each host/client, accordant to the
|
|
setting of B<--group-by>, see above), with filenames composed by adding
|
|
year and month (or host/client names) to the I<filename template>, for
|
|
example with B<--filetemplate> I<stats>:
|
|
|
|
stats-2012-01
|
|
stats-2012-02
|
|
... and so on
|
|
|
|
B<--nocomments> is enforced, see above.
|
|
|
|
=item B<--db> I<database table>
|
|
|
|
Override I<DBTableHosts> or I<DBTableClnts> from F<newsstats.conf>.
|
|
|
|
=item B<--conffile> I<filename>
|
|
|
|
Load configuration from I<filename> instead of F<newsstats.conf>.
|
|
|
|
=back
|
|
|
|
=head1 INSTALLATION
|
|
|
|
See L<doc/INSTALL>.
|
|
|
|
=head1 EXAMPLES
|
|
|
|
Show number of postings per group for lasth month in I<pretty> format:
|
|
|
|
cliservstats --type host
|
|
|
|
Show that report for January of 2010 and *.inka plus individual.net:
|
|
|
|
cliservstats --type host --month 2010-01 --names *.inka:individual.net:
|
|
|
|
Only show clients with 30 postings or less last month, ordered
|
|
by number of postings, descending, in I<pretty> format:
|
|
|
|
cliservstats --type client --upper 30 --order-by postings-desc
|
|
|
|
List number of postings per host for each month of 2010 and redirect
|
|
output to one file for each month, named hosts-2010-01 and so on, in
|
|
machine-readable form (without formatting):
|
|
|
|
cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts
|
|
|
|
|
|
=head1 FILES
|
|
|
|
=over 4
|
|
|
|
=item F<bin/cliservstats.pl>
|
|
|
|
The script itself.
|
|
|
|
=item F<lib/NewsStats.pm>
|
|
|
|
Library functions for the NewsStats package.
|
|
|
|
=item F<etc/newsstats.conf>
|
|
|
|
Runtime configuration file.
|
|
|
|
=back
|
|
|
|
=head1 BUGS
|
|
|
|
Please report any bugs or feature requests to the author or use the
|
|
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
|
|
|
=head1 SEE ALSO
|
|
|
|
=over 2
|
|
|
|
=item -
|
|
|
|
L<doc/README>
|
|
|
|
=item -
|
|
|
|
l>doc/INSTALL>
|
|
|
|
=item -
|
|
|
|
gatherstats -h
|
|
|
|
=back
|
|
|
|
This script is part of the B<NewsStats> package.
|
|
|
|
=head1 AUTHOR
|
|
|
|
Thomas Hochstein <thh@thh.name>
|
|
|
|
=head1 COPYRIGHT AND LICENSE
|
|
|
|
Copyright (c) 2025 Thomas Hochstein <thh@thh.name>
|
|
|
|
This program is free software; you may redistribute it and/or modify it
|
|
under the same terms as Perl itself.
|
|
|
|
=cut
|