diff --git a/bin/cliservstats.pl b/bin/cliservstats.pl new file mode 100644 index 0000000..6d76779 --- /dev/null +++ b/bin/cliservstats.pl @@ -0,0 +1,534 @@ +#! /usr/bin/perl +# +# cliservstats.pl +# +# This script will get statistical data on client (newsreader) and +# server (host) usage from a database. +# +# It is part of the NewsStats package. +# +# Copyright (c) 2025 Thomas Hochstein +# +# It can be redistributed and/or modified under the same terms under +# which Perl itself is published. + +BEGIN { + use File::Basename; + # we're in .../bin, so our module is in ../lib + push(@INC, dirname($0).'/../lib'); +} +use strict; +use warnings; + +use NewsStats qw(:DEFAULT :TimePeriods :Output :SQLHelper ReadGroupList); + +use DBI; +use Getopt::Long qw(GetOptions); +Getopt::Long::config ('bundling'); + +################################# Main program ################################# + +### read commandline options +my ($OptCaptions,$OptComments,$OptDB,$OptFileTemplate,$OptFormat, + $OptGroupBy,$LowBound,$OptMonth,$OptNames,$OptOrderBy, + $OptReportType,$OptSums,$OptType,$UppBound,$OptConfFile); +GetOptions ('c|captions!' => \$OptCaptions, + 'comments!' => \$OptComments, + 'db=s' => \$OptDB, + 'filetemplate=s' => \$OptFileTemplate, + 'f|format=s' => \$OptFormat, + 'g|group-by=s' => \$OptGroupBy, + 'l|lower=i' => \$LowBound, + 'm|month=s' => \$OptMonth, + 'n|names=s' => \$OptNames, + 'o|order-by=s' => \$OptOrderBy, + 'r|report=s' => \$OptReportType, + 's|sums!' => \$OptSums, + 't|type=s' => \$OptType, + 'u|upper=i' => \$UppBound, + 'conffile=s' => \$OptConfFile, + 'h|help' => \&ShowPOD, + 'V|version' => \&ShowVersion) or exit 1; +# parse parameters +# TODO: $OptSums is currently a no-op +# $OptComments defaults to TRUE +$OptComments = 1 if (!defined($OptComments)); +# force --nocomments when --filetemplate is used +$OptComments = 0 if ($OptFileTemplate); +# parse $OptType +if ($OptType) { + if ($OptType =~ /(host|server)s?/i) { + $OptType = 'host'; + } elsif ($OptType =~ /(newsreader|client)s?/i) { + $OptType = 'client'; + } +} +&Bleat(2, "Please use '--type server' or '-type newsreader'.") if !$OptType; +# parse $OptReportType +if ($OptReportType) { + if ($OptReportType =~ /sums?/i) { + $OptReportType = 'sum'; + } else { + $OptReportType = 'default'; + } +} + +### read configuration +my %Conf = %{ReadConfig($OptConfFile)}; + +### set DBTable +if ($OptDB) { + $Conf{'DBTable'} = $OptDB; +} +elsif ($OptType eq 'host') { + $Conf{'DBTable'} = $Conf{'DBTableHosts'}; +} else { + $Conf{'DBTable'} = $Conf{'DBTableClnts'}; +} + +### init database +my $DBHandle = InitDB(\%Conf,1); + +### get time period and newsgroups, prepare SQL 'WHERE' clause +# get time period +# and set caption for output and expression for SQL 'WHERE' clause +my ($CaptionPeriod,$SQLWherePeriod) = &GetTimePeriod($OptMonth); +# bail out if --month is invalid +&Bleat(2,"--month option has an invalid format - ". + "please use 'YYYY-MM', 'YYYY-MM:YYYY-MM' or 'ALL'!") if !$CaptionPeriod; +# get list of hosts and set expression for SQL 'WHERE' clause +# with placeholders as well as a list of newsgroup to bind to them +my ($SQLWhereNames,@SQLBindNames); +if ($OptNames) { + ($SQLWhereNames,@SQLBindNames) = &SQLGroupList($OptNames,$OptType); + # bail out if --names is invalid + &Bleat(2,"--names option has an invalid format!") + if !$SQLWhereNames; +} + +### build SQL WHERE clause +my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, + &SQLSetBounds('default',$LowBound,$UppBound)); + +### get sort order and build SQL 'ORDER BY' clause +# force to 'month' for $OptReportType 'sum' +$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default'); +# default to 'name' if $OptGroupBy is not set and +# just one name is requested, but more than one month +$OptGroupBy = 'name' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/ + and $OptNames and $OptNames !~ /[:*%]/); +# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause +# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month' +my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy, $OptType); +# $GroupBy will contain 'month' or 'host'/'client' (parsed result of $OptGroupBy) +# set it to 'month' or 'key' for OutputData() +$GroupBy = ($GroupBy eq 'month') ? 'month' : 'key'; + +### get report type and build SQL 'SELECT' query +my $SQLSelect; +my $SQLGroupClause = ''; +my $Precision = 0; # number of digits right of decimal point for output +if ($OptReportType and $OptReportType ne 'default') { + $SQLGroupClause = "GROUP BY $OptType"; + # change $SQLOrderClause: replace everything before 'postings' + $SQLOrderClause =~ s/BY.+postings/BY postings/; + $SQLSelect = "'All months',$OptType,SUM(postings)"; + # change $SQLOrderClause: replace 'postings' with 'SUM(postings)' + $SQLOrderClause =~ s/postings/SUM(postings)/; + } else { + $SQLSelect = "month,$OptType,postings"; +}; + +### get length of longest newsgroup name delivered by query +### for formatting purposes +my $Field = ($GroupBy eq 'month') ? $OptType : 'month'; +my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, + $Field,'postings',$SQLWhereClause, + @SQLBindNames); + +### build and execute SQL query +my ($DBQuery); +# prepare query +$DBQuery = $DBHandle->prepare(sprintf('SELECT %s FROM %s.%s %s %s %s', + $SQLSelect, + $Conf{'DBDatabase'},$Conf{'DBTable'}, + $SQLWhereClause,$SQLGroupClause, + $SQLOrderClause)); +# execute query +$DBQuery->execute(@SQLBindNames) + or &Bleat(2,sprintf("Can't get %ss data for %s from %s.%s: %s\n", + $OptType,$CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, + $DBI::errstr)); + +### output results +# set default to 'pretty' +$OptFormat = 'pretty' if !$OptFormat; +# print captions if --caption is set +if ($OptCaptions && $OptComments) { + # print time period with report type + my $CaptionReportType = '(number of postings for each month)'; + if ($OptReportType and $OptReportType ne 'default') { + $CaptionReportType = '(number of all postings for that time period)'; + } + printf("# ----- Report for %s %s\n",$CaptionPeriod,$CaptionReportType); + # print name list if --names is set + printf("# ----- Names: %s\n",join(',',split(/:/,$OptNames))) + if $OptNames; + # print boundaries, if set + my $CaptionBoundary= '(counting only month fulfilling this condition)'; + printf("# ----- Threshold: %s %s x %s %s %s\n", + $LowBound ? $LowBound : '',$LowBound ? '=>' : '', + $UppBound ? '<=' : '',$UppBound ? $UppBound : '',$CaptionBoundary) + if ($LowBound or $UppBound); + # print primary and secondary sort order + printf("# ----- Grouped by %s (%s), sorted %s%s\n", + ($GroupBy eq 'month') ? 'Months' : 'Names', + ($OptGroupBy and $OptGroupBy =~ /-?desc$/i) ? 'descending' : 'ascending', + ($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '', + ($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending'); +} + +# output data +&OutputData($OptFormat,$OptComments,$GroupBy,$Precision,'', + $OptFileTemplate,$DBQuery,$MaxLength,$MaxValLength); + +### close handles +$DBHandle->disconnect; + +__END__ + +################################ Documentation ################################# + +=head1 NAME + +cliservstats - create reports on host or client usage + +=head1 SYNOPSIS + +B B<-t> I [B<-Vhcs> B<--comments>] [B<-m> I[:I] | I] [B<-n> I] [B<-r> I] [B<-l> I] [B<-u> I] [B<-g> I] [B<-o> I] [B<-f> I] [B<--filetemplate> I] [B<--db> I] [B<--conffile> I] + +=head1 REQUIREMENTS + +See L. + +=head1 DESCRIPTION + +This script create reports on newsgroup usage (number of postings from +each host or using each client per month) taken from result tables +created by B. + +=head2 Features and options + +=head3 Time period and names + +The time period to act on defaults to last month; you can assign another +time period or a single month (or drop all time constraints) via the +B<--month> option (see below). + +B will process all hosts or clients by default; you can +limit processing to only some hosts or clients by supplying a list of +those names by using the B<--names> option (see below). + +=head3 Report type + +You can choose between different B<--report> types: postings per month +or all postings summed up; for details, see below. + +=head3 Upper and lower boundaries + +Furthermore you can set an upper and/or lower boundary to exclude some +results from output via the B<--lower> and B<--upper> options, +respectively. By default, all hosts/clients with more and/or less +postings per month will be excluded from the result set (i.e. not +shown and not considered forsum reports). + +=head3 Sorting and formatting the output + +By default, all results are grouped by month; you can group results by +hosts/clients instead via the B<--group-by> option. Within those +groups, the list of hosts/clients (or months) is sorted alphabetically +(or chronologically, respectively) ascending. You can change that order +(and sort by number of postings) with the B<--order-by> option. For +details and exceptions, please see below. + +The results will be formatted as a kind of table; you can change the +output format to a simple list or just a list of names and number of +postings with the B<--format> option. Captions will be added by means of +the B<--caption> option; all comments (and captions) can be supressed by +using B<--nocomments>. + +Last but not least you can redirect all output to a number of files, e.g. +one for each month, by submitting the B<--filetemplate> option, see below. +Captions and comments are automatically disabled in this case. + +=head2 Configuration + +B will read its configuration from F +which should be present in etc/ via Config::Auto or from a configuration file +submitted by the B<--conffile> option. + +See doc/INSTALL for an overview of possible configuration options. + +You can override some configuration options via the B<--db> option. + +=head1 OPTIONS + +=over 3 + +=item B<-V>, B<--version> + +Print out version and copyright information and exit. + +=item B<-h>, B<--help> + +Print this man page and exit. + +=item B<-t>, B<--type> I + +Create report for hosts (servers) or clients (newsreaders), using +I or I respectively. + +=item B<-m>, B<--month> I + +Set processing period to a single month in YYYY-MM format or to a time +period between two month in YYYY-MM:YYYY-MM format (two month, separated +by a colon). By using the keyword I instead, you can set no +processing period to process the whole database. + +=item B<-n>, B<--names> I + +Limit processing to a certain set of host or client names. I +can be a single name (eternal-september.org), a group of names +(*.inka.de) or a list of either of these, separated by colons, for +example + + eternal-september.org:solani.org:*.inka.de + +=item B<-r>, B<--report> I + +Choose the report type: I or I + +By default, B will report the number of postings for each +host/client in each month. But it can also report the total sum of postings +per host/client for all months. + +For report type I, the B option has no meaning and +will be silently ignored (see below). + +=item B<-l>, B<--lower> I + +Set the lower boundary. See below. + +=item B<-l>, B<--upper> I + +Set the upper boundary. + +By default, all hosts/clients with more postings per month than the +upper boundary and/or less postings per month than the lower boundary +will be excluded from further processing. For the default report that +means each month only hosts/clients with a number of postings between +the boundaries will be displayed. For the sums report, hosts/clients +with a number of postings exceeding the boundaries in all (!) months +will not be considered. + +=item B<-g>, B<--group-by> I + +By default, all results are grouped by month, sorted chronologically in +ascending order, like this: + + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + news.albasani.net: 9063 + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + news.albasani.net: 7879 + +The results can be grouped by host/client instead via +B<--group-by> I: + + ----- individual.net + 2012-01: 19525 + 2012-02: 16768 + ----- arcor-online.net + 2012-01: 9379 + 2012-02: 8606 + ----- news.albasani.net + 2012-01: 9063 + 2012-02: 7879 + +By appending I<-desc> to the group-by option parameter, you can reverse +the sort order - e.g. B<--group-by> I will give: + + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + news.albasani.net: 7879 + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + news.albasani.net: 9063 + +Sums reports (see above) will always be grouped by months; this option +will therefore be ignored. + +=item B<-o>, B<--order-by> I + +Within each group (a single month or single host/client, see above), +the report will be sorted by name (or month) in ascending alphabetical +order by default. You can change the sort order to descending or sort +by number of postings instead. + +=item B<-f>, B<--format> I + +Select the output format, I being the default: + + # ----- 2012-01: + arcor-online.net : 9379 + individual.net : 19525 + # ----- 2012-02: + arcor-online.net : 8606 + individual.net : 16768 + +I format looks like this: + + 2012-01 arcor-online.net 9379 + 2012-01 individual.net 19525 + 2012-02 arcor-online.net 8606 + 2012-02 individual.net 16768 + +And I format looks like this: + + # 2012-01: + arcor-online.net 9379 + individual.net 19525 + # 2012-02: + arcor-online.net 8606 + individual.net 16768 + +You can remove the comments by using B<--nocomments>, see below. + +=item B<-c>, B<--captions|--nocaptions> + +Add captions to output, like this: + + ----- Report for 2012-01 to 2012-02 (number of postings for each month) + ----- Names: individual.net + ----- Threshold: 8000 => x (counting only month fulfilling this condition) + ----- Grouped by Month (ascending), sorted by number of postings descending + +False by default. + +=item B<--comments|--nocomments> + +Add comments (group headers) to I and I output. True by default. + +Use I<--nocomments> to suppress anything except newsgroup names/months and +numbers of postings. This is enforced when using B<--filetemplate>, see below. + +=item B<--filetemplate> I + +Save output to file(s) instead of dumping it to STDOUT. B will +create one file for each month (or each host/client, accordant to the +setting of B<--group-by>, see above), with filenames composed by adding +year and month (or host/client names) to the I, for +example with B<--filetemplate> I: + + stats-2012-01 + stats-2012-02 + ... and so on + +B<--nocomments> is enforced, see above. + +=item B<--db> I + +Override I or I from F. + +=item B<--conffile> I + +Load configuration from I instead of F. + +=back + +=head1 INSTALLATION + +See L. + +=head1 EXAMPLES + +Show number of postings per group for lasth month in I format: + + cliservstats --type host + +Show that report for January of 2010 and *.inka plus individual.net: + + cliservstats --type host --month 2010-01 --names *.inka:individual.net: + +Only show clients with 30 postings or less last month, ordered +by number of postings, descending, in I format: + + cliservstats --type client --upper 30 --order-by postings-desc + +List number of postings per host for each month of 2010 and redirect +output to one file for each month, named hosts-2010-01 and so on, in +machine-readable form (without formatting): + + cliservstats -t host -m 2010-01:2010-12 -f dump --filetemplate hosts + + +=head1 FILES + +=over 4 + +=item F + +The script itself. + +=item F + +Library functions for the NewsStats package. + +=item F + +Runtime configuration file. + +=back + +=head1 BUGS + +Please report any bugs or feature requests to the author or use the +bug tracker at L! + +=head1 SEE ALSO + +=over 2 + +=item - + +L + +=item - + +l>doc/INSTALL> + +=item - + +gatherstats -h + +=back + +This script is part of the B package. + +=head1 AUTHOR + +Thomas Hochstein + +=head1 COPYRIGHT AND LICENSE + +Copyright (c) 2025 Thomas Hochstein + +This program is free software; you may redistribute it and/or modify it +under the same terms as Perl itself. + +=cut