thh/newsstats
1
0
Fork 0
Code Issues 4 Pull requests Packages Projects Releases Activity

Merge branch 'thh-restructure' into next

Browse source 
* thh-restructure:
  Make configuration file configurable.
  Fix some whitespace.
  Redo directory structure.
This commit is contained in:
Thomas Hochstein 2013-09-03 14:55:42 +02:00
parent 07c0b2589a 23ab67a099
commit 1af57a5390
7 changed files with 80 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,3 +1,3 @@
tmp/ tmp/
tmp/* tmp/*
newsstats.conf etc/newsstats.conf

26
feedlog.pl → bin/feedlog.pl
View file

@ -4,18 +4,19 @@
# #
# This script will log headers and other data to a database # This script will log headers and other data to a database
# for further analysis by parsing a feed from INN. # for further analysis by parsing a feed from INN.
# #
# It is part of the NewsStats package. # It is part of the NewsStats package.
# #
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net> # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
# #
# It can be redistributed and/or modified under the same terms under # It can be redistributed and/or modified under the same terms under
# which Perl itself is published. # which Perl itself is published.
BEGIN { BEGIN {
our $VERSION = "0.01"; our $VERSION = "0.01";
use File::Basename; use File::Basename;
push(@INC, dirname($0)); # we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
} }
use strict; use strict;
use warnings; use warnings;
@ -68,14 +69,15 @@ sub PrepareDB {
################################# Main program ################################# ################################# Main program #################################
### read commandline options ### read commandline options
my ($OptDebug,$OptQuiet); my ($OptDebug,$OptQuiet,$OptConfFile);
GetOptions ('d|debug!' => \$OptDebug, GetOptions ('d|debug!' => \$OptDebug,
'q|test!' => \$OptQuiet, 'q|test!' => \$OptQuiet,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD, 'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1; 'V|version' => \&ShowVersion) or exit 1;
### read configuration ### read configuration
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')}; my %Conf = %{ReadConfig($OptConfFile)};
### init syslog ### init syslog
openlog($0, 'nofatal,pid', LOG_NEWS); openlog($0, 'nofatal,pid', LOG_NEWS);
@ -129,7 +131,7 @@ while (<>) {
}; };
}; };
$DBQuery->finish; $DBQuery->finish;
warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n". warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
"Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n", "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
@ -151,7 +153,7 @@ feedlog - log data from an INN feed to a database
=head1 SYNOPSIS =head1 SYNOPSIS
B<feedlog> [B<-Vhdq>] B<feedlog> [B<-Vhdq>] [--conffile I<filename>]
=head1 REQUIREMENTS =head1 REQUIREMENTS
@ -197,6 +199,10 @@ find that information most probably in your B<INN> F<errlog> file.
Suppress logging to syslog. Suppress logging to syslog.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back =back
=head1 INSTALLATION =head1 INSTALLATION
@ -218,15 +224,15 @@ See L<doc/INSTALL> for further information.
=over 4 =over 4
=item F<feedlog.pl> =item F<bin/feedlog.pl>
The script itself. The script itself.
=item F<NewsStats.pm> =item F<lib/NewsStats.pm>
Library functions for the NewsStats package. Library functions for the NewsStats package.
=item F<newsstats.conf> =item F<etc/newsstats.conf>
Runtime configuration file. Runtime configuration file.

28
gatherstats.pl → bin/gatherstats.pl
View file

@ -4,18 +4,19 @@
# #
# This script will gather statistical information from a database # This script will gather statistical information from a database
# containing headers and other information from a INN feed. # containing headers and other information from a INN feed.
# #
# It is part of the NewsStats package. # It is part of the NewsStats package.
# #
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net> # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
# #
# It can be redistributed and/or modified under the same terms under # It can be redistributed and/or modified under the same terms under
# which Perl itself is published. # which Perl itself is published.
BEGIN { BEGIN {
our $VERSION = "0.01"; our $VERSION = "0.01";
use File::Basename; use File::Basename;
push(@INC, dirname($0)); # we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
} }
use strict; use strict;
use warnings; use warnings;
@ -37,7 +38,7 @@ my %LegalStats;
### read commandline options ### read commandline options
my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH, my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest); $OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest,$OptConfFile);
GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile, GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
'clientsdb=s' => \$OptClientsDB, 'clientsdb=s' => \$OptClientsDB,
'd|debug!' => \$OptDebug, 'd|debug!' => \$OptDebug,
@ -48,11 +49,12 @@ GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
'rawdb=s' => \$OptRawDB, 'rawdb=s' => \$OptRawDB,
's|stats=s' => \$OptStatsType, 's|stats=s' => \$OptStatsType,
't|test!' => \$OptTest, 't|test!' => \$OptTest,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD, 'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1; 'V|version' => \&ShowVersion) or exit 1;
### read configuration ### read configuration
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')}; my %Conf = %{ReadConfig($OptConfFile)};
### override configuration via commandline options ### override configuration via commandline options
my %ConfOverride; my %ConfOverride;
@ -160,7 +162,7 @@ foreach my $Month (&ListMonth($Period)) {
} }
}; };
}; };
# delete old data for that month # delete old data for that month
if (!$OptTest) { if (!$OptTest) {
$DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?", $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?",
@ -206,7 +208,7 @@ gatherstats - process statistical data from a raw source
=head1 SYNOPSIS =head1 SYNOPSIS
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] [--conffile I<filename>]
=head1 REQUIREMENTS =head1 REQUIREMENTS
@ -291,7 +293,7 @@ conjunction with B<--test> ... everything else seems a bit pointless.
Set processing period to a single month in YYYY-MM format or to a time Set processing period to a single month in YYYY-MM format or to a time
period between two month in YYYY-MM:YYYY-MM format (two month, separated period between two month in YYYY-MM:YYYY-MM format (two month, separated
by a colon). by a colon).
=item B<-s>, B<--stats> I<type> =item B<-s>, B<--stats> I<type>
@ -339,6 +341,10 @@ Override I<DBTableClnts> from F<newsstats.conf>.
Override I<DBTableHosts> from F<newsstats.conf>. Override I<DBTableHosts> from F<newsstats.conf>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back =back
=head1 INSTALLATION =head1 INSTALLATION
@ -368,15 +374,15 @@ checking against checkgroups-*:
=over 4 =over 4
=item F<gatherstats.pl> =item F<bin/gatherstats.pl>
The script itself. The script itself.
=item F<NewsStats.pm> =item F<lib/NewsStats.pm>
Library functions for the NewsStats package. Library functions for the NewsStats package.
=item F<newsstats.conf> =item F<etc/newsstats.conf>
Runtime configuration file. Runtime configuration file.

28
groupstats.pl → bin/groupstats.pl
View file

@ -4,18 +4,19 @@
# #
# This script will get statistical data on newgroup usage # This script will get statistical data on newgroup usage
# from a database. # from a database.
# #
# It is part of the NewsStats package. # It is part of the NewsStats package.
# #
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net> # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
# #
# It can be redistributed and/or modified under the same terms under # It can be redistributed and/or modified under the same terms under
# which Perl itself is published. # which Perl itself is published.
BEGIN { BEGIN {
our $VERSION = "0.01"; our $VERSION = "0.01";
use File::Basename; use File::Basename;
push(@INC, dirname($0)); # we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
} }
use strict; use strict;
use warnings; use warnings;
@ -31,7 +32,7 @@ Getopt::Long::config ('bundling');
### read commandline options ### read commandline options
my ($OptBoundType,$OptCaptions,$OptCheckgroupsFile,$OptComments, my ($OptBoundType,$OptCaptions,$OptCheckgroupsFile,$OptComments,
$OptFileTemplate,$OptFormat,$OptGroupBy,$OptGroupsDB,$LowBound,$OptMonth, $OptFileTemplate,$OptFormat,$OptGroupBy,$OptGroupsDB,$LowBound,$OptMonth,
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound); $OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound,$OptConfFile);
GetOptions ('b|boundary=s' => \$OptBoundType, GetOptions ('b|boundary=s' => \$OptBoundType,
'c|captions!' => \$OptCaptions, 'c|captions!' => \$OptCaptions,
'checkgroups=s' => \$OptCheckgroupsFile, 'checkgroups=s' => \$OptCheckgroupsFile,
@ -47,6 +48,7 @@ GetOptions ('b|boundary=s' => \$OptBoundType,
'r|report=s' => \$OptReportType, 'r|report=s' => \$OptReportType,
's|sums!' => \$OptSums, 's|sums!' => \$OptSums,
'u|upper=i' => \$UppBound, 'u|upper=i' => \$UppBound,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD, 'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1; 'V|version' => \&ShowVersion) or exit 1;
# parse parameters # parse parameters
@ -81,7 +83,7 @@ if ($OptReportType) {
my $ValidGroups = &ReadGroupList($OptCheckgroupsFile) if $OptCheckgroupsFile; my $ValidGroups = &ReadGroupList($OptCheckgroupsFile) if $OptCheckgroupsFile;
### read configuration ### read configuration
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')}; my %Conf = %{ReadConfig($OptConfFile)};
### override configuration via commandline options ### override configuration via commandline options
my %ConfOverride; my %ConfOverride;
@ -244,7 +246,7 @@ if ($OptCaptions && $OptComments) {
($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '', ($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending'); ($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
} }
# output data # output data
&OutputData($OptFormat,$OptComments,$GroupBy,$Precision, &OutputData($OptFormat,$OptComments,$GroupBy,$Precision,
$OptCheckgroupsFile ? $ValidGroups : '', $OptCheckgroupsFile ? $ValidGroups : '',
@ -263,7 +265,7 @@ groupstats - create reports on newsgroup usage
=head1 SYNOPSIS =head1 SYNOPSIS
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>] B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>] [--conffile I<filename>]
=head1 REQUIREMENTS =head1 REQUIREMENTS
@ -346,7 +348,7 @@ Print out version and copyright information and exit.
Print this man page and exit. Print this man page and exit.
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all> =item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
Set processing period to a single month in YYYY-MM format or to a time Set processing period to a single month in YYYY-MM format or to a time
period between two month in YYYY-MM:YYYY-MM format (two month, separated period between two month in YYYY-MM:YYYY-MM format (two month, separated
@ -592,6 +594,10 @@ B<--nocomments> is enforced, see above.
Override I<DBTableGrps> from F<newsstats.conf>. Override I<DBTableGrps> from F<newsstats.conf>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back =back
=head1 INSTALLATION =head1 INSTALLATION
@ -635,15 +641,15 @@ machine-readable form (without formatting):
=over 4 =over 4
=item F<groupstats.pl> =item F<bin/groupstats.pl>
The script itself. The script itself.
=item F<NewsStats.pm> =item F<lib/NewsStats.pm>
Library functions for the NewsStats package. Library functions for the NewsStats package.
=item F<newsstats.conf> =item F<etc/newsstats.conf>
Runtime configuration file. Runtime configuration file.

0
newsstats.conf.sample → etc/newsstats.conf.sample
View file

37
install/install.pl
View file

@ -3,19 +3,19 @@
# install.pl # install.pl
# #
# This script will create database tables as necessary. # This script will create database tables as necessary.
# #
# It is part of the NewsStats package. # It is part of the NewsStats package.
# #
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net> # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
# #
# It can be redistributed and/or modified under the same terms under # It can be redistributed and/or modified under the same terms under
# which Perl itself is published. # which Perl itself is published.
BEGIN { BEGIN {
our $VERSION = "0.01"; our $VERSION = "0.01";
use File::Basename; use File::Basename;
# we're in .../install, so our module is in .. # we're in .../install, so our module is in ../lib
push(@INC, dirname($0).'/..'); push(@INC, dirname($0).'/../lib');
} }
use strict; use strict;
use warnings; use warnings;
@ -31,8 +31,9 @@ Getopt::Long::config ('bundling');
################################# Main program ################################# ################################# Main program #################################
### read commandline options ### read commandline options
my ($OptUpdate); my ($OptUpdate,$OptConfFile);
GetOptions ('u|update=s' => \$OptUpdate, GetOptions ('u|update=s' => \$OptUpdate,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD, 'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1; 'V|version' => \&ShowVersion) or exit 1;
@ -42,7 +43,7 @@ my $Path = cwd();
### read configuration ### read configuration
print("Reading configuration.\n"); print("Reading configuration.\n");
my %Conf = %{ReadConfig($Path.'/newsstats.conf')}; my %Conf = %{ReadConfig($OptConfFile)};
##### -------------------------------------------------------------------------- ##### --------------------------------------------------------------------------
##### Database table definitions ##### Database table definitions
@ -53,9 +54,9 @@ CREATE DATABASE IF NOT EXISTS `$Conf{'DBDatabase'}` DEFAULT CHARSET=utf8;
SQLDB SQLDB
my %DBCreate = ('DBTableRaw' => <<RAW, 'DBTableGrps' => <<GRPS); my %DBCreate = ('DBTableRaw' => <<RAW, 'DBTableGrps' => <<GRPS);
-- --
-- Table structure for table DBTableRaw -- Table structure for table DBTableRaw
-- --
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` ( CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
`id` bigint(20) unsigned NOT NULL auto_increment, `id` bigint(20) unsigned NOT NULL auto_increment,
@ -76,9 +77,9 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
KEY `peer` (`peer`) KEY `peer` (`peer`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Raw data'; ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Raw data';
RAW RAW
-- --
-- Table structure for table DBTableGrps -- Table structure for table DBTableGrps
-- --
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableGrps'}` ( CREATE TABLE IF NOT EXISTS `$Conf{'DBTableGrps'}` (
`id` bigint(20) unsigned NOT NULL auto_increment, `id` bigint(20) unsigned NOT NULL auto_increment,
@ -167,7 +168,7 @@ if (!$OptUpdate) {
my $DBQuery = $DBHandle->prepare($DBCreate); my $DBQuery = $DBHandle->prepare($DBCreate);
$DBQuery->execute() or &Bleat(2, sprintf("Can't create database %s: %s%\n", $DBQuery->execute() or &Bleat(2, sprintf("Can't create database %s: %s%\n",
$Conf{'DBDatabase'}, $DBI::errstr)); $Conf{'DBDatabase'}, $DBI::errstr));
printf("Database table %s created succesfully.\n",$Conf{'DBDatabase'}); printf("Database table %s created succesfully.\n",$Conf{'DBDatabase'});
$DBHandle->disconnect; $DBHandle->disconnect;
}; };
@ -185,7 +186,7 @@ if (!$OptUpdate) {
&CreateTable($Table); &CreateTable($Table);
}; };
print "Database table generation done.\n"; print "Database table generation done.\n";
# Display install instructions # Display install instructions
print $Install; print $Install;
} else { } else {
@ -255,7 +256,7 @@ install - installation script
=head1 SYNOPSIS =head1 SYNOPSIS
B<install> [B<-Vh> [--update I<version>] B<install> [B<-Vh> [--update I<version>] [--conffile I<filename>]
=head1 REQUIREMENTS =head1 REQUIREMENTS
@ -288,21 +289,25 @@ Print this man page and exit.
Don't do a fresh install, but update from I<version>. Don't do a fresh install, but update from I<version>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back =back
=head1 FILES =head1 FILES
=over 4 =over 4
=item F<install.pl> =item F<install/install.pl>
The script itself. The script itself.
=item F<NewsStats.pm> =item F<lib/NewsStats.pm>
Library functions for the NewsStats package. Library functions for the NewsStats package.
=item F<newsstats.conf> =item F<etc/newsstats.conf>
Runtime configuration file. Runtime configuration file.

10
NewsStats.pm → lib/NewsStats.pm
View file

@ -54,15 +54,19 @@ our $PackageVersion = '0.01';
use Data::Dumper; use Data::Dumper;
use File::Basename; use File::Basename;
use Cwd qw(realpath);
use Config::Auto; use Config::Auto;
use DBI; use DBI;
#####-------------------------------- Vars --------------------------------##### #####-------------------------------- Vars --------------------------------#####
# trim the path # save $0 in $FullPath
our $FullPath = $0; our $FullPath = $0;
our $HomePath = dirname($0); # strip filename and /bin or /install directory to create the $HomePath
our $HomePath = dirname(realpath($0));
$HomePath =~ s/\/(bin|install)//;
# trim $0
$0 =~ s%.*/%%; $0 =~ s%.*/%%;
# set version string # set version string
our $MyVersion = "$0 $::VERSION (NewsStats.pm $VERSION)"; our $MyVersion = "$0 $::VERSION (NewsStats.pm $VERSION)";
@ -99,6 +103,8 @@ sub ReadConfig {
### IN : $ConfFile: config filename ### IN : $ConfFile: config filename
### OUT: reference to a hash containing the configuration ### OUT: reference to a hash containing the configuration
my ($ConfFile) = @_; my ($ConfFile) = @_;
# set default
$ConfFile = $HomePath . '/etc/newsstats.conf' if !$ConfFile;
# mandatory configuration options # mandatory configuration options
my @Mandatory = ('DBDriver','DBHost','DBUser','DBPw','DBDatabase', my @Mandatory = ('DBDriver','DBHost','DBUser','DBPw','DBDatabase',
'DBTableRaw','DBTableGrps'); 'DBTableRaw','DBTableGrps');