Merge branch 'thh-restructure' into next
* thh-restructure: Make configuration file configurable. Fix some whitespace. Redo directory structure.
This commit is contained in:
Thomas Hochstein
commit
1af57a5390
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -1,3 +1,3 @@
|
|||
tmp/
|
||||
tmp/*
|
||||
newsstats.conf
|
||||
etc/newsstats.conf
|
||||
|
|
|
|||
|
|
@ -4,18 +4,19 @@
|
|||
#
|
||||
# This script will log headers and other data to a database
|
||||
# for further analysis by parsing a feed from INN.
|
||||
#
|
||||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
|
@ -68,14 +69,15 @@ sub PrepareDB {
|
|||
################################# Main program #################################
|
||||
|
||||
### read commandline options
|
||||
my ($OptDebug,$OptQuiet);
|
||||
my ($OptDebug,$OptQuiet,$OptConfFile);
|
||||
GetOptions ('d|debug!' => \$OptDebug,
|
||||
'q|test!' => \$OptQuiet,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### init syslog
|
||||
openlog($0, 'nofatal,pid', LOG_NEWS);
|
||||
|
|
@ -129,7 +131,7 @@ while (<>) {
|
|||
};
|
||||
};
|
||||
$DBQuery->finish;
|
||||
|
||||
|
||||
warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
|
||||
"Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
|
||||
$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
|
||||
|
|
@ -151,7 +153,7 @@ feedlog - log data from an INN feed to a database
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<feedlog> [B<-Vhdq>]
|
||||
B<feedlog> [B<-Vhdq>] [--conffile I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
|
@ -197,6 +199,10 @@ find that information most probably in your B<INN> F<errlog> file.
|
|||
|
||||
Suppress logging to syslog.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
|
@ -218,15 +224,15 @@ See L<doc/INSTALL> for further information.
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<feedlog.pl>
|
||||
=item F<bin/feedlog.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
|
@ -4,18 +4,19 @@
|
|||
#
|
||||
# This script will gather statistical information from a database
|
||||
# containing headers and other information from a INN feed.
|
||||
#
|
||||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
|
@ -37,7 +38,7 @@ my %LegalStats;
|
|||
|
||||
### read commandline options
|
||||
my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
|
||||
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest);
|
||||
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest,$OptConfFile);
|
||||
GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
|
||||
'clientsdb=s' => \$OptClientsDB,
|
||||
'd|debug!' => \$OptDebug,
|
||||
|
|
@ -48,11 +49,12 @@ GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
|
|||
'rawdb=s' => \$OptRawDB,
|
||||
's|stats=s' => \$OptStatsType,
|
||||
't|test!' => \$OptTest,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### override configuration via commandline options
|
||||
my %ConfOverride;
|
||||
|
|
@ -160,7 +162,7 @@ foreach my $Month (&ListMonth($Period)) {
|
|||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
# delete old data for that month
|
||||
if (!$OptTest) {
|
||||
$DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?",
|
||||
|
|
@ -206,7 +208,7 @@ gatherstats - process statistical data from a raw source
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>]
|
||||
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] [--conffile I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
|
@ -291,7 +293,7 @@ conjunction with B<--test> ... everything else seems a bit pointless.
|
|||
|
||||
Set processing period to a single month in YYYY-MM format or to a time
|
||||
period between two month in YYYY-MM:YYYY-MM format (two month, separated
|
||||
by a colon).
|
||||
by a colon).
|
||||
|
||||
=item B<-s>, B<--stats> I<type>
|
||||
|
||||
|
|
@ -339,6 +341,10 @@ Override I<DBTableClnts> from F<newsstats.conf>.
|
|||
|
||||
Override I<DBTableHosts> from F<newsstats.conf>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
|
@ -368,15 +374,15 @@ checking against checkgroups-*:
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<gatherstats.pl>
|
||||
=item F<bin/gatherstats.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
|
@ -4,18 +4,19 @@
|
|||
#
|
||||
# This script will get statistical data on newgroup usage
|
||||
# from a database.
|
||||
#
|
||||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
|
@ -31,7 +32,7 @@ Getopt::Long::config ('bundling');
|
|||
### read commandline options
|
||||
my ($OptBoundType,$OptCaptions,$OptCheckgroupsFile,$OptComments,
|
||||
$OptFileTemplate,$OptFormat,$OptGroupBy,$OptGroupsDB,$LowBound,$OptMonth,
|
||||
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound);
|
||||
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound,$OptConfFile);
|
||||
GetOptions ('b|boundary=s' => \$OptBoundType,
|
||||
'c|captions!' => \$OptCaptions,
|
||||
'checkgroups=s' => \$OptCheckgroupsFile,
|
||||
|
|
@ -47,6 +48,7 @@ GetOptions ('b|boundary=s' => \$OptBoundType,
|
|||
'r|report=s' => \$OptReportType,
|
||||
's|sums!' => \$OptSums,
|
||||
'u|upper=i' => \$UppBound,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
# parse parameters
|
||||
|
|
@ -81,7 +83,7 @@ if ($OptReportType) {
|
|||
my $ValidGroups = &ReadGroupList($OptCheckgroupsFile) if $OptCheckgroupsFile;
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### override configuration via commandline options
|
||||
my %ConfOverride;
|
||||
|
|
@ -244,7 +246,7 @@ if ($OptCaptions && $OptComments) {
|
|||
($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
|
||||
($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
|
||||
}
|
||||
|
||||
|
||||
# output data
|
||||
&OutputData($OptFormat,$OptComments,$GroupBy,$Precision,
|
||||
$OptCheckgroupsFile ? $ValidGroups : '',
|
||||
|
|
@ -263,7 +265,7 @@ groupstats - create reports on newsgroup usage
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>]
|
||||
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>] [--conffile I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
|
@ -346,7 +348,7 @@ Print out version and copyright information and exit.
|
|||
|
||||
Print this man page and exit.
|
||||
|
||||
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
|
||||
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
|
||||
|
||||
Set processing period to a single month in YYYY-MM format or to a time
|
||||
period between two month in YYYY-MM:YYYY-MM format (two month, separated
|
||||
|
|
@ -592,6 +594,10 @@ B<--nocomments> is enforced, see above.
|
|||
|
||||
Override I<DBTableGrps> from F<newsstats.conf>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
|
@ -635,15 +641,15 @@ machine-readable form (without formatting):
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<groupstats.pl>
|
||||
=item F<bin/groupstats.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
|
@ -3,19 +3,19 @@
|
|||
# install.pl
|
||||
#
|
||||
# This script will create database tables as necessary.
|
||||
#
|
||||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
use File::Basename;
|
||||
# we're in .../install, so our module is in ..
|
||||
push(@INC, dirname($0).'/..');
|
||||
# we're in .../install, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
|
@ -31,8 +31,9 @@ Getopt::Long::config ('bundling');
|
|||
################################# Main program #################################
|
||||
|
||||
### read commandline options
|
||||
my ($OptUpdate);
|
||||
my ($OptUpdate,$OptConfFile);
|
||||
GetOptions ('u|update=s' => \$OptUpdate,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
|
|
@ -42,7 +43,7 @@ my $Path = cwd();
|
|||
|
||||
### read configuration
|
||||
print("Reading configuration.\n");
|
||||
my %Conf = %{ReadConfig($Path.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
##### --------------------------------------------------------------------------
|
||||
##### Database table definitions
|
||||
|
|
@ -53,9 +54,9 @@ CREATE DATABASE IF NOT EXISTS `$Conf{'DBDatabase'}` DEFAULT CHARSET=utf8;
|
|||
SQLDB
|
||||
|
||||
my %DBCreate = ('DBTableRaw' => <<RAW, 'DBTableGrps' => <<GRPS);
|
||||
--
|
||||
--
|
||||
-- Table structure for table DBTableRaw
|
||||
--
|
||||
--
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
|
||||
`id` bigint(20) unsigned NOT NULL auto_increment,
|
||||
|
|
@ -76,9 +77,9 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
|
|||
KEY `peer` (`peer`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Raw data';
|
||||
RAW
|
||||
--
|
||||
--
|
||||
-- Table structure for table DBTableGrps
|
||||
--
|
||||
--
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableGrps'}` (
|
||||
`id` bigint(20) unsigned NOT NULL auto_increment,
|
||||
|
|
@ -167,7 +168,7 @@ if (!$OptUpdate) {
|
|||
my $DBQuery = $DBHandle->prepare($DBCreate);
|
||||
$DBQuery->execute() or &Bleat(2, sprintf("Can't create database %s: %s%\n",
|
||||
$Conf{'DBDatabase'}, $DBI::errstr));
|
||||
|
||||
|
||||
printf("Database table %s created succesfully.\n",$Conf{'DBDatabase'});
|
||||
$DBHandle->disconnect;
|
||||
};
|
||||
|
|
@ -185,7 +186,7 @@ if (!$OptUpdate) {
|
|||
&CreateTable($Table);
|
||||
};
|
||||
print "Database table generation done.\n";
|
||||
|
||||
|
||||
# Display install instructions
|
||||
print $Install;
|
||||
} else {
|
||||
|
|
@ -255,7 +256,7 @@ install - installation script
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<install> [B<-Vh> [--update I<version>]
|
||||
B<install> [B<-Vh> [--update I<version>] [--conffile I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
|
@ -288,21 +289,25 @@ Print this man page and exit.
|
|||
|
||||
Don't do a fresh install, but update from I<version>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 FILES
|
||||
|
||||
=over 4
|
||||
|
||||
=item F<install.pl>
|
||||
=item F<install/install.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
|
|
|||
|
|
@ -54,15 +54,19 @@ our $PackageVersion = '0.01';
|
|||
|
||||
use Data::Dumper;
|
||||
use File::Basename;
|
||||
use Cwd qw(realpath);
|
||||
|
||||
use Config::Auto;
|
||||
use DBI;
|
||||
|
||||
#####-------------------------------- Vars --------------------------------#####
|
||||
|
||||
# trim the path
|
||||
# save $0 in $FullPath
|
||||
our $FullPath = $0;
|
||||
our $HomePath = dirname($0);
|
||||
# strip filename and /bin or /install directory to create the $HomePath
|
||||
our $HomePath = dirname(realpath($0));
|
||||
$HomePath =~ s/\/(bin|install)//;
|
||||
# trim $0
|
||||
$0 =~ s%.*/%%;
|
||||
# set version string
|
||||
our $MyVersion = "$0 $::VERSION (NewsStats.pm $VERSION)";
|
||||
|
|
@ -99,6 +103,8 @@ sub ReadConfig {
|
|||
### IN : $ConfFile: config filename
|
||||
### OUT: reference to a hash containing the configuration
|
||||
my ($ConfFile) = @_;
|
||||
# set default
|
||||
$ConfFile = $HomePath . '/etc/newsstats.conf' if !$ConfFile;
|
||||
# mandatory configuration options
|
||||
my @Mandatory = ('DBDriver','DBHost','DBUser','DBPw','DBDatabase',
|
||||
'DBTableRaw','DBTableGrps');
|
||||
Loading…
Reference in a new issue