thh/newsstats
1
0
Fork 0
Code Issues 4 Pull requests Packages Projects Releases Activity

Compare commits

base: thh:next
Branches Tags
thh:master
thh:next
thh:pu
thh:0.01
..
compare: thh:master
Branches Tags
thh:next
thh:pu
thh:master
thh:0.01

No commits in common. "next" and "master" have entirely different histories.
next ... master

11 changed files with 122 additions and 191 deletions
Show stats Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,3 +1,3 @@
tmp/
tmp/*
etc/newsstats.conf
newsstats.conf

38
lib/NewsStats.pm → NewsStats.pm
View file

@ -4,7 +4,7 @@
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# This module can be redistributed and/or modified under the same terms under
# This module can be redistributed and/or modified under the same terms under
# which Perl itself is published.
package NewsStats;
@ -49,24 +49,20 @@ require Exporter;
Output => [qw(OutputData FormatOutput)],
SQLHelper => [qw(SQLHierarchies SQLSortOrder SQLGroupList
SQLSetBounds SQLBuildClause GetMaxLength)]);
$VERSION = '0.02';
our $PackageVersion = '0.02';
$VERSION = '0.01';
our $PackageVersion = '0.01';
use Data::Dumper;
use File::Basename;
use Cwd qw(realpath);
use Config::Auto;
use DBI;
#####-------------------------------- Vars --------------------------------#####
# save $0 in $FullPath
# trim the path
our $FullPath = $0;
# strip filename and /bin or /install directory to create the $HomePath
our $HomePath = dirname(realpath($0));
$HomePath =~ s/\/(bin|install)//;
# trim $0
our $HomePath = dirname($0);
$0 =~ s%.*/%%;
# set version string
our $MyVersion = "$0 $::VERSION (NewsStats.pm $VERSION)";
@ -80,7 +76,7 @@ sub ShowVersion {
################################################################################
### display version and exit
print "NewsStats v$PackageVersion\n$MyVersion\n";
print "Copyright (c) 2010-2013 Thomas Hochstein <thh\@inter.net>\n";
print "Copyright (c) 2010-2012 Thomas Hochstein <thh\@inter.net>\n";
print "This program is free software; you may redistribute it ".
"and/or modify it under the same terms as Perl itself.\n";
exit(100);
@ -103,8 +99,6 @@ sub ReadConfig {
### IN : $ConfFile: config filename
### OUT: reference to a hash containing the configuration
my ($ConfFile) = @_;
# set default
$ConfFile = $HomePath . '/etc/newsstats.conf' if !$ConfFile;
# mandatory configuration options
my @Mandatory = ('DBDriver','DBHost','DBUser','DBPw','DBDatabase',
'DBTableRaw','DBTableGrps');
@ -244,7 +238,7 @@ sub ReadGroupList {
### ignoring everything after the first whitespace and so accepting files
### in checkgroups format as well as (parts of) an INN active file)
### IN : $Filename : file to read
### OUT: \%ValidGroups: reference to a hash containing all valid newsgroups
### OUT: \%ValidGroups: hash containing all valid newsgroups
my ($Filename) = @_;
my %ValidGroups;
open (my $LIST,"<$Filename") or &Bleat(2,"Cannot read $Filename: $!");
@ -275,12 +269,12 @@ sub GetTimePeriod {
my ($Verbal, $SQL);
# define a regular expression for a month
my $REMonth = '\d{4}-\d{2}';
# default to last month if option is not set
if(!$Month) {
$Month = &LastMonth;
}
# check for valid input
if ($Month =~ /^$REMonth$/) {
# single month (YYYY-MM)
@ -299,7 +293,7 @@ sub GetTimePeriod {
# invalid input
return (undef,undef);
}
return ($Verbal,$SQL);
};
@ -407,7 +401,7 @@ sub OutputData {
my %ValidKeys = %{$ValidKeys} if $ValidKeys;
my ($FileName, $Handle, $OUT);
our $LastIteration;
# define output types
my %LegalOutput;
@LegalOutput{('dump','list','pretty')} = ();
@ -439,7 +433,7 @@ sub OutputData {
# safeguards for filename creation:
# replace potential problem characters with '_'
$FileName = sprintf('%s-%s',$FileTempl,$Caption);
$FileName =~ s/[^a-zA-Z0-9_-]+/_/g;
$FileName =~ s/[^a-zA-Z0-9_-]+/_/g;
open ($OUT,">$FileName")
or &Bleat(2,sprintf("Cannot open output file '%s': $!",
$FileName));
@ -674,7 +668,7 @@ sub SQLSetBounds {
### OUT: SQL code to become part of a WHERE or HAVING clause
my ($Type,$LowBound,$UppBound) = @_;
($LowBound,$UppBound) = SQLCheckNumber($LowBound,$UppBound);
if($LowBound and defined($UppBound) and $LowBound > $UppBound) {
if($LowBound and $UppBound and $LowBound > $UppBound) {
&Bleat(1,"Lower boundary $LowBound is larger than Upper boundary ".
"$UppBound, exchanging boundaries.");
($LowBound,$UppBound) = ($UppBound,$LowBound);
@ -690,7 +684,7 @@ sub SQLSetBounds {
} elsif ($Type eq 'sum') {
$WhereHavingFunction = 'SUM(postings)'
}
$LowBound = sprintf('%s >= '.$LowBound,$WhereHavingFunction) if defined($LowBound);
$LowBound = sprintf('%s >= '.$LowBound,$WhereHavingFunction) if ($LowBound);
# set $LowBound to SQL statement:
# 'WHERE postings <=', 'HAVING MAX(postings) <=' or 'HAVING AVG(postings) <='
if ($Type eq 'level') {
@ -700,7 +694,7 @@ sub SQLSetBounds {
} elsif ($Type eq 'sum') {
$WhereHavingFunction = 'SUM(postings)'
}
$UppBound = sprintf('%s <= '.$UppBound,$WhereHavingFunction) if defined($UppBound);
$UppBound = sprintf('%s <= '.$UppBound,$WhereHavingFunction) if ($UppBound);
return ($LowBound,$UppBound);
};
@ -776,3 +770,5 @@ sub CheckValidNewsgroups {
#####------------------------------- done ---------------------------------#####
1;

15
README.md
View file

@ -1,15 +0,0 @@
# NewsStats
**NewsStats** is a software package to extract live data from an INN newsfeed and generate statistics from it.
## Description
**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in a database which in turn can be used to generate various reports.
The software package is still under development.
It is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy.
## More information
Please see the [distribution page](https://th-h.de/net/software/newsstats/) (in German).

14
doc/INSTALL
View file

@ -28,21 +28,21 @@ INSTALLATION INSTRUCTIONS
* Copy the sample configuration file newsstats.conf.sample to
newsstats.conf and modify it for your purposes:
# cp etc/newsstats.conf.sample etc/newsstats.conf
# vim etc/newsstats.conf
# cp newsstats.conf.sample newsstats.conf
# vim newsstats.conf
a) Mandatory configuration options
* DBDriver = mysql
Database driver used; currently only mysql is supported.
* DBHost = localhost
The host your mysql server is running on.
* DBUser =
The username to connect to the database server.
* DBPw =
* DBPw =
Matching password for your username.
* DBDatabase = newsstats
@ -61,17 +61,17 @@ INSTALLATION INSTRUCTIONS
* TLH = de
Limit examination to that top-level hierarchy.
3) Database (mysql) setup
* Setup your database server with a username, password and
database matching the NewsStats configuration (see 2 a).
* Start the installation script:
# install/install.pl
It will setup the necessary database tables and display some
It will setup the necessary database tables and display some
information on the next steps.
4) Feed (INN) setup

14
doc/README
View file

@ -73,15 +73,17 @@ Getting Started
Reporting Bugs
You can report bugs or feature requests to the author using the
issue tracker at <https://code.virtcomm.de/thh/newsstats/issues>.
bug tracker at <http://bugs.th-h.de/>.
Please have a look at the TODO list before suggesting
improvements.
Development
More Information
This program is maintained using the Git version control system at
<https://code.virtcomm.de/thh/newsstats/>.
This program is maintained using the Git version control system.
You may clone <git://code.th-h.de/usenet/newsstats.git> to check
out the current development tree or browse it on the web via
<http://code.th-h.de/?p=usenet/newsstats.git>.
Related projects
@ -91,6 +93,6 @@ Related projects
Author
Thomas Hochstein <thh@thh.name>
<https://th-h.de/>
Thomas Hochstein <thh@inter.net>
<http://th-h.de/>

17
doc/TODO
View file

@ -4,6 +4,8 @@
This is a list of planned bug fixes, improvements and enhancements for
NewsStats.
Bug numbers refer to the Mantis issue tracker at <http://bugs.th-h.de/>.
* General
- Improve Documentation
The documentation is rather sparse and could use some improvement.
@ -16,7 +18,7 @@ NewsStats.
to /usr/local/news/etc or /etc/news and so on
* Additional features
- Add hierarchy information (GroupInfo)
- Add hierarchy information (GroupInfo - Bugs #19 #20 #21 #22 #23 #24 #25 #26)
NewsStats should be able to recognize invalid (i.e. officially not existing)
newsgroups and - optionally - drop them from the list of groups. On the
other hand, it should recognize existing, but empty groups and add them with
@ -35,19 +37,6 @@ NewsStats.
NewsStats should offer tools e.g. to inject postings into the 'raw' database,
or to split databases.
* GroupInfo project
- Create a hierarchy information database, containing information on each
newsgroup, its creation and removal time, its tagline, charter and
moderation status, including the moderator contact address.
- Automatically update hierarchy information, e.g. by parsing control messages
(with verification!).
- Track changes in meta information (changes to tagline, charter, moderation
status etc.)
- Add tools to query for hierarchy information:
- canonical list of newsgroups for any given time
- generate list of changes for a time period
- find newsgroups (including wildcards) and display their history
* Individual improvements
+ NewsStats.pm
- Improve error handling when reading config

39
bin/feedlog.pl → feedlog.pl
View file

@ -4,19 +4,18 @@
#
# This script will log headers and other data to a database
# for further analysis by parsing a feed from INN.
#
#
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.
BEGIN {
our $VERSION = "0.02";
our $VERSION = "0.01";
use File::Basename;
# we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
push(@INC, dirname($0));
}
use strict;
use warnings;
@ -69,15 +68,14 @@ sub PrepareDB {
################################# Main program #################################
### read commandline options
my ($OptDebug,$OptQuiet,$OptConfFile);
my ($OptDebug,$OptQuiet);
GetOptions ('d|debug!' => \$OptDebug,
'q|test!' => \$OptQuiet,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1;
### read configuration
my %Conf = %{ReadConfig($OptConfFile)};
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
### init syslog
openlog($0, 'nofatal,pid', LOG_NEWS);
@ -131,7 +129,7 @@ while (<>) {
};
};
$DBQuery->finish;
warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
"Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
@ -153,7 +151,7 @@ feedlog - log data from an INN feed to a database
=head1 SYNOPSIS
B<feedlog> [B<-Vhdq>] [B<--conffile> I<filename>]
B<feedlog> [B<-Vhdq>]
=head1 REQUIREMENTS
@ -174,8 +172,7 @@ terminating would only result in a rapid respawn.
=head2 Configuration
B<feedlog> will read its configuration from F<newsstats.conf> which
should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option.
should be present in the same directory via Config::Auto.
See L<doc/INSTALL> for an overview of possible configuration options.
@ -200,10 +197,6 @@ find that information most probably in your B<INN> F<errlog> file.
Suppress logging to syslog.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back
=head1 INSTALLATION
@ -225,15 +218,15 @@ See L<doc/INSTALL> for further information.
=over 4
=item F<bin/feedlog.pl>
=item F<feedlog.pl>
The script itself.
=item F<lib/NewsStats.pm>
=item F<NewsStats.pm>
Library functions for the NewsStats package.
=item F<etc/newsstats.conf>
=item F<newsstats.conf>
Runtime configuration file.
@ -242,7 +235,7 @@ Runtime configuration file.
=head1 BUGS
Please report any bugs or feature requests to the author or use the
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
bug tracker at L<http://bugs.th-h.de/>!
=head1 SEE ALSO
@ -262,11 +255,11 @@ This script is part of the B<NewsStats> package.
=head1 AUTHOR
Thomas Hochstein <thh@thh.name>
Thomas Hochstein <thh@inter.net>
=head1 COPYRIGHT AND LICENSE
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

43
bin/gatherstats.pl → gatherstats.pl
View file

@ -4,19 +4,18 @@
#
# This script will gather statistical information from a database
# containing headers and other information from a INN feed.
#
#
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.
BEGIN {
our $VERSION = "0.02";
our $VERSION = "0.01";
use File::Basename;
# we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
push(@INC, dirname($0));
}
use strict;
use warnings;
@ -38,7 +37,7 @@ my %LegalStats;
### read commandline options
my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest,$OptConfFile);
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest);
GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
'clientsdb=s' => \$OptClientsDB,
'd|debug!' => \$OptDebug,
@ -49,12 +48,11 @@ GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
'rawdb=s' => \$OptRawDB,
's|stats=s' => \$OptStatsType,
't|test!' => \$OptTest,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1;
### read configuration
my %Conf = %{ReadConfig($OptConfFile)};
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
### override configuration via commandline options
my %ConfOverride;
@ -73,8 +71,6 @@ $OptStatsType = 'all' if !$OptStatsType;
### get time period from --month
# get verbal description of time period, drop SQL code
my ($Period) = &GetTimePeriod($OptMonth);
# bail out if --month is invalid or set to 'ALL';
# we don't support the latter
&Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ".
"'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time');
@ -164,7 +160,7 @@ foreach my $Month (&ListMonth($Period)) {
}
};
};
# delete old data for that month
if (!$OptTest) {
$DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?",
@ -210,7 +206,7 @@ gatherstats - process statistical data from a raw source
=head1 SYNOPSIS
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] [B<--conffile> I<filename>]
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>]
=head1 REQUIREMENTS
@ -261,8 +257,7 @@ override that default through the B<--groupsdb> option.
=head2 Configuration
B<gatherstats> will read its configuration from F<newsstats.conf>
which should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option.
which should be present in the same directory via Config::Auto.
See L<doc/INSTALL> for an overview of possible configuration options.
@ -296,7 +291,7 @@ conjunction with B<--test> ... everything else seems a bit pointless.
Set processing period to a single month in YYYY-MM format or to a time
period between two month in YYYY-MM:YYYY-MM format (two month, separated
by a colon).
by a colon).
=item B<-s>, B<--stats> I<type>
@ -344,10 +339,6 @@ Override I<DBTableClnts> from F<newsstats.conf>.
Override I<DBTableHosts> from F<newsstats.conf>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back
=head1 INSTALLATION
@ -377,15 +368,15 @@ checking against checkgroups-*:
=over 4
=item F<bin/gatherstats.pl>
=item F<gatherstats.pl>
The script itself.
=item F<lib/NewsStats.pm>
=item F<NewsStats.pm>
Library functions for the NewsStats package.
=item F<etc/newsstats.conf>
=item F<newsstats.conf>
Runtime configuration file.
@ -394,7 +385,7 @@ Runtime configuration file.
=head1 BUGS
Please report any bugs or feature requests to the author or use the
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
bug tracker at L<http://bugs.th-h.de/>!
=head1 SEE ALSO
@ -414,11 +405,11 @@ This script is part of the B<NewsStats> package.
=head1 AUTHOR
Thomas Hochstein <thh@thh.name>
Thomas Hochstein <thh@inter.net>
=head1 COPYRIGHT AND LICENSE
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

69
bin/groupstats.pl → groupstats.pl
View file

@ -4,19 +4,18 @@
#
# This script will get statistical data on newgroup usage
# from a database.
#
#
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.
BEGIN {
our $VERSION = "0.02";
our $VERSION = "0.01";
use File::Basename;
# we're in .../bin, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
push(@INC, dirname($0));
}
use strict;
use warnings;
@ -32,7 +31,7 @@ Getopt::Long::config ('bundling');
### read commandline options
my ($OptBoundType,$OptCaptions,$OptCheckgroupsFile,$OptComments,
$OptFileTemplate,$OptFormat,$OptGroupBy,$OptGroupsDB,$LowBound,$OptMonth,
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound,$OptConfFile);
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound);
GetOptions ('b|boundary=s' => \$OptBoundType,
'c|captions!' => \$OptCaptions,
'checkgroups=s' => \$OptCheckgroupsFile,
@ -48,7 +47,6 @@ GetOptions ('b|boundary=s' => \$OptBoundType,
'r|report=s' => \$OptReportType,
's|sums!' => \$OptSums,
'u|upper=i' => \$UppBound,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1;
# parse parameters
@ -78,19 +76,12 @@ if ($OptReportType) {
$OptReportType = 'default';
}
}
# honor $OptCheckgroupsFile,
# warn for $OptSums if set concurrently
my $ValidGroups;
if ($OptCheckgroupsFile) {
# read list of newsgroups from --checkgroups
# into a hash reference
$ValidGroups = &ReadGroupList($OptCheckgroupsFile);
&Bleat(1,"--sums option can't possibly work with --checkgroups option set")
if $OptSums;
}
# read list of newsgroups from --checkgroups
# into a hash reference
my $ValidGroups = &ReadGroupList($OptCheckgroupsFile) if $OptCheckgroupsFile;
### read configuration
my %Conf = %{ReadConfig($OptConfFile)};
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
### override configuration via commandline options
my %ConfOverride;
@ -133,17 +124,12 @@ if ($OptBoundType and $OptBoundType ne 'default') {
}
### get sort order and build SQL 'ORDER BY' clause
# force to 'month' for $OptReportType 'average' or 'sum'
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
# default to 'newsgroup' for $OptBoundType 'level' or 'average'
$OptGroupBy = 'newsgroup' if (!$OptGroupBy and
$OptBoundType and $OptBoundType ne 'default');
# default to 'newsgroup' if $OptGroupBy is not set and
# just one newsgroup is requested, but more than one month
$OptGroupBy = 'newsgroup' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
and $OptNewsgroups and $OptNewsgroups !~ /[:*%]/);
# force to 'month' for $OptReportType 'average' or 'sum'
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy);
# $GroupBy will contain 'month' or 'newsgroup' (parsed result of $OptGroupBy)
# set it to 'month' or 'key' for OutputData()
@ -258,7 +244,7 @@ if ($OptCaptions && $OptComments) {
($OptOrderBy and $OptOrderBy =~ /posting/i) ? 'by number of postings ' : '',
($OptOrderBy and $OptOrderBy =~ /-?desc$/i) ? 'descending' : 'ascending');
}
# output data
&OutputData($OptFormat,$OptComments,$GroupBy,$Precision,
$OptCheckgroupsFile ? $ValidGroups : '',
@ -277,7 +263,7 @@ groupstats - create reports on newsgroup usage
=head1 SYNOPSIS
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>] [B<--conffile> I<filename>]
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>]
=head1 REQUIREMENTS
@ -342,8 +328,7 @@ Captions and comments are automatically disabled in this case.
=head2 Configuration
B<groupstats> will read its configuration from F<newsstats.conf>
which should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option.
which should be present in the same directory via Config::Auto.
See doc/INSTALL for an overview of possible configuration options.
@ -361,7 +346,7 @@ Print out version and copyright information and exit.
Print this man page and exit.
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]|all>
Set processing period to a single month in YYYY-MM format or to a time
period between two month in YYYY-MM:YYYY-MM format (two month, separated
@ -388,9 +373,6 @@ example:
See the B<gatherstats> man page for details.
This option does not work together with the B<--checkgroups> option as
all "virtual" groups will not be present in the checkgroups file.
=item B<--checkgroups> I<filename>
Restrict output to those newgroups present in a file in checkgroups format
@ -400,9 +382,6 @@ line is ignored). All other newsgroups will be removed from output.
Contrary to B<gatherstats>, I<filename> is not a template, but refers to
a single file in checkgroups format.
The B<--sums> option will not work together with this option as "virtual"
groups will not be present in the checkgroups file.
=item B<-r>, B<--report> I<default|average|sums>
Choose the report type: I<default>, I<average> or I<sums>
@ -613,10 +592,6 @@ B<--nocomments> is enforced, see above.
Override I<DBTableGrps> from F<newsstats.conf>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back
=head1 INSTALLATION
@ -660,15 +635,15 @@ machine-readable form (without formatting):
=over 4
=item F<bin/groupstats.pl>
=item F<groupstats.pl>
The script itself.
=item F<lib/NewsStats.pm>
=item F<NewsStats.pm>
Library functions for the NewsStats package.
=item F<etc/newsstats.conf>
=item F<newsstats.conf>
Runtime configuration file.
@ -677,7 +652,7 @@ Runtime configuration file.
=head1 BUGS
Please report any bugs or feature requests to the author or use the
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
bug tracker at L<http://bugs.th-h.de/>!
=head1 SEE ALSO
@ -701,11 +676,11 @@ This script is part of the B<NewsStats> package.
=head1 AUTHOR
Thomas Hochstein <thh@thh.name>
Thomas Hochstein <thh@inter.net>
=head1 COPYRIGHT AND LICENSE
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

62
install/install.pl
View file

@ -3,25 +3,27 @@
# install.pl
#
# This script will create database tables as necessary.
#
#
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.
BEGIN {
our $VERSION = "0.02";
our $VERSION = "0.01";
use File::Basename;
# we're in .../install, so our module is in ../lib
push(@INC, dirname($0).'/../lib');
# we're in .../install, so our module is in ..
push(@INC, dirname($0).'/..');
}
use strict;
use warnings;
use NewsStats qw(:DEFAULT);
use Cwd;
use DBI;
use Getopt::Long qw(GetOptions);
Getopt::Long::config ('bundling');
@ -29,15 +31,18 @@ Getopt::Long::config ('bundling');
################################# Main program #################################
### read commandline options
my ($OptUpdate,$OptConfFile);
my ($OptUpdate);
GetOptions ('u|update=s' => \$OptUpdate,
'conffile=s' => \$OptConfFile,
'h|help' => \&ShowPOD,
'V|version' => \&ShowVersion) or exit 1;
### change working directory to .. (as we're in .../install)
chdir dirname($FullPath).'/..';
my $Path = cwd();
### read configuration
print("Reading configuration.\n");
my %Conf = %{ReadConfig($OptConfFile)};
my %Conf = %{ReadConfig($Path.'/newsstats.conf')};
##### --------------------------------------------------------------------------
##### Database table definitions
@ -48,9 +53,9 @@ CREATE DATABASE IF NOT EXISTS `$Conf{'DBDatabase'}` DEFAULT CHARSET=utf8;
SQLDB
my %DBCreate = ('DBTableRaw' => <<RAW, 'DBTableGrps' => <<GRPS);
--
--
-- Table structure for table DBTableRaw
--
--
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
`id` bigint(20) unsigned NOT NULL auto_increment,
@ -71,9 +76,9 @@ CREATE TABLE IF NOT EXISTS `$Conf{'DBTableRaw'}` (
KEY `peer` (`peer`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Raw data';
RAW
--
--
-- Table structure for table DBTableGrps
--
--
CREATE TABLE IF NOT EXISTS `$Conf{'DBTableGrps'}` (
`id` bigint(20) unsigned NOT NULL auto_increment,
@ -103,7 +108,7 @@ Things left to do:
## gather statistics for NewsStats
newsstats!\\
:!*,de.*\\
:Tc,WmtfbsPNH,Ac:$HomePath/bin/feedlog.pl
:Tc,WmtfbsPNH,Ac:$Path/feedlog.pl
Please
@ -131,7 +136,7 @@ Things left to do:
Enjoy!
-thh <thh\@thh.name>
-thh <thh\@inter.net>
INSTALL
my $Upgrade ='';
@ -162,7 +167,7 @@ if (!$OptUpdate) {
my $DBQuery = $DBHandle->prepare($DBCreate);
$DBQuery->execute() or &Bleat(2, sprintf("Can't create database %s: %s%\n",
$Conf{'DBDatabase'}, $DBI::errstr));
printf("Database table %s created succesfully.\n",$Conf{'DBDatabase'});
$DBHandle->disconnect;
};
@ -180,7 +185,7 @@ if (!$OptUpdate) {
&CreateTable($Table);
};
print "Database table generation done.\n";
# Display install instructions
print $Install;
} else {
@ -250,7 +255,7 @@ install - installation script
=head1 SYNOPSIS
B<install> [B<-Vh> [--update I<version>] [B<--conffile> I<filename>]
B<install> [B<-Vh> [--update I<version>]
=head1 REQUIREMENTS
@ -262,9 +267,8 @@ This script will create database tables as necessary and configured.
=head2 Configuration
B<install> will read its configuration from F<newsstats.conf> which should
be present in etc/ via Config::Auto or from a configuration file submitted
by the B<--conffile> option.
B<install> will read its configuration from F<newsstats.conf> via
Config::Auto.
See L<doc/INSTALL> for an overview of possible configuration options.
@ -284,25 +288,21 @@ Print this man page and exit.
Don't do a fresh install, but update from I<version>.
=item B<--conffile> I<filename>
Load configuration from I<filename> instead of F<newsstats.conf>.
=back
=head1 FILES
=over 4
=item F<install/install.pl>
=item F<install.pl>
The script itself.
=item F<lib/NewsStats.pm>
=item F<NewsStats.pm>
Library functions for the NewsStats package.
=item F<etc/newsstats.conf>
=item F<newsstats.conf>
Runtime configuration file.
@ -311,7 +311,7 @@ Runtime configuration file.
=head1 BUGS
Please report any bugs or feature requests to the author or use the
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
bug tracker at L<http://bugs.th-h.de/>!
=head1 SEE ALSO
@ -331,11 +331,11 @@ This script is part of the B<NewsStats> package.
=head1 AUTHOR
Thomas Hochstein <thh@thh.name>
Thomas Hochstein <thh@inter.net>
=head1 COPYRIGHT AND LICENSE
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

0
etc/newsstats.conf.sample → newsstats.conf.sample
View file