Compare commits
22 commits
Author | SHA1 | Date | |
---|---|---|---|
Thomas Hochstein | d49ef1d302 | ||
Thomas Hochstein | 09b45fc369 | ||
Thomas Hochstein | ee29be18c8 | ||
Thomas Hochstein | cd6f153a9e | ||
b5ef572664 | |||
91c674c4fe | |||
fd0717a15c | |||
b3b170c357 | |||
44c197097b | |||
e39d4207a6 | |||
24d2011f32 | |||
2871792120 | |||
22d3d70a72 | |||
599fefbf6a | |||
7624accb6e | |||
8dc6823e98 | |||
17ef44085f | |||
ea91003a99 | |||
1af57a5390 | |||
23ab67a099 | |||
dfc2b81c37 | |||
2ad99c20bc |
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -1,3 +1,3 @@
|
|||
tmp/
|
||||
tmp/*
|
||||
newsstats.conf
|
||||
etc/newsstats.conf
|
||||
|
|
15
README.md
Normal file
15
README.md
Normal file
|
@ -0,0 +1,15 @@
|
|||
# NewsStats
|
||||
|
||||
**NewsStats** is a software package to extract live data from an INN newsfeed and generate statistics from it.
|
||||
|
||||
## Description
|
||||
|
||||
**NewsStats** stores overview data and complete headers of all incoming postings (in one or more specific Usenet hierarchies) in real time in a MySQL database. This raw dataset can then be analysed regularly, e.g. monthly, for instance in terms of postings per group and month. The analysis results will also be stored in a database which in turn can be used to generate various reports.
|
||||
|
||||
The software package is still under development.
|
||||
|
||||
It is currently used to generate the monthly statistics posted to `de.admin.news.lists` for the de.\* hierarchy.
|
||||
|
||||
## More information
|
||||
|
||||
Please see the [distribution page](https://th-h.de/net/software/newsstats/) (in German).
|
|
@ -7,15 +7,16 @@
|
|||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
our $VERSION = "0.02";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
@ -68,14 +69,15 @@ sub PrepareDB {
|
|||
################################# Main program #################################
|
||||
|
||||
### read commandline options
|
||||
my ($OptDebug,$OptQuiet);
|
||||
my ($OptDebug,$OptQuiet,$OptConfFile);
|
||||
GetOptions ('d|debug!' => \$OptDebug,
|
||||
'q|test!' => \$OptQuiet,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### init syslog
|
||||
openlog($0, 'nofatal,pid', LOG_NEWS);
|
||||
|
@ -151,7 +153,7 @@ feedlog - log data from an INN feed to a database
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<feedlog> [B<-Vhdq>]
|
||||
B<feedlog> [B<-Vhdq>] [B<--conffile> I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
@ -172,7 +174,8 @@ terminating would only result in a rapid respawn.
|
|||
=head2 Configuration
|
||||
|
||||
B<feedlog> will read its configuration from F<newsstats.conf> which
|
||||
should be present in the same directory via Config::Auto.
|
||||
should be present in etc/ via Config::Auto or from a configuration file
|
||||
submitted by the B<--conffile> option.
|
||||
|
||||
See L<doc/INSTALL> for an overview of possible configuration options.
|
||||
|
||||
|
@ -197,6 +200,10 @@ find that information most probably in your B<INN> F<errlog> file.
|
|||
|
||||
Suppress logging to syslog.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
@ -218,15 +225,15 @@ See L<doc/INSTALL> for further information.
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<feedlog.pl>
|
||||
=item F<bin/feedlog.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
@ -235,7 +242,7 @@ Runtime configuration file.
|
|||
=head1 BUGS
|
||||
|
||||
Please report any bugs or feature requests to the author or use the
|
||||
bug tracker at L<http://bugs.th-h.de/>!
|
||||
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
|
@ -255,11 +262,11 @@ This script is part of the B<NewsStats> package.
|
|||
|
||||
=head1 AUTHOR
|
||||
|
||||
Thomas Hochstein <thh@inter.net>
|
||||
Thomas Hochstein <thh@thh.name>
|
||||
|
||||
=head1 COPYRIGHT AND LICENSE
|
||||
|
||||
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
|
||||
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
|
||||
This program is free software; you may redistribute it and/or modify it
|
||||
under the same terms as Perl itself.
|
|
@ -7,15 +7,16 @@
|
|||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
our $VERSION = "0.02";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
@ -37,7 +38,7 @@ my %LegalStats;
|
|||
|
||||
### read commandline options
|
||||
my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
|
||||
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest);
|
||||
$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest,$OptConfFile);
|
||||
GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
|
||||
'clientsdb=s' => \$OptClientsDB,
|
||||
'd|debug!' => \$OptDebug,
|
||||
|
@ -48,11 +49,12 @@ GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
|
|||
'rawdb=s' => \$OptRawDB,
|
||||
's|stats=s' => \$OptStatsType,
|
||||
't|test!' => \$OptTest,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### override configuration via commandline options
|
||||
my %ConfOverride;
|
||||
|
@ -71,6 +73,8 @@ $OptStatsType = 'all' if !$OptStatsType;
|
|||
### get time period from --month
|
||||
# get verbal description of time period, drop SQL code
|
||||
my ($Period) = &GetTimePeriod($OptMonth);
|
||||
# bail out if --month is invalid or set to 'ALL';
|
||||
# we don't support the latter
|
||||
&Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ".
|
||||
"'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time');
|
||||
|
||||
|
@ -206,7 +210,7 @@ gatherstats - process statistical data from a raw source
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>]
|
||||
B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats>] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>] [B<--conffile> I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
@ -257,7 +261,8 @@ override that default through the B<--groupsdb> option.
|
|||
=head2 Configuration
|
||||
|
||||
B<gatherstats> will read its configuration from F<newsstats.conf>
|
||||
which should be present in the same directory via Config::Auto.
|
||||
which should be present in etc/ via Config::Auto or from a configuration file
|
||||
submitted by the B<--conffile> option.
|
||||
|
||||
See L<doc/INSTALL> for an overview of possible configuration options.
|
||||
|
||||
|
@ -339,6 +344,10 @@ Override I<DBTableClnts> from F<newsstats.conf>.
|
|||
|
||||
Override I<DBTableHosts> from F<newsstats.conf>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
@ -368,15 +377,15 @@ checking against checkgroups-*:
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<gatherstats.pl>
|
||||
=item F<bin/gatherstats.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
@ -385,7 +394,7 @@ Runtime configuration file.
|
|||
=head1 BUGS
|
||||
|
||||
Please report any bugs or feature requests to the author or use the
|
||||
bug tracker at L<http://bugs.th-h.de/>!
|
||||
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
|
@ -405,11 +414,11 @@ This script is part of the B<NewsStats> package.
|
|||
|
||||
=head1 AUTHOR
|
||||
|
||||
Thomas Hochstein <thh@inter.net>
|
||||
Thomas Hochstein <thh@thh.name>
|
||||
|
||||
=head1 COPYRIGHT AND LICENSE
|
||||
|
||||
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
|
||||
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
|
||||
This program is free software; you may redistribute it and/or modify it
|
||||
under the same terms as Perl itself.
|
|
@ -7,15 +7,16 @@
|
|||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
our $VERSION = "0.02";
|
||||
use File::Basename;
|
||||
push(@INC, dirname($0));
|
||||
# we're in .../bin, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
@ -31,7 +32,7 @@ Getopt::Long::config ('bundling');
|
|||
### read commandline options
|
||||
my ($OptBoundType,$OptCaptions,$OptCheckgroupsFile,$OptComments,
|
||||
$OptFileTemplate,$OptFormat,$OptGroupBy,$OptGroupsDB,$LowBound,$OptMonth,
|
||||
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound);
|
||||
$OptNewsgroups,$OptOrderBy,$OptReportType,$OptSums,$UppBound,$OptConfFile);
|
||||
GetOptions ('b|boundary=s' => \$OptBoundType,
|
||||
'c|captions!' => \$OptCaptions,
|
||||
'checkgroups=s' => \$OptCheckgroupsFile,
|
||||
|
@ -47,6 +48,7 @@ GetOptions ('b|boundary=s' => \$OptBoundType,
|
|||
'r|report=s' => \$OptReportType,
|
||||
's|sums!' => \$OptSums,
|
||||
'u|upper=i' => \$UppBound,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
# parse parameters
|
||||
|
@ -76,12 +78,19 @@ if ($OptReportType) {
|
|||
$OptReportType = 'default';
|
||||
}
|
||||
}
|
||||
# read list of newsgroups from --checkgroups
|
||||
# into a hash reference
|
||||
my $ValidGroups = &ReadGroupList($OptCheckgroupsFile) if $OptCheckgroupsFile;
|
||||
# honor $OptCheckgroupsFile,
|
||||
# warn for $OptSums if set concurrently
|
||||
my $ValidGroups;
|
||||
if ($OptCheckgroupsFile) {
|
||||
# read list of newsgroups from --checkgroups
|
||||
# into a hash reference
|
||||
$ValidGroups = &ReadGroupList($OptCheckgroupsFile);
|
||||
&Bleat(1,"--sums option can't possibly work with --checkgroups option set")
|
||||
if $OptSums;
|
||||
}
|
||||
|
||||
### read configuration
|
||||
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
### override configuration via commandline options
|
||||
my %ConfOverride;
|
||||
|
@ -124,12 +133,17 @@ if ($OptBoundType and $OptBoundType ne 'default') {
|
|||
}
|
||||
|
||||
### get sort order and build SQL 'ORDER BY' clause
|
||||
# force to 'month' for $OptReportType 'average' or 'sum'
|
||||
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
|
||||
# default to 'newsgroup' for $OptBoundType 'level' or 'average'
|
||||
$OptGroupBy = 'newsgroup' if (!$OptGroupBy and
|
||||
$OptBoundType and $OptBoundType ne 'default');
|
||||
# force to 'month' for $OptReportType 'average' or 'sum'
|
||||
$OptGroupBy = 'month' if ($OptReportType and $OptReportType ne 'default');
|
||||
# default to 'newsgroup' if $OptGroupBy is not set and
|
||||
# just one newsgroup is requested, but more than one month
|
||||
$OptGroupBy = 'newsgroup' if (!$OptGroupBy and $OptMonth and $OptMonth =~ /:/
|
||||
and $OptNewsgroups and $OptNewsgroups !~ /[:*%]/);
|
||||
# parse $OptGroupBy to $GroupBy, create ORDER BY clause $SQLOrderClause
|
||||
# if $OptGroupBy is still not set, SQLSortOrder() will default to 'month'
|
||||
my ($GroupBy,$SQLOrderClause) = SQLSortOrder($OptGroupBy, $OptOrderBy);
|
||||
# $GroupBy will contain 'month' or 'newsgroup' (parsed result of $OptGroupBy)
|
||||
# set it to 'month' or 'key' for OutputData()
|
||||
|
@ -263,7 +277,7 @@ groupstats - create reports on newsgroup usage
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>]
|
||||
B<groupstats> [B<-Vhcs> B<--comments>] [B<-m> I<YYYY-MM>[:I<YYYY-MM>] | I<all>] [B<-n> I<newsgroup(s)>] [B<--checkgroups> I<checkgroups file>] [B<-r> I<report type>] [B<-l> I<lower boundary>] [B<-u> I<upper boundary>] [B<-b> I<boundary type>] [B<-g> I<group by>] [B<-o> I<order by>] [B<-f> I<output format>] [B<--filetemplate> I<filename template>] [B<--groupsdb> I<database table>] [B<--conffile> I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
@ -328,7 +342,8 @@ Captions and comments are automatically disabled in this case.
|
|||
=head2 Configuration
|
||||
|
||||
B<groupstats> will read its configuration from F<newsstats.conf>
|
||||
which should be present in the same directory via Config::Auto.
|
||||
which should be present in etc/ via Config::Auto or from a configuration file
|
||||
submitted by the B<--conffile> option.
|
||||
|
||||
See doc/INSTALL for an overview of possible configuration options.
|
||||
|
||||
|
@ -373,6 +388,9 @@ example:
|
|||
|
||||
See the B<gatherstats> man page for details.
|
||||
|
||||
This option does not work together with the B<--checkgroups> option as
|
||||
all "virtual" groups will not be present in the checkgroups file.
|
||||
|
||||
=item B<--checkgroups> I<filename>
|
||||
|
||||
Restrict output to those newgroups present in a file in checkgroups format
|
||||
|
@ -382,6 +400,9 @@ line is ignored). All other newsgroups will be removed from output.
|
|||
Contrary to B<gatherstats>, I<filename> is not a template, but refers to
|
||||
a single file in checkgroups format.
|
||||
|
||||
The B<--sums> option will not work together with this option as "virtual"
|
||||
groups will not be present in the checkgroups file.
|
||||
|
||||
=item B<-r>, B<--report> I<default|average|sums>
|
||||
|
||||
Choose the report type: I<default>, I<average> or I<sums>
|
||||
|
@ -592,6 +613,10 @@ B<--nocomments> is enforced, see above.
|
|||
|
||||
Override I<DBTableGrps> from F<newsstats.conf>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 INSTALLATION
|
||||
|
@ -635,15 +660,15 @@ machine-readable form (without formatting):
|
|||
|
||||
=over 4
|
||||
|
||||
=item F<groupstats.pl>
|
||||
=item F<bin/groupstats.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
@ -652,7 +677,7 @@ Runtime configuration file.
|
|||
=head1 BUGS
|
||||
|
||||
Please report any bugs or feature requests to the author or use the
|
||||
bug tracker at L<http://bugs.th-h.de/>!
|
||||
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
|
@ -676,11 +701,11 @@ This script is part of the B<NewsStats> package.
|
|||
|
||||
=head1 AUTHOR
|
||||
|
||||
Thomas Hochstein <thh@inter.net>
|
||||
Thomas Hochstein <thh@thh.name>
|
||||
|
||||
=head1 COPYRIGHT AND LICENSE
|
||||
|
||||
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
|
||||
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
|
||||
This program is free software; you may redistribute it and/or modify it
|
||||
under the same terms as Perl itself.
|
|
@ -28,8 +28,8 @@ INSTALLATION INSTRUCTIONS
|
|||
* Copy the sample configuration file newsstats.conf.sample to
|
||||
newsstats.conf and modify it for your purposes:
|
||||
|
||||
# cp newsstats.conf.sample newsstats.conf
|
||||
# vim newsstats.conf
|
||||
# cp etc/newsstats.conf.sample etc/newsstats.conf
|
||||
# vim etc/newsstats.conf
|
||||
|
||||
a) Mandatory configuration options
|
||||
|
||||
|
|
14
doc/README
14
doc/README
|
@ -73,17 +73,15 @@ Getting Started
|
|||
Reporting Bugs
|
||||
|
||||
You can report bugs or feature requests to the author using the
|
||||
bug tracker at <http://bugs.th-h.de/>.
|
||||
issue tracker at <https://code.virtcomm.de/thh/newsstats/issues>.
|
||||
|
||||
Please have a look at the TODO list before suggesting
|
||||
improvements.
|
||||
|
||||
More Information
|
||||
Development
|
||||
|
||||
This program is maintained using the Git version control system.
|
||||
You may clone <git://code.th-h.de/usenet/newsstats.git> to check
|
||||
out the current development tree or browse it on the web via
|
||||
<http://code.th-h.de/?p=usenet/newsstats.git>.
|
||||
This program is maintained using the Git version control system at
|
||||
<https://code.virtcomm.de/thh/newsstats/>.
|
||||
|
||||
Related projects
|
||||
|
||||
|
@ -93,6 +91,6 @@ Related projects
|
|||
|
||||
Author
|
||||
|
||||
Thomas Hochstein <thh@inter.net>
|
||||
<http://th-h.de/>
|
||||
Thomas Hochstein <thh@thh.name>
|
||||
<https://th-h.de/>
|
||||
|
||||
|
|
17
doc/TODO
17
doc/TODO
|
@ -4,8 +4,6 @@
|
|||
This is a list of planned bug fixes, improvements and enhancements for
|
||||
NewsStats.
|
||||
|
||||
Bug numbers refer to the Mantis issue tracker at <http://bugs.th-h.de/>.
|
||||
|
||||
* General
|
||||
- Improve Documentation
|
||||
The documentation is rather sparse and could use some improvement.
|
||||
|
@ -18,7 +16,7 @@ Bug numbers refer to the Mantis issue tracker at <http://bugs.th-h.de/>.
|
|||
to /usr/local/news/etc or /etc/news and so on
|
||||
|
||||
* Additional features
|
||||
- Add hierarchy information (GroupInfo - Bugs #19 #20 #21 #22 #23 #24 #25 #26)
|
||||
- Add hierarchy information (GroupInfo)
|
||||
NewsStats should be able to recognize invalid (i.e. officially not existing)
|
||||
newsgroups and - optionally - drop them from the list of groups. On the
|
||||
other hand, it should recognize existing, but empty groups and add them with
|
||||
|
@ -37,6 +35,19 @@ Bug numbers refer to the Mantis issue tracker at <http://bugs.th-h.de/>.
|
|||
NewsStats should offer tools e.g. to inject postings into the 'raw' database,
|
||||
or to split databases.
|
||||
|
||||
* GroupInfo project
|
||||
- Create a hierarchy information database, containing information on each
|
||||
newsgroup, its creation and removal time, its tagline, charter and
|
||||
moderation status, including the moderator contact address.
|
||||
- Automatically update hierarchy information, e.g. by parsing control messages
|
||||
(with verification!).
|
||||
- Track changes in meta information (changes to tagline, charter, moderation
|
||||
status etc.)
|
||||
- Add tools to query for hierarchy information:
|
||||
- canonical list of newsgroups for any given time
|
||||
- generate list of changes for a time period
|
||||
- find newsgroups (including wildcards) and display their history
|
||||
|
||||
* Individual improvements
|
||||
+ NewsStats.pm
|
||||
- Improve error handling when reading config
|
||||
|
|
|
@ -6,24 +6,22 @@
|
|||
#
|
||||
# It is part of the NewsStats package.
|
||||
#
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
|
||||
# Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
#
|
||||
# It can be redistributed and/or modified under the same terms under
|
||||
# which Perl itself is published.
|
||||
|
||||
BEGIN {
|
||||
our $VERSION = "0.01";
|
||||
our $VERSION = "0.02";
|
||||
use File::Basename;
|
||||
# we're in .../install, so our module is in ..
|
||||
push(@INC, dirname($0).'/..');
|
||||
# we're in .../install, so our module is in ../lib
|
||||
push(@INC, dirname($0).'/../lib');
|
||||
}
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use NewsStats qw(:DEFAULT);
|
||||
|
||||
use Cwd;
|
||||
|
||||
use DBI;
|
||||
use Getopt::Long qw(GetOptions);
|
||||
Getopt::Long::config ('bundling');
|
||||
|
@ -31,18 +29,15 @@ Getopt::Long::config ('bundling');
|
|||
################################# Main program #################################
|
||||
|
||||
### read commandline options
|
||||
my ($OptUpdate);
|
||||
my ($OptUpdate,$OptConfFile);
|
||||
GetOptions ('u|update=s' => \$OptUpdate,
|
||||
'conffile=s' => \$OptConfFile,
|
||||
'h|help' => \&ShowPOD,
|
||||
'V|version' => \&ShowVersion) or exit 1;
|
||||
|
||||
### change working directory to .. (as we're in .../install)
|
||||
chdir dirname($FullPath).'/..';
|
||||
my $Path = cwd();
|
||||
|
||||
### read configuration
|
||||
print("Reading configuration.\n");
|
||||
my %Conf = %{ReadConfig($Path.'/newsstats.conf')};
|
||||
my %Conf = %{ReadConfig($OptConfFile)};
|
||||
|
||||
##### --------------------------------------------------------------------------
|
||||
##### Database table definitions
|
||||
|
@ -108,7 +103,7 @@ Things left to do:
|
|||
## gather statistics for NewsStats
|
||||
newsstats!\\
|
||||
:!*,de.*\\
|
||||
:Tc,WmtfbsPNH,Ac:$Path/feedlog.pl
|
||||
:Tc,WmtfbsPNH,Ac:$HomePath/bin/feedlog.pl
|
||||
|
||||
Please
|
||||
|
||||
|
@ -136,7 +131,7 @@ Things left to do:
|
|||
|
||||
Enjoy!
|
||||
|
||||
-thh <thh\@inter.net>
|
||||
-thh <thh\@thh.name>
|
||||
INSTALL
|
||||
|
||||
my $Upgrade ='';
|
||||
|
@ -255,7 +250,7 @@ install - installation script
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<install> [B<-Vh> [--update I<version>]
|
||||
B<install> [B<-Vh> [--update I<version>] [B<--conffile> I<filename>]
|
||||
|
||||
=head1 REQUIREMENTS
|
||||
|
||||
|
@ -267,8 +262,9 @@ This script will create database tables as necessary and configured.
|
|||
|
||||
=head2 Configuration
|
||||
|
||||
B<install> will read its configuration from F<newsstats.conf> via
|
||||
Config::Auto.
|
||||
B<install> will read its configuration from F<newsstats.conf> which should
|
||||
be present in etc/ via Config::Auto or from a configuration file submitted
|
||||
by the B<--conffile> option.
|
||||
|
||||
See L<doc/INSTALL> for an overview of possible configuration options.
|
||||
|
||||
|
@ -288,21 +284,25 @@ Print this man page and exit.
|
|||
|
||||
Don't do a fresh install, but update from I<version>.
|
||||
|
||||
=item B<--conffile> I<filename>
|
||||
|
||||
Load configuration from I<filename> instead of F<newsstats.conf>.
|
||||
|
||||
=back
|
||||
|
||||
=head1 FILES
|
||||
|
||||
=over 4
|
||||
|
||||
=item F<install.pl>
|
||||
=item F<install/install.pl>
|
||||
|
||||
The script itself.
|
||||
|
||||
=item F<NewsStats.pm>
|
||||
=item F<lib/NewsStats.pm>
|
||||
|
||||
Library functions for the NewsStats package.
|
||||
|
||||
=item F<newsstats.conf>
|
||||
=item F<etc/newsstats.conf>
|
||||
|
||||
Runtime configuration file.
|
||||
|
||||
|
@ -311,7 +311,7 @@ Runtime configuration file.
|
|||
=head1 BUGS
|
||||
|
||||
Please report any bugs or feature requests to the author or use the
|
||||
bug tracker at L<http://bugs.th-h.de/>!
|
||||
bug tracker at L<https://code.virtcomm.de/thh/newsstats/issues>!
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
|
@ -331,11 +331,11 @@ This script is part of the B<NewsStats> package.
|
|||
|
||||
=head1 AUTHOR
|
||||
|
||||
Thomas Hochstein <thh@inter.net>
|
||||
Thomas Hochstein <thh@thh.name>
|
||||
|
||||
=head1 COPYRIGHT AND LICENSE
|
||||
|
||||
Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
|
||||
Copyright (c) 2010-2013 Thomas Hochstein <thh@thh.name>
|
||||
|
||||
This program is free software; you may redistribute it and/or modify it
|
||||
under the same terms as Perl itself.
|
||||
|
|
|
@ -49,20 +49,24 @@ require Exporter;
|
|||
Output => [qw(OutputData FormatOutput)],
|
||||
SQLHelper => [qw(SQLHierarchies SQLSortOrder SQLGroupList
|
||||
SQLSetBounds SQLBuildClause GetMaxLength)]);
|
||||
$VERSION = '0.01';
|
||||
our $PackageVersion = '0.01';
|
||||
$VERSION = '0.02';
|
||||
our $PackageVersion = '0.02';
|
||||
|
||||
use Data::Dumper;
|
||||
use File::Basename;
|
||||
use Cwd qw(realpath);
|
||||
|
||||
use Config::Auto;
|
||||
use DBI;
|
||||
|
||||
#####-------------------------------- Vars --------------------------------#####
|
||||
|
||||
# trim the path
|
||||
# save $0 in $FullPath
|
||||
our $FullPath = $0;
|
||||
our $HomePath = dirname($0);
|
||||
# strip filename and /bin or /install directory to create the $HomePath
|
||||
our $HomePath = dirname(realpath($0));
|
||||
$HomePath =~ s/\/(bin|install)//;
|
||||
# trim $0
|
||||
$0 =~ s%.*/%%;
|
||||
# set version string
|
||||
our $MyVersion = "$0 $::VERSION (NewsStats.pm $VERSION)";
|
||||
|
@ -76,7 +80,7 @@ sub ShowVersion {
|
|||
################################################################################
|
||||
### display version and exit
|
||||
print "NewsStats v$PackageVersion\n$MyVersion\n";
|
||||
print "Copyright (c) 2010-2012 Thomas Hochstein <thh\@inter.net>\n";
|
||||
print "Copyright (c) 2010-2013 Thomas Hochstein <thh\@inter.net>\n";
|
||||
print "This program is free software; you may redistribute it ".
|
||||
"and/or modify it under the same terms as Perl itself.\n";
|
||||
exit(100);
|
||||
|
@ -99,6 +103,8 @@ sub ReadConfig {
|
|||
### IN : $ConfFile: config filename
|
||||
### OUT: reference to a hash containing the configuration
|
||||
my ($ConfFile) = @_;
|
||||
# set default
|
||||
$ConfFile = $HomePath . '/etc/newsstats.conf' if !$ConfFile;
|
||||
# mandatory configuration options
|
||||
my @Mandatory = ('DBDriver','DBHost','DBUser','DBPw','DBDatabase',
|
||||
'DBTableRaw','DBTableGrps');
|
||||
|
@ -238,7 +244,7 @@ sub ReadGroupList {
|
|||
### ignoring everything after the first whitespace and so accepting files
|
||||
### in checkgroups format as well as (parts of) an INN active file)
|
||||
### IN : $Filename : file to read
|
||||
### OUT: \%ValidGroups: hash containing all valid newsgroups
|
||||
### OUT: \%ValidGroups: reference to a hash containing all valid newsgroups
|
||||
my ($Filename) = @_;
|
||||
my %ValidGroups;
|
||||
open (my $LIST,"<$Filename") or &Bleat(2,"Cannot read $Filename: $!");
|
||||
|
@ -668,7 +674,7 @@ sub SQLSetBounds {
|
|||
### OUT: SQL code to become part of a WHERE or HAVING clause
|
||||
my ($Type,$LowBound,$UppBound) = @_;
|
||||
($LowBound,$UppBound) = SQLCheckNumber($LowBound,$UppBound);
|
||||
if($LowBound and $UppBound and $LowBound > $UppBound) {
|
||||
if($LowBound and defined($UppBound) and $LowBound > $UppBound) {
|
||||
&Bleat(1,"Lower boundary $LowBound is larger than Upper boundary ".
|
||||
"$UppBound, exchanging boundaries.");
|
||||
($LowBound,$UppBound) = ($UppBound,$LowBound);
|
||||
|
@ -684,7 +690,7 @@ sub SQLSetBounds {
|
|||
} elsif ($Type eq 'sum') {
|
||||
$WhereHavingFunction = 'SUM(postings)'
|
||||
}
|
||||
$LowBound = sprintf('%s >= '.$LowBound,$WhereHavingFunction) if ($LowBound);
|
||||
$LowBound = sprintf('%s >= '.$LowBound,$WhereHavingFunction) if defined($LowBound);
|
||||
# set $LowBound to SQL statement:
|
||||
# 'WHERE postings <=', 'HAVING MAX(postings) <=' or 'HAVING AVG(postings) <='
|
||||
if ($Type eq 'level') {
|
||||
|
@ -694,7 +700,7 @@ sub SQLSetBounds {
|
|||
} elsif ($Type eq 'sum') {
|
||||
$WhereHavingFunction = 'SUM(postings)'
|
||||
}
|
||||
$UppBound = sprintf('%s <= '.$UppBound,$WhereHavingFunction) if ($UppBound);
|
||||
$UppBound = sprintf('%s <= '.$UppBound,$WhereHavingFunction) if defined($UppBound);
|
||||
return ($LowBound,$UppBound);
|
||||
};
|
||||
|
||||
|
@ -770,5 +776,3 @@ sub CheckValidNewsgroups {
|
|||
|
||||
#####------------------------------- done ---------------------------------#####
|
||||
1;
|
||||
|
||||
|
Loading…
Reference in a new issue