From ed3fb3cda0f2d5cd805be92ce4c8521532504830 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 01:06:15 +0200 Subject: [PATCH 1/4] Truncate overlong clients or versions. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 3 +++ doc/ChangeLog | 1 + 2 files changed, 4 insertions(+) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 9b02bfa..9cb0a9b 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -500,6 +500,9 @@ sub ClientStats { # encode to utf-8, if necessary $_->{'agent'} = encode('UTF-8', $_->{'agent'}) if $_->{'agent'} =~ /[\x80-\x{ffff}]/; $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/; + # truncate overlong clients or versions + $_->{'agent'} = substr($_->{'agent'}, 0, 150) if length($_->{'agent'}) > 150; + $_->{'version'} = substr($_->{'version'}, 0, 50) if $_->{'version'} and length($_->{'version'}) > 50; # special cases # Mozilla $_->{'agent'} = 'Mozilla' if $_->{'agent'} eq '•Mozilla'; diff --git a/doc/ChangeLog b/doc/ChangeLog index c829e19..aa498ca 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -10,6 +10,7 @@ NewsStats 0.4.0 (unreleased) * Add ClientStats to postingstats. * gatherstats: Don't die on parsing errors. * DBClnts: set version length to to 50. + * gatherstats: Truncate overlong clients or versions. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 0102b72971a5889173d67906aa799ea49cc8bcc6 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:48:23 +0200 Subject: [PATCH 2/4] Remove whitespace from client and version. Signed-off-by: Thomas Hochstein --- bin/gatherstats.pl | 6 ++++-- doc/ChangeLog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/gatherstats.pl b/bin/gatherstats.pl index 9cb0a9b..55c2dad 100755 --- a/bin/gatherstats.pl +++ b/bin/gatherstats.pl @@ -497,6 +497,9 @@ sub ClientStats { foreach (@Clients) { # filter agents for User-Agent with multiple agents next if $#Clients && exists($DropAgent{lc($_->{'agent'})}); + # remove whitespace + $_->{'agent'} =~ s/^\s+|\s+$//g; + $_->{'version'} =~ s/^\s+|\s+$//g if $_->{'version'}; # encode to utf-8, if necessary $_->{'agent'} = encode('UTF-8', $_->{'agent'}) if $_->{'agent'} =~ /[\x80-\x{ffff}]/; $_->{'version'} = encode('UTF-8', $_->{'version'}) if $_->{'version'} and $_->{'version'} =~ /[\x80-\x{ffff}]/; @@ -664,8 +667,7 @@ sub RemoveComments { # remove superfluous whitespace in header # and whitespace around header $Header =~ s/\s+/ /g; - $Header =~ s/^\s+//; - $Header =~ s/\s+$//; + $Header =~ s/^\s+|\s+$//g; return $Header; } diff --git a/doc/ChangeLog b/doc/ChangeLog index aa498ca..6bf2a46 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -11,6 +11,7 @@ NewsStats 0.4.0 (unreleased) * gatherstats: Don't die on parsing errors. * DBClnts: set version length to to 50. * gatherstats: Truncate overlong clients or versions. + * gatherstats: Remove whitespace from client and version. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine. From 07e45437174ef6551a6657df0f257186a061d6a5 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 00:58:23 +0200 Subject: [PATCH 3/4] Fix typos. Signed-off-by: Thomas Hochstein --- bin/postingstats.pl | 2 +- doc/ChangeLog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/postingstats.pl b/bin/postingstats.pl index a9d1e9a..e517a45 100644 --- a/bin/postingstats.pl +++ b/bin/postingstats.pl @@ -134,7 +134,7 @@ wenn Sie ermittelbar sind; daher kann die Summe der Newsreader-Versionen kleiner sein als die Postingzahl fuer den Newsreader. Ausserdem koennen an einem Beitrag mehrere Clients beteiligt sein, bspw. der Newsreader und ein lokaler Server wie der Hamster. Daher kann die Summe aller -Newsreader groesser sein als die Summer der Postings; auch ergeben die +Newsreader groesser sein als die Summe der Postings; auch ergeben die Prozentzahlen dementsprechend in der Summe mehr als 100%. CLIENTSOUT diff --git a/doc/ChangeLog b/doc/ChangeLog index 6bf2a46..a6e7ed9 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -9,7 +9,7 @@ NewsStats 0.4.0 (unreleased) * Add clientstats (for clients). * Add ClientStats to postingstats. * gatherstats: Don't die on parsing errors. - * DBClnts: set version length to to 50. + * DBClnts: set version length to 50. * gatherstats: Truncate overlong clients or versions. * gatherstats: Remove whitespace from client and version. From d02ae5e2ff3f013febc4d84d22651610eb7a88b2 Mon Sep 17 00:00:00 2001 From: Thomas Hochstein Date: Sat, 31 May 2025 09:38:41 +0200 Subject: [PATCH 4/4] Fix version queries. Add month to WHERE clause, use bind values. Signed-off-by: Thomas Hochstein --- bin/clientstats.pl | 12 ++++++++---- doc/ChangeLog | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/clientstats.pl b/bin/clientstats.pl index 0f7ae20..b94fa16 100644 --- a/bin/clientstats.pl +++ b/bin/clientstats.pl @@ -215,10 +215,14 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { # output client versions if ($OptVersions) { ### get client versions - # $SQLWhereClause without 'ALL' version + # $SQLWhereClause without 'ALL' version, with client and month set $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,$SQLWhereNames, - $ExcludeSums,"version != 'ALL'","client = '$Client'", + $ExcludeSums,"version != 'ALL'", + 'client = ?','month = ?', &SQLSetBounds('default',$LowBound,$UppBound)); + # push client and month to @SQLVersBindNames + my @SQLVersBindNames = @SQLBindNames; + push (@SQLVersBindNames, ($Client, $Month)); # save length of longest client my $ClientMaxLenght = $MaxLength; @@ -227,7 +231,7 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { # for formatting purposes my ($MaxLength,$MaxValLength) = &GetMaxLength($DBHandle,$Conf{'DBTable'}, 'version','postings',$SQLWhereClause, - '',@SQLBindNames); + '',@SQLVersBindNames); if ($MaxLength) { # add lenght of '- ' $MaxLength += 2; @@ -242,7 +246,7 @@ while (my ($Month, $Key, $Value) = $DBQuery->fetchrow_array) { $SQLWhereClause,$SQLGroupClause, $SQLOrderClause)); # execute query - $DBVersQuery->execute(@SQLBindNames) + $DBVersQuery->execute(@SQLVersBindNames) or &Bleat(2,sprintf("Can't get version data for %s from %s.%s: %s\n", $CaptionPeriod,$Conf{'DBDatabase'},$Conf{'DBTable'}, $DBI::errstr)); diff --git a/doc/ChangeLog b/doc/ChangeLog index a6e7ed9..ef17bfc 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -12,6 +12,7 @@ NewsStats 0.4.0 (unreleased) * DBClnts: set version length to 50. * gatherstats: Truncate overlong clients or versions. * gatherstats: Remove whitespace from client and version. + * Fix version queries. NewsStats 0.3.0 (2025-05-18) * Extract GroupStats (in gatherstats) to subroutine.