Skip to content

bugzilla
bugzilla
Commit af38912d authored by Marc Schumann's avatar Marc Schumann
Browse files
Options

Bug 604388 - Filenames used to store data for Old Charts should be based on…

Bug 604388 - Filenames used to store data for Old Charts should be based on product IDs, not names (avoid dataloss when renaming products). r/a=LpSolit
parent b60afed2
Hide whitespace changes
Inline Side-by-side
Showing with 118 additions and 71 deletions
Bugzilla/Install/Filesystem.pm
View file @ af38912d
......@@ -29,6 +29,7 @@ use File::Find;
use File::Path;
use File::Basename;
use File::Copy qw(move);
use File::Spec;
use IO::File;
use POSIX ();
......@@ -397,6 +398,13 @@ sub update_filesystem {
_update_old_charts($datadir);
}
# If there is a file named '-All-' in $datadir/mining, then we're still
# having mining files named by product name, and we need to convert them to
# files named by product ID.
if (-e File::Spec->catfile($datadir, 'mining', '-All-')) {
_update_old_mining_filenames(File::Spec->catdir($datadir, 'mining'));
}
# By sorting the dirs, we assure that shorter-named directories
# (meaning parent directories) are always created before their
# child directories.
......@@ -638,6 +646,59 @@ sub _update_old_charts {
}
}
# The old naming scheme has product names as mining file names; we rename them
# to product IDs.
sub _update_old_mining_filenames {
my ($miningdir) = @_;
my @conversion_errors;
require Bugzilla::Product;
# We use a dummy product instance with ID 0, representing all products
my $product_all = {id => 0, name => '-All-'};
bless($product_all, 'Bugzilla::Product');
print "Updating old charting data file names...";
my @products = Bugzilla::Product->get_all();
push(@products, $product_all);
foreach my $product (@products) {
if (-e File::Spec->catfile($miningdir, $product->id)) {
push(@conversion_errors,
{ product => $product,
message => 'A file named "' . $product->id .
'" already exists.' });
}
}
if (! @conversion_errors) {
# Renaming mining files should work now without a hitch.
foreach my $product (@products) {
if (! rename(File::Spec->catfile($miningdir, $product->name),
File::Spec->catfile($miningdir, $product->id))) {
push(@conversion_errors,
{ product => $product,
message => $! });
}
}
}
# Error reporting
if (! @conversion_errors) {
print " done.\n";
}
else {
print " FAILED:\n";
foreach my $error (@conversion_errors) {
printf "Cannot rename charting data file for product %d (%s): %s\n",
$error->{product}->id, $error->{product}->name,
$error->{message};
}
print "You need to empty the \"$miningdir\" directory, then run\n",
" collectstats.pl --regenerate\n",
"in order to clean this up.\n";
}
}
sub fix_dir_permissions {
my ($dir) = @_;
return if ON_WINDOWS;
......
collectstats.pl
View file @ af38912d
......@@ -38,6 +38,10 @@ $| = 1;
my $datadir = bz_locations()->{'datadir'};
my $graphsdir = bz_locations()->{'graphsdir'};
# We use a dummy product instance with ID 0, representing all products
my $product_all = {id => 0, name => '-All-'};
bless($product_all, 'Bugzilla::Product');
# Tidy up after graphing module
my $cwd = Cwd::getcwd();
if (chdir($graphsdir)) {
......@@ -120,7 +124,7 @@ if ($switch{'regenerate'}) {
my $tstart = time;
my @myproducts = Bugzilla::Product->get_all;
unshift(@myproducts, "-All-");
unshift(@myproducts, $product_all);
my $dir = "$datadir/mining";
if (!-d $dir) {
......@@ -149,18 +153,8 @@ sub collect_stats {
my $product = shift;
my $when = localtime (time);
my $dbh = Bugzilla->dbh;
my $product_id;
if (ref $product) {
$product_id = $product->id;
$product = $product->name;
}
# NB: Need to mangle the product for the filename, but use the real
# product name in the query
my $file_product = $product;
$file_product =~ s/\//-/gs;
my $file = join '/', $dir, $file_product;
my $file = join '/', $dir, $product->id;
my $exists = -f $file;
# if the file exists, get the old status and resolution list for that product.
......@@ -186,7 +180,7 @@ sub collect_stats {
my $status_sql = q{SELECT COUNT(*) FROM bugs WHERE bug_status = ?};
my $reso_sql = q{SELECT COUNT(*) FROM bugs WHERE resolution = ?};
if ($product ne '-All-') {
if ($product->id) {
$status_sql .= q{ AND product_id = ?};
$reso_sql .= q{ AND product_id = ?};
}
......@@ -197,13 +191,13 @@ sub collect_stats {
my @values ;
foreach my $status (@statuses) {
@values = ($status);
push (@values, $product_id) if ($product ne '-All-');
push (@values, $product->id) if ($product->id);
my $count = $dbh->selectrow_array($sth_status, undef, @values);
push(@row, $count);
}
foreach my $resolution (@resolutions) {
@values = ($resolution);
push (@values, $product_id) if ($product ne '-All-');
push (@values, $product->id) if ($product->id);
my $count = $dbh->selectrow_array($sth_reso, undef, @values);
push(@row, $count);
}
......@@ -216,7 +210,7 @@ sub collect_stats {
# Do not edit me! This file is generated.
#
# fields: $fields
# Product: $product
# Product: $product->name
# Created: $when
FIN
}
......@@ -285,24 +279,14 @@ sub regenerate_stats {
my $when = localtime(time());
my $tstart = time();
# NB: Need to mangle the product for the filename, but use the real
# product name in the query
if (ref $product) {
$product = $product->name;
}
my $file_product = $product;
$file_product =~ s/\//-/gs;
my $file = join '/', $dir, $file_product;
my $file = join '/', $dir, $product->id;
my $and_product = "";
my $from_product = "";
my @values = ();
if ($product ne '-All-') {
$and_product = q{ AND products.name = ?};
$from_product = q{ INNER JOIN products
ON bugs.product_id = products.id};
push (@values, $product);
if ($product->id) {
$and_product = q{ AND product_id = ?};
push (@values, $product->id);
}
# Determine the start date from the date the first bug in the
......@@ -312,7 +296,7 @@ sub regenerate_stats {
$dbh->sql_to_days('creation_ts') . q{ AS start_day, } .
$dbh->sql_to_days('current_date') . q{ AS end_day, } .
$dbh->sql_to_days("'1970-01-01'") .
qq{ FROM bugs $from_product
qq{ FROM bugs
WHERE } . $dbh->sql_to_days('creation_ts') .
qq{ IS NOT NULL $and_product
ORDER BY start_day } . $dbh->sql_limit(1);
......@@ -330,7 +314,7 @@ sub regenerate_stats {
# Do not edit me! This file is generated.
#
# fields: $fields
# Product: $product
# Product: $product->name
# Created: $when
FIN
# For each day, generate a line of statistics.
......@@ -339,12 +323,13 @@ FIN
for (my $day = $start + 1; $day <= $end; $day++) {
# Some output feedback
my $percent_done = ($day - $start - 1) * 100 / $total_days;
printf "\rRegenerating $product \[\%.1f\%\%]", $percent_done;
printf "\rRegenerating %s \[\%.1f\%\%]", $product->name,
$percent_done;
# Get a list of bugs that were created the previous day, and
# add those bugs to the list of bugs for this product.
$query = qq{SELECT bug_id
FROM bugs $from_product
FROM bugs
WHERE bugs.creation_ts < } .
$dbh->sql_from_days($day - 1) .
q{ AND bugs.creation_ts >= } .
......@@ -387,7 +372,8 @@ FIN
# Finish up output feedback for this product.
my $tend = time;
say "\rRegenerating $product \[100.0\%] - " . delta_time($tstart, $tend);
say "\rRegenerating " . $product->name . ' [100.0%] - ' .
delta_time($tstart, $tend);
close DATA;
}
......
reports.cgi
View file @ af38912d
......@@ -26,6 +26,10 @@ my $cgi = Bugzilla->cgi;
my $template = Bugzilla->template;
my $vars = {};
# We use a dummy product instance with ID 0, representing all products
my $product_all = {id => 0};
bless($product_all, 'Bugzilla::Product');
if (!Bugzilla->feature('old_charts')) {
ThrowCodeError('feature_disabled', { feature => 'old_charts' });
}
......@@ -33,11 +37,11 @@ if (!Bugzilla->feature('old_charts')) {
my $dir = bz_locations()->{'datadir'} . "/mining";
my $graph_dir = bz_locations()->{'graphsdir'};
my $graph_url = basename($graph_dir);
my $product_name = $cgi->param('product') || '';
my $product_id = $cgi->param('product_id');
Bugzilla->switch_to_shadow_db();
if (!$product_name) {
if (! defined($product_id)) {
# Can we do bug charts?
(-d $dir && -d $graph_dir)
|| ThrowCodeError('chart_dir_nonexistent',
......@@ -55,10 +59,10 @@ if (!$product_name) {
push(@datasets, $datasets);
}
# We only want those products that the user has permissions for.
my @myproducts = ('-All-');
# Extract product names from objects and add them to the list.
push( @myproducts, map { $_->name } @{$user->get_selectable_products} );
# Start our product list with an entry for all products, then add those
# products that the user has permissions for.
my @myproducts = ($product_all);
push( @myproducts, @{$user->get_selectable_products} );
$vars->{'datasets'} = \@datasets;
$vars->{'products'} = \@myproducts;
......@@ -66,16 +70,21 @@ if (!$product_name) {
print $cgi->header();
}
else {
# For security and correctness, validate the value of the "product" form variable.
# Valid values are those products for which the user has permissions which appear
# in the "product" drop-down menu on the report generation form.
my ($product) = grep { $_->name eq $product_name } @{$user->get_selectable_products};
($product || $product_name eq '-All-')
|| ThrowUserError('invalid_product_name', {product => $product_name});
# Product names can change over time. Their ID cannot; so use the ID
# to generate the filename.
my $prod_id = $product ? $product->id : 0;
my $product;
# For security and correctness, validate the value of the "product_id" form
# variable. Valid values are IDs of those products for which the user has
# permissions which appear in the "product_id" drop-down menu on the report
# generation form. The product_id 0 is a special case, meaning "All
# Products".
if ($product_id) {
$product = Bugzilla::Product->new($product_id);
$product && $user->can_see_product($product->name)
|| ThrowUserError('product_access_denied',
{id => $product_id});
}
else {
$product = $product_all;
}
# Make sure there is something to plot.
my @datasets = $cgi->param('datasets');
......@@ -87,9 +96,9 @@ else {
# Filenames must not be guessable as they can point to products
# you are not allowed to see. Also, different projects can have
# the same product names.
# the same product IDs.
my $project = bz_locations()->{'project'} || '';
my $image_file = join(':', ($project, $prod_id, @datasets));
my $image_file = join(':', ($project, $product->id, @datasets));
my $key = Bugzilla->localconfig->{'site_wide_secret'};
$image_file = hmac_sha256_base64($image_file, $key) . '.png';
$image_file =~ s/\+/-/g;
......@@ -116,8 +125,8 @@ sub get_data {
my $dir = shift;
my @datasets;
open(DATA, '<', "$dir/-All-")
|| ThrowCodeError('chart_file_open_fail', {filename => "$dir/-All-"});
open(DATA, '<', "$dir/0")
|| ThrowCodeError('chart_file_open_fail', {filename => "$dir/0"});
while (<DATA>) {
if (/^# fields?: (.+)\s*$/) {
......@@ -131,18 +140,13 @@ sub get_data {
sub generate_chart {
my ($dir, $image_file, $product, $datasets) = @_;
$product = $product ? $product->name : '-All-';
my $data_file = $product;
$data_file =~ s/\//-/gs;
$data_file = $dir . '/' . $data_file;
my $data_file = $dir . '/' . $product->id;
if (! open FILE, $data_file) {
if ($product eq '-All-') {
$product = '';
}
ThrowCodeError('chart_data_not_generated', {'product' => $product});
}
my $product_in_title = $product->id ? $product->name : 'All Products';
my @fields;
my @labels = qw(DATE);
my %datasets = map { $_ => 1 } @$datasets;
......@@ -205,7 +209,7 @@ sub generate_chart {
my %settings =
(
"title" => "Status Counts for $product",
"title" => "Status Counts for $product_in_title",
"x_label" => "Dates",
"y_label" => "Bug Counts",
"legend_labels" => \@labels,
......
template/en/default/global/code-error.html.tmpl
View file @ af38912d
......@@ -55,8 +55,8 @@
[% ELSIF error == "chart_data_not_generated" %]
[% admindocslinks = {'extraconfig.html' => 'Setting up Charting'} %]
[% IF product %]
Charts for the <em>[% product FILTER html %]</em> product are not
[% IF product.id %]
Charts for the <em>[% product.name FILTER html %]</em> product are not
available yet because no charting data has been collected for it since it
was created.
[% ELSE %]
......
template/en/default/global/user-error.html.tmpl
View file @ af38912d
......@@ -962,10 +962,6 @@
[% title = "Invalid Parameter" %]
The new value for [% name FILTER html %] is invalid: [% err FILTER html %].
[% ELSIF error == "invalid_product_name" %]
[% title = "Invalid Product Name" %]
The product name '[% product FILTER html %]' is invalid or does not exist.
[% ELSIF error == "invalid_regexp" %]
[% title = "Invalid regular expression" %]
The regular expression you entered is invalid.
......
template/en/default/reports/old-charts.html.tmpl
View file @ af38912d
......@@ -28,9 +28,9 @@
<tr>
<th>Product:</th>
<td align="center">
<select id="product" name="product">
<select id="product_id" name="product_id">
[% FOREACH product = products %]
<option value="[% product FILTER html %]">[% product FILTER html %]</option>
<option value="[% product.id FILTER html %]">[% product.name || '-All-' FILTER html %]</option>
[% END %]
</select>
</td>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment