Bug 519584: Implement a framework for migrating from other bug-trackers, and…

Bugzilla.pm

@@ -67,6 +67,7 @@ our $_request_cache = {};
 use constant SHUTDOWNHTML_EXEMPT => [
     'editparams.cgi',
     'checksetup.pl',
+    'migrate.pl',
     'recode.pl',
 ];
 

Bugzilla/DB.pm

@@ -873,6 +873,16 @@ sub bz_rename_table {
     $self->_bz_store_real_schema;
 }
 
+sub bz_set_next_serial_value {
+    my ($self, $table, $column, $value) = @_;
+    if (!$value) {
+        $value = $self->selectrow_array("SELECT MAX($column) FROM $table") || 0;
+        $value++;
+    }
+    my @sql = $self->_bz_real_schema->get_set_serial_sql($table, $column, $value);
+    $self->do($_) foreach @sql;
+}
+
 #####################################################################
 # Schema Information Methods
 #####################################################################

Bugzilla/DB/Schema/Mysql.pm

@@ -263,6 +263,11 @@ sub get_rename_indexes_ddl {
     return ($sql);
 }
 
+sub get_set_serial_sql {
+    my ($self, $table, $column, $value) = @_;
+    return ("ALTER TABLE $table AUTO_INCREMENT = $value");
+}
+
 # Converts a DBI column_info output to an abstract column definition.
 # Expects to only be called by Bugzila::DB::Mysql::_bz_build_schema_from_disk,
 # although there's a chance that it will also work properly if called

Bugzilla/DB/Schema/Oracle.pm

@@ -403,4 +403,13 @@ sub _get_create_seq_ddl {
     return @ddl;
 }
 
+sub get_set_serial_sql { 
+    my ($self, $table, $column, $value) = @_; 
+    my @sql;
+    my $seq_name = "${table}_${column}_SEQ";
+    push(@sql, "DROP SEQUENCE ${seq_name}");
+    push(@sql, $self->_get_create_seq_ddl($table, $column, $value));       
+    return @sql;
+} 
+
 1;

Bugzilla/DB/Schema/Pg.pm

@@ -119,6 +119,12 @@ sub get_rename_table_sql {
     return ("ALTER TABLE $old_name RENAME TO $new_name");
 }
 
+sub get_set_serial_sql {
+    my ($self, $table, $column, $value) = @_;
+    return ("SELECT setval('${table}_${column}_seq', $value, false)
+               FROM $table");
+}
+
 sub _get_alter_type_sql {
     my ($self, $table, $column, $new_def, $old_def) = @_;
     my @statements;

Bugzilla/Install/Filesystem.pm

@@ -118,6 +118,7 @@ sub FILESYSTEM {
         'email_in.pl'     => { perms => $ws_executable },
         'sanitycheck.pl'  => { perms => $ws_executable },
         'jobqueue.pl'     => { perms => $owner_executable },
+        'migrate.pl'      => { perms => $owner_executable },
         'install-module.pl' => { perms => $owner_executable },
 
         "$localconfig.old" => { perms => $owner_readable },

Bugzilla/Install/Requirements.pm

@@ -462,7 +462,9 @@ sub print_module_instructions {
         }
     }
 
-    if ($output && $check_results->{any_missing} && !ON_WINDOWS) {
+    if ($output && $check_results->{any_missing} && !ON_WINDOWS
+        && !$check_results->{hide_all}) 
+    {
         print install_string('install_all', { perl => $^X });
     }
     if (!$check_results->{pass}) {

contrib/bzdbcopy.pl

@@ -193,8 +193,7 @@ foreach my $table (@table_list) {
                 # PostgreSQL doesn't like it when you insert values into
                 # a serial field; it doesn't increment the counter 
                 # automatically.
-                $target_db->do("SELECT pg_catalog.setval 
-                                ('${table}_${column}_seq', $max_val, false)");
+                $target_db->bz_set_next_serial_value($table, $column);
             }
             elsif ($target_db->isa('Bugzilla::DB::Oracle')) {
                 # Oracle increments the counter on every insert, and *always*

migrate.pl

+#!/usr/bin/perl -w
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is The Bugzilla Migration Tool.
+#
+# The Initial Developer of the Original Code is Lambda Research
+# Corporation. Portions created by the Initial Developer are Copyright
+# (C) 2009 the Initial Developer. All Rights Reserved.
+#
+# Contributor(s): 
+#   Max Kanat-Alexander <mkanat@bugzilla.org>
+
+use strict;
+use File::Basename;
+BEGIN { chdir dirname($0); }
+use lib qw(. lib);
+use Bugzilla;
+use Bugzilla::Migrate;
+
+use Getopt::Long;
+use Pod::Usage;
+
+my %switch;
+GetOptions(\%switch, 'help|h|?', 'from=s', 'verbose|v+', 'dry-run');
+
+# Print the help message if that switch was selected or if --from
+# wasn't specified.
+if (!$switch{'from'} or $switch{'help'}) {
+    pod2usage({-exitval => 1});
+}
+
+my $migrator = Bugzilla::Migrate->load($switch{'from'});
+$migrator->verbose($switch{'verbose'});
+$migrator->dry_run($switch{'dry-run'});
+$migrator->check_requirements();
+$migrator->do_migration();
+
+# Even if there's an error, we want to be sure that the serial values
+# get reset properly.
+END {
+    if ($migrator and $migrator->dry_run) {
+        my $dbh = Bugzilla->dbh;
+        if ($dbh->bz_in_transaction) {
+            $dbh->bz_rollback_transaction();
+        }
+        $migrator->reset_serial_values();
+    }
+}
+
+__END__
+
+=head1 NAME
+
+migrate.pl - A script to migrate from other bug-trackers to Bugzilla.
+
+=head1 SYNOPSIS
+
+ ./migrate.pl --from=<tracker> [--verbose] [--dry-run]
+
+ Migrates from another bug-tracker to Bugzilla. If you want
+ to upgrade Bugzilla, use checksetup.pl instead.
+
+ Always test this on a backup copy of your database before
+ running it on your live Bugzilla.
+
+=head1 OPTIONS
+
+=over
+
+=item B<--from=tracker>
+
+Specifies what bug-tracker you're migrating from. To see what values
+are valid, see the contents of the F<Bugzilla/Migrate/> directory.
+
+=item B<--dry-run>
+
+Don't modify the Bugzilla database at all, just test the import.
+Note that this could cause significant slowdown and other strange effects
+on a live Bugzilla, so only use it on a test instance.
+
+=item B<--verbose>
+
+If specified, this script will output extra debugging information
+to STDERR. Specify multiple times (up to three) for more information.
+
+=back
+
+=head1 DESCRIPTION
+
+This script copies data from another bug-tracker into Bugzilla. It migrates
+users, products, and bugs from the other bug-tracker into this Bugzilla,
+without removing any of the data currently in this Bugzilla.
+
+Note that you will need enough space in your temporary directory to hold
+the size of all attachments in your current bug-tracker.
+
+You may also need to increase the number of file handles a process is allowed
+to hold open (as the migrator will create a file handle for each attachment
+in your database). On Linux and simliar systems, you can do this as root
+by typing C<ulimit -n 65535> before running your script.
\ No newline at end of file

template/en/default/global/messages.html.tmpl

@@ -491,6 +491,44 @@
     [% title = "$terms.Bugzilla Login Changed" %]
     Your [% terms.Bugzilla %] login has been changed.
 
+  [% ELSIF message_tag == "migrate_component_created" %]
+    Component created: [% comp.name FILTER html %]
+    (in [% product.name FILTER html %])
+
+  [% ELSIF message_tag == "migrate_creating_bugs" %]
+    Creating [% terms.bugs %]...
+
+  [% ELSIF message_tag == "migrate_field_created" %]
+    New custom field: [% field.description FILTER html %]
+    ([% field.name FILTER html %])
+
+  [% ELSIF message_tag == "migrate_product_created" %]
+    Product created: [% created.name FILTER html %]
+
+  [% ELSIF message_tag == "migrate_reading_bugs" %]
+    Reading [% terms.bugs %]...
+
+  [% ELSIF message_tag == "migrate_reading_products" %]
+    Reading products...
+
+  [% ELSIF message_tag == "migrate_reading_users" %]
+    Reading users...
+
+  [% ELSIF message_tag == "migrate_translating_bugs" %]
+    Converting [% terms.bug %] values to be appropriate for 
+    [%+ terms.Bugzilla %]...
+
+  [% ELSIF message_tag == "migrate_user_created" %]
+    User created: [% created.email FILTER html %]
+    [% IF password %] Password: [% password FILTER html %][% END %]
+
+  [% ELSIF message_tag == "migrate_value_created" %]
+    [% IF product.defined %]
+      [% product.name FILTER html %]
+    [% END %]
+    [%+ field_descs.${field.name} FILTER html %] value
+    created: [% value FILTER html %]
+
   [% ELSIF message_tag == "milestone_created" %]
     [% title = "Milestone Created" %]
     The milestone <em>[% milestone.name FILTER html %]</em> has been created.

template/en/default/global/user-error.html.tmpl

@@ -989,6 +989,15 @@
     You can't use %user% without being logged in, because %user% refers
     to your login name, which we don't know.
 
+  [% ELSIF error == "migrate_config_created" %]
+    The file <kbd>[% file FILTER html %]</kbd> contains configuration
+    variables that must be set before continuing with the migration.
+
+  [% ELSIF error == "migrate_from_invalid" %]
+    '[% from FILTER html %]' is not a valid type of [% terms.bug %]-tracker
+    to migrate from. See the contents of the <kbd>B[% %]ugzilla/Migrate/</kbd>
+    directory for a list of valid [% terms.bug %]-trackers.
+
   [% ELSIF error == "milestone_already_exists" %]
     [% title = "Milestone Already Exists" %]
     [% admindocslinks = {'products.html' => 'Administering products',

template/en/default/setup/strings.txt.pl

@@ -42,7 +42,7 @@ EOT
     commands_optional => 'COMMANDS TO INSTALL OPTIONAL MODULES:',
     commands_required => <<EOT,
 COMMANDS TO INSTALL REQUIRED MODULES (You *must* run all these commands
-and then re-run checksetup.pl):
+and then re-run this script):
 EOT
     done => 'done.',