Bug 180870 - Remove old shadowdb manual replication code

r, a=myk
parent a40d927b
......@@ -820,7 +820,7 @@ if ($my_create_htaccess) {
open HTACCESS, ">.htaccess";
print HTACCESS <<'END';
# don't allow people to retrieve non-cgi executable files or our private data
<FilesMatch ^(.*\.pl|localconfig|processmail|syncshadowdb|runtests.sh)$>
<FilesMatch ^(.*\.pl|localconfig|processmail|runtests.sh)$>
deny from all
</FilesMatch>
END
......@@ -1085,7 +1085,7 @@ WriteParams();
# These are the files which need to be marked executable
my @executable_files = ('processmail', 'whineatnews.pl', 'collectstats.pl',
'checksetup.pl', 'syncshadowdb', 'importxml.pl', 'runtests.sh');
'checksetup.pl', 'importxml.pl', 'runtests.sh');
# tell me if a file is executable. All CGI files and those in @executable_files
# are executable
......@@ -1648,13 +1648,6 @@ $table{milestones} =
sortkey smallint not null,
unique (product_id, value)';
$table{shadowlog} =
'id int not null auto_increment primary key,
ts timestamp,
reflected tinyint not null,
command mediumtext not null,
index(reflected)';
# GRM
$table{duplicates} =
'dupe_of mediumint(9) not null,
......@@ -3828,6 +3821,11 @@ if ($sth->rows == 0) {
}
}
# 2002-11-XX Bug 180870 - remove manual shadowdb replication code
if (TableExists('shadowlog')) {
print "Removing shadowlog table\n";
$dbh->do("DROP TABLE shadowlog");
}
#
# Final checks...
......
......@@ -67,30 +67,14 @@ sub check_shadowdb {
if ($value eq "") {
return "";
}
if (!Param("updateshadowdb")) {
# Can't test this, because ConnectToDatabase uses the param, but
# we can't set this before testing....
return "";
}
&::SendSQL("SHOW DATABASES");
while (&::MoreSQLData()) {
my $n = &::FetchOneColumn();
if (lc($n) eq lc($value)) {
return "The $n database already exists. If that's really the name you want to use for the backup, please CAREFULLY make the existing database go away somehow, and then try again.";
}
}
# We trust the admin....
trick_taint($value);
&::SendSQL("CREATE DATABASE $value");
&::SendSQL("INSERT INTO shadowlog (command) VALUES ('SYNCUP')", 1);
return "";
}
sub check_shadowdbhost {
my ($value) = (@_);
if ($value && Param("updateshadowdb")) {
return "Sorry, you can't have the shadowdb on a different connection to the main database if you want Bugzilla to handle the replication for you.";
if (!Param('shadowdbhost')) {
return "You need to specify a host when using a shadow database";
}
# Can't test existance of this because ConnectToDatabase uses the param,
# but we can't set this before testing....
# This can really only be fixed after we can use the DBI more openly
return "";
}
......@@ -271,50 +255,10 @@ sub check_netmask {
},
{
name => 'queryagainstshadowdb',
desc => 'If this is on, and the <tt>shadowdb</tt> parameter is set, then ' .
'certain queries will happen against the shadow database.',
type => 'b',
default => 0,
},
{
name => 'updateshadowdb',
desc => 'If this is on, and the <tt>shadowdb</tt> parameter is set, then ' .
'Bugzilla will use the old style of shadow database in which it ' .
'manually propogates changes to the shadow database. Otherwise, ' .
'Bugzilla will assume that the <tt>shadowdb</tt> database (if ' .
'any) is being updated via replication. <b>WARNING! This ' .
'manual replication is deprecated and is going away soon ' .
'(<u>BEFORE</u> the next stable Bugzilla release).</b> It has ' .
'several problems with data consistency, and replication is the ' .
'preferred option. If this parameter is on, and you disable it, ' .
'make sure that the shadow database is already set up for ' .
'replication, or queries will return stale data.',
type => 'b',
default => 1,
},
# This entry must be _after_ updateshadowdb, because check_shadowdbhost uses
# that
{
name => 'shadowdbhost',
desc => 'The host the shadow database is on. If blank, then then we ' .
'assume it\'s on the main database host (as defined in ' .
'localconfig) and ingore the <tt>shadowdbport</tt> and ' .
'<tt>shadowdbsock</tt> parameters below, which means that this ' .
'parameter <em>must be filled in</em> if your shadow database is ' .
'on a different instance of the mysql server, even if that ' .
'instance runs on the same machine as the main database. Note ' .
'that <tt>updateshadowdb</tt> must be off if the shadow database ' .
'is on a difference mysql instance, since Bugzilla can\'t ' .
'propogate changes between instances itself, and this should be ' .
'left blank if the shadow database is on the same instance, ' .
'since Bugzilla can then reuse the same database connection for '.
'better performance.',
desc => 'The host the shadow database is on.',
type => 't',
default => '',
checker => \&check_shadowdbhost,
},
{
......@@ -346,11 +290,12 @@ sub check_netmask {
{
name => 'shadowdb',
desc => 'If non-empty, then this is the name of another database in ' .
'which Bugzilla will keep a shadow read-only copy of everything. ' .
'which Bugzilla will use as a read-only copy of everything. ' .
'This is done so that long slow read-only operations can be used ' .
'against this db, and not lock up things for everyone else. ' .
'Turning on this parameter will create the given database ; be ' .
'careful not to use the name of an existing database with useful ' . 'data in it!',
'against this db, and not lock up things for everyone else. This ' .
'database is on the <tt>shadowdbhost</tt>, and must exist. ' .
'Bugzilla does not update it, if you use this paramater, then ' .
'you need to set up replication for your database',
type => 't',
default => '',
checker => \&check_shadowdb
......
......@@ -69,8 +69,14 @@
write locking. What this means is that if someone needs to make a
change to a bug, they will lock the entire table until the operation
is complete. Locking for write also blocks reads until the write is
complete. The
<quote>shadowdb</quote>
complete. Note that more recent versions of mysql support row level
locking using different table types. These types are slower than the
standard type, and Bugzilla does not yet take advantage of features
such as transactions which would justify this speed decrease. The
Bugzilla team are, however, happy to hear about any experiences with
row level locking and Bugzilla</para>
<para>The <quote>shadowdb</quote>
parameter was designed to get around this limitation. While only a
single user is allowed to write to a table at a time, reads can
continue unimpeded on a read-only shadow copy of the database.
......@@ -85,23 +91,10 @@
Bugzilla bug changes and comments per day.</para>
<para>The value of the parameter defines the name of the
shadow bug database.
Set "shadowdb" to e.g. "bug_shadowdb" if you will be running a
*very* large installation of Bugzilla.
<note>
<para>Enabling "shadowdb" can adversely affect the stability of
your installation of Bugzilla. You should regularly check that your
database is in sync. It is often advisable to force a shadow
database sync nightly via
<quote>cron</quote>.
</para>
</note>
</para>
<para>If you use the "shadowdb" option, it is only natural that you
should turn the "queryagainstshadowdb" option on as well. Otherwise
you are replicating data into a shadow database for no reason!</para>
shadow bug database. You will need to set the host and port settings
from the params page, and set up replication in your database server
so that updates reach this readonly mirror. Consult your database
documentation for more detail.</para>
</step>
<step>
......
......@@ -198,7 +198,6 @@
| products |
| profiles |
| profiles_activity |
| shadowlog |
| tokens |
| versions |
| votes |
......@@ -290,10 +289,6 @@ sshh... don't tell your users!)
profiles_activity: Need to know who did what when to who's profile? This'll
tell you, it's a pretty complete history.
shadowlog: I could be mistaken here, but I believe this table tells you when
your shadow database is updated and what commands were used to update it. We
don't use a shadow database at our site yet, so it's pretty empty for us.
versions: Version information for every product
votes: Who voted for what when
......
......@@ -559,9 +559,7 @@ AllowOverride Limit
<para>There are important files and directories that should not be a
served by the HTTP server - most files in the
<quote>data</quote>
and
<quote>shadow</quote>
directories and the
directory and the
<quote>localconfig</quote>
file. You should configure your HTTP server to not serve
these files. Failure to do so will expose critical passwords and
......
......@@ -69,8 +69,14 @@
write locking. What this means is that if someone needs to make a
change to a bug, they will lock the entire table until the operation
is complete. Locking for write also blocks reads until the write is
complete. The
<quote>shadowdb</quote>
complete. Note that more recent versions of mysql support row level
locking using different table types. These types are slower than the
standard type, and Bugzilla does not yet take advantage of features
such as transactions which would justify this speed decrease. The
Bugzilla team are, however, happy to hear about any experiences with
row level locking and Bugzilla</para>
<para>The <quote>shadowdb</quote>
parameter was designed to get around this limitation. While only a
single user is allowed to write to a table at a time, reads can
continue unimpeded on a read-only shadow copy of the database.
......@@ -85,23 +91,10 @@
Bugzilla bug changes and comments per day.</para>
<para>The value of the parameter defines the name of the
shadow bug database.
Set "shadowdb" to e.g. "bug_shadowdb" if you will be running a
*very* large installation of Bugzilla.
<note>
<para>Enabling "shadowdb" can adversely affect the stability of
your installation of Bugzilla. You should regularly check that your
database is in sync. It is often advisable to force a shadow
database sync nightly via
<quote>cron</quote>.
</para>
</note>
</para>
<para>If you use the "shadowdb" option, it is only natural that you
should turn the "queryagainstshadowdb" option on as well. Otherwise
you are replicating data into a shadow database for no reason!</para>
shadow bug database. You will need to set the host and port settings
from the params page, and set up replication in your database server
so that updates reach this readonly mirror. Consult your database
documentation for more detail.</para>
</step>
<step>
......
......@@ -198,7 +198,6 @@
| products |
| profiles |
| profiles_activity |
| shadowlog |
| tokens |
| versions |
| votes |
......@@ -290,10 +289,6 @@ sshh... don't tell your users!)
profiles_activity: Need to know who did what when to who's profile? This'll
tell you, it's a pretty complete history.
shadowlog: I could be mistaken here, but I believe this table tells you when
your shadow database is updated and what commands were used to update it. We
don't use a shadow database at our site yet, so it's pretty empty for us.
versions: Version information for every product
votes: Who voted for what when
......
......@@ -559,9 +559,7 @@ AllowOverride Limit
<para>There are important files and directories that should not be a
served by the HTTP server - most files in the
<quote>data</quote>
and
<quote>shadow</quote>
directories and the
directory and the
<quote>localconfig</quote>
file. You should configure your HTTP server to not serve
these files. Failure to do so will expose critical passwords and
......
......@@ -101,11 +101,6 @@ foreach my $i (GetParamList()) {
WriteParams();
unlink "data/versioncache";
if (Param("updateshadowdb")) {
print "<PRE>";
system("./syncshadowdb", "-v");
print "</PRE>";
}
print "OK, done.<p>\n";
print "<a href=editparams.cgi>Edit the params some more.</a><p>\n";
......
......@@ -109,19 +109,16 @@ $::dbwritesallowed = 1;
sub ConnectToDatabase {
my ($useshadow) = (@_);
$::dbwritesallowed = !$useshadow;
$useshadow = ($useshadow && Param("shadowdb") &&
Param("queryagainstshadowdb"));
my $useshadow_dbh = ($useshadow && Param("shadowdbhost") ne "");
my $name = $useshadow ? Param("shadowdb") : $::db_name;
$useshadow &&= Param("shadowdb");
my $connectstring;
if ($useshadow_dbh) {
if ($useshadow) {
if (defined $::shadow_dbh) {
$::db = $::shadow_dbh;
return;
}
$connectstring="DBI:mysql:host=" . Param("shadowdbhost") .
";database=$name;port=" . Param("shadowdbport");
";database=" . Param('shadowdb') . ";port=" . Param("shadowdbport");
if (Param("shadowdbsock") ne "") {
$connectstring .= ";mysql_socket=" . Param("shadowdbsock");
}
......@@ -130,7 +127,7 @@ sub ConnectToDatabase {
$::db = $::main_dbh;
return;
}
$connectstring="DBI:mysql:host=$::db_host;database=$name;port=$::db_port";
$connectstring="DBI:mysql:host=$::db_host;database=$::db_name;port=$::db_port";
if ($::db_sock ne "") {
$connectstring .= ";mysql_socket=$::db_sock";
}
......@@ -141,7 +138,7 @@ sub ConnectToDatabase {
Param("maintainer") . ". The error you should quote is: " .
$DBI::errstr;
if ($useshadow_dbh) {
if ($useshadow) {
$::shadow_dbh = $::db;
} else {
$::main_dbh = $::db;
......@@ -149,58 +146,17 @@ sub ConnectToDatabase {
}
sub ReconnectToShadowDatabase {
# This will connect us to the shadowdb if we're not already connected,
# but if we're using the same dbh for both the main db and the shadowdb,
# be sure to USE the correct db
if (Param("shadowdb") && Param("queryagainstshadowdb")) {
if (Param("shadowdb")) {
ConnectToDatabase(1);
if (!Param("shadowdbhost")) {
SendSQL("USE " . Param("shadowdb"));
}
}
}
sub ReconnectToMainDatabase {
if (Param("shadowdb") && Param("queryagainstshadowdb")) {
if (Param("shadowdb")) {
ConnectToDatabase();
if (!Param("shadowdbhost")) {
SendSQL("USE $::db_name");
}
}
}
my $shadowchanges = 0;
sub SyncAnyPendingShadowChanges {
if ($shadowchanges && Param("updateshadowdb")) {
my $pid;
FORK: {
if ($pid = fork) { # create a fork
# parent code runs here
$shadowchanges = 0;
return;
} elsif (defined $pid) {
# child process code runs here
my $redir = ($^O =~ /MSWin32/i) ? "NUL" : "/dev/null";
open STDOUT,">$redir";
open STDERR,">$redir";
exec("./syncshadowdb","--") or die "Unable to exec syncshadowdb: $!";
# the idea was that passing the second parameter tricks it into
# using execvp instead of running a shell. Not really necessary since
# there are no shell meta-characters, but it passes our tinderbox
# test that way. :) http://bugzilla.mozilla.org/show_bug.cgi?id=21253
} elsif ($! =~ /No more process/) {
# recoverable fork error, try again in 5 seconds
sleep 5;
redo FORK;
} else {
# something weird went wrong
die "Can't create background process to run syncshadowdb: $!";
}
}
}
}
# This is used to manipulate global state used by SendSQL(),
# MoreSQLData() and FetchSQLData(). It provides a way to do another
# SQL query without losing any as-yet-unfetched data from an existing
......@@ -248,7 +204,7 @@ sub SqlLog {
}
sub SendSQL {
my ($str, $dontshadow) = (@_);
my ($str) = (@_);
# Don't use DBI's taint stuff yet, because:
# a) We don't want out vars to be tainted (yet)
......@@ -262,12 +218,10 @@ sub SendSQL {
if ($iswrite && !$::dbwritesallowed) {
die "Evil code attempted to write '$str' to the shadow database";
}
if ($str =~ /^LOCK TABLES/i && $str !~ /shadowlog/ && $::dbwritesallowed) {
$str =~ s/^LOCK TABLES/LOCK TABLES shadowlog WRITE, /i;
}
# If we are shutdown, we don't want to run queries except in special cases
if (Param('shutdownhtml')) {
if ($0 =~ m:[\\/]((do)?editparams.cgi|syncshadowdb)$:) {
if ($0 =~ m:[\\/]((do)?editparams.cgi)$:) {
$::ignorequery = 0;
} else {
$::ignorequery = 1;
......@@ -284,19 +238,6 @@ sub SendSQL {
die "$str: " . $errstr;
}
SqlLog("Done");
if (!$dontshadow && $iswrite && Param("shadowdb") && Param("updateshadowdb")) {
my $q = SqlQuote($str);
my $insertid;
if ($str =~ /^(INSERT|REPLACE)/i) {
SendSQL("SELECT LAST_INSERT_ID()");
$insertid = FetchOneColumn();
}
SendSQL("INSERT INTO shadowlog (command) VALUES ($q)", 1);
if ($insertid) {
SendSQL("SET LAST_INSERT_ID = $insertid");
}
$shadowchanges++;
}
}
sub MoreSQLData {
......
......@@ -395,14 +395,6 @@ while (MoreSQLData()) {
push(@groupstoadd, $id)
}
}
# Lock tables before inserting records for the new bug into the database
# if we are using a shadow database to prevent shadow database corruption
# when two bugs get created at the same time.
SendSQL("LOCK TABLES bugs WRITE, bug_group_map WRITE, longdescs WRITE, " .
"cc WRITE, keywords WRITE, dependencies WRITE, bugs_activity WRITE, " .
"fielddefs READ, profiles READ, keyworddefs READ") if Param("shadowdb");
# Add the bug report to the DB.
SendSQL($sql);
......@@ -465,8 +457,6 @@ if (UserInGroup("editbugs")) {
}
}
SendSQL("UNLOCK TABLES") if Param("shadowdb");
# Assemble the -force* strings so this counts as "Added to this capacity"
my @ARGLIST = ();
if (@cc) {
......
......@@ -53,5 +53,3 @@
</body>
</html>
[% CALL SyncAnyPendingShadowChanges() IF SyncAnyPendingShadowChanges %]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment