Bug 237498: Add memcached integration

Bugzilla.pm

@@ -19,23 +19,24 @@ BEGIN {
     }
 }
 
-use Bugzilla::Config;
-use Bugzilla::Constants;
 use Bugzilla::Auth;
 use Bugzilla::Auth::Persist::Cookie;
 use Bugzilla::CGI;
-use Bugzilla::Extension;
+use Bugzilla::Config;
+use Bugzilla::Constants;
 use Bugzilla::DB;
+use Bugzilla::Error;
+use Bugzilla::Extension;
+use Bugzilla::Field;
+use Bugzilla::Flag;
 use Bugzilla::Install::Localconfig qw(read_localconfig);
 use Bugzilla::Install::Requirements qw(OPTIONAL_MODULES);
 use Bugzilla::Install::Util qw(init_console include_languages);
+use Bugzilla::Memcached;
 use Bugzilla::Template;
+use Bugzilla::Token;
 use Bugzilla::User;
-use Bugzilla::Error;
 use Bugzilla::Util;
-use Bugzilla::Field;
-use Bugzilla::Flag;
-use Bugzilla::Token;
 
 use File::Basename;
 use File::Spec::Functions;
@@ -641,6 +642,12 @@ sub process_cache {
     return $_process_cache;
 }
 
+# This is a memcached wrapper, which provides cross-process and cross-system
+# caching.
+sub memcached {
+    return $_[0]->process_cache->{memcached} ||= Bugzilla::Memcached->_new();
+}
+
 # Private methods
 
 # Per-process cleanup. Note that this is a plain subroutine, not a method,
@@ -949,17 +956,89 @@ this Bugzilla installation.
 Tells you whether or not a specific feature is enabled. For names
 of features, see C<OPTIONAL_MODULES> in C<Bugzilla::Install::Requirements>.
 
+=back
+
+=head1 B<CACHING>
+
+Bugzilla has several different caches available which provide different
+capabilities and lifetimes.
+
+The keys of all caches are unregulated; use of prefixes is suggested to avoid
+collisions.
+
+=over
+
+=item B<Request Cache>
+
+The request cache is a hashref which supports caching any perl variable for the
+duration of the current request. At the end of the current request the contents
+of this cache are cleared.
+
+Examples of its use include caching objects to avoid re-fetching the same data
+from the database, and passing data between otherwise unconnected parts of
+Bugzilla.
+
+=over
+
+=item C<request_cache>
+
+Returns a hashref which can be checked and modified to store any perl variable
+for the duration of the current request.
+
 =item C<clear_request_cache>
 
 Removes all entries from the C<request_cache>.
 
 =back
 
-=head1 B<Methods in need of POD>
+=item B<Process Cache>
+
+The process cache is a hashref which support caching of any perl variable. If
+Bugzilla is configured to run using Apache mod_perl, the contents of this cache
+are persisted across requests for the lifetime of the Apache worker process
+(which varies depending on the SizeLimit configuration in mod_perl.pl).
+
+If Bugzilla isn't running under mod_perl, the process cache's contents are
+cleared at the end of the request.
+
+The process cache is only suitable for items which never change while Bugzilla
+is running (for example the path where Bugzilla is installed).
 
 =over
 
-=item process_cache
+=item C<process_cache>
+
+Returns a hashref which can be checked and modified to store any perl variable
+for the duration of the current process (mod_perl) or request (mod_cgi).
+
+=back
+
+=item B<Memcached>
+
+If Memcached is installed and configured, Bugzilla can use it to cache data
+across requests and between webheads. Unlike the request and process caches,
+only scalars, hashrefs, and arrayrefs can be stored in Memcached.
+
+Memcached integration is only required for large installations of Bugzilla -- if
+you have multiple webheads then configuring Memcached is recommended.
+
+=over
+
+=item C<memcached>
+
+Returns a C<Bugzilla::Memcached> object. An object is always returned even if
+Memcached is not available.
+
+See the documentation for the C<Bugzilla::Memcached> module for more
+information.
+
+=back
+
+=back
+
+=head1 B<Methods in need of POD>
+
+=over
 
 =item init_page
 
@@ -971,8 +1050,6 @@ Removes all entries from the C<request_cache>.
 
 =item active_custom_fields
 
-=item request_cache
-
 =item has_flags
 
 =back

Bugzilla/Bug.pm

@@ -353,9 +353,9 @@ sub initialize {
     $_[0]->_create_cf_accessors();
 }
 
-sub cache_key {
+sub object_cache_key {
     my $class = shift;
-    my $key = $class->SUPER::cache_key(@_)
+    my $key = $class->SUPER::object_cache_key(@_)
       || return;
     return $key . ',' . Bugzilla->user->id;
 }
@@ -4422,7 +4422,7 @@ Ensures the accessors for custom fields are always created.
 
 =item set_op_sys
 
-=item cache_key
+=item object_cache_key
 
 =item bug_group
 

Bugzilla/Config/Memcached.pm

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::Config::Memcached;
+
+use 5.10.1;
+use strict;
+
+use Bugzilla::Config::Common;
+
+our $sortkey = 1550;
+
+sub get_param_list {
+  return (
+    {
+        name    => 'memcached_servers',
+        type    => 't',
+        default => ''
+    },
+    {
+        name    => 'memcached_namespace',
+        type    => 't',
+        default => 'bugzilla:',
+    },
+  );
+}
+
+1;

Bugzilla/DB.pm

@@ -1362,14 +1362,19 @@ sub _bz_real_schema {
     my ($self) = @_;
     return $self->{private_real_schema} if exists $self->{private_real_schema};
 
-    my ($data, $version) = $self->selectrow_array(
-        "SELECT schema_data, version FROM bz_schema");
+    my $bz_schema;
+    unless ($bz_schema = Bugzilla->memcached->get({ key => 'bz_schema' })) {
+        $bz_schema = $self->selectrow_arrayref(
+            "SELECT schema_data, version FROM bz_schema"
+        );
+        Bugzilla->memcached->set({ key => 'bz_schema', value => $bz_schema });
+    }
 
     (die "_bz_real_schema tried to read the bz_schema table but it's empty!")
-        if !$data;
+        if !$bz_schema;
 
-    $self->{private_real_schema} = 
-        $self->_bz_schema->deserialize_abstract($data, $version);
+    $self->{private_real_schema} =
+        $self->_bz_schema->deserialize_abstract($bz_schema->[0], $bz_schema->[1]);
 
     return $self->{private_real_schema};
 }
@@ -1411,6 +1416,8 @@ sub _bz_store_real_schema {
     $sth->bind_param(1, $store_me, $self->BLOB_TYPE);
     $sth->bind_param(2, $schema_version);
     $sth->execute();
+
+    Bugzilla->memcached->clear({ key => 'bz_schema' });
 }
 
 # For bz_populate_enum_tables

Bugzilla/Install/Requirements.pm

@@ -394,6 +394,14 @@ sub OPTIONAL_MODULES {
         version => '0',
         feature => ['typesniffer'],
     },
+
+    # memcached
+    {
+        package => 'Cache-Memcached',
+        module  => 'Cache::Memcached',
+        version => '0',
+        feature => ['memcached'],
+    },
     );
 
     my $extra_modules = _get_extension_requirements('OPTIONAL_MODULES');
@@ -417,6 +425,7 @@ use constant FEATURE_FILES => (
                       'Bugzilla/JobQueue/*', 'jobqueue.pl'],
     patch_viewer  => ['Bugzilla/Attachment/PatchReader.pm'],
     updates       => ['Bugzilla/Update.pm'],
+    memcached     => ['Bugzilla/Memcache.pm'],
 );
 
 # This implements the REQUIRED_MODULES and OPTIONAL_MODULES stuff

Bugzilla/Object.pm

@@ -34,6 +34,11 @@ use constant AUDIT_CREATES => 1;
 use constant AUDIT_UPDATES => 1;
 use constant AUDIT_REMOVES => 1;
 
+# When USE_MEMCACHED is true, the class is suitable for serialisation to
+# Memcached. This will be flipped to true by default once the majority of
+# Bugzilla Object have been tested with Memcached.
+use constant USE_MEMCACHED => 0;
+
 # This allows the JSON-RPC interface to return Bugzilla::Object instances
 # as though they were hashes. In the future, this may be modified to return
 # less information.
@@ -48,11 +53,41 @@ sub new {
     my $class    = ref($invocant) || $invocant;
     my $param    = shift;
 
-    my $object = $class->_cache_get($param);
+    my $object = $class->_object_cache_get($param);
     return $object if $object;
 
-    $object = $class->new_from_hash($class->_load_from_db($param));
-    $class->_cache_set($param, $object);
+    my ($data, $set_memcached);
+    if (Bugzilla->feature('memcached')
+        && $class->USE_MEMCACHED
+        && ref($param) eq 'HASH' && $param->{cache})
+    {
+        if (defined $param->{id}) {
+            $data = Bugzilla->memcached->get({
+                table => $class->DB_TABLE,
+                id    => $param->{id},
+            });
+        }
+        elsif (defined $param->{name}) {
+            $data = Bugzilla->memcached->get({
+                table => $class->DB_TABLE,
+                name  => $param->{name},
+            });
+        }
+        $set_memcached = $data ? 0 : 1;
+    }
+    $data ||= $class->_load_from_db($param);
+
+    if ($data && $set_memcached) {
+        Bugzilla->memcached->set({
+            table => $class->DB_TABLE,
+            id    => $data->{$class->ID_FIELD},
+            name  => $data->{$class->NAME_FIELD},
+            data  => $data,
+        });
+    }
+
+    $object = $class->new_from_hash($data);
+    $class->_object_cache_set($param, $object);
 
     return $object;
 }
@@ -157,32 +192,32 @@ sub initialize {
 }
 
 # Provides a mechanism for objects to be cached in the request_cache
-sub _cache_get {
+sub _object_cache_get {
     my $class = shift;
     my ($param) = @_;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     return Bugzilla->request_cache->{$cache_key};
 }
 
-sub _cache_set {
+sub _object_cache_set {
     my $class = shift;
     my ($param, $object) = @_;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     Bugzilla->request_cache->{$cache_key} = $object;
 }
 
-sub _cache_remove {
+sub _object_cache_remove {
     my $class = shift;
     my ($param) = @_;
     $param->{cache} = 1;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     delete Bugzilla->request_cache->{$cache_key};
 }
 
-sub cache_key {
+sub object_cache_key {
     my $class = shift;
     my ($param) = @_;
     if (ref($param) && $param->{cache} && ($param->{id} || $param->{name})) {
@@ -461,8 +496,9 @@ sub update {
     $self->audit_log(\%changes) if $self->AUDIT_UPDATES;
 
     $dbh->bz_commit_transaction();
-    $self->_cache_remove({ id => $self->id });
-    $self->_cache_remove({ name => $self->name }) if $self->name;
+    Bugzilla->memcached->clear({ table => $table, id => $self->id });
+    $self->_object_cache_remove({ id => $self->id });
+    $self->_object_cache_remove({ name => $self->name }) if $self->name;
 
     if (wantarray) {
         return (\%changes, $old_self);
@@ -481,8 +517,9 @@ sub remove_from_db {
     $self->audit_log(AUDIT_REMOVE) if $self->AUDIT_REMOVES;
     $dbh->do("DELETE FROM $table WHERE $id_field = ?", undef, $self->id);
     $dbh->bz_commit_transaction();
-    $self->_cache_remove({ id => $self->id });
-    $self->_cache_remove({ name => $self->name }) if $self->name;
+    Bugzilla->memcached->clear({ table => $table, id => $self->id });
+    $self->_object_cache_remove({ id => $self->id });
+    $self->_object_cache_remove({ name => $self->name }) if $self->name;
     undef $self;
 }
 
@@ -1399,7 +1436,7 @@ C<0> otherwise.
 
 =over
 
-=item cache_key
+=item object_cache_key
 
 =item check_time
 

checksetup.pl

@@ -206,6 +206,9 @@ Bugzilla::Hook::process('install_before_final_checks', { silent => $silent });
 # Final checks
 ###########################################################################
 
+# Clear all keys from Memcached
+Bugzilla->memcached->clear_all();
+
 # Check if the default parameter for urlbase is still set, and if so, give
 # notification that they should go and visit editparams.cgi 
 if (Bugzilla->params->{'urlbase'} eq '') {

template/en/default/admin/params/memcached.html.tmpl

+[%# This Source Code Form is subject to the terms of the Mozilla Public
+  # License, v. 2.0. If a copy of the MPL was not distributed with this
+  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  #
+  # This Source Code Form is "Incompatible With Secondary Licenses", as
+  # defined by the Mozilla Public License, v. 2.0.
+  #%]
+[%
+   title = "Memcached"
+   desc = "Set up Memcached integration"
+%]
+
+[% param_descs = {
+    memcached_servers =>
+      "If this option is set, $terms.Bugzilla will integrate with Memcached. " _
+      "Specify one of more server, separated by spaces, using hostname:port " _
+      "notation (for example: 127.0.0.1:11211).",
+
+    memcached_namespace =>
+      "Specify a string to prefix to each key on Memcached.",
+  }
+%]

template/en/default/setup/strings.txt.pl

@@ -91,6 +91,7 @@ END
     feature_jsonrpc_faster    => 'Make JSON-RPC Faster',
     feature_new_charts        => 'New Charts',
     feature_old_charts        => 'Old Charts',
+    feature_memcached         => 'Memcached Support',
     feature_mod_perl          => 'mod_perl',
     feature_moving            => 'Move Bugs Between Installations',
     feature_patch_viewer      => 'Patch Viewer',