Commit 2ccf81de authored by Frédéric Buclin's avatar Frédéric Buclin

Bug 902395: Enforce utf8 = true for all installations and remove the utf8 parameter

r=dkl a=sgreen
parent 1d96fa16
......@@ -77,7 +77,7 @@ sub init_page {
if (Bugzilla->usage_mode == USAGE_MODE_CMDLINE) {
init_console();
}
elsif (Bugzilla->params->{'utf8'}) {
else {
binmode STDOUT, ':utf8';
}
......
......@@ -73,12 +73,10 @@ sub process_diff {
$vars->{'other_patches'} = \@other_patches;
setup_template_patch_reader($reader, $vars);
# The patch is going to be displayed in a HTML page and if the utf8
# param is enabled, we have to encode attachment data as utf8.
if (Bugzilla->params->{'utf8'}) {
# The patch is going to be displayed in a HTML page, so we have
# to encode attachment data as utf8.
$attachment->data; # Populate ->{data}
utf8::decode($attachment->{data});
}
$reader->iterate_string('Attachment ' . $attachment->id, $attachment->data);
}
}
......@@ -91,9 +89,8 @@ sub process_interdiff {
require PatchReader::Raw;
# Encode attachment data as utf8 if it's going to be displayed in a HTML
# page using the UTF-8 encoding.
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) {
# Encode attachment data as utf8 if it's going to be displayed in a HTML page.
if ($format ne 'raw') {
$old_attachment->data; # Populate ->{data}
utf8::decode($old_attachment->{data});
$new_attachment->data; # Populate ->{data}
......@@ -130,7 +127,7 @@ sub process_interdiff {
$use_select = 1;
}
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) {
if ($format ne 'raw') {
binmode $interdiff_stdout, ':utf8';
binmode $interdiff_stderr, ':utf8';
} else {
......@@ -230,7 +227,7 @@ sub get_unified_diff {
# Prints out to temporary file.
my ($fh, $filename) = File::Temp::tempfile();
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) {
if ($format ne 'raw') {
# The HTML page will be displayed with the UTF-8 encoding.
binmode $fh, ':utf8';
}
......
......@@ -472,7 +472,7 @@ sub _generate_bugmail {
} else {
$email->content_type_set('multipart/alternative');
# Some mail clients need same encoding for each part, even empty ones.
$email->charset_set('UTF-8') if Bugzilla->params->{'utf8'};
$email->charset_set('UTF-8');
}
$email->parts_set(\@parts);
return $email;
......
......@@ -73,7 +73,7 @@ sub new {
}
# Send appropriate charset
$self->charset(Bugzilla->params->{'utf8'} ? 'UTF-8' : '');
$self->charset('UTF-8');
# Redirect to urlbase/sslbase if we are not viewing an attachment.
if ($self->url_is_attachment_base and $script ne 'attachment.cgi') {
......@@ -378,9 +378,7 @@ sub param {
}
# Fix UTF-8-ness of input parameters.
if (Bugzilla->params->{'utf8'}) {
@result = map { _fix_utf8($_) } @result;
}
return wantarray ? @result : $result[0];
}
......
......@@ -231,8 +231,6 @@ sub update_params {
}
}
$param->{'utf8'} = 1 if $new_install;
# Bug 452525: OR based groups are on by default for new installations
$param->{'or_groups'} = 1 if $new_install;
......
......@@ -25,7 +25,7 @@ use parent qw(Exporter);
qw(check_multi check_numeric check_regexp check_group
check_sslbase check_priority check_severity check_platform
check_opsys check_shadowdb check_urlbase check_user_verify_class
check_ip check_mail_delivery_method check_notification check_utf8
check_ip check_mail_delivery_method check_notification
check_bug_status check_smtp_auth check_theschwartz_available
check_maxattachmentsize check_email check_smtp_ssl
check_comment_taggers_group check_smtp_server
......@@ -115,18 +115,6 @@ sub check_ip {
return "";
}
sub check_utf8 {
my $utf8 = shift;
# You cannot turn off the UTF-8 parameter if you've already converted
# your tables to utf-8.
my $dbh = Bugzilla->dbh;
if ($dbh->isa('Bugzilla::DB::Mysql') && $dbh->bz_db_is_utf8 && !$utf8) {
return "You cannot disable UTF-8 support, because your MySQL database"
. " is encoded in UTF-8";
}
return "";
}
sub check_priority {
my ($value) = (@_);
my $legal_priorities = get_legal_field_values('priority');
......@@ -490,8 +478,6 @@ valid group is provided.
=item check_maxattachmentsize
=item check_utf8
=item check_group
=item check_opsys
......
......@@ -25,13 +25,6 @@ use constant get_param_list => (
},
{
name => 'utf8',
type => 'b',
default => '0',
checker => \&check_utf8
},
{
name => 'shutdownhtml',
type => 'l',
default => ''
......
......@@ -54,7 +54,7 @@ sub new {
$dsn .= ";mysql_socket=$sock" if $sock;
my %attrs = (
mysql_enable_utf8 => Bugzilla->params->{'utf8'},
mysql_enable_utf8 => 1,
# Needs to be explicitly specified for command-line processes.
mysql_auto_reconnect => 1,
);
......@@ -74,9 +74,8 @@ sub new {
my $self = $class->db_new({ dsn => $dsn, user => $user,
pass => $pass, attrs => \%attrs });
# This makes sure that if the tables are encoded as UTF-8, we
# return their data correctly.
$self->do("SET NAMES utf8") if Bugzilla->params->{'utf8'};
# This makes sure that we return table names correctly.
$self->do("SET NAMES utf8");
# all class local variables stored in DBI derived class needs to have
# a prefix 'private_'. See DBI documentation.
......@@ -543,13 +542,11 @@ sub bz_setup_database {
# If there are no tables, but the DB isn't utf8 and it should be,
# then we should alter the database to be utf8. We know it should be
# if the utf8 parameter is true or there are no params at all.
# if there are no params at all.
# This kind of situation happens when people create the database
# themselves, and if we don't do this they will get the big
# scary WARNING statement about conversion to UTF8.
if ( !$self->bz_db_is_utf8 && !@tables
&& (Bugzilla->params->{'utf8'} || !scalar keys %{Bugzilla->params}) )
{
if (!$self->bz_db_is_utf8 && !@tables) {
$self->_alter_db_charset_to_utf8();
}
......@@ -653,7 +650,7 @@ sub bz_setup_database {
MAX_ROWS=100000");
}
# Convert the database to UTF-8 if the utf8 parameter is on.
# Convert the database to UTF-8.
# We check if any table isn't utf8, because lots of crazy
# partial-conversion situations can happen, and this handles anything
# that could come up (including having the DB charset be utf8 but not
......@@ -666,7 +663,7 @@ sub bz_setup_database {
AND TABLE_COLLATION NOT LIKE 'utf8%'
LIMIT 1", undef, $db_name);
if (Bugzilla->params->{'utf8'} && $non_utf8_tables) {
if ($non_utf8_tables) {
print "\n", install_string('mysql_utf8_conversion');
if (!Bugzilla->installation_answers->{NO_PAUSE}) {
......@@ -681,8 +678,7 @@ sub bz_setup_database {
}
}
print "Converting table storage format to UTF-8. This may take a",
" while.\n";
say 'Converting table storage format to UTF-8. This may take a while.';
foreach my $table ($self->bz_table_list_real) {
my $info_sth = $self->prepare("SHOW FULL COLUMNS FROM $table");
$info_sth->execute();
......@@ -699,7 +695,7 @@ sub bz_setup_database {
{
my $name = $column->{Field};
print "$table.$name needs to be converted to UTF-8...\n";
say "$table.$name needs to be converted to UTF-8...";
# These will be automatically re-created at the end
# of checksetup.
......@@ -737,7 +733,7 @@ sub bz_setup_database {
}
}
print "Converting the $table table to UTF-8...\n";
say "Converting the $table table to UTF-8...";
my $bin = "ALTER TABLE $table " . join(', ', @binary_sql);
my $utf = "ALTER TABLE $table " . join(', ', @utf8_sql,
'DEFAULT CHARACTER SET utf8');
......@@ -761,9 +757,7 @@ sub bz_setup_database {
# a mysqldump.) So we have this change outside of the above block,
# so that it just happens silently if no actual *table* conversion
# needs to happen.
if (Bugzilla->params->{'utf8'} && !$self->bz_db_is_utf8) {
$self->_alter_db_charset_to_utf8();
}
$self->_alter_db_charset_to_utf8() unless $self->bz_db_is_utf8;
$self->_fix_defaults();
......
......@@ -55,7 +55,7 @@ sub new {
$dbname ||= Bugzilla->localconfig->{db_name};
# Set the language enviroment
$ENV{'NLS_LANG'} = '.AL32UTF8' if Bugzilla->params->{'utf8'};
$ENV{'NLS_LANG'} = '.AL32UTF8';
# construct the DSN from the parameters we got
my $dsn = "dbi:Oracle:host=$host;sid=$dbname";
......@@ -74,8 +74,7 @@ sub new {
# Set the session's default date format to match MySQL
$self->do("ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'");
$self->do("ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS'");
$self->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'")
if Bugzilla->params->{'utf8'};
$self->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'");
# To allow case insensitive query.
$self->do("ALTER SESSION SET NLS_COMP='ANSI'");
$self->do("ALTER SESSION SET NLS_SORT='BINARY_AI'");
......
......@@ -53,7 +53,7 @@ sub new {
# creating tables.
$dsn .= ";options='-c client_min_messages=warning'";
my $attrs = { pg_enable_utf8 => Bugzilla->params->{'utf8'} };
my $attrs = { pg_enable_utf8 => 1 };
my $self = $class->db_new({ dsn => $dsn, user => $user,
pass => $pass, attrs => $attrs });
......
......@@ -148,12 +148,7 @@ sub _get_create_index_ddl {
sub get_create_database_sql {
my ($self, $name) = @_;
# We only create as utf8 if we have no params (meaning we're doing
# a new installation) or if the utf8 param is on.
my $create_utf8 = Bugzilla->params->{'utf8'}
|| !defined Bugzilla->params->{'utf8'};
my $charset = $create_utf8 ? "CHARACTER SET utf8" : '';
return ("CREATE DATABASE `$name` $charset");
return ("CREATE DATABASE `$name` CHARACTER SET utf8");
}
# MySQL has a simpler ALTER TABLE syntax than ANSI.
......
......@@ -79,12 +79,7 @@ sub _initialize {
sub get_create_database_sql {
my ($self, $name) = @_;
# We only create as utf8 if we have no params (meaning we're doing
# a new installation) or if the utf8 param is on.
my $create_utf8 = Bugzilla->params->{'utf8'}
|| !defined Bugzilla->params->{'utf8'};
my $charset = $create_utf8 ? "ENCODING 'UTF8' TEMPLATE template0" : '';
return ("CREATE DATABASE \"$name\" $charset");
return ("CREATE DATABASE \"$name\" ENCODING 'UTF8' TEMPLATE template0");
}
sub get_rename_column_ddl {
......
......@@ -93,8 +93,7 @@ sub new {
my $dsn = "dbi:SQLite:dbname=$db_name";
my $attrs = {
# XXX Should we just enforce this to be always on?
sqlite_unicode => Bugzilla->params->{'utf8'},
sqlite_unicode => 1,
};
my $self = $class->db_new({ dsn => $dsn, user => '',
......
......@@ -125,9 +125,7 @@ sub MessageToMTA {
# We don't recode headers that happen multiple times.
next if scalar(@values) > 1;
if (my $value = $values[0]) {
if (Bugzilla->params->{'utf8'} && !utf8::is_utf8($value)) {
utf8::decode($value);
}
utf8::decode($value) unless utf8::is_utf8($value);
# avoid excessive line wrapping done by Encode.
local $Encode::Encoding{'MIME-Q'}->{'bpl'} = 998;
......@@ -187,11 +185,10 @@ sub MessageToMTA {
my $content_type = $part->content_type || '';
$content_type =~ /charset=['"](.+)['"]/;
# If no charset is defined or is the default us-ascii,
# then we encode the email to UTF-8 if Bugzilla has utf8 enabled.
# then we encode the email to UTF-8.
# XXX - This is a hack to workaround bug 723944.
if (!$1 || $1 eq 'us-ascii') {
my $body = $part->body;
if (Bugzilla->params->{'utf8'}) {
$part->charset_set('UTF-8');
# encoding_set works only with bytes, not with utf8 strings.
my $raw = $part->body_raw;
......@@ -199,7 +196,6 @@ sub MessageToMTA {
utf8::encode($raw);
$part->body_set($raw);
}
}
$part->encoding_set('quoted-printable') if !is_7bit_clean($body);
}
});
......
......@@ -752,7 +752,7 @@ sub create {
# Initialize templates (f.e. by loading plugins like Hook).
PRE_PROCESS => ["global/variables.none.tmpl"],
ENCODING => Bugzilla->params->{'utf8'} ? 'UTF-8' : undef,
ENCODING => 'UTF-8',
# Functions for processing text within templates in various ways.
# IMPORTANT! When adding a filter here that does not override a
......@@ -783,12 +783,7 @@ sub create {
# Strips out control characters excepting whitespace
strip_control_chars => sub {
my ($data) = @_;
state $use_utf8 = Bugzilla->params->{'utf8'};
# Only run for utf8 to avoid issues with other multibyte encodings
# that may be reassigning meaning to ascii characters.
if ($use_utf8) {
$data =~ s/(?![\t\r\n])[[:cntrl:]]//g;
}
return $data;
},
......
......@@ -185,9 +185,7 @@ sub issue_hash_token {
my $token = join('*', @args);
# Wide characters cause Digest::SHA to die.
if (Bugzilla->params->{'utf8'}) {
utf8::encode($token) if utf8::is_utf8($token);
}
$token = hmac_sha256_base64($token, Bugzilla->localconfig->{'site_wide_secret'});
$token =~ s/\+/-/g;
$token =~ s/\//_/g;
......
......@@ -73,11 +73,7 @@ sub html_quote {
# Obscure '@'.
$var =~ s/\@/\@/g;
state $use_utf8 = Bugzilla->params->{'utf8'};
if ($use_utf8) {
# Remove control characters if the encoding is utf8.
# Other multibyte encodings may be using this range; so ignore if not utf8.
# Remove control characters.
$var =~ s/(?![\t\r\n])[[:cntrl:]]//g;
# Remove the following characters because they're
......@@ -102,7 +98,6 @@ sub html_quote {
# |U+200f|Right-To-Left Mark |0xe2 0x80 0x8f |
# --------------------------------------------------------
$var =~ tr/\x{202a}-\x{202e}//d;
}
return $var;
}
......@@ -203,8 +198,8 @@ sub email_filter {
# This originally came from CGI.pm, by Lincoln D. Stein
sub url_quote {
my ($toencode) = (@_);
utf8::encode($toencode) # The below regex works only on bytes
if Bugzilla->params->{'utf8'} && utf8::is_utf8($toencode);
# The below regex works only on bytes
utf8::encode($toencode) if utf8::is_utf8($toencode);
$toencode =~ s/([^a-zA-Z0-9_\-.])/uc sprintf("%%%02x",ord($1))/eg;
return $toencode;
}
......@@ -635,9 +630,7 @@ sub bz_crypt {
}
# Wide characters cause crypt and Digest to die.
if (Bugzilla->params->{'utf8'}) {
utf8::encode($password) if utf8::is_utf8($password);
}
my $crypted_password;
if (!$algorithm) {
......@@ -785,9 +778,8 @@ sub display_value {
}
sub disable_utf8 {
if (Bugzilla->params->{'utf8'}) {
binmode STDOUT, ':bytes'; # Turn off UTF8 encoding.
}
# Turn off UTF8 encoding.
binmode STDOUT, ':bytes';
}
use constant UTF8_ACCIDENTAL => qw(shiftjis big5-eten euc-kr euc-jp);
......
......@@ -54,7 +54,7 @@ sub create_json_coder {
# This may seem a little backwards, but what this really means is
# "don't convert our utf8 into byte strings, just leave it as a
# utf8 string."
$json->utf8(0) if Bugzilla->params->{'utf8'};
$json->utf8(0);
return $json;
}
......
......@@ -173,9 +173,7 @@ sub collect_stats {
|| ThrowCodeError('chart_file_open_fail', {'filename' => $file});
}
if (Bugzilla->params->{'utf8'}) {
binmode DATA, ':utf8';
}
# Now collect current data.
my @row = (today());
......@@ -234,9 +232,7 @@ sub get_old_data {
open(DATA, '<', $file)
|| ThrowCodeError('chart_file_open_fail', {'filename' => $file});
if (Bugzilla->params->{'utf8'}) {
binmode DATA, ':utf8';
}
my @data;
my @columns;
......
......@@ -59,16 +59,6 @@ maintainer
responsible for maintaining this Bugzilla installation.
The address need not be that of a valid Bugzilla account.
utf8
Use UTF-8 (Unicode) encoding for all text in Bugzilla. Installations where
this parameter is set to :paramval:`off` should set it to :paramval:`on` only
after the data has been converted from existing legacy character
encodings to UTF-8, using the
:file:`contrib/recode.pl` script.
.. note:: If you turn this parameter from :paramval:`off` to :paramval:`on`,
you must re-run :file:`checksetup.pl` immediately afterward.
shutdownhtml
If there is any text in this field, this Bugzilla installation will
be completely disabled and this text will appear instead of all
......
......@@ -398,10 +398,7 @@ sub get_text_alternative {
sub _decode_body {
my ($charset, $body) = @_;
if (Bugzilla->params->{'utf8'} && !utf8::is_utf8($body)) {
return Encode::decode($charset, $body);
}
return $body;
return utf8::is_utf8($body) ? $body : Encode::decode($charset, $body);
}
sub remove_leading_blank_lines {
......
......@@ -191,9 +191,7 @@ foreach my $k (@bug_ids) {
if ($summary ne "" && $cgi->param('showsummary')) {
# Wide characters cause GraphViz to die.
if (Bugzilla->params->{'utf8'}) {
utf8::encode($summary) if utf8::is_utf8($summary);
}
$summary = wrap_comment($summary);
$summary =~ s/([\\\"])/\\$1/g;
push(@params, qq{label="$k\\n$summary"});
......
......@@ -16,17 +16,6 @@
"The email address of the person who maintains this installation "
_ " of Bugzilla.",
utf8 =>
"Use UTF-8 (Unicode) encoding for all text in ${terms.Bugzilla}. New"
_ " installations should set this to true to avoid character encoding"
_ " problems. <strong>Existing databases should set this to true"
_ " only after the data has been converted from existing legacy"
_ " character encodings to UTF-8, using the <kbd>contrib/recode.pl</kbd>"
_ " script</strong>."
_ " <p>Note that if you turn this parameter from &quot;off&quot; to"
_ " &quot;on&quot;, you must re-run <kbd>checksetup.pl</kbd> immediately"
_ " afterward.</p>",
shutdownhtml =>
"If this field is non-empty, then Bugzilla will be completely"
_ " disabled and this text will be displayed instead of all the"
......
......@@ -8,7 +8,7 @@
[% PROCESS bug/time.html.tmpl %]
[% USE Bugzilla %]
[% cgi = Bugzilla.cgi %]
<?xml version="1.0" [% IF Param('utf8') %]encoding="UTF-8" [% END %]standalone="yes" ?>
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
<!DOCTYPE bugzilla [% IF cgi.param('dtd') %][[% PROCESS pages/bugzilla.dtd.tmpl %]][% ELSE %]SYSTEM "[% urlbase FILTER xml %]page.cgi?id=bugzilla.dtd"[% END %]>
<bugzilla version="[% constants.BUGZILLA_VERSION %]"
......
......@@ -10,7 +10,7 @@
# So we cache it here for better performance.
%]
[% escaped_urlbase = BLOCK %][% urlbase FILTER xml %][% END %]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?>
<?xml version="1.0" encoding="UTF-8"?>
<!-- Note: this interface is experimental and under development.
- We may and probably will make breaking changes to it in the future. -->
......
......@@ -103,10 +103,7 @@
<head>
[% Hook.process("start") %]
<title>[% title %]</title>
[% IF Param('utf8') %]
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
[% END %]
[% IF NOT no_yui %]
[% SET yui = yui_resolve_deps(yui, yui_deps) %]
......
......@@ -543,9 +543,6 @@
[% IF param_changed.size > 0 %]
[% FOREACH param = param_changed %]
Changed <em>[% param FILTER html %]</em><br>
[% IF param == 'utf8' && Param('utf8') %]
<strong>You must now re-run <kbd>checksetup.pl</kbd>.</strong><br>
[% END %]
[% END %]
[% ELSE %]
No changes made.
......
......@@ -8,7 +8,7 @@
[% DEFAULT title = "$terms.Bugzilla $terms.Bugs" %]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?>
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>[% title FILTER xml %]</title>
<link rel="alternate" type="text/html"
......
......@@ -6,7 +6,7 @@
# defined by the Mozilla Public License, v. 2.0.
#%]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?>
<?xml version="1.0" encoding="UTF-8"?>
<!-- [% template_version %] -->
<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment