Commit 2ccf81de authored by Frédéric Buclin's avatar Frédéric Buclin

Bug 902395: Enforce utf8 = true for all installations and remove the utf8 parameter

r=dkl a=sgreen
parent 1d96fa16
...@@ -77,7 +77,7 @@ sub init_page { ...@@ -77,7 +77,7 @@ sub init_page {
if (Bugzilla->usage_mode == USAGE_MODE_CMDLINE) { if (Bugzilla->usage_mode == USAGE_MODE_CMDLINE) {
init_console(); init_console();
} }
elsif (Bugzilla->params->{'utf8'}) { else {
binmode STDOUT, ':utf8'; binmode STDOUT, ':utf8';
} }
......
...@@ -73,12 +73,10 @@ sub process_diff { ...@@ -73,12 +73,10 @@ sub process_diff {
$vars->{'other_patches'} = \@other_patches; $vars->{'other_patches'} = \@other_patches;
setup_template_patch_reader($reader, $vars); setup_template_patch_reader($reader, $vars);
# The patch is going to be displayed in a HTML page and if the utf8 # The patch is going to be displayed in a HTML page, so we have
# param is enabled, we have to encode attachment data as utf8. # to encode attachment data as utf8.
if (Bugzilla->params->{'utf8'}) { $attachment->data; # Populate ->{data}
$attachment->data; # Populate ->{data} utf8::decode($attachment->{data});
utf8::decode($attachment->{data});
}
$reader->iterate_string('Attachment ' . $attachment->id, $attachment->data); $reader->iterate_string('Attachment ' . $attachment->id, $attachment->data);
} }
} }
...@@ -91,9 +89,8 @@ sub process_interdiff { ...@@ -91,9 +89,8 @@ sub process_interdiff {
require PatchReader::Raw; require PatchReader::Raw;
# Encode attachment data as utf8 if it's going to be displayed in a HTML # Encode attachment data as utf8 if it's going to be displayed in a HTML page.
# page using the UTF-8 encoding. if ($format ne 'raw') {
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) {
$old_attachment->data; # Populate ->{data} $old_attachment->data; # Populate ->{data}
utf8::decode($old_attachment->{data}); utf8::decode($old_attachment->{data});
$new_attachment->data; # Populate ->{data} $new_attachment->data; # Populate ->{data}
...@@ -130,7 +127,7 @@ sub process_interdiff { ...@@ -130,7 +127,7 @@ sub process_interdiff {
$use_select = 1; $use_select = 1;
} }
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) { if ($format ne 'raw') {
binmode $interdiff_stdout, ':utf8'; binmode $interdiff_stdout, ':utf8';
binmode $interdiff_stderr, ':utf8'; binmode $interdiff_stderr, ':utf8';
} else { } else {
...@@ -230,7 +227,7 @@ sub get_unified_diff { ...@@ -230,7 +227,7 @@ sub get_unified_diff {
# Prints out to temporary file. # Prints out to temporary file.
my ($fh, $filename) = File::Temp::tempfile(); my ($fh, $filename) = File::Temp::tempfile();
if ($format ne 'raw' && Bugzilla->params->{'utf8'}) { if ($format ne 'raw') {
# The HTML page will be displayed with the UTF-8 encoding. # The HTML page will be displayed with the UTF-8 encoding.
binmode $fh, ':utf8'; binmode $fh, ':utf8';
} }
......
...@@ -472,7 +472,7 @@ sub _generate_bugmail { ...@@ -472,7 +472,7 @@ sub _generate_bugmail {
} else { } else {
$email->content_type_set('multipart/alternative'); $email->content_type_set('multipart/alternative');
# Some mail clients need same encoding for each part, even empty ones. # Some mail clients need same encoding for each part, even empty ones.
$email->charset_set('UTF-8') if Bugzilla->params->{'utf8'}; $email->charset_set('UTF-8');
} }
$email->parts_set(\@parts); $email->parts_set(\@parts);
return $email; return $email;
......
...@@ -73,7 +73,7 @@ sub new { ...@@ -73,7 +73,7 @@ sub new {
} }
# Send appropriate charset # Send appropriate charset
$self->charset(Bugzilla->params->{'utf8'} ? 'UTF-8' : ''); $self->charset('UTF-8');
# Redirect to urlbase/sslbase if we are not viewing an attachment. # Redirect to urlbase/sslbase if we are not viewing an attachment.
if ($self->url_is_attachment_base and $script ne 'attachment.cgi') { if ($self->url_is_attachment_base and $script ne 'attachment.cgi') {
...@@ -378,9 +378,7 @@ sub param { ...@@ -378,9 +378,7 @@ sub param {
} }
# Fix UTF-8-ness of input parameters. # Fix UTF-8-ness of input parameters.
if (Bugzilla->params->{'utf8'}) { @result = map { _fix_utf8($_) } @result;
@result = map { _fix_utf8($_) } @result;
}
return wantarray ? @result : $result[0]; return wantarray ? @result : $result[0];
} }
......
...@@ -231,8 +231,6 @@ sub update_params { ...@@ -231,8 +231,6 @@ sub update_params {
} }
} }
$param->{'utf8'} = 1 if $new_install;
# Bug 452525: OR based groups are on by default for new installations # Bug 452525: OR based groups are on by default for new installations
$param->{'or_groups'} = 1 if $new_install; $param->{'or_groups'} = 1 if $new_install;
......
...@@ -25,7 +25,7 @@ use parent qw(Exporter); ...@@ -25,7 +25,7 @@ use parent qw(Exporter);
qw(check_multi check_numeric check_regexp check_group qw(check_multi check_numeric check_regexp check_group
check_sslbase check_priority check_severity check_platform check_sslbase check_priority check_severity check_platform
check_opsys check_shadowdb check_urlbase check_user_verify_class check_opsys check_shadowdb check_urlbase check_user_verify_class
check_ip check_mail_delivery_method check_notification check_utf8 check_ip check_mail_delivery_method check_notification
check_bug_status check_smtp_auth check_theschwartz_available check_bug_status check_smtp_auth check_theschwartz_available
check_maxattachmentsize check_email check_smtp_ssl check_maxattachmentsize check_email check_smtp_ssl
check_comment_taggers_group check_smtp_server check_comment_taggers_group check_smtp_server
...@@ -115,18 +115,6 @@ sub check_ip { ...@@ -115,18 +115,6 @@ sub check_ip {
return ""; return "";
} }
sub check_utf8 {
my $utf8 = shift;
# You cannot turn off the UTF-8 parameter if you've already converted
# your tables to utf-8.
my $dbh = Bugzilla->dbh;
if ($dbh->isa('Bugzilla::DB::Mysql') && $dbh->bz_db_is_utf8 && !$utf8) {
return "You cannot disable UTF-8 support, because your MySQL database"
. " is encoded in UTF-8";
}
return "";
}
sub check_priority { sub check_priority {
my ($value) = (@_); my ($value) = (@_);
my $legal_priorities = get_legal_field_values('priority'); my $legal_priorities = get_legal_field_values('priority');
...@@ -490,8 +478,6 @@ valid group is provided. ...@@ -490,8 +478,6 @@ valid group is provided.
=item check_maxattachmentsize =item check_maxattachmentsize
=item check_utf8
=item check_group =item check_group
=item check_opsys =item check_opsys
......
...@@ -25,13 +25,6 @@ use constant get_param_list => ( ...@@ -25,13 +25,6 @@ use constant get_param_list => (
}, },
{ {
name => 'utf8',
type => 'b',
default => '0',
checker => \&check_utf8
},
{
name => 'shutdownhtml', name => 'shutdownhtml',
type => 'l', type => 'l',
default => '' default => ''
......
...@@ -54,7 +54,7 @@ sub new { ...@@ -54,7 +54,7 @@ sub new {
$dsn .= ";mysql_socket=$sock" if $sock; $dsn .= ";mysql_socket=$sock" if $sock;
my %attrs = ( my %attrs = (
mysql_enable_utf8 => Bugzilla->params->{'utf8'}, mysql_enable_utf8 => 1,
# Needs to be explicitly specified for command-line processes. # Needs to be explicitly specified for command-line processes.
mysql_auto_reconnect => 1, mysql_auto_reconnect => 1,
); );
...@@ -74,9 +74,8 @@ sub new { ...@@ -74,9 +74,8 @@ sub new {
my $self = $class->db_new({ dsn => $dsn, user => $user, my $self = $class->db_new({ dsn => $dsn, user => $user,
pass => $pass, attrs => \%attrs }); pass => $pass, attrs => \%attrs });
# This makes sure that if the tables are encoded as UTF-8, we # This makes sure that we return table names correctly.
# return their data correctly. $self->do("SET NAMES utf8");
$self->do("SET NAMES utf8") if Bugzilla->params->{'utf8'};
# all class local variables stored in DBI derived class needs to have # all class local variables stored in DBI derived class needs to have
# a prefix 'private_'. See DBI documentation. # a prefix 'private_'. See DBI documentation.
...@@ -543,13 +542,11 @@ sub bz_setup_database { ...@@ -543,13 +542,11 @@ sub bz_setup_database {
# If there are no tables, but the DB isn't utf8 and it should be, # If there are no tables, but the DB isn't utf8 and it should be,
# then we should alter the database to be utf8. We know it should be # then we should alter the database to be utf8. We know it should be
# if the utf8 parameter is true or there are no params at all. # if there are no params at all.
# This kind of situation happens when people create the database # This kind of situation happens when people create the database
# themselves, and if we don't do this they will get the big # themselves, and if we don't do this they will get the big
# scary WARNING statement about conversion to UTF8. # scary WARNING statement about conversion to UTF8.
if ( !$self->bz_db_is_utf8 && !@tables if (!$self->bz_db_is_utf8 && !@tables) {
&& (Bugzilla->params->{'utf8'} || !scalar keys %{Bugzilla->params}) )
{
$self->_alter_db_charset_to_utf8(); $self->_alter_db_charset_to_utf8();
} }
...@@ -653,7 +650,7 @@ sub bz_setup_database { ...@@ -653,7 +650,7 @@ sub bz_setup_database {
MAX_ROWS=100000"); MAX_ROWS=100000");
} }
# Convert the database to UTF-8 if the utf8 parameter is on. # Convert the database to UTF-8.
# We check if any table isn't utf8, because lots of crazy # We check if any table isn't utf8, because lots of crazy
# partial-conversion situations can happen, and this handles anything # partial-conversion situations can happen, and this handles anything
# that could come up (including having the DB charset be utf8 but not # that could come up (including having the DB charset be utf8 but not
...@@ -665,8 +662,8 @@ sub bz_setup_database { ...@@ -665,8 +662,8 @@ sub bz_setup_database {
WHERE TABLE_SCHEMA = ? AND TABLE_COLLATION IS NOT NULL WHERE TABLE_SCHEMA = ? AND TABLE_COLLATION IS NOT NULL
AND TABLE_COLLATION NOT LIKE 'utf8%' AND TABLE_COLLATION NOT LIKE 'utf8%'
LIMIT 1", undef, $db_name); LIMIT 1", undef, $db_name);
if (Bugzilla->params->{'utf8'} && $non_utf8_tables) { if ($non_utf8_tables) {
print "\n", install_string('mysql_utf8_conversion'); print "\n", install_string('mysql_utf8_conversion');
if (!Bugzilla->installation_answers->{NO_PAUSE}) { if (!Bugzilla->installation_answers->{NO_PAUSE}) {
...@@ -681,8 +678,7 @@ sub bz_setup_database { ...@@ -681,8 +678,7 @@ sub bz_setup_database {
} }
} }
print "Converting table storage format to UTF-8. This may take a", say 'Converting table storage format to UTF-8. This may take a while.';
" while.\n";
foreach my $table ($self->bz_table_list_real) { foreach my $table ($self->bz_table_list_real) {
my $info_sth = $self->prepare("SHOW FULL COLUMNS FROM $table"); my $info_sth = $self->prepare("SHOW FULL COLUMNS FROM $table");
$info_sth->execute(); $info_sth->execute();
...@@ -699,7 +695,7 @@ sub bz_setup_database { ...@@ -699,7 +695,7 @@ sub bz_setup_database {
{ {
my $name = $column->{Field}; my $name = $column->{Field};
print "$table.$name needs to be converted to UTF-8...\n"; say "$table.$name needs to be converted to UTF-8...";
# These will be automatically re-created at the end # These will be automatically re-created at the end
# of checksetup. # of checksetup.
...@@ -737,7 +733,7 @@ sub bz_setup_database { ...@@ -737,7 +733,7 @@ sub bz_setup_database {
} }
} }
print "Converting the $table table to UTF-8...\n"; say "Converting the $table table to UTF-8...";
my $bin = "ALTER TABLE $table " . join(', ', @binary_sql); my $bin = "ALTER TABLE $table " . join(', ', @binary_sql);
my $utf = "ALTER TABLE $table " . join(', ', @utf8_sql, my $utf = "ALTER TABLE $table " . join(', ', @utf8_sql,
'DEFAULT CHARACTER SET utf8'); 'DEFAULT CHARACTER SET utf8');
...@@ -761,11 +757,9 @@ sub bz_setup_database { ...@@ -761,11 +757,9 @@ sub bz_setup_database {
# a mysqldump.) So we have this change outside of the above block, # a mysqldump.) So we have this change outside of the above block,
# so that it just happens silently if no actual *table* conversion # so that it just happens silently if no actual *table* conversion
# needs to happen. # needs to happen.
if (Bugzilla->params->{'utf8'} && !$self->bz_db_is_utf8) { $self->_alter_db_charset_to_utf8() unless $self->bz_db_is_utf8;
$self->_alter_db_charset_to_utf8();
}
$self->_fix_defaults(); $self->_fix_defaults();
# Bug 451735 highlighted a bug in bz_drop_index() which didn't # Bug 451735 highlighted a bug in bz_drop_index() which didn't
# check for FKs before trying to delete an index. Consequently, # check for FKs before trying to delete an index. Consequently,
......
...@@ -55,7 +55,7 @@ sub new { ...@@ -55,7 +55,7 @@ sub new {
$dbname ||= Bugzilla->localconfig->{db_name}; $dbname ||= Bugzilla->localconfig->{db_name};
# Set the language enviroment # Set the language enviroment
$ENV{'NLS_LANG'} = '.AL32UTF8' if Bugzilla->params->{'utf8'}; $ENV{'NLS_LANG'} = '.AL32UTF8';
# construct the DSN from the parameters we got # construct the DSN from the parameters we got
my $dsn = "dbi:Oracle:host=$host;sid=$dbname"; my $dsn = "dbi:Oracle:host=$host;sid=$dbname";
...@@ -74,8 +74,7 @@ sub new { ...@@ -74,8 +74,7 @@ sub new {
# Set the session's default date format to match MySQL # Set the session's default date format to match MySQL
$self->do("ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"); $self->do("ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'");
$self->do("ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS'"); $self->do("ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS'");
$self->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'") $self->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'");
if Bugzilla->params->{'utf8'};
# To allow case insensitive query. # To allow case insensitive query.
$self->do("ALTER SESSION SET NLS_COMP='ANSI'"); $self->do("ALTER SESSION SET NLS_COMP='ANSI'");
$self->do("ALTER SESSION SET NLS_SORT='BINARY_AI'"); $self->do("ALTER SESSION SET NLS_SORT='BINARY_AI'");
......
...@@ -53,7 +53,7 @@ sub new { ...@@ -53,7 +53,7 @@ sub new {
# creating tables. # creating tables.
$dsn .= ";options='-c client_min_messages=warning'"; $dsn .= ";options='-c client_min_messages=warning'";
my $attrs = { pg_enable_utf8 => Bugzilla->params->{'utf8'} }; my $attrs = { pg_enable_utf8 => 1 };
my $self = $class->db_new({ dsn => $dsn, user => $user, my $self = $class->db_new({ dsn => $dsn, user => $user,
pass => $pass, attrs => $attrs }); pass => $pass, attrs => $attrs });
......
...@@ -148,12 +148,7 @@ sub _get_create_index_ddl { ...@@ -148,12 +148,7 @@ sub _get_create_index_ddl {
sub get_create_database_sql { sub get_create_database_sql {
my ($self, $name) = @_; my ($self, $name) = @_;
# We only create as utf8 if we have no params (meaning we're doing return ("CREATE DATABASE `$name` CHARACTER SET utf8");
# a new installation) or if the utf8 param is on.
my $create_utf8 = Bugzilla->params->{'utf8'}
|| !defined Bugzilla->params->{'utf8'};
my $charset = $create_utf8 ? "CHARACTER SET utf8" : '';
return ("CREATE DATABASE `$name` $charset");
} }
# MySQL has a simpler ALTER TABLE syntax than ANSI. # MySQL has a simpler ALTER TABLE syntax than ANSI.
......
...@@ -79,12 +79,7 @@ sub _initialize { ...@@ -79,12 +79,7 @@ sub _initialize {
sub get_create_database_sql { sub get_create_database_sql {
my ($self, $name) = @_; my ($self, $name) = @_;
# We only create as utf8 if we have no params (meaning we're doing return ("CREATE DATABASE \"$name\" ENCODING 'UTF8' TEMPLATE template0");
# a new installation) or if the utf8 param is on.
my $create_utf8 = Bugzilla->params->{'utf8'}
|| !defined Bugzilla->params->{'utf8'};
my $charset = $create_utf8 ? "ENCODING 'UTF8' TEMPLATE template0" : '';
return ("CREATE DATABASE \"$name\" $charset");
} }
sub get_rename_column_ddl { sub get_rename_column_ddl {
......
...@@ -93,8 +93,7 @@ sub new { ...@@ -93,8 +93,7 @@ sub new {
my $dsn = "dbi:SQLite:dbname=$db_name"; my $dsn = "dbi:SQLite:dbname=$db_name";
my $attrs = { my $attrs = {
# XXX Should we just enforce this to be always on? sqlite_unicode => 1,
sqlite_unicode => Bugzilla->params->{'utf8'},
}; };
my $self = $class->db_new({ dsn => $dsn, user => '', my $self = $class->db_new({ dsn => $dsn, user => '',
......
...@@ -125,9 +125,7 @@ sub MessageToMTA { ...@@ -125,9 +125,7 @@ sub MessageToMTA {
# We don't recode headers that happen multiple times. # We don't recode headers that happen multiple times.
next if scalar(@values) > 1; next if scalar(@values) > 1;
if (my $value = $values[0]) { if (my $value = $values[0]) {
if (Bugzilla->params->{'utf8'} && !utf8::is_utf8($value)) { utf8::decode($value) unless utf8::is_utf8($value);
utf8::decode($value);
}
# avoid excessive line wrapping done by Encode. # avoid excessive line wrapping done by Encode.
local $Encode::Encoding{'MIME-Q'}->{'bpl'} = 998; local $Encode::Encoding{'MIME-Q'}->{'bpl'} = 998;
...@@ -187,18 +185,16 @@ sub MessageToMTA { ...@@ -187,18 +185,16 @@ sub MessageToMTA {
my $content_type = $part->content_type || ''; my $content_type = $part->content_type || '';
$content_type =~ /charset=['"](.+)['"]/; $content_type =~ /charset=['"](.+)['"]/;
# If no charset is defined or is the default us-ascii, # If no charset is defined or is the default us-ascii,
# then we encode the email to UTF-8 if Bugzilla has utf8 enabled. # then we encode the email to UTF-8.
# XXX - This is a hack to workaround bug 723944. # XXX - This is a hack to workaround bug 723944.
if (!$1 || $1 eq 'us-ascii') { if (!$1 || $1 eq 'us-ascii') {
my $body = $part->body; my $body = $part->body;
if (Bugzilla->params->{'utf8'}) { $part->charset_set('UTF-8');
$part->charset_set('UTF-8'); # encoding_set works only with bytes, not with utf8 strings.
# encoding_set works only with bytes, not with utf8 strings. my $raw = $part->body_raw;
my $raw = $part->body_raw; if (utf8::is_utf8($raw)) {
if (utf8::is_utf8($raw)) { utf8::encode($raw);
utf8::encode($raw); $part->body_set($raw);
$part->body_set($raw);
}
} }
$part->encoding_set('quoted-printable') if !is_7bit_clean($body); $part->encoding_set('quoted-printable') if !is_7bit_clean($body);
} }
......
...@@ -752,7 +752,7 @@ sub create { ...@@ -752,7 +752,7 @@ sub create {
# Initialize templates (f.e. by loading plugins like Hook). # Initialize templates (f.e. by loading plugins like Hook).
PRE_PROCESS => ["global/variables.none.tmpl"], PRE_PROCESS => ["global/variables.none.tmpl"],
ENCODING => Bugzilla->params->{'utf8'} ? 'UTF-8' : undef, ENCODING => 'UTF-8',
# Functions for processing text within templates in various ways. # Functions for processing text within templates in various ways.
# IMPORTANT! When adding a filter here that does not override a # IMPORTANT! When adding a filter here that does not override a
...@@ -783,15 +783,10 @@ sub create { ...@@ -783,15 +783,10 @@ sub create {
# Strips out control characters excepting whitespace # Strips out control characters excepting whitespace
strip_control_chars => sub { strip_control_chars => sub {
my ($data) = @_; my ($data) = @_;
state $use_utf8 = Bugzilla->params->{'utf8'}; $data =~ s/(?![\t\r\n])[[:cntrl:]]//g;
# Only run for utf8 to avoid issues with other multibyte encodings
# that may be reassigning meaning to ascii characters.
if ($use_utf8) {
$data =~ s/(?![\t\r\n])[[:cntrl:]]//g;
}
return $data; return $data;
}, },
# HTML collapses newlines in element attributes to a single space, # HTML collapses newlines in element attributes to a single space,
# so form elements which may have whitespace (ie comments) need # so form elements which may have whitespace (ie comments) need
# to be encoded using 
 # to be encoded using 
......
...@@ -185,9 +185,7 @@ sub issue_hash_token { ...@@ -185,9 +185,7 @@ sub issue_hash_token {
my $token = join('*', @args); my $token = join('*', @args);
# Wide characters cause Digest::SHA to die. # Wide characters cause Digest::SHA to die.
if (Bugzilla->params->{'utf8'}) { utf8::encode($token) if utf8::is_utf8($token);
utf8::encode($token) if utf8::is_utf8($token);
}
$token = hmac_sha256_base64($token, Bugzilla->localconfig->{'site_wide_secret'}); $token = hmac_sha256_base64($token, Bugzilla->localconfig->{'site_wide_secret'});
$token =~ s/\+/-/g; $token =~ s/\+/-/g;
$token =~ s/\//_/g; $token =~ s/\//_/g;
......
...@@ -73,36 +73,31 @@ sub html_quote { ...@@ -73,36 +73,31 @@ sub html_quote {
# Obscure '@'. # Obscure '@'.
$var =~ s/\@/\@/g; $var =~ s/\@/\@/g;
state $use_utf8 = Bugzilla->params->{'utf8'}; # Remove control characters.
$var =~ s/(?![\t\r\n])[[:cntrl:]]//g;
if ($use_utf8) {
# Remove control characters if the encoding is utf8. # Remove the following characters because they're
# Other multibyte encodings may be using this range; so ignore if not utf8. # influencing BiDi:
$var =~ s/(?![\t\r\n])[[:cntrl:]]//g; # --------------------------------------------------------
# |Code |Name |UTF-8 representation|
# Remove the following characters because they're # |------|--------------------------|--------------------|
# influencing BiDi: # |U+202a|Left-To-Right Embedding |0xe2 0x80 0xaa |
# -------------------------------------------------------- # |U+202b|Right-To-Left Embedding |0xe2 0x80 0xab |
# |Code |Name |UTF-8 representation| # |U+202c|Pop Directional Formatting|0xe2 0x80 0xac |
# |------|--------------------------|--------------------| # |U+202d|Left-To-Right Override |0xe2 0x80 0xad |
# |U+202a|Left-To-Right Embedding |0xe2 0x80 0xaa | # |U+202e|Right-To-Left Override |0xe2 0x80 0xae |
# |U+202b|Right-To-Left Embedding |0xe2 0x80 0xab | # --------------------------------------------------------
# |U+202c|Pop Directional Formatting|0xe2 0x80 0xac | #
# |U+202d|Left-To-Right Override |0xe2 0x80 0xad | # The following are characters influencing BiDi, too, but
# |U+202e|Right-To-Left Override |0xe2 0x80 0xae | # they can be spared from filtering because they don't
# -------------------------------------------------------- # influence more than one character right or left:
# # --------------------------------------------------------
# The following are characters influencing BiDi, too, but # |Code |Name |UTF-8 representation|
# they can be spared from filtering because they don't # |------|--------------------------|--------------------|
# influence more than one character right or left: # |U+200e|Left-To-Right Mark |0xe2 0x80 0x8e |
# -------------------------------------------------------- # |U+200f|Right-To-Left Mark |0xe2 0x80 0x8f |
# |Code |Name |UTF-8 representation| # --------------------------------------------------------
# |------|--------------------------|--------------------| $var =~ tr/\x{202a}-\x{202e}//d;
# |U+200e|Left-To-Right Mark |0xe2 0x80 0x8e |
# |U+200f|Right-To-Left Mark |0xe2 0x80 0x8f |
# --------------------------------------------------------
$var =~ tr/\x{202a}-\x{202e}//d;
}
return $var; return $var;
} }
...@@ -203,8 +198,8 @@ sub email_filter { ...@@ -203,8 +198,8 @@ sub email_filter {
# This originally came from CGI.pm, by Lincoln D. Stein # This originally came from CGI.pm, by Lincoln D. Stein
sub url_quote { sub url_quote {
my ($toencode) = (@_); my ($toencode) = (@_);
utf8::encode($toencode) # The below regex works only on bytes # The below regex works only on bytes
if Bugzilla->params->{'utf8'} && utf8::is_utf8($toencode); utf8::encode($toencode) if utf8::is_utf8($toencode);
$toencode =~ s/([^a-zA-Z0-9_\-.])/uc sprintf("%%%02x",ord($1))/eg; $toencode =~ s/([^a-zA-Z0-9_\-.])/uc sprintf("%%%02x",ord($1))/eg;
return $toencode; return $toencode;
} }
...@@ -635,9 +630,7 @@ sub bz_crypt { ...@@ -635,9 +630,7 @@ sub bz_crypt {
} }
# Wide characters cause crypt and Digest to die. # Wide characters cause crypt and Digest to die.
if (Bugzilla->params->{'utf8'}) { utf8::encode($password) if utf8::is_utf8($password);
utf8::encode($password) if utf8::is_utf8($password);
}
my $crypted_password; my $crypted_password;
if (!$algorithm) { if (!$algorithm) {
...@@ -785,9 +778,8 @@ sub display_value { ...@@ -785,9 +778,8 @@ sub display_value {
} }
sub disable_utf8 { sub disable_utf8 {
if (Bugzilla->params->{'utf8'}) { # Turn off UTF8 encoding.
binmode STDOUT, ':bytes'; # Turn off UTF8 encoding. binmode STDOUT, ':bytes';
}
} }
use constant UTF8_ACCIDENTAL => qw(shiftjis big5-eten euc-kr euc-jp); use constant UTF8_ACCIDENTAL => qw(shiftjis big5-eten euc-kr euc-jp);
......
...@@ -54,7 +54,7 @@ sub create_json_coder { ...@@ -54,7 +54,7 @@ sub create_json_coder {
# This may seem a little backwards, but what this really means is # This may seem a little backwards, but what this really means is
# "don't convert our utf8 into byte strings, just leave it as a # "don't convert our utf8 into byte strings, just leave it as a
# utf8 string." # utf8 string."
$json->utf8(0) if Bugzilla->params->{'utf8'}; $json->utf8(0);
return $json; return $json;
} }
......
...@@ -173,9 +173,7 @@ sub collect_stats { ...@@ -173,9 +173,7 @@ sub collect_stats {
|| ThrowCodeError('chart_file_open_fail', {'filename' => $file}); || ThrowCodeError('chart_file_open_fail', {'filename' => $file});
} }
if (Bugzilla->params->{'utf8'}) { binmode DATA, ':utf8';
binmode DATA, ':utf8';
}
# Now collect current data. # Now collect current data.
my @row = (today()); my @row = (today());
...@@ -234,9 +232,7 @@ sub get_old_data { ...@@ -234,9 +232,7 @@ sub get_old_data {
open(DATA, '<', $file) open(DATA, '<', $file)
|| ThrowCodeError('chart_file_open_fail', {'filename' => $file}); || ThrowCodeError('chart_file_open_fail', {'filename' => $file});
if (Bugzilla->params->{'utf8'}) { binmode DATA, ':utf8';
binmode DATA, ':utf8';
}
my @data; my @data;
my @columns; my @columns;
......
...@@ -59,16 +59,6 @@ maintainer ...@@ -59,16 +59,6 @@ maintainer
responsible for maintaining this Bugzilla installation. responsible for maintaining this Bugzilla installation.
The address need not be that of a valid Bugzilla account. The address need not be that of a valid Bugzilla account.
utf8
Use UTF-8 (Unicode) encoding for all text in Bugzilla. Installations where
this parameter is set to :paramval:`off` should set it to :paramval:`on` only
after the data has been converted from existing legacy character
encodings to UTF-8, using the
:file:`contrib/recode.pl` script.
.. note:: If you turn this parameter from :paramval:`off` to :paramval:`on`,
you must re-run :file:`checksetup.pl` immediately afterward.
shutdownhtml shutdownhtml
If there is any text in this field, this Bugzilla installation will If there is any text in this field, this Bugzilla installation will
be completely disabled and this text will appear instead of all be completely disabled and this text will appear instead of all
......
...@@ -398,10 +398,7 @@ sub get_text_alternative { ...@@ -398,10 +398,7 @@ sub get_text_alternative {
sub _decode_body { sub _decode_body {
my ($charset, $body) = @_; my ($charset, $body) = @_;
if (Bugzilla->params->{'utf8'} && !utf8::is_utf8($body)) { return utf8::is_utf8($body) ? $body : Encode::decode($charset, $body);
return Encode::decode($charset, $body);
}
return $body;
} }
sub remove_leading_blank_lines { sub remove_leading_blank_lines {
......
...@@ -191,9 +191,7 @@ foreach my $k (@bug_ids) { ...@@ -191,9 +191,7 @@ foreach my $k (@bug_ids) {
if ($summary ne "" && $cgi->param('showsummary')) { if ($summary ne "" && $cgi->param('showsummary')) {
# Wide characters cause GraphViz to die. # Wide characters cause GraphViz to die.
if (Bugzilla->params->{'utf8'}) { utf8::encode($summary) if utf8::is_utf8($summary);
utf8::encode($summary) if utf8::is_utf8($summary);
}
$summary = wrap_comment($summary); $summary = wrap_comment($summary);
$summary =~ s/([\\\"])/\\$1/g; $summary =~ s/([\\\"])/\\$1/g;
push(@params, qq{label="$k\\n$summary"}); push(@params, qq{label="$k\\n$summary"});
......
...@@ -16,17 +16,6 @@ ...@@ -16,17 +16,6 @@
"The email address of the person who maintains this installation " "The email address of the person who maintains this installation "
_ " of Bugzilla.", _ " of Bugzilla.",
utf8 =>
"Use UTF-8 (Unicode) encoding for all text in ${terms.Bugzilla}. New"
_ " installations should set this to true to avoid character encoding"
_ " problems. <strong>Existing databases should set this to true"
_ " only after the data has been converted from existing legacy"
_ " character encodings to UTF-8, using the <kbd>contrib/recode.pl</kbd>"
_ " script</strong>."
_ " <p>Note that if you turn this parameter from &quot;off&quot; to"
_ " &quot;on&quot;, you must re-run <kbd>checksetup.pl</kbd> immediately"
_ " afterward.</p>",
shutdownhtml => shutdownhtml =>
"If this field is non-empty, then Bugzilla will be completely" "If this field is non-empty, then Bugzilla will be completely"
_ " disabled and this text will be displayed instead of all the" _ " disabled and this text will be displayed instead of all the"
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
[% PROCESS bug/time.html.tmpl %] [% PROCESS bug/time.html.tmpl %]
[% USE Bugzilla %] [% USE Bugzilla %]
[% cgi = Bugzilla.cgi %] [% cgi = Bugzilla.cgi %]
<?xml version="1.0" [% IF Param('utf8') %]encoding="UTF-8" [% END %]standalone="yes" ?> <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
<!DOCTYPE bugzilla [% IF cgi.param('dtd') %][[% PROCESS pages/bugzilla.dtd.tmpl %]][% ELSE %]SYSTEM "[% urlbase FILTER xml %]page.cgi?id=bugzilla.dtd"[% END %]> <!DOCTYPE bugzilla [% IF cgi.param('dtd') %][[% PROCESS pages/bugzilla.dtd.tmpl %]][% ELSE %]SYSTEM "[% urlbase FILTER xml %]page.cgi?id=bugzilla.dtd"[% END %]>
<bugzilla version="[% constants.BUGZILLA_VERSION %]" <bugzilla version="[% constants.BUGZILLA_VERSION %]"
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
# So we cache it here for better performance. # So we cache it here for better performance.
%] %]
[% escaped_urlbase = BLOCK %][% urlbase FILTER xml %][% END %] [% escaped_urlbase = BLOCK %][% urlbase FILTER xml %][% END %]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?> <?xml version="1.0" encoding="UTF-8"?>
<!-- Note: this interface is experimental and under development. <!-- Note: this interface is experimental and under development.
- We may and probably will make breaking changes to it in the future. --> - We may and probably will make breaking changes to it in the future. -->
......
...@@ -103,10 +103,7 @@ ...@@ -103,10 +103,7 @@
<head> <head>
[% Hook.process("start") %] [% Hook.process("start") %]
<title>[% title %]</title> <title>[% title %]</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
[% IF Param('utf8') %]
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
[% END %]
[% IF NOT no_yui %] [% IF NOT no_yui %]
[% SET yui = yui_resolve_deps(yui, yui_deps) %] [% SET yui = yui_resolve_deps(yui, yui_deps) %]
......
...@@ -543,9 +543,6 @@ ...@@ -543,9 +543,6 @@
[% IF param_changed.size > 0 %] [% IF param_changed.size > 0 %]
[% FOREACH param = param_changed %] [% FOREACH param = param_changed %]
Changed <em>[% param FILTER html %]</em><br> Changed <em>[% param FILTER html %]</em><br>
[% IF param == 'utf8' && Param('utf8') %]
<strong>You must now re-run <kbd>checksetup.pl</kbd>.</strong><br>
[% END %]
[% END %] [% END %]
[% ELSE %] [% ELSE %]
No changes made. No changes made.
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
[% DEFAULT title = "$terms.Bugzilla $terms.Bugs" %] [% DEFAULT title = "$terms.Bugzilla $terms.Bugs" %]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?> <?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"> <feed xmlns="http://www.w3.org/2005/Atom">
<title>[% title FILTER xml %]</title> <title>[% title FILTER xml %]</title>
<link rel="alternate" type="text/html" <link rel="alternate" type="text/html"
......
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
# defined by the Mozilla Public License, v. 2.0. # defined by the Mozilla Public License, v. 2.0.
#%] #%]
<?xml version="1.0"[% IF Param('utf8') %] encoding="UTF-8"[% END %]?> <?xml version="1.0" encoding="UTF-8"?>
<!-- [% template_version %] --> <!-- [% template_version %] -->
<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" <RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment