kolan/Redmine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

scm: replace invalid utf-8 sequences in comments instead of stripping on Ruby 1.8.

Browse Source 
git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@5373 e93f8b46-1217-0410-a6f0-8f06a7374b81
This commit is contained in:
Toshi MARUYAMA 2011-04-09 09:31:14 +00:00
parent b9ce061319
commit 6536c53e09
2 changed files with 24 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/models/changeset.rb
View File

@ -255,8 +255,8 @@ class Changeset < ActiveRecord::Base
str.force_encoding("UTF-8") if str.respond_to?(:force_encoding) str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
return str return str
end end
enc = encoding.blank? ? "UTF-8" : encoding
if str.respond_to?(:force_encoding) if str.respond_to?(:force_encoding)
enc = encoding.blank? ? "UTF-8" : encoding
if enc != "UTF-8" if enc != "UTF-8"
str.force_encoding(enc) str.force_encoding(enc)
str = str.encode("UTF-8", :invalid => :replace, str = str.encode("UTF-8", :invalid => :replace,
@ -269,19 +269,18 @@ class Changeset < ActiveRecord::Base
end end
end end
else else
unless encoding.blank? || encoding == 'UTF-8' ic = Iconv.new('UTF-8', enc)
begin txtar = ""
str = Iconv.conv('UTF-8', encoding, str)
rescue Iconv::Failure
# do nothing here
end
end
# removes invalid UTF8 sequences
begin begin
str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + ' ')[0..-3] txtar += ic.iconv(str)
rescue Iconv::InvalidEncoding rescue Iconv::IllegalSequence
# "UTF-8//IGNORE" is not supported on some OS txtar += $!.success
str = '?' + $!.failed[1,$!.failed.length]
retry
rescue
txtar += $!.success
end end
str = txtar
end end
str str
end end

32
test/unit/changeset_test.rb
View File

@ -21,7 +21,8 @@ require File.expand_path('../../test_helper', __FILE__)
class ChangesetTest < ActiveSupport::TestCase class ChangesetTest < ActiveSupport::TestCase
fixtures :projects, :repositories, :issues, :issue_statuses, fixtures :projects, :repositories, :issues, :issue_statuses,
:changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers :changesets, :changes, :issue_categories, :enumerations,
:custom_fields, :custom_values, :users, :members, :member_roles, :trackers
def setup def setup
end end
@ -250,29 +251,26 @@ class ChangesetTest < ActiveSupport::TestCase
assert_equal str_utf8, c.comments assert_equal str_utf8, c.comments
end end
def test_invalid_utf8_sequences_in_comments_should_be_stripped def test_invalid_utf8_sequences_in_comments_should_be_replaced_latin1
proj = Project.find(3) proj = Project.find(3)
# str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt") # str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
str = "Texte encod\xe9 en ISO-8859-1." str = "Texte encod\xe9 en ISO-8859-1."
str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding) str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
r = Repository::Bazaar.create!( r = Repository::Bazaar.create!(
:project => proj, :url => '/tmp/test/bazaar', :project => proj,
:url => '/tmp/test/bazaar',
:log_encoding => 'UTF-8' ) :log_encoding => 'UTF-8' )
assert r assert r
c = Changeset.new(:repository => r, c = Changeset.new(:repository => r,
:committed_on => Time.now, :committed_on => Time.now,
:revision => '123', :revision => '123',
:scmid => '12345', :scmid => '12345',
:comments => str) :comments => str)
assert( c.save ) assert( c.save )
if str.respond_to?(:force_encoding) assert_equal "Texte encod? en ISO-8859-1.", c.comments
assert_equal "Texte encod? en ISO-8859-1.", c.comments
else
assert_equal "Texte encod en ISO-8859-1.", c.comments
end
end end
def test_invalid_utf8_sequences_in_comments_should_be_stripped_ja_jis def test_invalid_utf8_sequences_in_comments_should_be_replaced_ja_jis
proj = Project.find(3) proj = Project.find(3)
str = "test\xb5\xfetest\xb5\xfe" str = "test\xb5\xfetest\xb5\xfe"
if str.respond_to?(:force_encoding) if str.respond_to?(:force_encoding)
@ -280,7 +278,7 @@ class ChangesetTest < ActiveSupport::TestCase
end end
r = Repository::Bazaar.create!( r = Repository::Bazaar.create!(
:project => proj, :project => proj,
:url => '/tmp/test/bazaar', :url => '/tmp/test/bazaar',
:log_encoding => 'ISO-2022-JP' ) :log_encoding => 'ISO-2022-JP' )
assert r assert r
c = Changeset.new(:repository => r, c = Changeset.new(:repository => r,
@ -289,11 +287,7 @@ class ChangesetTest < ActiveSupport::TestCase
:scmid => '12345', :scmid => '12345',
:comments => str) :comments => str)
assert( c.save ) assert( c.save )
if str.respond_to?(:force_encoding) assert_equal "test??test??", c.comments
assert_equal "test??test??", c.comments
else
assert_equal "testtest", c.comments
end
end end
def test_comments_should_be_converted_all_latin1_to_utf8 def test_comments_should_be_converted_all_latin1_to_utf8