Merge branch 'ticket/release-v2.0.0/444-scm-encoding' into release-v2.0.0

app/models/changeset.rb

@@ -239,28 +239,45 @@ class Changeset < ActiveRecord::Base
   private
 
   def self.to_utf8(str, encoding)
-    return str if str.blank?
+    return str if str.nil?
-    unless encoding.blank? || encoding == 'UTF-8'
+    str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
-      begin
+    if str.empty?
-        str = Iconv.conv('UTF-8', encoding, str)
+      str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
-      rescue Iconv::Failure
+      return str
-        # do nothing here
-      end
     end
+    normalized_encoding = encoding.blank? ? "UTF-8" : encoding
     if str.respond_to?(:force_encoding)
-      str.force_encoding('UTF-8')
+      if normalized_encoding.upcase != "UTF-8"
-      if ! str.valid_encoding?
+        str.force_encoding(normalized_encoding)
-        str = str.encode("US-ASCII", :invalid => :replace,
+        str = str.encode("UTF-8", :invalid => :replace,
-              :undef => :replace, :replace => '?').encode("UTF-8")
+              :undef => :replace, :replace => '?')
+      else
+        str.force_encoding("UTF-8")
+        unless str.valid_encoding?
+          str = str.encode("US-ASCII", :invalid => :replace,
+                :undef => :replace, :replace => '?').encode("UTF-8")
+        end
       end
     else
-      # removes invalid UTF8 sequences
+      
+      txtar = ""
       begin
-        str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+        txtar += Iconv.new('UTF-8', normalized_encoding).iconv(str)
-      rescue Iconv::InvalidEncoding
+      rescue Iconv::IllegalSequence
-        # "UTF-8//IGNORE" is not supported on some OS
+        txtar += $!.success
+        str = '?' + $!.failed[1,$!.failed.length]
+        retry
+      rescue
+        txtar += $!.success
       end
+      str = txtar
+    end
+    # removes invalid UTF8 sequences
+    begin
+      Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+    rescue Iconv::InvalidEncoding
+      # "UTF-8//IGNORE" is not supported on some OS
+      str
     end
-    str
   end
 end

test/unit/changeset_test.rb

@@ -14,7 +14,9 @@
 require File.expand_path('../../test_helper', __FILE__)
 
 class ChangesetTest < ActiveSupport::TestCase
-  fixtures :projects, :repositories, :issues, :issue_statuses, :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
+  fixtures :projects, :repositories, :issues, :issue_statuses,
+           :changesets, :changes, :issue_categories, :enumerations,
+           :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
 
   def setup
   end
@@ -235,32 +237,49 @@ class ChangesetTest < ActiveSupport::TestCase
       assert_equal "Texte encodé en ISO-8859-1.", c.comments
   end
 
-  def test_invalid_utf8_sequences_in_comments_should_be_stripped
+  def test_invalid_utf8_sequences_in_comments_should_be_replaced_latin1
       proj = Project.find(3)
       str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
       r = Repository::Bazaar.create!(
-            :project => proj, :url => '/tmp/test/bazaar',
+            :project => proj,
+            :url => '/tmp/test/bazaar',
             :log_encoding => 'UTF-8' )
       assert r
-      c = Changeset.new(:repository => r,
+      c = Changeset.new(:repository   => r,
                         :committed_on => Time.now,
-                        :revision => '123',
+                        :revision     => '123',
-                        :scmid => '12345',
+                        :scmid        => '12345',
-                        :comments => str)
+                        :comments     => str)
       assert( c.save )
+      assert_equal "Texte encod? en ISO-8859-1.", c.comments
+  end
+
+  def test_invalid_utf8_sequences_in_comments_should_be_replaced_ja_jis
+      proj = Project.find(3)
+      str = "test\xb5\xfetest\xb5\xfe"
       if str.respond_to?(:force_encoding)
-        assert_equal "Texte encod? en ISO-8859-1.", c.comments
+        str.force_encoding('ASCII-8BIT')
-      else
-        assert_equal "Texte encod en ISO-8859-1.", c.comments
       end
+      r = Repository::Bazaar.create!(
+            :project => proj,
+            :url     => '/tmp/test/bazaar',
+            :log_encoding => 'ISO-2022-JP' )
+      assert r
+      c = Changeset.new(:repository   => r,
+                        :committed_on => Time.now,
+                        :revision     => '123',
+                        :scmid        => '12345',
+                        :comments     => str)
+      assert( c.save )
+      assert_equal "test??test??", c.comments
   end
 
   def test_comments_should_be_converted_all_latin1_to_utf8
       s1 = "\xC2\x80"
       s2 = "\xc3\x82\xc2\x80"
+      s4 = s2.dup
       if s1.respond_to?(:force_encoding)
         s3 = s1.dup
-        s4 = s2.dup
         s1.force_encoding('ASCII-8BIT')
         s2.force_encoding('ASCII-8BIT')
         s3.force_encoding('ISO-8859-1')
@@ -278,7 +297,43 @@ class ChangesetTest < ActiveSupport::TestCase
                         :scmid => '12345',
                         :comments => s1)
       assert( c.save )
-      assert_equal s2, c.comments
+      assert_equal s4, c.comments
+  end
+
+  def test_comments_nil
+      proj = Project.find(3)
+      r = Repository::Bazaar.create!(
+            :project => proj, :url => '/tmp/test/bazaar',
+            :log_encoding => 'ISO-8859-1' )
+      assert r
+      c = Changeset.new(:repository => r,
+                        :committed_on => Time.now,
+                        :revision => '123',
+                        :scmid => '12345',
+                        :comments => nil)
+      assert( c.save )
+      assert_equal "", c.comments
+      if c.comments.respond_to?(:force_encoding)
+        assert_equal "UTF-8", c.comments.encoding.to_s
+      end
+  end
+
+  def test_comments_empty
+      proj = Project.find(3)
+      r = Repository::Bazaar.create!(
+            :project => proj, :url => '/tmp/test/bazaar',
+            :log_encoding => 'ISO-8859-1' )
+      assert r
+      c = Changeset.new(:repository => r,
+                        :committed_on => Time.now,
+                        :revision => '123',
+                        :scmid => '12345',
+                        :comments => "")
+      assert( c.save )
+      assert_equal "", c.comments
+      if c.comments.respond_to?(:force_encoding)
+        assert_equal "UTF-8", c.comments.encoding.to_s
+      end
   end
 
   def test_identifier