Character encoding detection in attachments is now automatic

Gemfile

@@ -11,6 +11,8 @@ gem "liquid", "~> 2.3.0"
 gem "acts-as-taggable-on", "= 2.1.0"
 # Needed only on RUBY_VERSION = 1.8, ruby 1.9+ compatible interpreters should bring their csv
 gem "fastercsv", "~> 1.5.0", :platforms => [:ruby_18, :jruby, :mingw_18]
+# need for automatic encoding detection in diff,annotate and cat
+gem "chardet", ">= 0.9.0"
 
 group :test do
   gem 'shoulda', '~> 2.10.3'

app/helpers/attachments_helper.rb

@@ -12,6 +12,9 @@
 # See doc/COPYRIGHT.rdoc for more details.
 #++
 
+require 'rubygems'
+require 'UniversalDetector'
+
 module AttachmentsHelper
   # Displays view/delete links to the attachments of the given object
   # Options:
@@ -26,18 +29,16 @@ module AttachmentsHelper
   end
 
   def to_utf8_for_attachments(str)
-    if str.respond_to?(:force_encoding)
-      str.force_encoding('UTF-8')
-      return str if str.valid_encoding?
-    else
-      return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
-    end
-
+      return nil if str.nil?
+      iconv_str = str
+      detected = UniversalDetector::chardet(str)
+      enc = detected['encoding']
+      if !enc.nil?
         begin
-      Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
-    rescue Iconv::InvalidEncoding
-      # "UTF-8//IGNORE" is not supported on some OS
-      str
+          iconv_str = Iconv.conv('UTF-8', enc, str)
+        rescue Iconv::Failure => err
         end
       end
+      iconv_str
+  end
 end

lib/redmine/scm/adapters/mercurial_adapter.rb

@@ -14,6 +14,8 @@
 
 require 'redmine/scm/adapters/abstract_adapter'
 require 'cgi'
+require 'rubygems'
+require 'UniversalDetector'
 
 module Redmine
   module Scm
@@ -239,7 +241,18 @@ module Redmine
           diff = []
           hg *hg_args do |io|
             io.each_line do |line|
-              diff << line
+              iconv_line = line
+              if !line.nil?
+                detected = UniversalDetector::chardet(line)
+                enc = detected['encoding']
+                if !enc.nil?
+                  begin
+                    iconv_line = Iconv.conv('UTF-8', enc, line)
+                  rescue Iconv::Failure => err
+                  end
+                end
+              end
+              diff << iconv_line
             end
           end
           diff
@@ -251,7 +264,18 @@ module Redmine
           p = CGI.escape(scm_iconv(@path_encoding, 'UTF-8', path))
           hg 'rhcat', '-r', CGI.escape(hgrev(identifier)), hgtarget(p) do |io|
             io.binmode
-            io.read
+            str = io.read
+            return nil if str.nil?
+            iconv_str = str
+            detected = UniversalDetector::chardet(str)
+            enc = detected['encoding']
+            if !enc.nil?
+              begin
+                iconv_str = Iconv.conv('UTF-8', enc, str)
+              rescue Iconv::Failure => err
+              end
+            end
+            return iconv_str
           end
         rescue HgCommandAborted
           nil  # means not found
@@ -266,7 +290,19 @@ module Redmine
               next unless line =~ %r{^([^:]+)\s(\d+)\s([0-9a-f]+):\s(.*)$}
               r = Revision.new(:author => $1.strip, :revision => $2, :scmid => $3,
                                :identifier => $3)
-              blame.add_line($4.rstrip, r)
+              str = $4.rstrip
+              iconv_str = str
+              if !str.nil?
+                detected = UniversalDetector::chardet(str)
+                enc = detected['encoding']
+                if !enc.nil?
+                  begin
+                    iconv_str = Iconv.conv('UTF-8', enc, str)
+                  rescue Iconv::Failure => err
+                  end
+                end
+              end
+              blame.add_line(iconv_str, r)
             end
           end
           blame