From 206230a4ecb5c7e1b4de445ccda14814ad5a0232 Mon Sep 17 00:00:00 2001 From: Saito Date: Tue, 29 May 2012 16:16:59 +0800 Subject: [PATCH 1/8] rewrite encode strategy. --- lib/gitlab/encode.rb | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/gitlab/encode.rb b/lib/gitlab/encode.rb index 1c95a9477b..cee3ace202 100644 --- a/lib/gitlab/encode.rb +++ b/lib/gitlab/encode.rb @@ -1,25 +1,29 @@ # Patch Strings to enable detect_encoding! on views require 'charlock_holmes/string' module Gitlab - module Encode + module Encode extend self def utf8 message + # return nil if message is nil return nil unless message - detect = CharlockHolmes::EncodingDetector.detect(message) rescue {} + # if message is utf-8 encoding, just return it + message.force_encoding("utf-8") + return message if message.valid_encoding? - # It's better to default to UTF-8 as sometimes it's wrongly detected as another charset - if detect[:encoding] && detect[:confidence] == 100 - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - else - message - end.force_encoding("utf-8") + # if message is not utf-8 encoding, detect and convert it + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect[:encoding] && detect[:confidence] > 60 + message.force_encoding(detect[:encoding]) + message.encode!("utf-8", detect[:encoding], :undef => :replace, :replace => "", :invalid => :replace) + end - # Prevent app from crash cause of - # encoding errors + message.valid_encoding? ? message : raise + + # Prevent app from crash cause of encoding errors rescue - "--broken encoding: #{encoding}" + "--broken encoding: #{detect[:encoding]}" end def detect_encoding message From e1d1673e74c35fc2d64e71320668d981d634b02a Mon Sep 17 00:00:00 2001 From: Saito Date: Tue, 29 May 2012 16:17:31 +0800 Subject: [PATCH 2/8] monkey patch grit to support utf8 encoding --- config/initializers/gitlabhq/20_grit_ext.rb | 27 +++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/config/initializers/gitlabhq/20_grit_ext.rb b/config/initializers/gitlabhq/20_grit_ext.rb index 33c33ccc77..f17cf4415e 100644 --- a/config/initializers/gitlabhq/20_grit_ext.rb +++ b/config/initializers/gitlabhq/20_grit_ext.rb @@ -1,27 +1,34 @@ require 'grit' require 'pygments' +Grit::Git.git_timeout = GIT_OPTS["git_timeout"] +Grit::Git.git_max_size = GIT_OPTS["git_max_size"] + Grit::Blob.class_eval do include Linguist::BlobHelper -end -#monkey patch raw_object from string -Grit::GitRuby::Internal::RawObject.class_eval do - def content - @content + def data + @data ||= @repo.git.cat_file({:p => true}, id) + Gitlab::Encode.utf8 @data end end +Grit::Commit.class_eval do + def message + Gitlab::Encode.utf8 @message + end +end Grit::Diff.class_eval do def old_path - Gitlab::Encode.utf8 a_path + Gitlab::Encode.utf8 @a_path end def new_path - Gitlab::Encode.utf8 b_path + Gitlab::Encode.utf8 @b_path + end + + def diff + Gitlab::Encode.utf8 @diff end end - -Grit::Git.git_timeout = GIT_OPTS["git_timeout"] -Grit::Git.git_max_size = GIT_OPTS["git_max_size"] From efd9a717c10f1cafd05fb20729eafb61226d9c1d Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 11:44:36 +0800 Subject: [PATCH 3/8] solve the binary problem --- lib/gitlab/encode.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/encode.rb b/lib/gitlab/encode.rb index cee3ace202..ba7fe27cb6 100644 --- a/lib/gitlab/encode.rb +++ b/lib/gitlab/encode.rb @@ -8,17 +8,21 @@ module Gitlab # return nil if message is nil return nil unless message + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message if detect[:type] == :binary + # if message is utf-8 encoding, just return it message.force_encoding("utf-8") return message if message.valid_encoding? - # if message is not utf-8 encoding, detect and convert it - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect[:encoding] && detect[:confidence] > 60 + # if message is not utf-8 encoding, convert it + if detect[:encoding] message.force_encoding(detect[:encoding]) message.encode!("utf-8", detect[:encoding], :undef => :replace, :replace => "", :invalid => :replace) end + # ensure message encoding is utf8 message.valid_encoding? ? message : raise # Prevent app from crash cause of encoding errors From c71a76e71a33315977797db3e72be1f76462183f Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 11:47:26 +0800 Subject: [PATCH 4/8] fix graph problem if authorname or message isnot utf8 encoding --- config/initializers/gitlabhq/20_grit_ext.rb | 19 +++++++++++++++++-- lib/graph_commit.rb | 4 ++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/config/initializers/gitlabhq/20_grit_ext.rb b/config/initializers/gitlabhq/20_grit_ext.rb index f17cf4415e..314553f795 100644 --- a/config/initializers/gitlabhq/20_grit_ext.rb +++ b/config/initializers/gitlabhq/20_grit_ext.rb @@ -14,8 +14,23 @@ Grit::Blob.class_eval do end Grit::Commit.class_eval do - def message - Gitlab::Encode.utf8 @message + def to_hash + { + 'id' => id, + 'parents' => parents.map { |p| { 'id' => p.id } }, + 'tree' => tree.id, + 'message' => Gitlab::Encode.utf8(message), + 'author' => { + 'name' => Gitlab::Encode.utf8(author.name), + 'email' => author.email + }, + 'committer' => { + 'name' => Gitlab::Encode.utf8(committer.name), + 'email' => committer.email + }, + 'authored_date' => authored_date.xmlschema, + 'committed_date' => committed_date.xmlschema, + } end end diff --git a/lib/graph_commit.rb b/lib/graph_commit.rb index 0080e85640..54550d99cf 100644 --- a/lib/graph_commit.rb +++ b/lib/graph_commit.rb @@ -96,13 +96,13 @@ class GraphCommit h[:parents] = self.parents.collect do |p| [p.id,0,0] end - h[:author] = author.name.force_encoding("UTF-8") + h[:author] = author.name h[:time] = time h[:space] = space h[:refs] = refs.collect{|r|r.name}.join(" ") unless refs.nil? h[:id] = sha h[:date] = date - h[:message] = message.force_encoding("UTF-8") + h[:message] = message h[:login] = author.email h end From 202807c21a61696ad50aadf43781ca15b9deaede Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 11:56:00 +0800 Subject: [PATCH 5/8] remove detect_enoding. detect_encoding means force_encoding to the file.data's encoding, not encoding to utf8. will cause encoding problem. --- app/views/refs/_tree.html.haml | 4 ++-- app/views/refs/_tree_file.html.haml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/refs/_tree.html.haml b/app/views/refs/_tree.html.haml index 0d9d2d7541..ee2f278693 100644 --- a/app/views/refs/_tree.html.haml +++ b/app/views/refs/_tree.html.haml @@ -42,9 +42,9 @@ .readme - if content.name =~ /\.(md|markdown)$/i = preserve do - = markdown(content.data.detect_encoding!) + = markdown(content.data) - else - = simple_format(content.data.detect_encoding!) + = simple_format(content.data) - if params[:path] - history_path = tree_file_project_ref_path(@project, @ref, params[:path]) diff --git a/app/views/refs/_tree_file.html.haml b/app/views/refs/_tree_file.html.haml index cb8a1193e7..201028dc2a 100644 --- a/app/views/refs/_tree_file.html.haml +++ b/app/views/refs/_tree_file.html.haml @@ -13,7 +13,7 @@ #tree-readme-holder .readme = preserve do - = markdown(file.data.detect_encoding!) + = markdown(file.data) - else .view_file_content - unless file.empty? From 34cc38b6f5942523820ce239b1d5877b0a97e25e Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 11:56:48 +0800 Subject: [PATCH 6/8] diff now no need to force_encoding to utf8 --- app/helpers/commits_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/helpers/commits_helper.rb b/app/helpers/commits_helper.rb index cf1395f20d..63f38eeda0 100644 --- a/app/helpers/commits_helper.rb +++ b/app/helpers/commits_helper.rb @@ -58,14 +58,14 @@ module CommitsHelper next if line.match(/^\-\-\- a/) next if line.match(/^\+\+\+ b/) - full_line = html_escape(line.gsub(/\n/, '')).force_encoding("UTF-8") + full_line = html_escape(line.gsub(/\n/, '')) if line.match(/^@@ -/) type = "match" line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - + next if line_old == 1 && line_new == 1 yield(full_line, type, nil, nil, nil) next From c62715acc920f92ce2f3febf4dc3f838e3097410 Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 12:11:03 +0800 Subject: [PATCH 7/8] now render the correct authorname and message --- config/initializers/gitlabhq/20_grit_ext.rb | 21 --------------------- lib/graph_commit.rb | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/config/initializers/gitlabhq/20_grit_ext.rb b/config/initializers/gitlabhq/20_grit_ext.rb index 314553f795..ad8ea1058f 100644 --- a/config/initializers/gitlabhq/20_grit_ext.rb +++ b/config/initializers/gitlabhq/20_grit_ext.rb @@ -13,27 +13,6 @@ Grit::Blob.class_eval do end end -Grit::Commit.class_eval do - def to_hash - { - 'id' => id, - 'parents' => parents.map { |p| { 'id' => p.id } }, - 'tree' => tree.id, - 'message' => Gitlab::Encode.utf8(message), - 'author' => { - 'name' => Gitlab::Encode.utf8(author.name), - 'email' => author.email - }, - 'committer' => { - 'name' => Gitlab::Encode.utf8(committer.name), - 'email' => committer.email - }, - 'authored_date' => authored_date.xmlschema, - 'committed_date' => committed_date.xmlschema, - } - end -end - Grit::Diff.class_eval do def old_path Gitlab::Encode.utf8 @a_path diff --git a/lib/graph_commit.rb b/lib/graph_commit.rb index 54550d99cf..64498d8b1d 100644 --- a/lib/graph_commit.rb +++ b/lib/graph_commit.rb @@ -96,13 +96,13 @@ class GraphCommit h[:parents] = self.parents.collect do |p| [p.id,0,0] end - h[:author] = author.name + h[:author] = Gitlab::Encode.utf8(author.name) h[:time] = time h[:space] = space h[:refs] = refs.collect{|r|r.name}.join(" ") unless refs.nil? h[:id] = sha h[:date] = date - h[:message] = message + h[:message] = Gitlab::Encode.utf8(message) h[:login] = author.email h end From e851cb07762aa464aae10e8b4b28de87c1a6f925 Mon Sep 17 00:00:00 2001 From: Saito Date: Wed, 30 May 2012 13:40:47 +0800 Subject: [PATCH 8/8] must force_encoding to utf-8 first --- lib/gitlab/encode.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/encode.rb b/lib/gitlab/encode.rb index ba7fe27cb6..8b6ffa3d49 100644 --- a/lib/gitlab/encode.rb +++ b/lib/gitlab/encode.rb @@ -8,12 +8,12 @@ module Gitlab # return nil if message is nil return nil unless message + message.force_encoding("utf-8") # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) return message if detect[:type] == :binary # if message is utf-8 encoding, just return it - message.force_encoding("utf-8") return message if message.valid_encoding? # if message is not utf-8 encoding, convert it