Merge pull request #26 from why404/release/v3.2.2

fix up 701

CHANGELOG.md

@@ -1,5 +1,11 @@
 ## CHANGE LOG
 
+### v3.2.2
+
+fixed E701 error
+
+断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ，规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。
+
 ### v3.2.1
 
 allow images uploaded auto-orient.
@@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and
 参考：
 
 1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression)
-2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
+2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    qiniu-rs (3.2.1)
+    qiniu-rs (3.2.2)
       json (~> 1.7)
       mime-types (~> 1.19)
       rest-client (~> 1.6)
@@ -14,17 +14,17 @@ GEM
     fakeweb (1.3.0)
     json (1.7.5)
     mime-types (1.19)
-    rake (0.9.2.2)
+    rake (10.0.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    rspec (2.11.0)
+    rspec (2.12.0)
-      rspec-core (~> 2.11.0)
+      rspec-core (~> 2.12.0)
-      rspec-expectations (~> 2.11.0)
+      rspec-expectations (~> 2.12.0)
-      rspec-mocks (~> 2.11.0)
+      rspec-mocks (~> 2.12.0)
-    rspec-core (2.11.1)
+    rspec-core (2.12.1)
-    rspec-expectations (2.11.3)
+    rspec-expectations (2.12.0)
       diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.3)
+    rspec-mocks (2.12.0)
     ruby-hmac (0.4.0)
 
 PLATFORMS
@@ -33,5 +33,5 @@ PLATFORMS
 DEPENDENCIES
   fakeweb (~> 1.3)
   qiniu-rs!
-  rake (~> 0.9)
+  rake (>= 0.9)
   rspec (~> 2.11)

lib/qiniu/rs/exceptions.rb

@@ -4,9 +4,6 @@ module Qiniu
   module RS
 
     class Exception < RuntimeError
-      def to_s
-        inspect
-      end
     end
 
     class ResponseError < Exception

lib/qiniu/rs/up.rb

@@ -14,6 +14,9 @@ module Qiniu
   module RS
     module UP
 
+      PROGRESS_TMP_FILE = 'progresses'
+      CHECKSUM_TMP_FILE = 'ctxes'
+
       module AbstractClass
         class ChunkProgressNotifier
           include Qiniu::RS::Abstract
@@ -29,78 +32,58 @@ module Qiniu
       end
 
       class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
+          attr_reader :tmpdata
           def initialize(id)
-              @data = UP::ProgressData.new(id)
+              @tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE)
           end
           def notify(index, progress)
-              @data.set_progresses(index, progress)
+              @tmpdata.set(index, progress)
-              logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" +
+              logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
-                       "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
               Utils.debug(logmsg)
           end
       end
 
       class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
+          attr_reader :tmpdata
           def initialize(id)
-              @data = UP::ProgressData.new(id)
+              @tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE)
           end
           def notify(index, checksum)
-              @data.set_checksums(index, checksum)
+              @tmpdata.set(index, checksum)
+              Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
               Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
           end
       end
 
-      class ProgressData
+      class TmpData
-          def initialize(id)
+          def initialize(dir, filename)
-              @id = id
+              @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir
-              @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
               FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir)
-              @checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
+              @tmpfile = @tmpdir + File::SEPARATOR + filename
-              @progress_file = @tmpdir + File::SEPARATOR + 'progresses'
           end
 
-          def get_checksums
+          def init(values)
-              File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
+              File.open(@tmpfile, "w") do |f|
-          end
+                  YAML::dump(values, f)
-
+                  Utils.debug %Q(Initializing tmpfile: #{@tmpfile})
-          def get_progresses
-              File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
-          end
-
-          def set_checksums(index, checksum)
-              checksums = get_checksums
-              checksums[index] = checksum
-              File.open(@checksum_file, "w") do |f|
-                  YAML::dump(checksums, f)
-                  Utils.debug %Q(Updating tmpfile: #{@checksum_file})
               end
           end
 
-          def set_progresses(index, progress)
+          def all
-              progresses = get_progresses
+              File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : []
-              progresses[index] = progress
-              File.open(@progress_file, "w") do |f|
-                  YAML::dump(progresses, f)
-                  Utils.debug %Q(Updating tmpfile: #{@progress_file})
-              end
           end
 
-          def init_checksums(checksums)
+          def set(index, value)
-              File.open(@checksum_file, "w") do |f|
+              values = all
-                  YAML::dump(checksums, f)
+              values[index] = value
-                  Utils.debug %Q(Initializing tmpfile: #{@checksum_file})
+              File.open(@tmpfile, "w") do |f|
-              end
+                  YAML::dump(values, f)
-          end
+                  Utils.debug %Q(Updating tmpfile: #{@tmpfile})
-
-          def init_progresses(progresses)
-              File.open(@progress_file, "w") do |f|
-                  YAML::dump(progresses, f)
-                  Utils.debug %Q(Initializing tmpfile: #{@progress_file})
               end
           end
 
           def sweep!
-              FileUtils.rm_r(@tmpdir)
+              FileUtils.rm_r(@tmpdir) if  Dir.exists?(@tmpdir)
           end
       end
 
@@ -162,20 +145,21 @@ module Qiniu
         end
 
         def _new_block_put_progress_data
-          {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
+          {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
         end
 
-        def _call_binary_with_token(uptoken, url, data, retry_times = 0)
+        def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
           options = {
               :method => :post,
               :content_type => 'application/octet-stream',
               :upload_signature_token => uptoken
           }
+          options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
           code, data = http_request url, data, options
           unless Utils.is_response_ok?(code)
               retry_times += 1
               if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
-                  return _call_binary_with_token(uptoken, url, data, retry_times)
+                  return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
               end
           end
           [code, data]
@@ -186,8 +170,8 @@ module Qiniu
             _call_binary_with_token(uptoken, url, body)
         end
 
-        def _putblock(uptoken, ctx, offset, body)
+        def _putblock(uphost, uptoken, ctx, offset, body)
-            url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
+            url = uphost + "/bput/#{ctx}/#{offset}"
             _call_binary_with_token(uptoken, url, body)
         end
 
@@ -214,6 +198,7 @@ module Qiniu
                         progress[:offset] = body_length
                         progress[:restsize] = block_size - body_length
                         progress[:status_code] = code
+                        progress[:host] = data["host"]
                         if !notifier.nil? && notifier.respond_to?("notify")
                             notifier.notify(block_index, progress)
                         end
@@ -236,13 +221,14 @@ module Qiniu
                     if result_length != body_length
                         raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
                     end
-                    code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
+                    code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
                     body_crc32 = Zlib.crc32(body)
                     if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
                         progress[:ctx] = data["ctx"]
                         progress[:offset] += body_length
                         progress[:restsize] -= body_length
                         progress[:status_code] = code
+                        progress[:host] = data["host"]
                         if !notifier.nil? && notifier.respond_to?("notify")
                             notifier.notify(block_index, progress)
                         end
@@ -280,7 +266,8 @@ module Qiniu
                     end
                     code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
                     if Utils.is_response_ok?(code)
-                        checksums[block_index] = data["checksum"]
+                        #checksums[block_index] = data["checksum"]
+                        checksums[block_index] = data["ctx"]
                         if !block_notifier.nil? && block_notifier.respond_to?("notify")
                             block_notifier.notify(block_index, checksums[block_index])
                         end
@@ -290,7 +277,7 @@ module Qiniu
             return [code, data]
         end
 
-        def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
+        def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
           path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
           path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
           path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
@@ -298,37 +285,39 @@ module Qiniu
           callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
           path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
           path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
-          url = Config.settings[:up_host] + path
+          url = uphost + path
-          body = ''
+          #body = ''
-          checksums.each do |checksum|
+          #checksums.each do |checksum|
-              body += Utils.urlsafe_base64_decode(checksum)
+          #    body += Utils.urlsafe_base64_decode(checksum)
-          end
+          #end
-          _call_binary_with_token(uptoken, url, body)
+          body = checksums.join(',')
+          _call_binary_with_token(uptoken, url, body, 'text/plain')
         end
 
         def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
           block_count = _block_count(fsize)
-          progress_data = ProgressData.new(key)
-          checksums = progress_data.get_checksums
-          progresses = progress_data.get_progresses
-          if checksums.empty?
-              block_count.times{checksums << ''}
-              progress_data.init_checksums(checksums)
-          end
-          if progresses.empty?
-              block_count.times{progresses << _new_block_put_progress_data}
-              progress_data.init_progresses(progresses)
-          end
           chunk_notifier = ChunkProgressNotifier.new(key)
           block_notifier = BlockProgressNotifier.new(key)
+          progresses = chunk_notifier.tmpdata.all
+          if progresses.empty?
+              block_count.times{progresses << _new_block_put_progress_data}
+              chunk_notifier.tmpdata.init(progresses)
+          end
+          checksums = block_notifier.tmpdata.all
+          if checksums.empty?
+              block_count.times{checksums << ''}
+              block_notifier.tmpdata.init(checksums)
+          end
           code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
           if Utils.is_response_ok?(code)
+            uphost = data["host"]
             entry_uri = bucket + ':' + key
-            code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
+            code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
           end
           if Utils.is_response_ok?(code)
             Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
-            progress_data.sweep!
+            chunk_notifier.tmpdata.sweep!
+            block_notifier.tmpdata.sweep!
           end
           [code, data]
         end

lib/qiniu/rs/version.rb

@@ -5,7 +5,7 @@ module Qiniu
     module Version
       MAJOR = 3
       MINOR = 2
-      PATCH = 1
+      PATCH = 2
       # Returns a version string by joining <tt>MAJOR</tt>, <tt>MINOR</tt>, and <tt>PATCH</tt> with <tt>'.'</tt>
       #
       # Example

qiniu-rs.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.version       = Qiniu::RS::Version.to_s
 
   # specify any dependencies here; for example:
-  gem.add_development_dependency "rake", "~> 0.9"
+  gem.add_development_dependency "rake", ">= 0.9"
   gem.add_development_dependency "rspec", "~> 2.11"
   gem.add_development_dependency "fakeweb", "~> 1.3"
   gem.add_runtime_dependency "json", "~> 1.7"

spec/qiniu/rs/up_spec.rb

@@ -11,7 +11,7 @@ module Qiniu
 
       before :all do
         @localfile = "bigfile.txt"
-        File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}}
+        File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}}
         @bucket = "up_test_bucket"
         @key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s)
 
@@ -21,10 +21,8 @@ module Qiniu
       end
 
       after :all do
-        @localfile = "bigfile.txt"
         File.unlink(@localfile) if File.exists?(@localfile)
 
-        @bucket = "up_test_bucket"
         code, data = Qiniu::RS::RS.drop(@bucket)
         puts [code, data].inspect
         code.should == 200

spec/spec_helper.rb

@@ -6,13 +6,7 @@ require 'rspec'
 
 RSpec.configure do |config|
   config.before :all do
-    Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
+    Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
-                                    :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
+                                    :secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j"
-                                    :auth_url   => "http://m1.qbox.me:13001/oauth2/token",
-                                    :rs_host    => "http://m1.qbox.me:13003",
-                                    :io_host    => "http://m1.qbox.me:13004",
-                                    :up_host    => "http://m1.qbox.me:13019",
-                                    :pub_host   => "http://m1.qbox.me:13012",
-                                    :eu_host    => "http://m1.qbox.me:13050"
   end
 end