From a8a8f63887220b34ca8fcf1faf3900f92cf6865c Mon Sep 17 00:00:00 2001 From: 404 Date: Wed, 5 Dec 2012 14:00:12 +0800 Subject: [PATCH] fix up 701 --- docs/NEWS.md => CHANGELOG.md | 8 ++- Gemfile.lock | 20 +++--- lib/qiniu/rs/up.rb | 133 ++++++++++++++++------------------- lib/qiniu/rs/version.rb | 2 +- qiniu-rs.gemspec | 2 +- spec/qiniu/rs/up_spec.rb | 4 +- spec/spec_helper.rb | 10 +-- 7 files changed, 83 insertions(+), 96 deletions(-) rename docs/NEWS.md => CHANGELOG.md (80%) diff --git a/docs/NEWS.md b/CHANGELOG.md similarity index 80% rename from docs/NEWS.md rename to CHANGELOG.md index a2e60f2..46a69b5 100644 --- a/docs/NEWS.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## CHANGE LOG +### v3.2.2 + +fixed E701 error + +断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ，规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。 + ### v3.2.1 allow images uploaded auto-orient. @@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and 参考： 1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression) -2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token) \ No newline at end of file +2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token) diff --git a/Gemfile.lock b/Gemfile.lock index badf1cd..9c5d6a1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - qiniu-rs (3.2.1) + qiniu-rs (3.2.2) json (~> 1.7) mime-types (~> 1.19) rest-client (~> 1.6) @@ -14,17 +14,17 @@ GEM fakeweb (1.3.0) json (1.7.5) mime-types (1.19) - rake (0.9.2.2) + rake (10.0.2) rest-client (1.6.7) mime-types (>= 1.16) - rspec (2.11.0) - rspec-core (~> 2.11.0) - rspec-expectations (~> 2.11.0) - rspec-mocks (~> 2.11.0) - rspec-core (2.11.1) - rspec-expectations (2.11.3) + rspec (2.12.0) + rspec-core (~> 2.12.0) + rspec-expectations (~> 2.12.0) + rspec-mocks (~> 2.12.0) + rspec-core (2.12.1) + rspec-expectations (2.12.0) diff-lcs (~> 1.1.3) - rspec-mocks (2.11.3) + rspec-mocks (2.12.0) ruby-hmac (0.4.0) PLATFORMS @@ -33,5 +33,5 @@ PLATFORMS DEPENDENCIES fakeweb (~> 1.3) qiniu-rs! - rake (~> 0.9) + rake (>= 0.9) rspec (~> 2.11) diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb index 5a02ea0..ea07541 100755 --- a/lib/qiniu/rs/up.rb +++ b/lib/qiniu/rs/up.rb @@ -14,6 +14,9 @@ module Qiniu module RS module UP + PROGRESS_TMP_FILE = 'progresses' + CHECKSUM_TMP_FILE = 'ctxes' + module AbstractClass class ChunkProgressNotifier include Qiniu::RS::Abstract @@ -29,78 +32,58 @@ module Qiniu end class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier + attr_reader :tmpdata def initialize(id) - @data = UP::ProgressData.new(id) + @tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE) end def notify(index, progress) - @data.set_progresses(index, progress) - logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + - "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" + @tmpdata.set(index, progress) + logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s Utils.debug(logmsg) end end class BlockProgressNotifier < AbstractClass::BlockProgressNotifier + attr_reader :tmpdata def initialize(id) - @data = UP::ProgressData.new(id) + @tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE) end def notify(index, checksum) - @data.set_checksums(index, checksum) + @tmpdata.set(index, checksum) + Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded." Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded." end end - class ProgressData - def initialize(id) - @id = id - @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id + class TmpData + def initialize(dir, filename) + @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir) - @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' - @progress_file = @tmpdir + File::SEPARATOR + 'progresses' + @tmpfile = @tmpdir + File::SEPARATOR + filename end - def get_checksums - File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] - end - - def get_progresses - File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] - end - - def set_checksums(index, checksum) - checksums = get_checksums - checksums[index] = checksum - File.open(@checksum_file, "w") do |f| - YAML::dump(checksums, f) - Utils.debug %Q(Updating tmpfile: #{@checksum_file}) + def init(values) + File.open(@tmpfile, "w") do |f| + YAML::dump(values, f) + Utils.debug %Q(Initializing tmpfile: #{@tmpfile}) end end - def set_progresses(index, progress) - progresses = get_progresses - progresses[index] = progress - File.open(@progress_file, "w") do |f| - YAML::dump(progresses, f) - Utils.debug %Q(Updating tmpfile: #{@progress_file}) - end + def all + File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : [] end - def init_checksums(checksums) - File.open(@checksum_file, "w") do |f| - YAML::dump(checksums, f) - Utils.debug %Q(Initializing tmpfile: #{@checksum_file}) - end - end - - def init_progresses(progresses) - File.open(@progress_file, "w") do |f| - YAML::dump(progresses, f) - Utils.debug %Q(Initializing tmpfile: #{@progress_file}) + def set(index, value) + values = all + values[index] = value + File.open(@tmpfile, "w") do |f| + YAML::dump(values, f) + Utils.debug %Q(Updating tmpfile: #{@tmpfile}) end end def sweep! - FileUtils.rm_r(@tmpdir) + FileUtils.rm_r(@tmpdir) if Dir.exists?(@tmpdir) end end @@ -162,20 +145,21 @@ module Qiniu end def _new_block_put_progress_data - {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil} + {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil} end - def _call_binary_with_token(uptoken, url, data, retry_times = 0) + def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0) options = { :method => :post, :content_type => 'application/octet-stream', :upload_signature_token => uptoken } + options[:content_type] = content_type if !content_type.nil? && !content_type.empty? code, data = http_request url, data, options unless Utils.is_response_ok?(code) retry_times += 1 if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] - return _call_binary_with_token(uptoken, url, data, retry_times) + return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times) end end [code, data] @@ -186,8 +170,8 @@ module Qiniu _call_binary_with_token(uptoken, url, body) end - def _putblock(uptoken, ctx, offset, body) - url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}" + def _putblock(uphost, uptoken, ctx, offset, body) + url = uphost + "/bput/#{ctx}/#{offset}" _call_binary_with_token(uptoken, url, body) end @@ -214,6 +198,7 @@ module Qiniu progress[:offset] = body_length progress[:restsize] = block_size - body_length progress[:status_code] = code + progress[:host] = data["host"] if !notifier.nil? && notifier.respond_to?("notify") notifier.notify(block_index, progress) end @@ -236,13 +221,14 @@ module Qiniu if result_length != body_length raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end - code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body) + code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body) body_crc32 = Zlib.crc32(body) if Utils.is_response_ok?(code) && data["crc32"] == body_crc32 progress[:ctx] = data["ctx"] progress[:offset] += body_length progress[:restsize] -= body_length progress[:status_code] = code + progress[:host] = data["host"] if !notifier.nil? && notifier.respond_to?("notify") notifier.notify(block_index, progress) end @@ -280,7 +266,8 @@ module Qiniu end code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier) if Utils.is_response_ok?(code) - checksums[block_index] = data["checksum"] + #checksums[block_index] = data["checksum"] + checksums[block_index] = data["ctx"] if !block_notifier.nil? && block_notifier.respond_to?("notify") block_notifier.notify(block_index, checksums[block_index]) end @@ -290,7 +277,7 @@ module Qiniu return [code, data] end - def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) + def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}" path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? @@ -298,37 +285,39 @@ module Qiniu callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty? path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty? path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0 - url = Config.settings[:up_host] + path - body = '' - checksums.each do |checksum| - body += Utils.urlsafe_base64_decode(checksum) - end - _call_binary_with_token(uptoken, url, body) + url = uphost + path + #body = '' + #checksums.each do |checksum| + # body += Utils.urlsafe_base64_decode(checksum) + #end + body = checksums.join(',') + _call_binary_with_token(uptoken, url, body, 'text/plain') end def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) block_count = _block_count(fsize) - progress_data = ProgressData.new(key) - checksums = progress_data.get_checksums - progresses = progress_data.get_progresses - if checksums.empty? - block_count.times{checksums << ''} - progress_data.init_checksums(checksums) - end - if progresses.empty? - block_count.times{progresses << _new_block_put_progress_data} - progress_data.init_progresses(progresses) - end chunk_notifier = ChunkProgressNotifier.new(key) block_notifier = BlockProgressNotifier.new(key) + progresses = chunk_notifier.tmpdata.all + if progresses.empty? + block_count.times{progresses << _new_block_put_progress_data} + chunk_notifier.tmpdata.init(progresses) + end + checksums = block_notifier.tmpdata.all + if checksums.empty? + block_count.times{checksums << ''} + block_notifier.tmpdata.init(checksums) + end code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) if Utils.is_response_ok?(code) + uphost = data["host"] entry_uri = bucket + ':' + key - code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate) + code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate) end if Utils.is_response_ok?(code) Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded." - progress_data.sweep! + chunk_notifier.tmpdata.sweep! + block_notifier.tmpdata.sweep! end [code, data] end diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb index 39161dc..dc7e222 100755 --- a/lib/qiniu/rs/version.rb +++ b/lib/qiniu/rs/version.rb @@ -5,7 +5,7 @@ module Qiniu module Version MAJOR = 3 MINOR = 2 - PATCH = 1 + PATCH = 2 # Returns a version string by joining MAJOR, MINOR, and PATCH with '.' # # Example diff --git a/qiniu-rs.gemspec b/qiniu-rs.gemspec index cae822f..868b259 100755 --- a/qiniu-rs.gemspec +++ b/qiniu-rs.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |gem| gem.version = Qiniu::RS::Version.to_s # specify any dependencies here; for example: - gem.add_development_dependency "rake", "~> 0.9" + gem.add_development_dependency "rake", ">= 0.9" gem.add_development_dependency "rspec", "~> 2.11" gem.add_development_dependency "fakeweb", "~> 1.3" gem.add_runtime_dependency "json", "~> 1.7" diff --git a/spec/qiniu/rs/up_spec.rb b/spec/qiniu/rs/up_spec.rb index a0e12c9..024d17b 100755 --- a/spec/qiniu/rs/up_spec.rb +++ b/spec/qiniu/rs/up_spec.rb @@ -11,7 +11,7 @@ module Qiniu before :all do @localfile = "bigfile.txt" - File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}} + File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}} @bucket = "up_test_bucket" @key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s) @@ -21,10 +21,8 @@ module Qiniu end after :all do - @localfile = "bigfile.txt" File.unlink(@localfile) if File.exists?(@localfile) - @bucket = "up_test_bucket" code, data = Qiniu::RS::RS.drop(@bucket) puts [code, data].inspect code.should == 200 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d886e24..4647738 100755 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,13 +6,7 @@ require 'rspec' RSpec.configure do |config| config.before :all do - Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh", - :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW", - :auth_url => "http://m1.qbox.me:13001/oauth2/token", - :rs_host => "http://m1.qbox.me:13003", - :io_host => "http://m1.qbox.me:13004", - :up_host => "http://m1.qbox.me:13019", - :pub_host => "http://m1.qbox.me:13012", - :eu_host => "http://m1.qbox.me:13050" + Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV", + :secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j" end end