diff --git a/docs/NEWS.md b/CHANGELOG.md
similarity index 80%
rename from docs/NEWS.md
rename to CHANGELOG.md
index a2e60f2..46a69b5 100644
--- a/docs/NEWS.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
## CHANGE LOG
+### v3.2.2
+
+fixed E701 error
+
+断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ，规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。
+
### v3.2.1
allow images uploaded auto-orient.
@@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and
参考：
1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression)
-2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
\ No newline at end of file
+2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
diff --git a/Gemfile.lock b/Gemfile.lock
index badf1cd..9c5d6a1 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
- qiniu-rs (3.2.1)
+ qiniu-rs (3.2.2)
json (~> 1.7)
mime-types (~> 1.19)
rest-client (~> 1.6)
@@ -14,17 +14,17 @@ GEM
fakeweb (1.3.0)
json (1.7.5)
mime-types (1.19)
- rake (0.9.2.2)
+ rake (10.0.2)
rest-client (1.6.7)
mime-types (>= 1.16)
- rspec (2.11.0)
- rspec-core (~> 2.11.0)
- rspec-expectations (~> 2.11.0)
- rspec-mocks (~> 2.11.0)
- rspec-core (2.11.1)
- rspec-expectations (2.11.3)
+ rspec (2.12.0)
+ rspec-core (~> 2.12.0)
+ rspec-expectations (~> 2.12.0)
+ rspec-mocks (~> 2.12.0)
+ rspec-core (2.12.1)
+ rspec-expectations (2.12.0)
diff-lcs (~> 1.1.3)
- rspec-mocks (2.11.3)
+ rspec-mocks (2.12.0)
ruby-hmac (0.4.0)
PLATFORMS
@@ -33,5 +33,5 @@ PLATFORMS
DEPENDENCIES
fakeweb (~> 1.3)
qiniu-rs!
- rake (~> 0.9)
+ rake (>= 0.9)
rspec (~> 2.11)
diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb
index 5a02ea0..ea07541 100755
--- a/lib/qiniu/rs/up.rb
+++ b/lib/qiniu/rs/up.rb
@@ -14,6 +14,9 @@ module Qiniu
module RS
module UP
+ PROGRESS_TMP_FILE = 'progresses'
+ CHECKSUM_TMP_FILE = 'ctxes'
+
module AbstractClass
class ChunkProgressNotifier
include Qiniu::RS::Abstract
@@ -29,78 +32,58 @@ module Qiniu
end
class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
+ attr_reader :tmpdata
def initialize(id)
- @data = UP::ProgressData.new(id)
+ @tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE)
end
def notify(index, progress)
- @data.set_progresses(index, progress)
- logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" +
- "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
+ @tmpdata.set(index, progress)
+ logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
Utils.debug(logmsg)
end
end
class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
+ attr_reader :tmpdata
def initialize(id)
- @data = UP::ProgressData.new(id)
+ @tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE)
end
def notify(index, checksum)
- @data.set_checksums(index, checksum)
+ @tmpdata.set(index, checksum)
+ Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
end
end
- class ProgressData
- def initialize(id)
- @id = id
- @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
+ class TmpData
+ def initialize(dir, filename)
+ @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir
FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir)
- @checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
- @progress_file = @tmpdir + File::SEPARATOR + 'progresses'
+ @tmpfile = @tmpdir + File::SEPARATOR + filename
end
- def get_checksums
- File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
- end
-
- def get_progresses
- File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
- end
-
- def set_checksums(index, checksum)
- checksums = get_checksums
- checksums[index] = checksum
- File.open(@checksum_file, "w") do |f|
- YAML::dump(checksums, f)
- Utils.debug %Q(Updating tmpfile: #{@checksum_file})
+ def init(values)
+ File.open(@tmpfile, "w") do |f|
+ YAML::dump(values, f)
+ Utils.debug %Q(Initializing tmpfile: #{@tmpfile})
end
end
- def set_progresses(index, progress)
- progresses = get_progresses
- progresses[index] = progress
- File.open(@progress_file, "w") do |f|
- YAML::dump(progresses, f)
- Utils.debug %Q(Updating tmpfile: #{@progress_file})
- end
+ def all
+ File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : []
end
- def init_checksums(checksums)
- File.open(@checksum_file, "w") do |f|
- YAML::dump(checksums, f)
- Utils.debug %Q(Initializing tmpfile: #{@checksum_file})
- end
- end
-
- def init_progresses(progresses)
- File.open(@progress_file, "w") do |f|
- YAML::dump(progresses, f)
- Utils.debug %Q(Initializing tmpfile: #{@progress_file})
+ def set(index, value)
+ values = all
+ values[index] = value
+ File.open(@tmpfile, "w") do |f|
+ YAML::dump(values, f)
+ Utils.debug %Q(Updating tmpfile: #{@tmpfile})
end
end
def sweep!
- FileUtils.rm_r(@tmpdir)
+ FileUtils.rm_r(@tmpdir) if Dir.exists?(@tmpdir)
end
end
@@ -162,20 +145,21 @@ module Qiniu
end
def _new_block_put_progress_data
- {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
+ {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
end
- def _call_binary_with_token(uptoken, url, data, retry_times = 0)
+ def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
options = {
:method => :post,
:content_type => 'application/octet-stream',
:upload_signature_token => uptoken
}
+ options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
code, data = http_request url, data, options
unless Utils.is_response_ok?(code)
retry_times += 1
if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
- return _call_binary_with_token(uptoken, url, data, retry_times)
+ return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
end
end
[code, data]
@@ -186,8 +170,8 @@ module Qiniu
_call_binary_with_token(uptoken, url, body)
end
- def _putblock(uptoken, ctx, offset, body)
- url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
+ def _putblock(uphost, uptoken, ctx, offset, body)
+ url = uphost + "/bput/#{ctx}/#{offset}"
_call_binary_with_token(uptoken, url, body)
end
@@ -214,6 +198,7 @@ module Qiniu
progress[:offset] = body_length
progress[:restsize] = block_size - body_length
progress[:status_code] = code
+ progress[:host] = data["host"]
if !notifier.nil? && notifier.respond_to?("notify")
notifier.notify(block_index, progress)
end
@@ -236,13 +221,14 @@ module Qiniu
if result_length != body_length
raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
end
- code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
+ code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
body_crc32 = Zlib.crc32(body)
if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
progress[:ctx] = data["ctx"]
progress[:offset] += body_length
progress[:restsize] -= body_length
progress[:status_code] = code
+ progress[:host] = data["host"]
if !notifier.nil? && notifier.respond_to?("notify")
notifier.notify(block_index, progress)
end
@@ -280,7 +266,8 @@ module Qiniu
end
code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
if Utils.is_response_ok?(code)
- checksums[block_index] = data["checksum"]
+ #checksums[block_index] = data["checksum"]
+ checksums[block_index] = data["ctx"]
if !block_notifier.nil? && block_notifier.respond_to?("notify")
block_notifier.notify(block_index, checksums[block_index])
end
@@ -290,7 +277,7 @@ module Qiniu
return [code, data]
end
- def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
+ def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
@@ -298,37 +285,39 @@ module Qiniu
callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
- url = Config.settings[:up_host] + path
- body = ''
- checksums.each do |checksum|
- body += Utils.urlsafe_base64_decode(checksum)
- end
- _call_binary_with_token(uptoken, url, body)
+ url = uphost + path
+ #body = ''
+ #checksums.each do |checksum|
+ # body += Utils.urlsafe_base64_decode(checksum)
+ #end
+ body = checksums.join(',')
+ _call_binary_with_token(uptoken, url, body, 'text/plain')
end
def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
block_count = _block_count(fsize)
- progress_data = ProgressData.new(key)
- checksums = progress_data.get_checksums
- progresses = progress_data.get_progresses
- if checksums.empty?
- block_count.times{checksums << ''}
- progress_data.init_checksums(checksums)
- end
- if progresses.empty?
- block_count.times{progresses << _new_block_put_progress_data}
- progress_data.init_progresses(progresses)
- end
chunk_notifier = ChunkProgressNotifier.new(key)
block_notifier = BlockProgressNotifier.new(key)
+ progresses = chunk_notifier.tmpdata.all
+ if progresses.empty?
+ block_count.times{progresses << _new_block_put_progress_data}
+ chunk_notifier.tmpdata.init(progresses)
+ end
+ checksums = block_notifier.tmpdata.all
+ if checksums.empty?
+ block_count.times{checksums << ''}
+ block_notifier.tmpdata.init(checksums)
+ end
code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
if Utils.is_response_ok?(code)
+ uphost = data["host"]
entry_uri = bucket + ':' + key
- code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
+ code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
end
if Utils.is_response_ok?(code)
Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
- progress_data.sweep!
+ chunk_notifier.tmpdata.sweep!
+ block_notifier.tmpdata.sweep!
end
[code, data]
end
diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb
index 39161dc..dc7e222 100755
--- a/lib/qiniu/rs/version.rb
+++ b/lib/qiniu/rs/version.rb
@@ -5,7 +5,7 @@ module Qiniu
module Version
MAJOR = 3
MINOR = 2
- PATCH = 1
+ PATCH = 2
# Returns a version string by joining MAJOR, MINOR, and PATCH with '.'
#
# Example
diff --git a/qiniu-rs.gemspec b/qiniu-rs.gemspec
index cae822f..868b259 100755
--- a/qiniu-rs.gemspec
+++ b/qiniu-rs.gemspec
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
gem.version = Qiniu::RS::Version.to_s
# specify any dependencies here; for example:
- gem.add_development_dependency "rake", "~> 0.9"
+ gem.add_development_dependency "rake", ">= 0.9"
gem.add_development_dependency "rspec", "~> 2.11"
gem.add_development_dependency "fakeweb", "~> 1.3"
gem.add_runtime_dependency "json", "~> 1.7"
diff --git a/spec/qiniu/rs/up_spec.rb b/spec/qiniu/rs/up_spec.rb
index a0e12c9..024d17b 100755
--- a/spec/qiniu/rs/up_spec.rb
+++ b/spec/qiniu/rs/up_spec.rb
@@ -11,7 +11,7 @@ module Qiniu
before :all do
@localfile = "bigfile.txt"
- File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}}
+ File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}}
@bucket = "up_test_bucket"
@key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s)
@@ -21,10 +21,8 @@ module Qiniu
end
after :all do
- @localfile = "bigfile.txt"
File.unlink(@localfile) if File.exists?(@localfile)
- @bucket = "up_test_bucket"
code, data = Qiniu::RS::RS.drop(@bucket)
puts [code, data].inspect
code.should == 200
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index d886e24..4647738 100755
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -6,13 +6,7 @@ require 'rspec'
RSpec.configure do |config|
config.before :all do
- Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
- :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
- :auth_url => "http://m1.qbox.me:13001/oauth2/token",
- :rs_host => "http://m1.qbox.me:13003",
- :io_host => "http://m1.qbox.me:13004",
- :up_host => "http://m1.qbox.me:13019",
- :pub_host => "http://m1.qbox.me:13012",
- :eu_host => "http://m1.qbox.me:13050"
+ Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
+ :secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j"
end
end