home/ruby-sdk
2
0
Fork 0
mirror of https://github.com/wahyd4/ruby-sdk.git synced 2026-08-09 12:56:21 +10:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26 from why404/release/v3.2.2

Browse Source 
fix up 701
This commit is contained in:
xushiwei
2012-12-04 22:32:53 -08:00
parent 64cf9b5d3f a8a8f63887
commit 2cbb175423
7 changed files with 83 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/NEWS.md → CHANGELOG.md
+7 -1
View File
@@ -1,5 +1,11 @@
## CHANGE LOG
### v3.2.2
fixed E701 error
断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ，规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。
### v3.2.1
allow images uploaded auto-orient.
@@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and
参考：
1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression)
2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
Gemfile.lock
+10 -10
View File
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
qiniu-rs (3.2.1)
qiniu-rs (3.2.2)
json (~> 1.7)
mime-types (~> 1.19)
rest-client (~> 1.6)
@@ -14,17 +14,17 @@ GEM
fakeweb (1.3.0)
json (1.7.5)
mime-types (1.19)
rake (0.9.2.2)
rake (10.0.2)
rest-client (1.6.7)
mime-types (>= 1.16)
rspec (2.11.0)
rspec-core (~> 2.11.0)
rspec-expectations (~> 2.11.0)
rspec-mocks (~> 2.11.0)
rspec-core (2.11.1)
rspec-expectations (2.11.3)
rspec (2.12.0)
rspec-core (~> 2.12.0)
rspec-expectations (~> 2.12.0)
rspec-mocks (~> 2.12.0)
rspec-core (2.12.1)
rspec-expectations (2.12.0)
diff-lcs (~> 1.1.3)
rspec-mocks (2.11.3)
rspec-mocks (2.12.0)
ruby-hmac (0.4.0)
PLATFORMS
@@ -33,5 +33,5 @@ PLATFORMS
DEPENDENCIES
fakeweb (~> 1.3)
qiniu-rs!
rake (~> 0.9)
rake (>= 0.9)
rspec (~> 2.11)
lib/qiniu/rs/up.rb
+61 -72
View File
@@ -14,6 +14,9 @@ module Qiniu
module RS
module UP
PROGRESS_TMP_FILE = 'progresses'
CHECKSUM_TMP_FILE = 'ctxes'
module AbstractClass
class ChunkProgressNotifier
include Qiniu::RS::Abstract
@@ -29,78 +32,58 @@ module Qiniu
end
class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
attr_reader :tmpdata
def initialize(id)
@data = UP::ProgressData.new(id)
@tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE)
end
def notify(index, progress)
@data.set_progresses(index, progress)
logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" +
"{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
@tmpdata.set(index, progress)
logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
Utils.debug(logmsg)
end
end
class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
attr_reader :tmpdata
def initialize(id)
@data = UP::ProgressData.new(id)
@tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE)
end
def notify(index, checksum)
@data.set_checksums(index, checksum)
@tmpdata.set(index, checksum)
Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
end
end
class ProgressData
def initialize(id)
@id = id
@tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
class TmpData
def initialize(dir, filename)
@tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir
FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir)
@checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
@progress_file = @tmpdir + File::SEPARATOR + 'progresses'
@tmpfile = @tmpdir + File::SEPARATOR + filename
end
def get_checksums
File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
end
def get_progresses
File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
end
def set_checksums(index, checksum)
checksums = get_checksums
checksums[index] = checksum
File.open(@checksum_file, "w") do |f|
YAML::dump(checksums, f)
Utils.debug %Q(Updating tmpfile: #{@checksum_file})
def init(values)
File.open(@tmpfile, "w") do |f|
YAML::dump(values, f)
Utils.debug %Q(Initializing tmpfile: #{@tmpfile})
end
end
def set_progresses(index, progress)
progresses = get_progresses
progresses[index] = progress
File.open(@progress_file, "w") do |f|
YAML::dump(progresses, f)
Utils.debug %Q(Updating tmpfile: #{@progress_file})
end
def all
File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : []
end
def init_checksums(checksums)
File.open(@checksum_file, "w") do |f|
YAML::dump(checksums, f)
Utils.debug %Q(Initializing tmpfile: #{@checksum_file})
end
end
def init_progresses(progresses)
File.open(@progress_file, "w") do |f|
YAML::dump(progresses, f)
Utils.debug %Q(Initializing tmpfile: #{@progress_file})
def set(index, value)
values = all
values[index] = value
File.open(@tmpfile, "w") do |f|
YAML::dump(values, f)
Utils.debug %Q(Updating tmpfile: #{@tmpfile})
end
end
def sweep!
FileUtils.rm_r(@tmpdir)
FileUtils.rm_r(@tmpdir) if Dir.exists?(@tmpdir)
end
end
@@ -162,20 +145,21 @@ module Qiniu
end
def _new_block_put_progress_data
{:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
{:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
end
def _call_binary_with_token(uptoken, url, data, retry_times = 0)
def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
options = {
:method => :post,
:content_type => 'application/octet-stream',
:upload_signature_token => uptoken
}
options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
code, data = http_request url, data, options
unless Utils.is_response_ok?(code)
retry_times += 1
if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
return _call_binary_with_token(uptoken, url, data, retry_times)
return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
end
end
[code, data]
@@ -186,8 +170,8 @@ module Qiniu
_call_binary_with_token(uptoken, url, body)
end
def _putblock(uptoken, ctx, offset, body)
url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
def _putblock(uphost, uptoken, ctx, offset, body)
url = uphost + "/bput/#{ctx}/#{offset}"
_call_binary_with_token(uptoken, url, body)
end
@@ -214,6 +198,7 @@ module Qiniu
progress[:offset] = body_length
progress[:restsize] = block_size - body_length
progress[:status_code] = code
progress[:host] = data["host"]
if !notifier.nil? && notifier.respond_to?("notify")
notifier.notify(block_index, progress)
end
@@ -236,13 +221,14 @@ module Qiniu
if result_length != body_length
raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
end
code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
body_crc32 = Zlib.crc32(body)
if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
progress[:ctx] = data["ctx"]
progress[:offset] += body_length
progress[:restsize] -= body_length
progress[:status_code] = code
progress[:host] = data["host"]
if !notifier.nil? && notifier.respond_to?("notify")
notifier.notify(block_index, progress)
end
@@ -280,7 +266,8 @@ module Qiniu
end
code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
if Utils.is_response_ok?(code)
checksums[block_index] = data["checksum"]
#checksums[block_index] = data["checksum"]
checksums[block_index] = data["ctx"]
if !block_notifier.nil? && block_notifier.respond_to?("notify")
block_notifier.notify(block_index, checksums[block_index])
end
@@ -290,7 +277,7 @@ module Qiniu
return [code, data]
end
def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
@@ -298,37 +285,39 @@ module Qiniu
callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
url = Config.settings[:up_host] + path
body = ''
checksums.each do |checksum|
body += Utils.urlsafe_base64_decode(checksum)
end
_call_binary_with_token(uptoken, url, body)
url = uphost + path
#body = ''
#checksums.each do |checksum|
# body += Utils.urlsafe_base64_decode(checksum)
#end
body = checksums.join(',')
_call_binary_with_token(uptoken, url, body, 'text/plain')
end
def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
block_count = _block_count(fsize)
progress_data = ProgressData.new(key)
checksums = progress_data.get_checksums
progresses = progress_data.get_progresses
if checksums.empty?
block_count.times{checksums << ''}
progress_data.init_checksums(checksums)
end
if progresses.empty?
block_count.times{progresses << _new_block_put_progress_data}
progress_data.init_progresses(progresses)
end
chunk_notifier = ChunkProgressNotifier.new(key)
block_notifier = BlockProgressNotifier.new(key)
progresses = chunk_notifier.tmpdata.all
if progresses.empty?
block_count.times{progresses << _new_block_put_progress_data}
chunk_notifier.tmpdata.init(progresses)
end
checksums = block_notifier.tmpdata.all
if checksums.empty?
block_count.times{checksums << ''}
block_notifier.tmpdata.init(checksums)
end
code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
if Utils.is_response_ok?(code)
uphost = data["host"]
entry_uri = bucket + ':' + key
code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
end
if Utils.is_response_ok?(code)
Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
progress_data.sweep!
chunk_notifier.tmpdata.sweep!
block_notifier.tmpdata.sweep!
end
[code, data]
end
lib/qiniu/rs/version.rb
+1 -1
View File
@@ -5,7 +5,7 @@ module Qiniu
module Version
MAJOR = 3
MINOR = 2
PATCH = 1
PATCH = 2
# Returns a version string by joining <tt>MAJOR</tt>, <tt>MINOR</tt>, and <tt>PATCH</tt> with <tt>'.'</tt>
#
# Example
qiniu-rs.gemspec
+1 -1
View File
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
gem.version = Qiniu::RS::Version.to_s
# specify any dependencies here; for example:
gem.add_development_dependency "rake", "~> 0.9"
gem.add_development_dependency "rake", ">= 0.9"
gem.add_development_dependency "rspec", "~> 2.11"
gem.add_development_dependency "fakeweb", "~> 1.3"
gem.add_runtime_dependency "json", "~> 1.7"
spec/qiniu/rs/up_spec.rb
+1 -3
View File
@@ -11,7 +11,7 @@ module Qiniu
before :all do
@localfile = "bigfile.txt"
File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}}
File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}}
@bucket = "up_test_bucket"
@key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s)
@@ -21,10 +21,8 @@ module Qiniu
end
after :all do
@localfile = "bigfile.txt"
File.unlink(@localfile) if File.exists?(@localfile)
@bucket = "up_test_bucket"
code, data = Qiniu::RS::RS.drop(@bucket)
puts [code, data].inspect
code.should == 200
spec/spec_helper.rb
+2 -8
View File
@@ -6,13 +6,7 @@ require 'rspec'
RSpec.configure do |config|
config.before :all do
Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
:secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
:auth_url => "http://m1.qbox.me:13001/oauth2/token",
:rs_host => "http://m1.qbox.me:13003",
:io_host => "http://m1.qbox.me:13004",
:up_host => "http://m1.qbox.me:13019",
:pub_host => "http://m1.qbox.me:13012",
:eu_host => "http://m1.qbox.me:13050"
Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
:secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j"
end
end