From fa40449b120df69a027679db7936a221a1a4081f Mon Sep 17 00:00:00 2001 From: 404 Date: Thu, 27 Sep 2012 20:22:51 +0800 Subject: [PATCH 1/5] added UploadFailedError --- lib/qiniu/rs/exceptions.rb | 10 ++++++++++ lib/qiniu/rs/version.rb | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index 2c282df..da43311 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -40,6 +40,16 @@ module Qiniu end end + class UploadFailedError < ResponseError + def message + "Uploading Failed. HTTP Status Code #{http_code}" + end + + def to_s + message + end + end + class MissingArgsError < Exception def initialize(missing_keys) key_list = missing_keys.map {|key| key.to_s}.join(' and the ') diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb index 01c1c6c..343ce15 100755 --- a/lib/qiniu/rs/version.rb +++ b/lib/qiniu/rs/version.rb @@ -2,6 +2,6 @@ module Qiniu module RS - VERSION = "3.0.6" + VERSION = "3.0.7" end end From bd34ea428e0abd89490bf2111babb3986c045ffd Mon Sep 17 00:00:00 2001 From: 404 Date: Tue, 2 Oct 2012 18:47:19 +0800 Subject: [PATCH 2/5] add Qiniu::RS::UP module --- Gemfile.lock | 4 +- lib/qiniu/rs.rb | 4 +- lib/qiniu/rs/abstract.rb | 24 +++ lib/qiniu/rs/auth.rb | 6 +- lib/qiniu/rs/config.rb | 13 +- lib/qiniu/rs/exceptions.rb | 18 ++ lib/qiniu/rs/up.rb | 294 +++++++++++++++++++++++++++++++++ lib/qiniu/rs/utils.rb | 14 +- lib/qiniu/rs/version.rb | 14 +- qiniu-rs.gemspec | 2 +- spec/qiniu/rs/abstract_spec.rb | 30 ++++ spec/qiniu/rs/rs_spec.rb | 6 +- spec/qiniu/rs/version_spec.rb | 4 +- spec/spec_helper.rb | 5 + 14 files changed, 416 insertions(+), 22 deletions(-) create mode 100644 lib/qiniu/rs/abstract.rb create mode 100755 lib/qiniu/rs/up.rb create mode 100755 spec/qiniu/rs/abstract_spec.rb diff --git a/Gemfile.lock b/Gemfile.lock index d2cffa8..45c38ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - qiniu-rs (3.0.6) + qiniu-rs (3.1.0) json (~> 1.7) mime-types (~> 1.19) rest-client (~> 1.6) @@ -24,7 +24,7 @@ GEM rspec-core (2.11.1) rspec-expectations (2.11.3) diff-lcs (~> 1.1.3) - rspec-mocks (2.11.2) + rspec-mocks (2.11.3) ruby-hmac (0.4.0) PLATFORMS diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb index 3c9dc72..aa1cda9 100755 --- a/lib/qiniu/rs.rb +++ b/lib/qiniu/rs.rb @@ -1,9 +1,8 @@ # -*- encoding: utf-8 -*- -require 'qiniu/rs/version' - module Qiniu module RS + autoload :Version, 'qiniu/rs/version' autoload :Config, 'qiniu/rs/config' autoload :Log, 'qiniu/rs/log' autoload :Exception, 'qiniu/rs/exceptions' @@ -17,6 +16,7 @@ module Qiniu autoload :AccessToken, 'qiniu/tokens/access_token' autoload :QboxToken, 'qiniu/tokens/qbox_token' autoload :UploadToken, 'qiniu/tokens/upload_token' + autoload :Abstract, 'qiniu/rs/abstract' class << self diff --git a/lib/qiniu/rs/abstract.rb b/lib/qiniu/rs/abstract.rb new file mode 100644 index 0000000..d9fc671 --- /dev/null +++ b/lib/qiniu/rs/abstract.rb @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- + +module Qiniu + module RS + module Abstract + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + def abstract_methods(*args) + args.each do |name| + class_eval <<-END + def #{name}(*args) + errmsg = %Q(class \#{self.class.name} must implement abstract method #{self.name}##{name}().) + raise NotImplementedError.new(errmsg) + end + END + end + end + end + end + end +end diff --git a/lib/qiniu/rs/auth.rb b/lib/qiniu/rs/auth.rb index 6a9a924..bd93a79 100755 --- a/lib/qiniu/rs/auth.rb +++ b/lib/qiniu/rs/auth.rb @@ -18,7 +18,7 @@ module Qiniu :password => password } code, data = http_request Config.settings[:auth_url], post_data - reset_token(data["access_token"], data["refresh_token"]) if code == 200 + reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code) [code, data] end @@ -29,7 +29,7 @@ module Qiniu :refresh_token => refresh_token } code, data = http_request Config.settings[:auth_url], post_data - reset_token(data["access_token"], data["refresh_token"]) if code == 200 + reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code) [code, data] end @@ -48,7 +48,7 @@ module Qiniu raise MissingUsernameOrPassword if (@username.nil? || @password.nil?) code, data = exchange_by_password!(@username, @password) end - if code == 200 + if Utils.is_response_ok?(code) retry_times += 1 if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] return call_with_logged_in(url, data, retry_times) diff --git a/lib/qiniu/rs/config.rb b/lib/qiniu/rs/config.rb index b667c91..59ff91a 100755 --- a/lib/qiniu/rs/config.rb +++ b/lib/qiniu/rs/config.rb @@ -8,7 +8,7 @@ # Qbox::Config.load "path/to/your_project/config/qiniu.yml" # -require "qiniu/rs/version" +require 'tmpdir' module Qiniu module RS @@ -16,11 +16,11 @@ module Qiniu class << self DEFAULT_OPTIONS = { - :user_agent => 'Qiniu-RS-Ruby-SDK-' + VERSION + '()', + :user_agent => 'Qiniu-RS-Ruby-SDK-' + Version.to_s + '()', :method => :post, :content_type => 'application/x-www-form-urlencoded', :auth_url => "https://acc.qbox.me/oauth2/token", - :rs_host => "http://rs.qbox.me:10100", + :rs_host => "http://rs.qbox.me", :io_host => "http://iovip.qbox.me", :up_host => "http://up.qbox.me", :pub_host => "http://pu.qbox.me:10200", @@ -30,10 +30,13 @@ module Qiniu :access_key => "", :secret_key => "", :auto_reconnect => true, - :max_retry_times => 5 + :max_retry_times => 3, + :block_size => 1024*1024*4, + :chunk_size => 1024*256, + :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK' } - REQUIRED_OPTION_KEYS = [:client_id, :client_secret, :auth_url, :rs_host, :io_host] + REQUIRED_OPTION_KEYS = [:access_key, :secret_key] attr_reader :settings, :default_params diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index da43311..8c5a975 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -50,6 +50,24 @@ module Qiniu end end + class ResumablePutBlockError < ResponseError + def initialize(message) + super(message) + end + end + + class ResumablePutError < ResponseError + def initialize(message) + super(message) + end + end + + class FileSeekReadError < ResponseError + def initialize(seek_pos, read_length, result_length) + super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length}) + end + end + class MissingArgsError < Exception def initialize(missing_keys) key_list = missing_keys.map {|key| key.to_s}.join(' and the ') diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb new file mode 100755 index 0000000..7010cfe --- /dev/null +++ b/lib/qiniu/rs/up.rb @@ -0,0 +1,294 @@ +# -*- encoding: utf-8 -*- + +require 'zlib' +require 'yaml' +require 'tmpdir' +require 'mime/types' +require 'digest/sha1' +require 'qiniu/rs/abstract' +require 'qiniu/rs/exceptions' + +module Qiniu + module RS + module UP + + module Abstract + class ChunkProgressNotifier + include Abstract + abstract_methods :notify + # def notify(block_index, block_put_progress); end + end + + class BlockProgressNotifier + include Abstract + abstract_methods :notify + # def notify(block_index, checksum); end + end + end + + class ChunkProgressNotifier < Abstract::ChunkProgressNotifier + def initialize(id) + @data = ProgressData.new(id) + end + def notify(index, progress) + @data.set_progresses(index, progress) + logmsg = "chunk #{index} successfully uploaded.\n" + + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" + Log.logger.info logmsg + end + end + + class BlockProgressNotifier < Abstract::BlockProgressNotifier + def initialize(id) + @data = ProgressData.new(id) + end + def notify(index, checksum) + @data.set_checksums(index, checksum) + logmsg = "block #{index}:#{checksum} successfully uploaded." + Log.logger.info logmsg + end + end + + + class << self + include Utils + + def upload(uptoken, + local_file, + bucket, + key = nil, + mime_type = nil, + custom_meta = nil, + customer = nil, + callback_params = nil) + raise NoSuchFileError, local_file unless File.exist?(local_file) + begin + ifile = File.open(local_file, 'rb') + fh = FileData.new(ifile) + key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? + entry_uri = bucket + ':' + key + if mime_type.nil? || mime_type.empty? + mime = MIME::Types.type_for local_file + mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type + end + fsize = fh.data_size + block_count = _block_count(fsize) + progress_data = ProgressData.new(key) + checksums = progress_data.get_checksums + progresses = progress_data.get_progresses + block_count.times{checksums << ''} if checksums.empty? + block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + chunk_notifier = ChunkProgressNotifier.new(key) + block_notifier = BlockProgressNotifier.new(key) + code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) + if Utils.is_response_ok?(code) + code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) + end + if Utils.is_response_ok?(code) + Log.logger.info "File #{local_file} successfully uploaded." + # progress_data.sweep! + end + [code, data] + ensure + ifile.close unless ifile.nil? + end + end + + private + + class FileData + attr_accessor :fh + def initialize(fh) + @fh = fh + end + def data_size + @fh.stat.size + end + def get_data(offset, length) + @fh.seek(offset) + @fh.read(length) + end + delegate :mtime, :to => :fh + end + + class ProgressData + def initialize(id) + @id = id + @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id + Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir) + @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' + @progress_file = @tmpdir + File::SEPARATOR + 'progresses' + end + + def get_checksums + File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] + end + + def get_progresses + File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] + end + + def set_checksums(index, checksum) + checksums = get_checksums + checksums[index] = checksum + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + end + end + + def set_progresses(index, progress) + progresses = get_progresses + progresses[index] = progress + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + end + end + + def sweep! + Dir.rmdir(@tmpdir) + end + end + + def _new_block_put_progress_data + {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil} + end + + def _call_binary_with_token(uptoken, url, data, retry_times = 0) + options = { + :method => :post, + :content_type => 'application/octet-stream', + :upload_signature_token => uptoken + } + code, data = http_request url, data, options + unless Utils.is_response_ok?(code) + retry_times += 1 + if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] + return _call_binary_with_token(uptoken, url, data, retry_times) + end + end + [code, data] + end + + def _mkblock(uptoken, block_size, body) + url = Config.settings[:up_host] + "/mkblk/#{block_size}" + _call_binary_with_token(uptoken, url, body) + end + + def _putblock(uptoken, ctx, offset, body) + url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}" + _call_binary_with_token(uptoken, url, body) + end + + def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier) + code, data = 0, {} + # this block has never been uploaded. + if progress[:ctx] == nil || progress[:ctx].empty? + progress[:offset] = 0 + progress[:restsize] = block_size + # choose the smaller one + body_length = [block_size, chunk_size].min + for i in 1..retry_times + seek_pos = block_index*Config.settings[:block_size] + body = fh.get_data(seek_pos, body_length) + result_length = body.length + if result_length != body_length + raise FileSeekReadError.new(seek_pos, body_length, result_length) + end + code, data = _mkblock(uptoken, block_size, body) + body_crc32 = Zlib.crc32(body) + if Utils.is_response_ok?(code) && data["crc32"] == body_crc32 + progress[:ctx] = data["ctx"] + progress[:offset] = body_length + progress[:restsize] = block_size - body_length + progress[:status_code] = code + if !notifier.nil? && notifier.respond_to?("notify") + notifier.notify(block_index, progress) + end + break + elsif i == retry_times && data["crc32"] != body_crc32 + Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + end + end + elsif progress[:offset] + progress[:restsize] != block_size + raise ResumablePutBlockError.new("Invalid arg. File length does not match.") + end + # loop uploading other chunks except the first one + while progress[:restsize].to_i > 0 && progress[:restsize] < block_size + # choose the smaller one + body_length = [progress[:restsize], chunk_size].min + for i in 1..retry_times + seek_pos = block_index*Config.settings[:block_size] + progress[:offset] + body = fh.get_data(seek_pos, body_length) + result_length = body.length + if result_length != body_length + raise FileSeekReadError.new(seek_pos, body_length, result_length) + end + code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body) + body_crc32 = Zlib.crc32(body) + if Utils.is_response_ok?(code) && data["crc32"] == body_crc32 + progress[:ctx] = data["ctx"] + progress[:offset] += body_length + progress[:restsize] -= body_length + progress[:status_code] = code + if !notifier.nil? && notifier.respond_to?("notify") + notifier.notify(block_index, progress) + end + break + elsif i == retry_times && data["crc32"] != body_crc32 + Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + end + end + end + # return + return [code, data] + end + + def _block_count(fsize) + ((fsize + Config.block_size - 1) / Config.block_size).to_i + end + + def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) + code, data = 0, {} + block_count = _block_count(fh.data_size) + if checksums.length != block_count || progresses.length != block_count + raise ResumablePutError.new("Invalid arg. Unexpected block count.") + end + 0.upto(block_count-1).each do |block_index| + if checksums[block_index].nil? || checksums[block_index].empty? + block_size = Config.settings[:block_size] + if block_index == block_count - 1 + block_size = fsize - block_index*Config.settings[:block_size] + end + if progresses[block_index].nil? + progresses[block_index] = _new_block_put_progress_data + end + code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier) + if Utils.is_response_ok?(code) + checksums[block_index] = data["checksum"] + if !block_notifier.nil? && block_notifier.respond_to?("notify") + block_notifier.notify(block_index, checksums[block_index]) + end + end + end + end + return [code, data] + end + + def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize + path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? + path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? + path += '/customer/' + customer if !customer.nil? && !customer.empty? + path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty? + url = Config.settings[:up_host] + path + body = '' + checksums.each do |checksum| + body += Utils.urlsafe_base64_decode(checksum) + end + _call_binary_with_token(uptoken, url, body) + end + + end + end + end +end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index f12c39c..341cd92 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -32,6 +32,14 @@ module Qiniu {} end + def is_response_ok?(status_code) + status_code/100 == 2 + end + + def response_error(status_code, errmsg) + [status_code, {"error" => errmsg}] + end + def send_request_with url, data = nil, options = {} options[:method] = Config.settings[:method] unless options[:method] options[:content_type] = Config.settings[:content_type] unless options[:content_type] @@ -42,8 +50,8 @@ module Qiniu auth_token = nil if !options[:qbox_signature_token].nil? && !options[:qbox_signature_token].empty? auth_token = 'QBox ' + options[:qbox_signature_token] - #elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty? - # auth_token = 'UpToken ' + options[:upload_signature_token] + elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty? + auth_token = 'UpToken ' + options[:upload_signature_token] elsif options[:access_token] auth_token = 'Bearer ' + options[:access_token] end @@ -56,7 +64,7 @@ module Qiniu response = RestClient.post(url, data, header_options) end code = response.respond_to?(:code) ? response.code.to_i : 0 - if code != 200 + unless is_response_ok?(code) raise RequestFailed.new(response) else data = {} diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb index 343ce15..eb073b7 100755 --- a/lib/qiniu/rs/version.rb +++ b/lib/qiniu/rs/version.rb @@ -2,6 +2,18 @@ module Qiniu module RS - VERSION = "3.0.7" + module Version + MAJOR = 3 + MINOR = 1 + PATCH = 0 + # Returns a version string by joining MAJOR, MINOR, and PATCH with '.' + # + # Example + # + # Version.to_s # '1.0.2' + def self.to_s + [MAJOR, MINOR, PATCH].join('.') + end + end end end diff --git a/qiniu-rs.gemspec b/qiniu-rs.gemspec index b82562f..cae822f 100755 --- a/qiniu-rs.gemspec +++ b/qiniu-rs.gemspec @@ -14,7 +14,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "qiniu-rs" gem.require_paths = ["lib"] - gem.version = Qiniu::RS::VERSION + gem.version = Qiniu::RS::Version.to_s # specify any dependencies here; for example: gem.add_development_dependency "rake", "~> 0.9" diff --git a/spec/qiniu/rs/abstract_spec.rb b/spec/qiniu/rs/abstract_spec.rb new file mode 100755 index 0000000..e69af62 --- /dev/null +++ b/spec/qiniu/rs/abstract_spec.rb @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- + +require 'spec_helper' +require 'qiniu/rs/abstract' + +describe Qiniu::RS::Abstract do + before(:each) do + @klass = Class.new do + include Qiniu::RS::Abstract + + abstract_methods :foo, :bar + end + end + + it "raises NotImplementedError" do + proc { + @klass.new.foo + }.should raise_error(NotImplementedError) + end + + it "can be overridden" do + subclass = Class.new(@klass) do + def foo + :overridden + end + end + + subclass.new.foo.should == :overridden + end +end diff --git a/spec/qiniu/rs/rs_spec.rb b/spec/qiniu/rs/rs_spec.rb index 7deff63..ec8e936 100755 --- a/spec/qiniu/rs/rs_spec.rb +++ b/spec/qiniu/rs/rs_spec.rb @@ -17,22 +17,22 @@ module Qiniu @domain = 'iovip.qbox.me/test' code, data = Qiniu::RS::RS.mkbucket(@bucket) - code.should == 200 puts data.inspect + code.should == 200 end context "IO.upload_file" do it "should works" do code, data = Qiniu::RS::IO.put_auth() + puts data.inspect code.should == 200 data["url"].should_not be_empty data["expiresIn"].should_not be_zero - puts data.inspect @put_url = data["url"] code2, data2 = Qiniu::RS::IO.upload_file(@put_url, __FILE__, @bucket, @key) - code2.should == 200 puts data2.inspect + code2.should == 200 end end diff --git a/spec/qiniu/rs/version_spec.rb b/spec/qiniu/rs/version_spec.rb index c18adde..4df286b 100755 --- a/spec/qiniu/rs/version_spec.rb +++ b/spec/qiniu/rs/version_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' require 'qiniu/rs/version' -describe Qiniu::RS do +describe Qiniu::RS::Version do it "should has a VERSION" do - Qiniu::RS::VERSION.should =~ /^\d+\.\d+\.\d+?$/ + Qiniu::RS::Version.to_s.should =~ /^\d+\.\d+\.\d+?$/ end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d886e24..74ed7f8 100755 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,6 +6,7 @@ require 'rspec' RSpec.configure do |config| config.before :all do +=begin Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh", :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW", :auth_url => "http://m1.qbox.me:13001/oauth2/token", @@ -14,5 +15,9 @@ RSpec.configure do |config| :up_host => "http://m1.qbox.me:13019", :pub_host => "http://m1.qbox.me:13012", :eu_host => "http://m1.qbox.me:13050" +=end + + Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr", + :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG" end end From 2451b5b51fbbff8f0cc34e77d9c14e1788be5b3e Mon Sep 17 00:00:00 2001 From: 404 Date: Tue, 2 Oct 2012 23:40:26 +0800 Subject: [PATCH 3/5] update qiniu/rs/exceptions --- lib/qiniu/rs.rb | 35 ++++++++++---- lib/qiniu/rs/exceptions.rb | 43 ++++++++++------- lib/qiniu/rs/up.rb | 95 ++++++++++++++++++++++---------------- lib/qiniu/rs/utils.rb | 2 +- 4 files changed, 108 insertions(+), 67 deletions(-) diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb index aa1cda9..1788afc 100755 --- a/lib/qiniu/rs.rb +++ b/lib/qiniu/rs.rb @@ -99,15 +99,32 @@ module Qiniu end def upload_file opts = {} - code, data = IO.upload_with_token(opts[:uptoken], - opts[:file], - opts[:bucket], - opts[:key], - opts[:mime_type], - opts[:note], - opts[:callback_params], - opts[:enable_crc32_check]) - code == StatusOK ? data : false + [:uptoken, :file, :bucket, :key].each do |opt| + raise MissingArgsError, [opt] unless opts.has_key?(opt) + end + source_file = opts[:file] + raise NoSuchFileError, source_file unless File.exist?(source_file) + if opts[:enable_resumable_upload] && File::size(source_file) > Config.settings[:block_size] + code, data = UP.upload_with_token(opts[:uptoken], + opts[:file], + opts[:bucket], + opts[:key], + opts[:mime_type], + opts[:note], + opts[:customer], + opts[:callback_params]) + else + code, data = IO.upload_with_token(opts[:uptoken], + opts[:file], + opts[:bucket], + opts[:key], + opts[:mime_type], + opts[:note], + opts[:callback_params], + opts[:enable_crc32_check]) + end + raise UploadFailedError.new(code, data) if code != StatusOK + return data end def stat(bucket, key) diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index 8c5a975..1b9df9c 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -32,7 +32,7 @@ module Qiniu class RequestFailed < ResponseError def message - "HTTP status code #{http_code}" + "HTTP status code: #{http_code}. Response body: #{http_body}" end def to_s @@ -40,31 +40,40 @@ module Qiniu end end - class UploadFailedError < ResponseError - def message - "Uploading Failed. HTTP Status Code #{http_code}" - end - - def to_s - message + class UploadFailedError < Exception + def initialize(status_code, response_data) + data_string = response_data.map { |key, value| %Q(:#{key.to_s} => #{value.to_s}) } + msg = %Q(Uploading Failed. HTTP Status Code: #{status_code}. HTTP response body: #{data_string.join(', ')}.) + super(msg) end end - class ResumablePutBlockError < ResponseError - def initialize(message) - super(message) + class FileSeekReadError < Exception + def initialize(fpath, block_index, seek_pos, read_length, result_length) + msg = "Reading file: #{fpath}, " + + "at block index: #{block_index}. " + + "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " + + "but got result_length: #{result_length}." + super(msg) end end - class ResumablePutError < ResponseError - def initialize(message) - super(message) + class BlockSizeNotMathchError < Exception + def initialize(fpath, block_index, offset, restsize, block_size) + msg = "Reading file: #{fpath}, " + + "at block index: #{block_index}. " + + "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " + + "but got offset+restsize=#{offset+restsize}." + super(msg) end end - class FileSeekReadError < ResponseError - def initialize(seek_pos, read_length, result_length) - super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length}) + class BlockCountNotMathchError < Exception + def initialize(fpath, block_count, checksum_count, progress_count) + msg = "Reading file: #{fpath}, " + + "Expected block_count, checksum_count, progress_count is: #{block_count}, " + + "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." + super(msg) end end diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb index 7010cfe..dae0f7d 100755 --- a/lib/qiniu/rs/up.rb +++ b/lib/qiniu/rs/up.rb @@ -7,6 +7,7 @@ require 'mime/types' require 'digest/sha1' require 'qiniu/rs/abstract' require 'qiniu/rs/exceptions' +require 'qiniu/rs/io' module Qiniu module RS @@ -53,42 +54,31 @@ module Qiniu class << self include Utils - def upload(uptoken, - local_file, - bucket, - key = nil, - mime_type = nil, - custom_meta = nil, - customer = nil, - callback_params = nil) + def upload_with_token(uptoken, + local_file, + bucket, + key = nil, + mime_type = nil, + custom_meta = nil, + customer = nil, + callback_params = nil) raise NoSuchFileError, local_file unless File.exist?(local_file) begin - ifile = File.open(local_file, 'rb') - fh = FileData.new(ifile) - key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? - entry_uri = bucket + ':' + key - if mime_type.nil? || mime_type.empty? - mime = MIME::Types.type_for local_file - mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type - end - fsize = fh.data_size - block_count = _block_count(fsize) - progress_data = ProgressData.new(key) - checksums = progress_data.get_checksums - progresses = progress_data.get_progresses - block_count.times{checksums << ''} if checksums.empty? - block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? - chunk_notifier = ChunkProgressNotifier.new(key) - block_notifier = BlockProgressNotifier.new(key) - code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) - if Utils.is_response_ok?(code) - code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) - end - if Utils.is_response_ok?(code) - Log.logger.info "File #{local_file} successfully uploaded." - # progress_data.sweep! - end - [code, data] + ifile = File.open(local_file, 'rb') + fh = FileData.new(ifile) + fsize = fh.data_size + key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? + entry_uri = bucket + ':' + key + if mime_type.nil? || mime_type.empty? + mime = MIME::Types.type_for local_file + mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type + end + if fsize > Config.settings[:block_size] + code, data = _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type, custom_meta, customer, callback_params) + else + code, data = IO.upload_with_token(uptoken, local_file, bucket, key, mime_type, custom_meta, callback_params, true) + end + [code, data] ensure ifile.close unless ifile.nil? end @@ -108,6 +98,7 @@ module Qiniu @fh.seek(offset) @fh.read(length) end + delegate :path, :to => :fh delegate :mtime, :to => :fh end @@ -181,6 +172,7 @@ module Qiniu def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier) code, data = 0, {} + fpath = fh.path # this block has never been uploaded. if progress[:ctx] == nil || progress[:ctx].empty? progress[:offset] = 0 @@ -192,7 +184,7 @@ module Qiniu body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length - raise FileSeekReadError.new(seek_pos, body_length, result_length) + raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data = _mkblock(uptoken, block_size, body) body_crc32 = Zlib.crc32(body) @@ -210,7 +202,7 @@ module Qiniu end end elsif progress[:offset] + progress[:restsize] != block_size - raise ResumablePutBlockError.new("Invalid arg. File length does not match.") + raise BlockSizeNotMathchError.new(fpath, block_index, progress[:offset], progress[:restsize], block_size) end # loop uploading other chunks except the first one while progress[:restsize].to_i > 0 && progress[:restsize] < block_size @@ -221,7 +213,7 @@ module Qiniu body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length - raise FileSeekReadError.new(seek_pos, body_length, result_length) + raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body) body_crc32 = Zlib.crc32(body) @@ -250,8 +242,10 @@ module Qiniu def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) code, data = 0, {} block_count = _block_count(fh.data_size) - if checksums.length != block_count || progresses.length != block_count - raise ResumablePutError.new("Invalid arg. Unexpected block count.") + checksum_count = checksums.length + progress_count = progresses.length + if checksum_count != block_count || progress_count != block_count + raise BlockCountNotMathchError.new(fh.path, block_count, checksum_count, progress_count) end 0.upto(block_count-1).each do |block_index| if checksums[block_index].nil? || checksums[block_index].empty? @@ -279,7 +273,8 @@ module Qiniu path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? path += '/customer/' + customer if !customer.nil? && !customer.empty? - path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty? + callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty? + path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty? url = Config.settings[:up_host] + path body = '' checksums.each do |checksum| @@ -288,6 +283,26 @@ module Qiniu _call_binary_with_token(uptoken, url, body) end + def _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + block_count = _block_count(fsize) + progress_data = ProgressData.new(key) + checksums = progress_data.get_checksums + progresses = progress_data.get_progresses + block_count.times{checksums << ''} if checksums.empty? + block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + chunk_notifier = ChunkProgressNotifier.new(key) + block_notifier = BlockProgressNotifier.new(key) + code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) + if Utils.is_response_ok?(code) + code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) + end + if Utils.is_response_ok?(code) + Log.logger.info "File #{local_file} successfully uploaded." + # progress_data.sweep! + end + [code, data] + end + end end end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index 341cd92..ee1a65e 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -65,7 +65,7 @@ module Qiniu end code = response.respond_to?(:code) ? response.code.to_i : 0 unless is_response_ok?(code) - raise RequestFailed.new(response) + raise RequestFailed.new("Request Failed", response) else data = {} body = response.respond_to?(:body) ? response.body : {} From 8011d6d837a73bd203bfec487df1a365cf0b4e96 Mon Sep 17 00:00:00 2001 From: 404 Date: Wed, 3 Oct 2012 12:02:38 +0800 Subject: [PATCH 4/5] all resumable uploading tests are passed --- lib/qiniu/rs/config.rb | 1 + lib/qiniu/rs/exceptions.rb | 22 +++--- lib/qiniu/rs/up.rb | 152 ++++++++++++++++++++++--------------- lib/qiniu/rs/utils.rb | 6 ++ spec/qiniu/rs/up_spec.rb | 51 +++++++++++++ spec/spec_helper.rb | 4 +- 6 files changed, 161 insertions(+), 75 deletions(-) create mode 100755 spec/qiniu/rs/up_spec.rb diff --git a/lib/qiniu/rs/config.rb b/lib/qiniu/rs/config.rb index 59ff91a..159447c 100755 --- a/lib/qiniu/rs/config.rb +++ b/lib/qiniu/rs/config.rb @@ -33,6 +33,7 @@ module Qiniu :max_retry_times => 3, :block_size => 1024*1024*4, :chunk_size => 1024*256, + :enable_debug => true, :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK' } diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index 1b9df9c..50d092b 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -50,29 +50,29 @@ module Qiniu class FileSeekReadError < Exception def initialize(fpath, block_index, seek_pos, read_length, result_length) - msg = "Reading file: #{fpath}, " - + "at block index: #{block_index}. " - + "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " - + "but got result_length: #{result_length}." + msg = "Reading file: #{fpath}, " + msg += "at block index: #{block_index}. " + msg += "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " + msg += "but got result_length: #{result_length}." super(msg) end end class BlockSizeNotMathchError < Exception def initialize(fpath, block_index, offset, restsize, block_size) - msg = "Reading file: #{fpath}, " - + "at block index: #{block_index}. " - + "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " - + "but got offset+restsize=#{offset+restsize}." + msg = "Reading file: #{fpath}, " + msg += "at block index: #{block_index}. " + msg += "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " + msg += "but got offset+restsize=#{offset+restsize}." super(msg) end end class BlockCountNotMathchError < Exception def initialize(fpath, block_count, checksum_count, progress_count) - msg = "Reading file: #{fpath}, " - + "Expected block_count, checksum_count, progress_count is: #{block_count}, " - + "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." + msg = "Reading file: #{fpath}, " + msg += "Expected block_count, checksum_count, progress_count is: #{block_count}, " + msg += "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." super(msg) end end diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb index dae0f7d..c54dbd4 100755 --- a/lib/qiniu/rs/up.rb +++ b/lib/qiniu/rs/up.rb @@ -3,6 +3,7 @@ require 'zlib' require 'yaml' require 'tmpdir' +require 'fileutils' require 'mime/types' require 'digest/sha1' require 'qiniu/rs/abstract' @@ -13,40 +14,93 @@ module Qiniu module RS module UP - module Abstract + module AbstractClass class ChunkProgressNotifier - include Abstract + include Qiniu::RS::Abstract abstract_methods :notify # def notify(block_index, block_put_progress); end end class BlockProgressNotifier - include Abstract + include Qiniu::RS::Abstract abstract_methods :notify # def notify(block_index, checksum); end end end - class ChunkProgressNotifier < Abstract::ChunkProgressNotifier + class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier def initialize(id) - @data = ProgressData.new(id) + @data = UP::ProgressData.new(id) end def notify(index, progress) @data.set_progresses(index, progress) - logmsg = "chunk #{index} successfully uploaded.\n" - + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" - Log.logger.info logmsg + logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" + Utils.debug(logmsg) end end - class BlockProgressNotifier < Abstract::BlockProgressNotifier + class BlockProgressNotifier < AbstractClass::BlockProgressNotifier def initialize(id) - @data = ProgressData.new(id) + @data = UP::ProgressData.new(id) end def notify(index, checksum) @data.set_checksums(index, checksum) - logmsg = "block #{index}:#{checksum} successfully uploaded." - Log.logger.info logmsg + Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded." + end + end + + class ProgressData + def initialize(id) + @id = id + @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id + FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir) + @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' + @progress_file = @tmpdir + File::SEPARATOR + 'progresses' + end + + def get_checksums + File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] + end + + def get_progresses + File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] + end + + def set_checksums(index, checksum) + checksums = get_checksums + checksums[index] = checksum + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + Utils.debug %Q(Updating tmpfile: #{@checksum_file}) + end + end + + def set_progresses(index, progress) + progresses = get_progresses + progresses[index] = progress + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + Utils.debug %Q(Updating tmpfile: #{@progress_file}) + end + end + + def init_checksums(checksums) + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + Utils.debug %Q(Initializing tmpfile: #{@checksum_file}) + end + end + + def init_progresses(progresses) + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + Utils.debug %Q(Initializing tmpfile: #{@progress_file}) + end + end + + def sweep! + FileUtils.rm_r(@tmpdir) end end @@ -68,13 +122,12 @@ module Qiniu fh = FileData.new(ifile) fsize = fh.data_size key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? - entry_uri = bucket + ':' + key if mime_type.nil? || mime_type.empty? mime = MIME::Types.type_for local_file mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type end if fsize > Config.settings[:block_size] - code, data = _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type, custom_meta, customer, callback_params) + code, data = _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type, custom_meta, customer, callback_params) else code, data = IO.upload_with_token(uptoken, local_file, bucket, key, mime_type, custom_meta, callback_params, true) end @@ -98,46 +151,13 @@ module Qiniu @fh.seek(offset) @fh.read(length) end - delegate :path, :to => :fh - delegate :mtime, :to => :fh - end - - class ProgressData - def initialize(id) - @id = id - @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id - Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir) - @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' - @progress_file = @tmpdir + File::SEPARATOR + 'progresses' - end - - def get_checksums - File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] - end - - def get_progresses - File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] - end - - def set_checksums(index, checksum) - checksums = get_checksums - checksums[index] = checksum - File.open(@checksum_file, "w") do |f| - YAML::dump(checksums, f) - end - end - - def set_progresses(index, progress) - progresses = get_progresses - progresses[index] = progress - File.open(@progress_file, "w") do |f| - YAML::dump(progresses, f) - end - end - - def sweep! - Dir.rmdir(@tmpdir) - end + def path + @fh.path + end + def mtime + @fh.mtime + end + #delegate :path, :mtime, :to => :fh end def _new_block_put_progress_data @@ -236,12 +256,13 @@ module Qiniu end def _block_count(fsize) - ((fsize + Config.block_size - 1) / Config.block_size).to_i + ((fsize + Config.settings[:block_size] - 1) / Config.settings[:block_size]).to_i end def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) code, data = 0, {} - block_count = _block_count(fh.data_size) + fsize = fh.data_size + block_count = _block_count(fsize) checksum_count = checksums.length progress_count = progresses.length if checksum_count != block_count || progress_count != block_count @@ -269,7 +290,7 @@ module Qiniu end def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) - path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize + path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}" path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? path += '/customer/' + customer if !customer.nil? && !customer.empty? @@ -283,22 +304,29 @@ module Qiniu _call_binary_with_token(uptoken, url, body) end - def _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) block_count = _block_count(fsize) progress_data = ProgressData.new(key) checksums = progress_data.get_checksums progresses = progress_data.get_progresses - block_count.times{checksums << ''} if checksums.empty? - block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + if checksums.empty? + block_count.times{checksums << ''} + progress_data.init_checksums(checksums) + end + if progresses.empty? + block_count.times{progresses << _new_block_put_progress_data} + progress_data.init_progresses(progresses) + end chunk_notifier = ChunkProgressNotifier.new(key) block_notifier = BlockProgressNotifier.new(key) code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) if Utils.is_response_ok?(code) + entry_uri = bucket + ':' + key code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) end if Utils.is_response_ok?(code) - Log.logger.info "File #{local_file} successfully uploaded." - # progress_data.sweep! + Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded." + progress_data.sweep! end [code, data] end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index ee1a65e..3c0d169 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -40,6 +40,12 @@ module Qiniu [status_code, {"error" => errmsg}] end + def debug(msg) + if Config.settings[:enable_debug] + Log.logger.debug(msg) + end + end + def send_request_with url, data = nil, options = {} options[:method] = Config.settings[:method] unless options[:method] options[:content_type] = Config.settings[:content_type] unless options[:content_type] diff --git a/spec/qiniu/rs/up_spec.rb b/spec/qiniu/rs/up_spec.rb new file mode 100755 index 0000000..ccd623e --- /dev/null +++ b/spec/qiniu/rs/up_spec.rb @@ -0,0 +1,51 @@ +# Utils.-*- encoding: utf-8 -*- + +require 'digest/sha1' +require 'spec_helper' +require 'qiniu/rs/rs' +require 'qiniu/rs/up' + +module Qiniu + module RS + describe UP do + + before :all do + @localfile = "bigfile.txt" + File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}} + @bucket = "test" + @key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s) + end + + after :all do + File.unlink(@localfile) if File.exists?(@localfile) + end + + context ".upload_with_token" do + it "should works" do + upopts = {:scope => @bucket, :expires_in => 3600, :customer => "awhy.xu@gmail.com"} + uptoken = Qiniu::RS.generate_upload_token(upopts) + code, data = Qiniu::RS::UP.upload_with_token(uptoken, @localfile, @bucket, @key) + puts data.inspect + (code/100).should == 2 + end + end + + context ".stat" do + it "should exists" do + code, data = Qiniu::RS::RS.stat(@bucket, @key) + puts data.inspect + code.should == 200 + end + end + + context ".delete" do + it "should works" do + code, data = Qiniu::RS::RS.delete(@bucket, @key) + puts data.inspect + code.should == 200 + end + end + + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 74ed7f8..b9fe532 100755 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,7 +6,6 @@ require 'rspec' RSpec.configure do |config| config.before :all do -=begin Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh", :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW", :auth_url => "http://m1.qbox.me:13001/oauth2/token", @@ -15,9 +14,10 @@ RSpec.configure do |config| :up_host => "http://m1.qbox.me:13019", :pub_host => "http://m1.qbox.me:13012", :eu_host => "http://m1.qbox.me:13050" -=end +=begin Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr", :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG" +=end end end From 9a5a27429debe757e110ee664f81ca67596b7fde Mon Sep 17 00:00:00 2001 From: 404 Date: Mon, 8 Oct 2012 12:59:30 +0800 Subject: [PATCH 5/5] add resumable upload and all tests(63 examples) passed --- docs/README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- lib/qiniu/rs.rb | 2 ++ spec/qiniu/rs_spec.rb | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 3768955..00f1e61 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ title: Ruby SDK 使用指南 | 七牛云存储 - [上传文件](#upload) - [获取用于上传文件的临时授权凭证](#generate-upload-token) - [服务端上传文件](#upload-server-side) + - [断点续上传](#resumable-upload) - [针对 NotFound 场景处理](#upload-file-for-not-found) - [客户端直传文件](#upload-client-side) - [查看文件属性信息](#stat) @@ -185,10 +186,50 @@ title: Ruby SDK 使用指南 | 七牛云存储 **返回值** -上传成功，返回如下一个 Hash，否则返回 `false`： +上传成功，返回如下一个 Hash： {"hash"=>"FgHk-_iqpnZji6PsNr4ghsK5qEwR"} +上传失败，会抛出 `UploadFailedError` 异常。 + + + +##### 断点续上传 + +无需任何额外改动，SDK 提供的 `Qiniu::RS.upload_file()` 方法缺省支持断点续上传。默认情况下，SDK 会自动启用断点续上传的方式来上传超过 4MB 大小的文件。您也可以在 [应用接入](/v3/sdk/ruby/#establish_connection!) 时通过修改缺省配置来设置该阀值： + + Qiniu::RS.establish_connection! :access_key => YOUR_APP_ACCESS_KEY, + :secret_key => YOUR_APP_SECRET_KEY, + :block_size => 1024*1024*4, + :chunk_size => 1024*256, + :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK', + :enable_debug => true, + :auto_reconnect => true, + :max_retry_times => 3 + +**参数详解** + +应用接入初始化时，以下配置参数均为可选： + +:block_size +: 整型，指定断点续上传针对大文件所使用的分块大小，缺省为 4MB ，小于该阀值的文件不启用断点续上传。 + +:chunk_size +: 整型，指定断点续上传每次http请求上传的数据块大小，缺省为 256KB。该设置尽量不要超过实际使用的上行带宽，且不能超过 `:block_size` 定义的值。 + +:tmpdir +: 字符串类型，指定持久化保存断点续上传进度状态临时文件的目录，缺省放置于操作系统的临时目录中。 + +:enable_debug +: 布尔值，是否启用调试模式，缺省启用（true），启用后会打印相关日志。该参数 SDK 全局有效。 + +:auto_reconnect +: 布尔值，指定每次 http 若请求失败是否启用重试，缺省启用（true）。该参数 SDK 全局有效。 + +:max_retry_times +: 整型，指定每次 http 若请求失败最多可以重试的次数，缺省为3次。该参数 SDK 全局有效。 + + ##### 针对 NotFound 场景处理 diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb index 1788afc..f826a44 100755 --- a/lib/qiniu/rs.rb +++ b/lib/qiniu/rs.rb @@ -9,6 +9,7 @@ module Qiniu autoload :Utils, 'qiniu/rs/utils' autoload :Auth, 'qiniu/rs/auth' autoload :IO, 'qiniu/rs/io' + autoload :UP, 'qiniu/rs/up' autoload :RS, 'qiniu/rs/rs' autoload :EU, 'qiniu/rs/eu' autoload :Pub, 'qiniu/rs/pub' @@ -104,6 +105,7 @@ module Qiniu end source_file = opts[:file] raise NoSuchFileError, source_file unless File.exist?(source_file) + opts[:enable_resumable_upload] = true unless opts.has_key?(:enable_resumable_upload) if opts[:enable_resumable_upload] && File::size(source_file) > Config.settings[:block_size] code, data = UP.upload_with_token(opts[:uptoken], opts[:file], diff --git a/spec/qiniu/rs_spec.rb b/spec/qiniu/rs_spec.rb index 8f2a32a..8204b18 100755 --- a/spec/qiniu/rs_spec.rb +++ b/spec/qiniu/rs_spec.rb @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- +require 'digest/sha1' require 'spec_helper' require 'qiniu/rs' @@ -129,6 +130,41 @@ module Qiniu end end + context ".resumable_upload_file" do + it "should works" do + # generate bigfile for testing + localfile = "test_bigfile" + File.open(localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}} + key = Digest::SHA1.hexdigest(localfile+Time.now.to_s) + # generate the upload token + uptoken_opts = {:scope => @bucket, :expires_in => 3600, :customer => "awhy.xu@gmail.com"} + uptoken = Qiniu::RS.generate_upload_token(uptoken_opts) + # uploading + upload_opts = { + :uptoken => uptoken, + :file => localfile, + :bucket => @bucket, + :key => key + } + #uploading + result1 = Qiniu::RS.upload_file(upload_opts) + #drop the bigfile + File.unlink(localfile) if File.exists?(localfile) + #expect + puts result1.inspect + result1.should_not be_false + result1.should_not be_empty + #stat + result2 = Qiniu::RS.stat(@bucket, key) + puts result2.inspect + result2.should_not be_false + #delete + result3 = Qiniu::RS.delete(@bucket, key) + puts result3.inspect + result3.should_not be_false + end + end + context ".stat" do it "should works" do result = Qiniu::RS.stat(@bucket, @key)