diff --git a/Gemfile.lock b/Gemfile.lock index d2cffa8..45c38ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - qiniu-rs (3.0.6) + qiniu-rs (3.1.0) json (~> 1.7) mime-types (~> 1.19) rest-client (~> 1.6) @@ -24,7 +24,7 @@ GEM rspec-core (2.11.1) rspec-expectations (2.11.3) diff-lcs (~> 1.1.3) - rspec-mocks (2.11.2) + rspec-mocks (2.11.3) ruby-hmac (0.4.0) PLATFORMS diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb index 3c9dc72..aa1cda9 100755 --- a/lib/qiniu/rs.rb +++ b/lib/qiniu/rs.rb @@ -1,9 +1,8 @@ # -*- encoding: utf-8 -*- -require 'qiniu/rs/version' - module Qiniu module RS + autoload :Version, 'qiniu/rs/version' autoload :Config, 'qiniu/rs/config' autoload :Log, 'qiniu/rs/log' autoload :Exception, 'qiniu/rs/exceptions' @@ -17,6 +16,7 @@ module Qiniu autoload :AccessToken, 'qiniu/tokens/access_token' autoload :QboxToken, 'qiniu/tokens/qbox_token' autoload :UploadToken, 'qiniu/tokens/upload_token' + autoload :Abstract, 'qiniu/rs/abstract' class << self diff --git a/lib/qiniu/rs/abstract.rb b/lib/qiniu/rs/abstract.rb new file mode 100644 index 0000000..d9fc671 --- /dev/null +++ b/lib/qiniu/rs/abstract.rb @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- + +module Qiniu + module RS + module Abstract + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + def abstract_methods(*args) + args.each do |name| + class_eval <<-END + def #{name}(*args) + errmsg = %Q(class \#{self.class.name} must implement abstract method #{self.name}##{name}().) + raise NotImplementedError.new(errmsg) + end + END + end + end + end + end + end +end diff --git a/lib/qiniu/rs/auth.rb b/lib/qiniu/rs/auth.rb index 6a9a924..bd93a79 100755 --- a/lib/qiniu/rs/auth.rb +++ b/lib/qiniu/rs/auth.rb @@ -18,7 +18,7 @@ module Qiniu :password => password } code, data = http_request Config.settings[:auth_url], post_data - reset_token(data["access_token"], data["refresh_token"]) if code == 200 + reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code) [code, data] end @@ -29,7 +29,7 @@ module Qiniu :refresh_token => refresh_token } code, data = http_request Config.settings[:auth_url], post_data - reset_token(data["access_token"], data["refresh_token"]) if code == 200 + reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code) [code, data] end @@ -48,7 +48,7 @@ module Qiniu raise MissingUsernameOrPassword if (@username.nil? || @password.nil?) code, data = exchange_by_password!(@username, @password) end - if code == 200 + if Utils.is_response_ok?(code) retry_times += 1 if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] return call_with_logged_in(url, data, retry_times) diff --git a/lib/qiniu/rs/config.rb b/lib/qiniu/rs/config.rb index b667c91..59ff91a 100755 --- a/lib/qiniu/rs/config.rb +++ b/lib/qiniu/rs/config.rb @@ -8,7 +8,7 @@ # Qbox::Config.load "path/to/your_project/config/qiniu.yml" # -require "qiniu/rs/version" +require 'tmpdir' module Qiniu module RS @@ -16,11 +16,11 @@ module Qiniu class << self DEFAULT_OPTIONS = { - :user_agent => 'Qiniu-RS-Ruby-SDK-' + VERSION + '()', + :user_agent => 'Qiniu-RS-Ruby-SDK-' + Version.to_s + '()', :method => :post, :content_type => 'application/x-www-form-urlencoded', :auth_url => "https://acc.qbox.me/oauth2/token", - :rs_host => "http://rs.qbox.me:10100", + :rs_host => "http://rs.qbox.me", :io_host => "http://iovip.qbox.me", :up_host => "http://up.qbox.me", :pub_host => "http://pu.qbox.me:10200", @@ -30,10 +30,13 @@ module Qiniu :access_key => "", :secret_key => "", :auto_reconnect => true, - :max_retry_times => 5 + :max_retry_times => 3, + :block_size => 1024*1024*4, + :chunk_size => 1024*256, + :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK' } - REQUIRED_OPTION_KEYS = [:client_id, :client_secret, :auth_url, :rs_host, :io_host] + REQUIRED_OPTION_KEYS = [:access_key, :secret_key] attr_reader :settings, :default_params diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index da43311..8c5a975 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -50,6 +50,24 @@ module Qiniu end end + class ResumablePutBlockError < ResponseError + def initialize(message) + super(message) + end + end + + class ResumablePutError < ResponseError + def initialize(message) + super(message) + end + end + + class FileSeekReadError < ResponseError + def initialize(seek_pos, read_length, result_length) + super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length}) + end + end + class MissingArgsError < Exception def initialize(missing_keys) key_list = missing_keys.map {|key| key.to_s}.join(' and the ') diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb new file mode 100755 index 0000000..7010cfe --- /dev/null +++ b/lib/qiniu/rs/up.rb @@ -0,0 +1,294 @@ +# -*- encoding: utf-8 -*- + +require 'zlib' +require 'yaml' +require 'tmpdir' +require 'mime/types' +require 'digest/sha1' +require 'qiniu/rs/abstract' +require 'qiniu/rs/exceptions' + +module Qiniu + module RS + module UP + + module Abstract + class ChunkProgressNotifier + include Abstract + abstract_methods :notify + # def notify(block_index, block_put_progress); end + end + + class BlockProgressNotifier + include Abstract + abstract_methods :notify + # def notify(block_index, checksum); end + end + end + + class ChunkProgressNotifier < Abstract::ChunkProgressNotifier + def initialize(id) + @data = ProgressData.new(id) + end + def notify(index, progress) + @data.set_progresses(index, progress) + logmsg = "chunk #{index} successfully uploaded.\n" + + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" + Log.logger.info logmsg + end + end + + class BlockProgressNotifier < Abstract::BlockProgressNotifier + def initialize(id) + @data = ProgressData.new(id) + end + def notify(index, checksum) + @data.set_checksums(index, checksum) + logmsg = "block #{index}:#{checksum} successfully uploaded." + Log.logger.info logmsg + end + end + + + class << self + include Utils + + def upload(uptoken, + local_file, + bucket, + key = nil, + mime_type = nil, + custom_meta = nil, + customer = nil, + callback_params = nil) + raise NoSuchFileError, local_file unless File.exist?(local_file) + begin + ifile = File.open(local_file, 'rb') + fh = FileData.new(ifile) + key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? + entry_uri = bucket + ':' + key + if mime_type.nil? || mime_type.empty? + mime = MIME::Types.type_for local_file + mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type + end + fsize = fh.data_size + block_count = _block_count(fsize) + progress_data = ProgressData.new(key) + checksums = progress_data.get_checksums + progresses = progress_data.get_progresses + block_count.times{checksums << ''} if checksums.empty? + block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + chunk_notifier = ChunkProgressNotifier.new(key) + block_notifier = BlockProgressNotifier.new(key) + code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) + if Utils.is_response_ok?(code) + code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) + end + if Utils.is_response_ok?(code) + Log.logger.info "File #{local_file} successfully uploaded." + # progress_data.sweep! + end + [code, data] + ensure + ifile.close unless ifile.nil? + end + end + + private + + class FileData + attr_accessor :fh + def initialize(fh) + @fh = fh + end + def data_size + @fh.stat.size + end + def get_data(offset, length) + @fh.seek(offset) + @fh.read(length) + end + delegate :mtime, :to => :fh + end + + class ProgressData + def initialize(id) + @id = id + @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id + Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir) + @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' + @progress_file = @tmpdir + File::SEPARATOR + 'progresses' + end + + def get_checksums + File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] + end + + def get_progresses + File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] + end + + def set_checksums(index, checksum) + checksums = get_checksums + checksums[index] = checksum + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + end + end + + def set_progresses(index, progress) + progresses = get_progresses + progresses[index] = progress + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + end + end + + def sweep! + Dir.rmdir(@tmpdir) + end + end + + def _new_block_put_progress_data + {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil} + end + + def _call_binary_with_token(uptoken, url, data, retry_times = 0) + options = { + :method => :post, + :content_type => 'application/octet-stream', + :upload_signature_token => uptoken + } + code, data = http_request url, data, options + unless Utils.is_response_ok?(code) + retry_times += 1 + if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] + return _call_binary_with_token(uptoken, url, data, retry_times) + end + end + [code, data] + end + + def _mkblock(uptoken, block_size, body) + url = Config.settings[:up_host] + "/mkblk/#{block_size}" + _call_binary_with_token(uptoken, url, body) + end + + def _putblock(uptoken, ctx, offset, body) + url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}" + _call_binary_with_token(uptoken, url, body) + end + + def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier) + code, data = 0, {} + # this block has never been uploaded. + if progress[:ctx] == nil || progress[:ctx].empty? + progress[:offset] = 0 + progress[:restsize] = block_size + # choose the smaller one + body_length = [block_size, chunk_size].min + for i in 1..retry_times + seek_pos = block_index*Config.settings[:block_size] + body = fh.get_data(seek_pos, body_length) + result_length = body.length + if result_length != body_length + raise FileSeekReadError.new(seek_pos, body_length, result_length) + end + code, data = _mkblock(uptoken, block_size, body) + body_crc32 = Zlib.crc32(body) + if Utils.is_response_ok?(code) && data["crc32"] == body_crc32 + progress[:ctx] = data["ctx"] + progress[:offset] = body_length + progress[:restsize] = block_size - body_length + progress[:status_code] = code + if !notifier.nil? && notifier.respond_to?("notify") + notifier.notify(block_index, progress) + end + break + elsif i == retry_times && data["crc32"] != body_crc32 + Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + end + end + elsif progress[:offset] + progress[:restsize] != block_size + raise ResumablePutBlockError.new("Invalid arg. File length does not match.") + end + # loop uploading other chunks except the first one + while progress[:restsize].to_i > 0 && progress[:restsize] < block_size + # choose the smaller one + body_length = [progress[:restsize], chunk_size].min + for i in 1..retry_times + seek_pos = block_index*Config.settings[:block_size] + progress[:offset] + body = fh.get_data(seek_pos, body_length) + result_length = body.length + if result_length != body_length + raise FileSeekReadError.new(seek_pos, body_length, result_length) + end + code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body) + body_crc32 = Zlib.crc32(body) + if Utils.is_response_ok?(code) && data["crc32"] == body_crc32 + progress[:ctx] = data["ctx"] + progress[:offset] += body_length + progress[:restsize] -= body_length + progress[:status_code] = code + if !notifier.nil? && notifier.respond_to?("notify") + notifier.notify(block_index, progress) + end + break + elsif i == retry_times && data["crc32"] != body_crc32 + Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + end + end + end + # return + return [code, data] + end + + def _block_count(fsize) + ((fsize + Config.block_size - 1) / Config.block_size).to_i + end + + def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) + code, data = 0, {} + block_count = _block_count(fh.data_size) + if checksums.length != block_count || progresses.length != block_count + raise ResumablePutError.new("Invalid arg. Unexpected block count.") + end + 0.upto(block_count-1).each do |block_index| + if checksums[block_index].nil? || checksums[block_index].empty? + block_size = Config.settings[:block_size] + if block_index == block_count - 1 + block_size = fsize - block_index*Config.settings[:block_size] + end + if progresses[block_index].nil? + progresses[block_index] = _new_block_put_progress_data + end + code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier) + if Utils.is_response_ok?(code) + checksums[block_index] = data["checksum"] + if !block_notifier.nil? && block_notifier.respond_to?("notify") + block_notifier.notify(block_index, checksums[block_index]) + end + end + end + end + return [code, data] + end + + def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize + path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? + path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? + path += '/customer/' + customer if !customer.nil? && !customer.empty? + path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty? + url = Config.settings[:up_host] + path + body = '' + checksums.each do |checksum| + body += Utils.urlsafe_base64_decode(checksum) + end + _call_binary_with_token(uptoken, url, body) + end + + end + end + end +end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index f12c39c..341cd92 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -32,6 +32,14 @@ module Qiniu {} end + def is_response_ok?(status_code) + status_code/100 == 2 + end + + def response_error(status_code, errmsg) + [status_code, {"error" => errmsg}] + end + def send_request_with url, data = nil, options = {} options[:method] = Config.settings[:method] unless options[:method] options[:content_type] = Config.settings[:content_type] unless options[:content_type] @@ -42,8 +50,8 @@ module Qiniu auth_token = nil if !options[:qbox_signature_token].nil? && !options[:qbox_signature_token].empty? auth_token = 'QBox ' + options[:qbox_signature_token] - #elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty? - # auth_token = 'UpToken ' + options[:upload_signature_token] + elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty? + auth_token = 'UpToken ' + options[:upload_signature_token] elsif options[:access_token] auth_token = 'Bearer ' + options[:access_token] end @@ -56,7 +64,7 @@ module Qiniu response = RestClient.post(url, data, header_options) end code = response.respond_to?(:code) ? response.code.to_i : 0 - if code != 200 + unless is_response_ok?(code) raise RequestFailed.new(response) else data = {} diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb index 343ce15..eb073b7 100755 --- a/lib/qiniu/rs/version.rb +++ b/lib/qiniu/rs/version.rb @@ -2,6 +2,18 @@ module Qiniu module RS - VERSION = "3.0.7" + module Version + MAJOR = 3 + MINOR = 1 + PATCH = 0 + # Returns a version string by joining MAJOR, MINOR, and PATCH with '.' + # + # Example + # + # Version.to_s # '1.0.2' + def self.to_s + [MAJOR, MINOR, PATCH].join('.') + end + end end end diff --git a/qiniu-rs.gemspec b/qiniu-rs.gemspec index b82562f..cae822f 100755 --- a/qiniu-rs.gemspec +++ b/qiniu-rs.gemspec @@ -14,7 +14,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "qiniu-rs" gem.require_paths = ["lib"] - gem.version = Qiniu::RS::VERSION + gem.version = Qiniu::RS::Version.to_s # specify any dependencies here; for example: gem.add_development_dependency "rake", "~> 0.9" diff --git a/spec/qiniu/rs/abstract_spec.rb b/spec/qiniu/rs/abstract_spec.rb new file mode 100755 index 0000000..e69af62 --- /dev/null +++ b/spec/qiniu/rs/abstract_spec.rb @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- + +require 'spec_helper' +require 'qiniu/rs/abstract' + +describe Qiniu::RS::Abstract do + before(:each) do + @klass = Class.new do + include Qiniu::RS::Abstract + + abstract_methods :foo, :bar + end + end + + it "raises NotImplementedError" do + proc { + @klass.new.foo + }.should raise_error(NotImplementedError) + end + + it "can be overridden" do + subclass = Class.new(@klass) do + def foo + :overridden + end + end + + subclass.new.foo.should == :overridden + end +end diff --git a/spec/qiniu/rs/rs_spec.rb b/spec/qiniu/rs/rs_spec.rb index 7deff63..ec8e936 100755 --- a/spec/qiniu/rs/rs_spec.rb +++ b/spec/qiniu/rs/rs_spec.rb @@ -17,22 +17,22 @@ module Qiniu @domain = 'iovip.qbox.me/test' code, data = Qiniu::RS::RS.mkbucket(@bucket) - code.should == 200 puts data.inspect + code.should == 200 end context "IO.upload_file" do it "should works" do code, data = Qiniu::RS::IO.put_auth() + puts data.inspect code.should == 200 data["url"].should_not be_empty data["expiresIn"].should_not be_zero - puts data.inspect @put_url = data["url"] code2, data2 = Qiniu::RS::IO.upload_file(@put_url, __FILE__, @bucket, @key) - code2.should == 200 puts data2.inspect + code2.should == 200 end end diff --git a/spec/qiniu/rs/version_spec.rb b/spec/qiniu/rs/version_spec.rb index c18adde..4df286b 100755 --- a/spec/qiniu/rs/version_spec.rb +++ b/spec/qiniu/rs/version_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' require 'qiniu/rs/version' -describe Qiniu::RS do +describe Qiniu::RS::Version do it "should has a VERSION" do - Qiniu::RS::VERSION.should =~ /^\d+\.\d+\.\d+?$/ + Qiniu::RS::Version.to_s.should =~ /^\d+\.\d+\.\d+?$/ end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d886e24..74ed7f8 100755 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,6 +6,7 @@ require 'rspec' RSpec.configure do |config| config.before :all do +=begin Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh", :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW", :auth_url => "http://m1.qbox.me:13001/oauth2/token", @@ -14,5 +15,9 @@ RSpec.configure do |config| :up_host => "http://m1.qbox.me:13019", :pub_host => "http://m1.qbox.me:13012", :eu_host => "http://m1.qbox.me:13050" +=end + + Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr", + :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG" end end