diff --git a/Gemfile.lock b/Gemfile.lock
index d2cffa8..45c38ca 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
- qiniu-rs (3.0.6)
+ qiniu-rs (3.1.0)
json (~> 1.7)
mime-types (~> 1.19)
rest-client (~> 1.6)
@@ -24,7 +24,7 @@ GEM
rspec-core (2.11.1)
rspec-expectations (2.11.3)
diff-lcs (~> 1.1.3)
- rspec-mocks (2.11.2)
+ rspec-mocks (2.11.3)
ruby-hmac (0.4.0)
PLATFORMS
diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb
index 3c9dc72..aa1cda9 100755
--- a/lib/qiniu/rs.rb
+++ b/lib/qiniu/rs.rb
@@ -1,9 +1,8 @@
# -*- encoding: utf-8 -*-
-require 'qiniu/rs/version'
-
module Qiniu
module RS
+ autoload :Version, 'qiniu/rs/version'
autoload :Config, 'qiniu/rs/config'
autoload :Log, 'qiniu/rs/log'
autoload :Exception, 'qiniu/rs/exceptions'
@@ -17,6 +16,7 @@ module Qiniu
autoload :AccessToken, 'qiniu/tokens/access_token'
autoload :QboxToken, 'qiniu/tokens/qbox_token'
autoload :UploadToken, 'qiniu/tokens/upload_token'
+ autoload :Abstract, 'qiniu/rs/abstract'
class << self
diff --git a/lib/qiniu/rs/abstract.rb b/lib/qiniu/rs/abstract.rb
new file mode 100644
index 0000000..d9fc671
--- /dev/null
+++ b/lib/qiniu/rs/abstract.rb
@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+module Qiniu
+ module RS
+ module Abstract
+ def self.included(base)
+ base.extend(ClassMethods)
+ end
+
+ module ClassMethods
+ def abstract_methods(*args)
+ args.each do |name|
+ class_eval <<-END
+ def #{name}(*args)
+ errmsg = %Q(class \#{self.class.name} must implement abstract method #{self.name}##{name}().)
+ raise NotImplementedError.new(errmsg)
+ end
+ END
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/qiniu/rs/auth.rb b/lib/qiniu/rs/auth.rb
index 6a9a924..bd93a79 100755
--- a/lib/qiniu/rs/auth.rb
+++ b/lib/qiniu/rs/auth.rb
@@ -18,7 +18,7 @@ module Qiniu
:password => password
}
code, data = http_request Config.settings[:auth_url], post_data
- reset_token(data["access_token"], data["refresh_token"]) if code == 200
+ reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code)
[code, data]
end
@@ -29,7 +29,7 @@ module Qiniu
:refresh_token => refresh_token
}
code, data = http_request Config.settings[:auth_url], post_data
- reset_token(data["access_token"], data["refresh_token"]) if code == 200
+ reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code)
[code, data]
end
@@ -48,7 +48,7 @@ module Qiniu
raise MissingUsernameOrPassword if (@username.nil? || @password.nil?)
code, data = exchange_by_password!(@username, @password)
end
- if code == 200
+ if Utils.is_response_ok?(code)
retry_times += 1
if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
return call_with_logged_in(url, data, retry_times)
diff --git a/lib/qiniu/rs/config.rb b/lib/qiniu/rs/config.rb
index b667c91..59ff91a 100755
--- a/lib/qiniu/rs/config.rb
+++ b/lib/qiniu/rs/config.rb
@@ -8,7 +8,7 @@
# Qbox::Config.load "path/to/your_project/config/qiniu.yml"
#
-require "qiniu/rs/version"
+require 'tmpdir'
module Qiniu
module RS
@@ -16,11 +16,11 @@ module Qiniu
class << self
DEFAULT_OPTIONS = {
- :user_agent => 'Qiniu-RS-Ruby-SDK-' + VERSION + '()',
+ :user_agent => 'Qiniu-RS-Ruby-SDK-' + Version.to_s + '()',
:method => :post,
:content_type => 'application/x-www-form-urlencoded',
:auth_url => "https://acc.qbox.me/oauth2/token",
- :rs_host => "http://rs.qbox.me:10100",
+ :rs_host => "http://rs.qbox.me",
:io_host => "http://iovip.qbox.me",
:up_host => "http://up.qbox.me",
:pub_host => "http://pu.qbox.me:10200",
@@ -30,10 +30,13 @@ module Qiniu
:access_key => "",
:secret_key => "",
:auto_reconnect => true,
- :max_retry_times => 5
+ :max_retry_times => 3,
+ :block_size => 1024*1024*4,
+ :chunk_size => 1024*256,
+ :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK'
}
- REQUIRED_OPTION_KEYS = [:client_id, :client_secret, :auth_url, :rs_host, :io_host]
+ REQUIRED_OPTION_KEYS = [:access_key, :secret_key]
attr_reader :settings, :default_params
diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb
index da43311..8c5a975 100755
--- a/lib/qiniu/rs/exceptions.rb
+++ b/lib/qiniu/rs/exceptions.rb
@@ -50,6 +50,24 @@ module Qiniu
end
end
+ class ResumablePutBlockError < ResponseError
+ def initialize(message)
+ super(message)
+ end
+ end
+
+ class ResumablePutError < ResponseError
+ def initialize(message)
+ super(message)
+ end
+ end
+
+ class FileSeekReadError < ResponseError
+ def initialize(seek_pos, read_length, result_length)
+ super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length})
+ end
+ end
+
class MissingArgsError < Exception
def initialize(missing_keys)
key_list = missing_keys.map {|key| key.to_s}.join(' and the ')
diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb
new file mode 100755
index 0000000..7010cfe
--- /dev/null
+++ b/lib/qiniu/rs/up.rb
@@ -0,0 +1,294 @@
+# -*- encoding: utf-8 -*-
+
+require 'zlib'
+require 'yaml'
+require 'tmpdir'
+require 'mime/types'
+require 'digest/sha1'
+require 'qiniu/rs/abstract'
+require 'qiniu/rs/exceptions'
+
+module Qiniu
+ module RS
+ module UP
+
+ module Abstract
+ class ChunkProgressNotifier
+ include Abstract
+ abstract_methods :notify
+ # def notify(block_index, block_put_progress); end
+ end
+
+ class BlockProgressNotifier
+ include Abstract
+ abstract_methods :notify
+ # def notify(block_index, checksum); end
+ end
+ end
+
+ class ChunkProgressNotifier < Abstract::ChunkProgressNotifier
+ def initialize(id)
+ @data = ProgressData.new(id)
+ end
+ def notify(index, progress)
+ @data.set_progresses(index, progress)
+ logmsg = "chunk #{index} successfully uploaded.\n"
+ + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
+ Log.logger.info logmsg
+ end
+ end
+
+ class BlockProgressNotifier < Abstract::BlockProgressNotifier
+ def initialize(id)
+ @data = ProgressData.new(id)
+ end
+ def notify(index, checksum)
+ @data.set_checksums(index, checksum)
+ logmsg = "block #{index}:#{checksum} successfully uploaded."
+ Log.logger.info logmsg
+ end
+ end
+
+
+ class << self
+ include Utils
+
+ def upload(uptoken,
+ local_file,
+ bucket,
+ key = nil,
+ mime_type = nil,
+ custom_meta = nil,
+ customer = nil,
+ callback_params = nil)
+ raise NoSuchFileError, local_file unless File.exist?(local_file)
+ begin
+ ifile = File.open(local_file, 'rb')
+ fh = FileData.new(ifile)
+ key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil?
+ entry_uri = bucket + ':' + key
+ if mime_type.nil? || mime_type.empty?
+ mime = MIME::Types.type_for local_file
+ mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type
+ end
+ fsize = fh.data_size
+ block_count = _block_count(fsize)
+ progress_data = ProgressData.new(key)
+ checksums = progress_data.get_checksums
+ progresses = progress_data.get_progresses
+ block_count.times{checksums << ''} if checksums.empty?
+ block_count.times{progresses << _new_block_put_progress_data} if progresses.empty?
+ chunk_notifier = ChunkProgressNotifier.new(key)
+ block_notifier = BlockProgressNotifier.new(key)
+ code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
+ if Utils.is_response_ok?(code)
+ code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params)
+ end
+ if Utils.is_response_ok?(code)
+ Log.logger.info "File #{local_file} successfully uploaded."
+ # progress_data.sweep!
+ end
+ [code, data]
+ ensure
+ ifile.close unless ifile.nil?
+ end
+ end
+
+ private
+
+ class FileData
+ attr_accessor :fh
+ def initialize(fh)
+ @fh = fh
+ end
+ def data_size
+ @fh.stat.size
+ end
+ def get_data(offset, length)
+ @fh.seek(offset)
+ @fh.read(length)
+ end
+ delegate :mtime, :to => :fh
+ end
+
+ class ProgressData
+ def initialize(id)
+ @id = id
+ @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
+ Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir)
+ @checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
+ @progress_file = @tmpdir + File::SEPARATOR + 'progresses'
+ end
+
+ def get_checksums
+ File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
+ end
+
+ def get_progresses
+ File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
+ end
+
+ def set_checksums(index, checksum)
+ checksums = get_checksums
+ checksums[index] = checksum
+ File.open(@checksum_file, "w") do |f|
+ YAML::dump(checksums, f)
+ end
+ end
+
+ def set_progresses(index, progress)
+ progresses = get_progresses
+ progresses[index] = progress
+ File.open(@progress_file, "w") do |f|
+ YAML::dump(progresses, f)
+ end
+ end
+
+ def sweep!
+ Dir.rmdir(@tmpdir)
+ end
+ end
+
+ def _new_block_put_progress_data
+ {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
+ end
+
+ def _call_binary_with_token(uptoken, url, data, retry_times = 0)
+ options = {
+ :method => :post,
+ :content_type => 'application/octet-stream',
+ :upload_signature_token => uptoken
+ }
+ code, data = http_request url, data, options
+ unless Utils.is_response_ok?(code)
+ retry_times += 1
+ if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
+ return _call_binary_with_token(uptoken, url, data, retry_times)
+ end
+ end
+ [code, data]
+ end
+
+ def _mkblock(uptoken, block_size, body)
+ url = Config.settings[:up_host] + "/mkblk/#{block_size}"
+ _call_binary_with_token(uptoken, url, body)
+ end
+
+ def _putblock(uptoken, ctx, offset, body)
+ url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
+ _call_binary_with_token(uptoken, url, body)
+ end
+
+ def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier)
+ code, data = 0, {}
+ # this block has never been uploaded.
+ if progress[:ctx] == nil || progress[:ctx].empty?
+ progress[:offset] = 0
+ progress[:restsize] = block_size
+ # choose the smaller one
+ body_length = [block_size, chunk_size].min
+ for i in 1..retry_times
+ seek_pos = block_index*Config.settings[:block_size]
+ body = fh.get_data(seek_pos, body_length)
+ result_length = body.length
+ if result_length != body_length
+ raise FileSeekReadError.new(seek_pos, body_length, result_length)
+ end
+ code, data = _mkblock(uptoken, block_size, body)
+ body_crc32 = Zlib.crc32(body)
+ if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+ progress[:ctx] = data["ctx"]
+ progress[:offset] = body_length
+ progress[:restsize] = block_size - body_length
+ progress[:status_code] = code
+ if !notifier.nil? && notifier.respond_to?("notify")
+ notifier.notify(block_index, progress)
+ end
+ break
+ elsif i == retry_times && data["crc32"] != body_crc32
+ Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+ end
+ end
+ elsif progress[:offset] + progress[:restsize] != block_size
+ raise ResumablePutBlockError.new("Invalid arg. File length does not match.")
+ end
+ # loop uploading other chunks except the first one
+ while progress[:restsize].to_i > 0 && progress[:restsize] < block_size
+ # choose the smaller one
+ body_length = [progress[:restsize], chunk_size].min
+ for i in 1..retry_times
+ seek_pos = block_index*Config.settings[:block_size] + progress[:offset]
+ body = fh.get_data(seek_pos, body_length)
+ result_length = body.length
+ if result_length != body_length
+ raise FileSeekReadError.new(seek_pos, body_length, result_length)
+ end
+ code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
+ body_crc32 = Zlib.crc32(body)
+ if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+ progress[:ctx] = data["ctx"]
+ progress[:offset] += body_length
+ progress[:restsize] -= body_length
+ progress[:status_code] = code
+ if !notifier.nil? && notifier.respond_to?("notify")
+ notifier.notify(block_index, progress)
+ end
+ break
+ elsif i == retry_times && data["crc32"] != body_crc32
+ Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+ end
+ end
+ end
+ # return
+ return [code, data]
+ end
+
+ def _block_count(fsize)
+ ((fsize + Config.block_size - 1) / Config.block_size).to_i
+ end
+
+ def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil)
+ code, data = 0, {}
+ block_count = _block_count(fh.data_size)
+ if checksums.length != block_count || progresses.length != block_count
+ raise ResumablePutError.new("Invalid arg. Unexpected block count.")
+ end
+ 0.upto(block_count-1).each do |block_index|
+ if checksums[block_index].nil? || checksums[block_index].empty?
+ block_size = Config.settings[:block_size]
+ if block_index == block_count - 1
+ block_size = fsize - block_index*Config.settings[:block_size]
+ end
+ if progresses[block_index].nil?
+ progresses[block_index] = _new_block_put_progress_data
+ end
+ code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
+ if Utils.is_response_ok?(code)
+ checksums[block_index] = data["checksum"]
+ if !block_notifier.nil? && block_notifier.respond_to?("notify")
+ block_notifier.notify(block_index, checksums[block_index])
+ end
+ end
+ end
+ end
+ return [code, data]
+ end
+
+ def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil)
+ path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize
+ path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
+ path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
+ path += '/customer/' + customer if !customer.nil? && !customer.empty?
+ path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty?
+ url = Config.settings[:up_host] + path
+ body = ''
+ checksums.each do |checksum|
+ body += Utils.urlsafe_base64_decode(checksum)
+ end
+ _call_binary_with_token(uptoken, url, body)
+ end
+
+ end
+ end
+ end
+end
diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb
index f12c39c..341cd92 100755
--- a/lib/qiniu/rs/utils.rb
+++ b/lib/qiniu/rs/utils.rb
@@ -32,6 +32,14 @@ module Qiniu
{}
end
+ def is_response_ok?(status_code)
+ status_code/100 == 2
+ end
+
+ def response_error(status_code, errmsg)
+ [status_code, {"error" => errmsg}]
+ end
+
def send_request_with url, data = nil, options = {}
options[:method] = Config.settings[:method] unless options[:method]
options[:content_type] = Config.settings[:content_type] unless options[:content_type]
@@ -42,8 +50,8 @@ module Qiniu
auth_token = nil
if !options[:qbox_signature_token].nil? && !options[:qbox_signature_token].empty?
auth_token = 'QBox ' + options[:qbox_signature_token]
- #elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty?
- # auth_token = 'UpToken ' + options[:upload_signature_token]
+ elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty?
+ auth_token = 'UpToken ' + options[:upload_signature_token]
elsif options[:access_token]
auth_token = 'Bearer ' + options[:access_token]
end
@@ -56,7 +64,7 @@ module Qiniu
response = RestClient.post(url, data, header_options)
end
code = response.respond_to?(:code) ? response.code.to_i : 0
- if code != 200
+ unless is_response_ok?(code)
raise RequestFailed.new(response)
else
data = {}
diff --git a/lib/qiniu/rs/version.rb b/lib/qiniu/rs/version.rb
index 343ce15..eb073b7 100755
--- a/lib/qiniu/rs/version.rb
+++ b/lib/qiniu/rs/version.rb
@@ -2,6 +2,18 @@
module Qiniu
module RS
- VERSION = "3.0.7"
+ module Version
+ MAJOR = 3
+ MINOR = 1
+ PATCH = 0
+ # Returns a version string by joining MAJOR, MINOR, and PATCH with '.'
+ #
+ # Example
+ #
+ # Version.to_s # '1.0.2'
+ def self.to_s
+ [MAJOR, MINOR, PATCH].join('.')
+ end
+ end
end
end
diff --git a/qiniu-rs.gemspec b/qiniu-rs.gemspec
index b82562f..cae822f 100755
--- a/qiniu-rs.gemspec
+++ b/qiniu-rs.gemspec
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = "qiniu-rs"
gem.require_paths = ["lib"]
- gem.version = Qiniu::RS::VERSION
+ gem.version = Qiniu::RS::Version.to_s
# specify any dependencies here; for example:
gem.add_development_dependency "rake", "~> 0.9"
diff --git a/spec/qiniu/rs/abstract_spec.rb b/spec/qiniu/rs/abstract_spec.rb
new file mode 100755
index 0000000..e69af62
--- /dev/null
+++ b/spec/qiniu/rs/abstract_spec.rb
@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+require 'spec_helper'
+require 'qiniu/rs/abstract'
+
+describe Qiniu::RS::Abstract do
+ before(:each) do
+ @klass = Class.new do
+ include Qiniu::RS::Abstract
+
+ abstract_methods :foo, :bar
+ end
+ end
+
+ it "raises NotImplementedError" do
+ proc {
+ @klass.new.foo
+ }.should raise_error(NotImplementedError)
+ end
+
+ it "can be overridden" do
+ subclass = Class.new(@klass) do
+ def foo
+ :overridden
+ end
+ end
+
+ subclass.new.foo.should == :overridden
+ end
+end
diff --git a/spec/qiniu/rs/rs_spec.rb b/spec/qiniu/rs/rs_spec.rb
index 7deff63..ec8e936 100755
--- a/spec/qiniu/rs/rs_spec.rb
+++ b/spec/qiniu/rs/rs_spec.rb
@@ -17,22 +17,22 @@ module Qiniu
@domain = 'iovip.qbox.me/test'
code, data = Qiniu::RS::RS.mkbucket(@bucket)
- code.should == 200
puts data.inspect
+ code.should == 200
end
context "IO.upload_file" do
it "should works" do
code, data = Qiniu::RS::IO.put_auth()
+ puts data.inspect
code.should == 200
data["url"].should_not be_empty
data["expiresIn"].should_not be_zero
- puts data.inspect
@put_url = data["url"]
code2, data2 = Qiniu::RS::IO.upload_file(@put_url, __FILE__, @bucket, @key)
- code2.should == 200
puts data2.inspect
+ code2.should == 200
end
end
diff --git a/spec/qiniu/rs/version_spec.rb b/spec/qiniu/rs/version_spec.rb
index c18adde..4df286b 100755
--- a/spec/qiniu/rs/version_spec.rb
+++ b/spec/qiniu/rs/version_spec.rb
@@ -3,8 +3,8 @@
require 'spec_helper'
require 'qiniu/rs/version'
-describe Qiniu::RS do
+describe Qiniu::RS::Version do
it "should has a VERSION" do
- Qiniu::RS::VERSION.should =~ /^\d+\.\d+\.\d+?$/
+ Qiniu::RS::Version.to_s.should =~ /^\d+\.\d+\.\d+?$/
end
end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index d886e24..74ed7f8 100755
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -6,6 +6,7 @@ require 'rspec'
RSpec.configure do |config|
config.before :all do
+=begin
Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
:secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
:auth_url => "http://m1.qbox.me:13001/oauth2/token",
@@ -14,5 +15,9 @@ RSpec.configure do |config|
:up_host => "http://m1.qbox.me:13019",
:pub_host => "http://m1.qbox.me:13012",
:eu_host => "http://m1.qbox.me:13050"
+=end
+
+ Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr",
+ :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG"
end
end