From 2451b5b51fbbff8f0cc34e77d9c14e1788be5b3e Mon Sep 17 00:00:00 2001 From: 404 Date: Tue, 2 Oct 2012 23:40:26 +0800 Subject: [PATCH] update qiniu/rs/exceptions --- lib/qiniu/rs.rb | 35 ++++++++++---- lib/qiniu/rs/exceptions.rb | 43 ++++++++++------- lib/qiniu/rs/up.rb | 95 ++++++++++++++++++++++---------------- lib/qiniu/rs/utils.rb | 2 +- 4 files changed, 108 insertions(+), 67 deletions(-) diff --git a/lib/qiniu/rs.rb b/lib/qiniu/rs.rb index aa1cda9..1788afc 100755 --- a/lib/qiniu/rs.rb +++ b/lib/qiniu/rs.rb @@ -99,15 +99,32 @@ module Qiniu end def upload_file opts = {} - code, data = IO.upload_with_token(opts[:uptoken], - opts[:file], - opts[:bucket], - opts[:key], - opts[:mime_type], - opts[:note], - opts[:callback_params], - opts[:enable_crc32_check]) - code == StatusOK ? data : false + [:uptoken, :file, :bucket, :key].each do |opt| + raise MissingArgsError, [opt] unless opts.has_key?(opt) + end + source_file = opts[:file] + raise NoSuchFileError, source_file unless File.exist?(source_file) + if opts[:enable_resumable_upload] && File::size(source_file) > Config.settings[:block_size] + code, data = UP.upload_with_token(opts[:uptoken], + opts[:file], + opts[:bucket], + opts[:key], + opts[:mime_type], + opts[:note], + opts[:customer], + opts[:callback_params]) + else + code, data = IO.upload_with_token(opts[:uptoken], + opts[:file], + opts[:bucket], + opts[:key], + opts[:mime_type], + opts[:note], + opts[:callback_params], + opts[:enable_crc32_check]) + end + raise UploadFailedError.new(code, data) if code != StatusOK + return data end def stat(bucket, key) diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index 8c5a975..1b9df9c 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -32,7 +32,7 @@ module Qiniu class RequestFailed < ResponseError def message - "HTTP status code #{http_code}" + "HTTP status code: #{http_code}. Response body: #{http_body}" end def to_s @@ -40,31 +40,40 @@ module Qiniu end end - class UploadFailedError < ResponseError - def message - "Uploading Failed. HTTP Status Code #{http_code}" - end - - def to_s - message + class UploadFailedError < Exception + def initialize(status_code, response_data) + data_string = response_data.map { |key, value| %Q(:#{key.to_s} => #{value.to_s}) } + msg = %Q(Uploading Failed. HTTP Status Code: #{status_code}. HTTP response body: #{data_string.join(', ')}.) + super(msg) end end - class ResumablePutBlockError < ResponseError - def initialize(message) - super(message) + class FileSeekReadError < Exception + def initialize(fpath, block_index, seek_pos, read_length, result_length) + msg = "Reading file: #{fpath}, " + + "at block index: #{block_index}. " + + "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " + + "but got result_length: #{result_length}." + super(msg) end end - class ResumablePutError < ResponseError - def initialize(message) - super(message) + class BlockSizeNotMathchError < Exception + def initialize(fpath, block_index, offset, restsize, block_size) + msg = "Reading file: #{fpath}, " + + "at block index: #{block_index}. " + + "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " + + "but got offset+restsize=#{offset+restsize}." + super(msg) end end - class FileSeekReadError < ResponseError - def initialize(seek_pos, read_length, result_length) - super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length}) + class BlockCountNotMathchError < Exception + def initialize(fpath, block_count, checksum_count, progress_count) + msg = "Reading file: #{fpath}, " + + "Expected block_count, checksum_count, progress_count is: #{block_count}, " + + "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." + super(msg) end end diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb index 7010cfe..dae0f7d 100755 --- a/lib/qiniu/rs/up.rb +++ b/lib/qiniu/rs/up.rb @@ -7,6 +7,7 @@ require 'mime/types' require 'digest/sha1' require 'qiniu/rs/abstract' require 'qiniu/rs/exceptions' +require 'qiniu/rs/io' module Qiniu module RS @@ -53,42 +54,31 @@ module Qiniu class << self include Utils - def upload(uptoken, - local_file, - bucket, - key = nil, - mime_type = nil, - custom_meta = nil, - customer = nil, - callback_params = nil) + def upload_with_token(uptoken, + local_file, + bucket, + key = nil, + mime_type = nil, + custom_meta = nil, + customer = nil, + callback_params = nil) raise NoSuchFileError, local_file unless File.exist?(local_file) begin - ifile = File.open(local_file, 'rb') - fh = FileData.new(ifile) - key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? - entry_uri = bucket + ':' + key - if mime_type.nil? || mime_type.empty? - mime = MIME::Types.type_for local_file - mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type - end - fsize = fh.data_size - block_count = _block_count(fsize) - progress_data = ProgressData.new(key) - checksums = progress_data.get_checksums - progresses = progress_data.get_progresses - block_count.times{checksums << ''} if checksums.empty? - block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? - chunk_notifier = ChunkProgressNotifier.new(key) - block_notifier = BlockProgressNotifier.new(key) - code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) - if Utils.is_response_ok?(code) - code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) - end - if Utils.is_response_ok?(code) - Log.logger.info "File #{local_file} successfully uploaded." - # progress_data.sweep! - end - [code, data] + ifile = File.open(local_file, 'rb') + fh = FileData.new(ifile) + fsize = fh.data_size + key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? + entry_uri = bucket + ':' + key + if mime_type.nil? || mime_type.empty? + mime = MIME::Types.type_for local_file + mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type + end + if fsize > Config.settings[:block_size] + code, data = _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type, custom_meta, customer, callback_params) + else + code, data = IO.upload_with_token(uptoken, local_file, bucket, key, mime_type, custom_meta, callback_params, true) + end + [code, data] ensure ifile.close unless ifile.nil? end @@ -108,6 +98,7 @@ module Qiniu @fh.seek(offset) @fh.read(length) end + delegate :path, :to => :fh delegate :mtime, :to => :fh end @@ -181,6 +172,7 @@ module Qiniu def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier) code, data = 0, {} + fpath = fh.path # this block has never been uploaded. if progress[:ctx] == nil || progress[:ctx].empty? progress[:offset] = 0 @@ -192,7 +184,7 @@ module Qiniu body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length - raise FileSeekReadError.new(seek_pos, body_length, result_length) + raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data = _mkblock(uptoken, block_size, body) body_crc32 = Zlib.crc32(body) @@ -210,7 +202,7 @@ module Qiniu end end elsif progress[:offset] + progress[:restsize] != block_size - raise ResumablePutBlockError.new("Invalid arg. File length does not match.") + raise BlockSizeNotMathchError.new(fpath, block_index, progress[:offset], progress[:restsize], block_size) end # loop uploading other chunks except the first one while progress[:restsize].to_i > 0 && progress[:restsize] < block_size @@ -221,7 +213,7 @@ module Qiniu body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length - raise FileSeekReadError.new(seek_pos, body_length, result_length) + raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body) body_crc32 = Zlib.crc32(body) @@ -250,8 +242,10 @@ module Qiniu def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) code, data = 0, {} block_count = _block_count(fh.data_size) - if checksums.length != block_count || progresses.length != block_count - raise ResumablePutError.new("Invalid arg. Unexpected block count.") + checksum_count = checksums.length + progress_count = progresses.length + if checksum_count != block_count || progress_count != block_count + raise BlockCountNotMathchError.new(fh.path, block_count, checksum_count, progress_count) end 0.upto(block_count-1).each do |block_index| if checksums[block_index].nil? || checksums[block_index].empty? @@ -279,7 +273,8 @@ module Qiniu path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? path += '/customer/' + customer if !customer.nil? && !customer.empty? - path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty? + callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty? + path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty? url = Config.settings[:up_host] + path body = '' checksums.each do |checksum| @@ -288,6 +283,26 @@ module Qiniu _call_binary_with_token(uptoken, url, body) end + def _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + block_count = _block_count(fsize) + progress_data = ProgressData.new(key) + checksums = progress_data.get_checksums + progresses = progress_data.get_progresses + block_count.times{checksums << ''} if checksums.empty? + block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + chunk_notifier = ChunkProgressNotifier.new(key) + block_notifier = BlockProgressNotifier.new(key) + code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) + if Utils.is_response_ok?(code) + code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) + end + if Utils.is_response_ok?(code) + Log.logger.info "File #{local_file} successfully uploaded." + # progress_data.sweep! + end + [code, data] + end + end end end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index 341cd92..ee1a65e 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -65,7 +65,7 @@ module Qiniu end code = response.respond_to?(:code) ? response.code.to_i : 0 unless is_response_ok?(code) - raise RequestFailed.new(response) + raise RequestFailed.new("Request Failed", response) else data = {} body = response.respond_to?(:body) ? response.body : {}