diff --git a/lib/qiniu/rs/config.rb b/lib/qiniu/rs/config.rb index 59ff91a..159447c 100755 --- a/lib/qiniu/rs/config.rb +++ b/lib/qiniu/rs/config.rb @@ -33,6 +33,7 @@ module Qiniu :max_retry_times => 3, :block_size => 1024*1024*4, :chunk_size => 1024*256, + :enable_debug => true, :tmpdir => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK' } diff --git a/lib/qiniu/rs/exceptions.rb b/lib/qiniu/rs/exceptions.rb index 1b9df9c..50d092b 100755 --- a/lib/qiniu/rs/exceptions.rb +++ b/lib/qiniu/rs/exceptions.rb @@ -50,29 +50,29 @@ module Qiniu class FileSeekReadError < Exception def initialize(fpath, block_index, seek_pos, read_length, result_length) - msg = "Reading file: #{fpath}, " - + "at block index: #{block_index}. " - + "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " - + "but got result_length: #{result_length}." + msg = "Reading file: #{fpath}, " + msg += "at block index: #{block_index}. " + msg += "Expected seek_pos:#{seek_pos} and read_length:#{read_length}, " + msg += "but got result_length: #{result_length}." super(msg) end end class BlockSizeNotMathchError < Exception def initialize(fpath, block_index, offset, restsize, block_size) - msg = "Reading file: #{fpath}, " - + "at block index: #{block_index}. " - + "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " - + "but got offset+restsize=#{offset+restsize}." + msg = "Reading file: #{fpath}, " + msg += "at block index: #{block_index}. " + msg += "Expected offset: #{offset}, restsize: #{restsize} and block_size: #{block_size}, " + msg += "but got offset+restsize=#{offset+restsize}." super(msg) end end class BlockCountNotMathchError < Exception def initialize(fpath, block_count, checksum_count, progress_count) - msg = "Reading file: #{fpath}, " - + "Expected block_count, checksum_count, progress_count is: #{block_count}, " - + "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." + msg = "Reading file: #{fpath}, " + msg += "Expected block_count, checksum_count, progress_count is: #{block_count}, " + msg += "but got checksum_count: #{checksum_count}, progress_count: #{progress_count}." super(msg) end end diff --git a/lib/qiniu/rs/up.rb b/lib/qiniu/rs/up.rb index dae0f7d..c54dbd4 100755 --- a/lib/qiniu/rs/up.rb +++ b/lib/qiniu/rs/up.rb @@ -3,6 +3,7 @@ require 'zlib' require 'yaml' require 'tmpdir' +require 'fileutils' require 'mime/types' require 'digest/sha1' require 'qiniu/rs/abstract' @@ -13,40 +14,93 @@ module Qiniu module RS module UP - module Abstract + module AbstractClass class ChunkProgressNotifier - include Abstract + include Qiniu::RS::Abstract abstract_methods :notify # def notify(block_index, block_put_progress); end end class BlockProgressNotifier - include Abstract + include Qiniu::RS::Abstract abstract_methods :notify # def notify(block_index, checksum); end end end - class ChunkProgressNotifier < Abstract::ChunkProgressNotifier + class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier def initialize(id) - @data = ProgressData.new(id) + @data = UP::ProgressData.new(id) end def notify(index, progress) @data.set_progresses(index, progress) - logmsg = "chunk #{index} successfully uploaded.\n" - + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" - Log.logger.info logmsg + logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}" + Utils.debug(logmsg) end end - class BlockProgressNotifier < Abstract::BlockProgressNotifier + class BlockProgressNotifier < AbstractClass::BlockProgressNotifier def initialize(id) - @data = ProgressData.new(id) + @data = UP::ProgressData.new(id) end def notify(index, checksum) @data.set_checksums(index, checksum) - logmsg = "block #{index}:#{checksum} successfully uploaded." - Log.logger.info logmsg + Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded." + end + end + + class ProgressData + def initialize(id) + @id = id + @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id + FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir) + @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' + @progress_file = @tmpdir + File::SEPARATOR + 'progresses' + end + + def get_checksums + File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] + end + + def get_progresses + File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] + end + + def set_checksums(index, checksum) + checksums = get_checksums + checksums[index] = checksum + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + Utils.debug %Q(Updating tmpfile: #{@checksum_file}) + end + end + + def set_progresses(index, progress) + progresses = get_progresses + progresses[index] = progress + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + Utils.debug %Q(Updating tmpfile: #{@progress_file}) + end + end + + def init_checksums(checksums) + File.open(@checksum_file, "w") do |f| + YAML::dump(checksums, f) + Utils.debug %Q(Initializing tmpfile: #{@checksum_file}) + end + end + + def init_progresses(progresses) + File.open(@progress_file, "w") do |f| + YAML::dump(progresses, f) + Utils.debug %Q(Initializing tmpfile: #{@progress_file}) + end + end + + def sweep! + FileUtils.rm_r(@tmpdir) end end @@ -68,13 +122,12 @@ module Qiniu fh = FileData.new(ifile) fsize = fh.data_size key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? - entry_uri = bucket + ':' + key if mime_type.nil? || mime_type.empty? mime = MIME::Types.type_for local_file mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type end if fsize > Config.settings[:block_size] - code, data = _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type, custom_meta, customer, callback_params) + code, data = _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type, custom_meta, customer, callback_params) else code, data = IO.upload_with_token(uptoken, local_file, bucket, key, mime_type, custom_meta, callback_params, true) end @@ -98,46 +151,13 @@ module Qiniu @fh.seek(offset) @fh.read(length) end - delegate :path, :to => :fh - delegate :mtime, :to => :fh - end - - class ProgressData - def initialize(id) - @id = id - @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id - Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir) - @checksum_file = @tmpdir + File::SEPARATOR + 'checksums' - @progress_file = @tmpdir + File::SEPARATOR + 'progresses' - end - - def get_checksums - File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : [] - end - - def get_progresses - File.exist?(@progress_file) ? YAML.load_file(@progress_file) : [] - end - - def set_checksums(index, checksum) - checksums = get_checksums - checksums[index] = checksum - File.open(@checksum_file, "w") do |f| - YAML::dump(checksums, f) - end - end - - def set_progresses(index, progress) - progresses = get_progresses - progresses[index] = progress - File.open(@progress_file, "w") do |f| - YAML::dump(progresses, f) - end - end - - def sweep! - Dir.rmdir(@tmpdir) - end + def path + @fh.path + end + def mtime + @fh.mtime + end + #delegate :path, :mtime, :to => :fh end def _new_block_put_progress_data @@ -236,12 +256,13 @@ module Qiniu end def _block_count(fsize) - ((fsize + Config.block_size - 1) / Config.block_size).to_i + ((fsize + Config.settings[:block_size] - 1) / Config.settings[:block_size]).to_i end def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) code, data = 0, {} - block_count = _block_count(fh.data_size) + fsize = fh.data_size + block_count = _block_count(fsize) checksum_count = checksums.length progress_count = progresses.length if checksum_count != block_count || progress_count != block_count @@ -269,7 +290,7 @@ module Qiniu end def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) - path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize + path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}" path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? path += '/customer/' + customer if !customer.nil? && !customer.empty? @@ -283,22 +304,29 @@ module Qiniu _call_binary_with_token(uptoken, url, body) end - def _resumable_upload(uptoken, fh, fsize, entry_uri, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) + def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil) block_count = _block_count(fsize) progress_data = ProgressData.new(key) checksums = progress_data.get_checksums progresses = progress_data.get_progresses - block_count.times{checksums << ''} if checksums.empty? - block_count.times{progresses << _new_block_put_progress_data} if progresses.empty? + if checksums.empty? + block_count.times{checksums << ''} + progress_data.init_checksums(checksums) + end + if progresses.empty? + block_count.times{progresses << _new_block_put_progress_data} + progress_data.init_progresses(progresses) + end chunk_notifier = ChunkProgressNotifier.new(key) block_notifier = BlockProgressNotifier.new(key) code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) if Utils.is_response_ok?(code) + entry_uri = bucket + ':' + key code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params) end if Utils.is_response_ok?(code) - Log.logger.info "File #{local_file} successfully uploaded." - # progress_data.sweep! + Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded." + progress_data.sweep! end [code, data] end diff --git a/lib/qiniu/rs/utils.rb b/lib/qiniu/rs/utils.rb index ee1a65e..3c0d169 100755 --- a/lib/qiniu/rs/utils.rb +++ b/lib/qiniu/rs/utils.rb @@ -40,6 +40,12 @@ module Qiniu [status_code, {"error" => errmsg}] end + def debug(msg) + if Config.settings[:enable_debug] + Log.logger.debug(msg) + end + end + def send_request_with url, data = nil, options = {} options[:method] = Config.settings[:method] unless options[:method] options[:content_type] = Config.settings[:content_type] unless options[:content_type] diff --git a/spec/qiniu/rs/up_spec.rb b/spec/qiniu/rs/up_spec.rb new file mode 100755 index 0000000..ccd623e --- /dev/null +++ b/spec/qiniu/rs/up_spec.rb @@ -0,0 +1,51 @@ +# Utils.-*- encoding: utf-8 -*- + +require 'digest/sha1' +require 'spec_helper' +require 'qiniu/rs/rs' +require 'qiniu/rs/up' + +module Qiniu + module RS + describe UP do + + before :all do + @localfile = "bigfile.txt" + File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}} + @bucket = "test" + @key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s) + end + + after :all do + File.unlink(@localfile) if File.exists?(@localfile) + end + + context ".upload_with_token" do + it "should works" do + upopts = {:scope => @bucket, :expires_in => 3600, :customer => "awhy.xu@gmail.com"} + uptoken = Qiniu::RS.generate_upload_token(upopts) + code, data = Qiniu::RS::UP.upload_with_token(uptoken, @localfile, @bucket, @key) + puts data.inspect + (code/100).should == 2 + end + end + + context ".stat" do + it "should exists" do + code, data = Qiniu::RS::RS.stat(@bucket, @key) + puts data.inspect + code.should == 200 + end + end + + context ".delete" do + it "should works" do + code, data = Qiniu::RS::RS.delete(@bucket, @key) + puts data.inspect + code.should == 200 + end + end + + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 74ed7f8..b9fe532 100755 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,7 +6,6 @@ require 'rspec' RSpec.configure do |config| config.before :all do -=begin Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh", :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW", :auth_url => "http://m1.qbox.me:13001/oauth2/token", @@ -15,9 +14,10 @@ RSpec.configure do |config| :up_host => "http://m1.qbox.me:13019", :pub_host => "http://m1.qbox.me:13012", :eu_host => "http://m1.qbox.me:13050" -=end +=begin Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr", :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG" +=end end end