add Qiniu::RS::UP module

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    qiniu-rs (3.0.6)
+    qiniu-rs (3.1.0)
       json (~> 1.7)
       mime-types (~> 1.19)
       rest-client (~> 1.6)
@@ -24,7 +24,7 @@ GEM
     rspec-core (2.11.1)
     rspec-expectations (2.11.3)
       diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.2)
+    rspec-mocks (2.11.3)
     ruby-hmac (0.4.0)
 
 PLATFORMS

lib/qiniu/rs.rb

@@ -1,9 +1,8 @@
 # -*- encoding: utf-8 -*-
 
-require 'qiniu/rs/version'
-
 module Qiniu
   module RS
+    autoload :Version, 'qiniu/rs/version'
     autoload :Config, 'qiniu/rs/config'
     autoload :Log, 'qiniu/rs/log'
     autoload :Exception, 'qiniu/rs/exceptions'
@@ -17,6 +16,7 @@ module Qiniu
     autoload :AccessToken, 'qiniu/tokens/access_token'
     autoload :QboxToken, 'qiniu/tokens/qbox_token'
     autoload :UploadToken, 'qiniu/tokens/upload_token'
+    autoload :Abstract, 'qiniu/rs/abstract'
 
     class << self
 

lib/qiniu/rs/abstract.rb

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+module Qiniu
+  module RS
+    module Abstract
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def abstract_methods(*args)
+          args.each do |name|
+            class_eval <<-END
+              def #{name}(*args)
+                errmsg = %Q(class \#{self.class.name} must implement abstract method #{self.name}##{name}().)
+                raise NotImplementedError.new(errmsg)
+              end
+            END
+          end
+        end
+      end
+    end
+  end
+end

lib/qiniu/rs/auth.rb

@@ -18,7 +18,7 @@ module Qiniu
             :password   => password
           }
           code, data = http_request Config.settings[:auth_url], post_data
-          reset_token(data["access_token"], data["refresh_token"]) if code == 200
+          reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code)
           [code, data]
         end
 
@@ -29,7 +29,7 @@ module Qiniu
             :refresh_token => refresh_token
           }
           code, data = http_request Config.settings[:auth_url], post_data
-          reset_token(data["access_token"], data["refresh_token"]) if code == 200
+          reset_token(data["access_token"], data["refresh_token"]) if Utils.is_response_ok?(code)
           [code, data]
         end
 
@@ -48,7 +48,7 @@ module Qiniu
               raise MissingUsernameOrPassword if (@username.nil? || @password.nil?)
               code, data = exchange_by_password!(@username, @password)
             end
-            if code == 200
+            if Utils.is_response_ok?(code)
               retry_times += 1
               if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
                 return call_with_logged_in(url, data, retry_times)

lib/qiniu/rs/config.rb

@@ -8,7 +8,7 @@
 # Qbox::Config.load "path/to/your_project/config/qiniu.yml"
 #
 
-require "qiniu/rs/version"
+require 'tmpdir'
 
 module Qiniu
   module RS
@@ -16,11 +16,11 @@ module Qiniu
       class << self
 
         DEFAULT_OPTIONS = {
-          :user_agent      => 'Qiniu-RS-Ruby-SDK-' + VERSION + '()',
+          :user_agent      => 'Qiniu-RS-Ruby-SDK-' + Version.to_s + '()',
           :method          => :post,
           :content_type    => 'application/x-www-form-urlencoded',
           :auth_url        => "https://acc.qbox.me/oauth2/token",
-          :rs_host         => "http://rs.qbox.me:10100",
+          :rs_host         => "http://rs.qbox.me",
           :io_host         => "http://iovip.qbox.me",
           :up_host         => "http://up.qbox.me",
           :pub_host        => "http://pu.qbox.me:10200",
@@ -30,10 +30,13 @@ module Qiniu
           :access_key      => "",
           :secret_key      => "",
           :auto_reconnect  => true,
-          :max_retry_times => 5
+          :max_retry_times => 3,
+          :block_size      => 1024*1024*4,
+          :chunk_size      => 1024*256,
+          :tmpdir          => Dir.tmpdir + File::SEPARATOR + 'Qiniu-RS-Ruby-SDK'
         }
 
-        REQUIRED_OPTION_KEYS = [:client_id, :client_secret, :auth_url, :rs_host, :io_host]
+        REQUIRED_OPTION_KEYS = [:access_key, :secret_key]
 
         attr_reader :settings, :default_params
 

lib/qiniu/rs/exceptions.rb

@@ -50,6 +50,24 @@ module Qiniu
       end
     end
 
+    class ResumablePutBlockError < ResponseError
+      def initialize(message)
+        super(message)
+      end
+    end
+
+    class ResumablePutError < ResponseError
+      def initialize(message)
+        super(message)
+      end
+    end
+
+    class FileSeekReadError < ResponseError
+      def initialize(seek_pos, read_length, result_length)
+        super %Q(Expected seek_pos:#{seek_pos} and read_length:#{read_length}, but got result_length: #{result_length})
+      end
+    end
+
     class MissingArgsError < Exception
       def initialize(missing_keys)
         key_list = missing_keys.map {|key| key.to_s}.join(' and the ')

lib/qiniu/rs/up.rb

@@ -0,0 +1,294 @@
+# -*- encoding: utf-8 -*-
+
+require 'zlib'
+require 'yaml'
+require 'tmpdir'
+require 'mime/types'
+require 'digest/sha1'
+require 'qiniu/rs/abstract'
+require 'qiniu/rs/exceptions'
+
+module Qiniu
+  module RS
+    module UP
+
+      module Abstract
+        class ChunkProgressNotifier
+          include Abstract
+          abstract_methods :notify
+          # def notify(block_index, block_put_progress); end
+        end
+
+        class BlockProgressNotifier
+          include Abstract
+          abstract_methods :notify
+          # def notify(block_index, checksum); end
+        end
+      end
+
+      class ChunkProgressNotifier < Abstract::ChunkProgressNotifier
+          def initialize(id)
+              @data = ProgressData.new(id)
+          end
+          def notify(index, progress)
+              @data.set_progresses(index, progress)
+              logmsg = "chunk #{index} successfully uploaded.\n"
+                     + "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
+              Log.logger.info logmsg
+          end
+      end
+
+      class BlockProgressNotifier < Abstract::BlockProgressNotifier
+          def initialize(id)
+              @data = ProgressData.new(id)
+          end
+          def notify(index, checksum)
+              @data.set_checksums(index, checksum)
+              logmsg = "block #{index}:#{checksum} successfully uploaded."
+              Log.logger.info logmsg
+          end
+      end
+
+
+      class << self
+        include Utils
+
+        def upload(uptoken,
+                   local_file,
+                   bucket,
+                   key = nil,
+                   mime_type = nil,
+                   custom_meta = nil,
+                   customer = nil,
+                   callback_params = nil)
+          raise NoSuchFileError, local_file unless File.exist?(local_file)
+          begin
+              ifile = File.open(local_file, 'rb')
+              fh = FileData.new(ifile)
+              key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil?
+              entry_uri = bucket + ':' + key
+              if mime_type.nil? || mime_type.empty?
+                mime = MIME::Types.type_for local_file
+                mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type
+              end
+              fsize = fh.data_size
+              block_count = _block_count(fsize)
+              progress_data = ProgressData.new(key)
+              checksums = progress_data.get_checksums
+              progresses = progress_data.get_progresses
+              block_count.times{checksums << ''} if checksums.empty?
+              block_count.times{progresses << _new_block_put_progress_data} if progresses.empty?
+              chunk_notifier = ChunkProgressNotifier.new(key)
+              block_notifier = BlockProgressNotifier.new(key)
+              code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
+              if Utils.is_response_ok?(code)
+                  code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params)
+              end
+              if Utils.is_response_ok?(code)
+                  Log.logger.info "File #{local_file} successfully uploaded."
+              #    progress_data.sweep!
+              end
+              [code, data]
+          ensure
+            ifile.close unless ifile.nil?
+          end
+        end
+
+        private
+
+        class FileData
+            attr_accessor :fh
+            def initialize(fh)
+                @fh = fh
+            end
+            def data_size
+                @fh.stat.size
+            end
+            def get_data(offset, length)
+                @fh.seek(offset)
+                @fh.read(length)
+            end
+            delegate :mtime, :to => :fh
+        end
+
+        class ProgressData
+          def initialize(id)
+              @id = id
+              @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
+              Dir.mkdir(@tmpdir) unless Dir.exists?(@tmpdir)
+              @checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
+              @progress_file = @tmpdir + File::SEPARATOR + 'progresses'
+          end
+
+          def get_checksums
+              File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
+          end
+
+          def get_progresses
+              File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
+          end
+
+          def set_checksums(index, checksum)
+              checksums = get_checksums
+              checksums[index] = checksum
+              File.open(@checksum_file, "w") do |f|
+                  YAML::dump(checksums, f)
+              end
+          end
+
+          def set_progresses(index, progress)
+              progresses = get_progresses
+              progresses[index] = progress
+              File.open(@progress_file, "w") do |f|
+                  YAML::dump(progresses, f)
+              end
+          end
+
+          def sweep!
+              Dir.rmdir(@tmpdir)
+          end
+        end
+
+        def _new_block_put_progress_data
+          {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
+        end
+
+        def _call_binary_with_token(uptoken, url, data, retry_times = 0)
+          options = {
+              :method => :post,
+              :content_type => 'application/octet-stream',
+              :upload_signature_token => uptoken
+          }
+          code, data = http_request url, data, options
+          unless Utils.is_response_ok?(code)
+              retry_times += 1
+              if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
+                  return _call_binary_with_token(uptoken, url, data, retry_times)
+              end
+          end
+          [code, data]
+        end
+
+        def _mkblock(uptoken, block_size, body)
+            url = Config.settings[:up_host] + "/mkblk/#{block_size}"
+            _call_binary_with_token(uptoken, url, body)
+        end
+
+        def _putblock(uptoken, ctx, offset, body)
+            url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
+            _call_binary_with_token(uptoken, url, body)
+        end
+
+        def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times = 1, notifier)
+            code, data = 0, {}
+            # this block has never been uploaded.
+            if progress[:ctx] == nil || progress[:ctx].empty?
+                progress[:offset] = 0
+                progress[:restsize] = block_size
+                # choose the smaller one
+                body_length = [block_size, chunk_size].min
+                for i in 1..retry_times
+                    seek_pos = block_index*Config.settings[:block_size]
+                    body = fh.get_data(seek_pos, body_length)
+                    result_length = body.length
+                    if result_length != body_length
+                        raise FileSeekReadError.new(seek_pos, body_length, result_length)
+                    end
+                    code, data = _mkblock(uptoken, block_size, body)
+                    body_crc32 = Zlib.crc32(body)
+                    if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+                        progress[:ctx] = data["ctx"]
+                        progress[:offset] = body_length
+                        progress[:restsize] = block_size - body_length
+                        progress[:status_code] = code
+                        if !notifier.nil? && notifier.respond_to?("notify")
+                            notifier.notify(block_index, progress)
+                        end
+                        break
+                    elsif i == retry_times && data["crc32"] != body_crc32
+                        Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+                    end
+                end
+            elsif progress[:offset] + progress[:restsize] != block_size
+                raise ResumablePutBlockError.new("Invalid arg. File length does not match.")
+            end
+            # loop uploading other chunks except the first one
+            while progress[:restsize].to_i > 0 && progress[:restsize] < block_size
+                # choose the smaller one
+                body_length = [progress[:restsize], chunk_size].min
+                for i in 1..retry_times
+                    seek_pos = block_index*Config.settings[:block_size] + progress[:offset]
+                    body = fh.get_data(seek_pos, body_length)
+                    result_length = body.length
+                    if result_length != body_length
+                        raise FileSeekReadError.new(seek_pos, body_length, result_length)
+                    end
+                    code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
+                    body_crc32 = Zlib.crc32(body)
+                    if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+                        progress[:ctx] = data["ctx"]
+                        progress[:offset] += body_length
+                        progress[:restsize] -= body_length
+                        progress[:status_code] = code
+                        if !notifier.nil? && notifier.respond_to?("notify")
+                            notifier.notify(block_index, progress)
+                        end
+                        break
+                    elsif i == retry_times && data["crc32"] != body_crc32
+                        Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+                    end
+                end
+            end
+            # return
+            return [code, data]
+        end
+
+        def _block_count(fsize)
+            ((fsize + Config.block_size - 1) / Config.block_size).to_i
+        end
+
+        def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil)
+            code, data = 0, {}
+            block_count = _block_count(fh.data_size)
+            if checksums.length != block_count || progresses.length != block_count
+                raise ResumablePutError.new("Invalid arg. Unexpected block count.")
+            end
+            0.upto(block_count-1).each do |block_index|
+                if checksums[block_index].nil? || checksums[block_index].empty?
+                    block_size = Config.settings[:block_size]
+                    if block_index == block_count - 1
+                        block_size = fsize - block_index*Config.settings[:block_size]
+                    end
+                    if progresses[block_index].nil?
+                        progresses[block_index] = _new_block_put_progress_data
+                    end
+                    code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
+                    if Utils.is_response_ok?(code)
+                        checksums[block_index] = data["checksum"]
+                        if !block_notifier.nil? && block_notifier.respond_to?("notify")
+                            block_notifier.notify(block_index, checksums[block_index])
+                        end
+                    end
+                end
+            end
+            return [code, data]
+        end
+
+        def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil)
+          path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + '/fsize/' + fsize
+          path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
+          path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
+          path += '/customer/' + customer if !customer.nil? && !customer.empty?
+          path += '/params/' + Utils.urlsafe_base64_encode(callback_params) if !callback_params.nil? && !callback_params.empty?
+          url = Config.settings[:up_host] + path
+          body = ''
+          checksums.each do |checksum|
+              body += Utils.urlsafe_base64_decode(checksum)
+          end
+          _call_binary_with_token(uptoken, url, body)
+        end
+
+      end
+    end
+  end
+end

lib/qiniu/rs/utils.rb

@@ -32,6 +32,14 @@ module Qiniu
         {}
       end
 
+      def is_response_ok?(status_code)
+          status_code/100 == 2
+      end
+
+      def response_error(status_code, errmsg)
+          [status_code, {"error" => errmsg}]
+      end
+
       def send_request_with url, data = nil, options = {}
         options[:method] = Config.settings[:method] unless options[:method]
         options[:content_type] = Config.settings[:content_type] unless options[:content_type]
@@ -42,8 +50,8 @@ module Qiniu
         auth_token = nil
         if !options[:qbox_signature_token].nil? && !options[:qbox_signature_token].empty?
           auth_token = 'QBox ' + options[:qbox_signature_token]
-        #elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty?
-        #  auth_token = 'UpToken ' + options[:upload_signature_token]
+        elsif !options[:upload_signature_token].nil? && !options[:upload_signature_token].empty?
+          auth_token = 'UpToken ' + options[:upload_signature_token]
         elsif options[:access_token]
           auth_token = 'Bearer ' + options[:access_token]
         end
@@ -56,7 +64,7 @@ module Qiniu
           response = RestClient.post(url, data, header_options)
         end
         code = response.respond_to?(:code) ? response.code.to_i : 0
-        if code != 200
+        unless is_response_ok?(code)
           raise RequestFailed.new(response)
         else
           data = {}

lib/qiniu/rs/version.rb

@@ -2,6 +2,18 @@
 
 module Qiniu
   module RS
-    VERSION = "3.0.7"
+    module Version
+      MAJOR = 3
+      MINOR = 1
+      PATCH = 0
+      # Returns a version string by joining <tt>MAJOR</tt>, <tt>MINOR</tt>, and <tt>PATCH</tt> with <tt>'.'</tt>
+      #
+      # Example
+      #
+      #   Version.to_s # '1.0.2'
+      def self.to_s
+        [MAJOR, MINOR, PATCH].join('.')
+      end
+    end
   end
 end

qiniu-rs.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "qiniu-rs"
   gem.require_paths = ["lib"]
-  gem.version       = Qiniu::RS::VERSION
+  gem.version       = Qiniu::RS::Version.to_s
 
   # specify any dependencies here; for example:
   gem.add_development_dependency "rake", "~> 0.9"

spec/qiniu/rs/abstract_spec.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+require 'spec_helper'
+require 'qiniu/rs/abstract'
+
+describe Qiniu::RS::Abstract do
+  before(:each) do
+    @klass = Class.new do
+      include Qiniu::RS::Abstract
+
+      abstract_methods :foo, :bar
+    end
+  end
+
+  it "raises NotImplementedError" do
+    proc {
+      @klass.new.foo
+    }.should raise_error(NotImplementedError)
+  end
+
+  it "can be overridden" do
+    subclass = Class.new(@klass) do
+      def foo
+        :overridden
+      end
+    end
+
+    subclass.new.foo.should == :overridden
+  end
+end

spec/qiniu/rs/rs_spec.rb

@@ -17,22 +17,22 @@ module Qiniu
         @domain = 'iovip.qbox.me/test'
 
         code, data = Qiniu::RS::RS.mkbucket(@bucket)
-        code.should == 200
         puts data.inspect
+        code.should == 200
       end
 
       context "IO.upload_file" do
         it "should works" do
           code, data = Qiniu::RS::IO.put_auth()
+          puts data.inspect
           code.should == 200
           data["url"].should_not be_empty
           data["expiresIn"].should_not be_zero
-          puts data.inspect
           @put_url = data["url"]
 
           code2, data2 = Qiniu::RS::IO.upload_file(@put_url, __FILE__, @bucket, @key)
-          code2.should == 200
           puts data2.inspect
+          code2.should == 200
         end
       end
 

spec/qiniu/rs/version_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 require 'qiniu/rs/version'
 
-describe Qiniu::RS do
+describe Qiniu::RS::Version do
   it "should has a VERSION" do
-    Qiniu::RS::VERSION.should =~ /^\d+\.\d+\.\d+?$/
+    Qiniu::RS::Version.to_s.should =~ /^\d+\.\d+\.\d+?$/
   end
 end

spec/spec_helper.rb

@@ -6,6 +6,7 @@ require 'rspec'
 
 RSpec.configure do |config|
   config.before :all do
+=begin
     Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
                                     :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
                                     :auth_url   => "http://m1.qbox.me:13001/oauth2/token",
@@ -14,5 +15,9 @@ RSpec.configure do |config|
                                     :up_host    => "http://m1.qbox.me:13019",
                                     :pub_host   => "http://m1.qbox.me:13012",
                                     :eu_host    => "http://m1.qbox.me:13050"
+=end
+
+    Qiniu::RS.establish_connection! :access_key => "aPoWOtE9EFca1fLxFCtlkeZAOV7aADVMTLdSydmr",
+                                    :secret_key => "L3ShtjCQTCagVCDPfHJoOix7JO_o3qHz3ScyflUG"
   end
 end