From 5cc4db22313d570e8871938fb89c707e79e22b63 Mon Sep 17 00:00:00 2001 From: Liang Tao Date: Sun, 6 Apr 2014 11:47:04 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=B7=BB=E5=8A=A0pfop=E7=9B=B8=E5=85=B3SDK?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/qiniu/adt.rb | 46 +++++++++++++++ lib/qiniu/fop.rb | 1 + lib/qiniu/http.rb | 8 +++ lib/qiniu/pfop.rb | 123 ++++++++++++++++++++++++++++++++++++++++ spec/qiniu/pfop_spec.rb | 89 +++++++++++++++++++++++++++++ 5 files changed, 267 insertions(+) create mode 100644 lib/qiniu/adt.rb create mode 100755 lib/qiniu/pfop.rb create mode 100644 spec/qiniu/pfop_spec.rb diff --git a/lib/qiniu/adt.rb b/lib/qiniu/adt.rb new file mode 100644 index 0000000..830206a --- /dev/null +++ b/lib/qiniu/adt.rb @@ -0,0 +1,46 @@ +# -*- encoding: utf-8 -*- +# vim: sw=2 ts=2 + +require 'uri' + +module Qiniu + module ADT + + class ApiSpecification + public + def to_str; return ""; end + end # class ApiSpecification + + module Policy + public + def to_json + args = {} + + self.params.each_pair do |key, fld| + val = self.__send__(key) + if !val.nil? then + args[fld] = val + end + end + + return args.to_json + end # to_json + + def to_query_string + args = [] + + self.params.each_pair do |key, fld| + val = self.__send__(key) + if !val.nil? then + new_fld = CGI.escape(fld.to_s) + new_val = CGI.escape(val.to_s).gsub('+', '%20') + args.push("#{new_fld}=#{new_val}") + end + end + + return args.join("&") + end # to_query_string + end # module Policy + + end # module ADT +end # module Qiniu diff --git a/lib/qiniu/fop.rb b/lib/qiniu/fop.rb index 0abe87e..a820de2 100755 --- a/lib/qiniu/fop.rb +++ b/lib/qiniu/fop.rb @@ -1,3 +1,4 @@ # -*- encoding: utf-8 -*- require 'qiniu/image' +require 'qiniu/pfop' diff --git a/lib/qiniu/http.rb b/lib/qiniu/http.rb index a03ff82..0f7004d 100755 --- a/lib/qiniu/http.rb +++ b/lib/qiniu/http.rb @@ -22,6 +22,7 @@ module Qiniu def get (url, opts = {}) ### 配置请求Header req_headers = { + :connection => 'close', :accept => '*/*', :user_agent => Config.settings[:user_agent] } @@ -36,6 +37,9 @@ module Qiniu return response.code.to_i, response.body, response.raw_headers rescue => e Log.logger.warn "#{e.message} => Qiniu::HTTP.get('#{url}')" + if e.respond_to?(:response) && e.response.respond_to?(:code) then + return e.response.code, e.response.body, e.response.raw_headers + end return nil, nil, nil end # get @@ -72,6 +76,7 @@ module Qiniu def post (url, req_body = nil, opts = {}) ### 配置请求Header req_headers = { + :connection => 'close', :accept => '*/*', :user_agent => Config.settings[:user_agent] } @@ -86,6 +91,9 @@ module Qiniu return response.code.to_i, response.body, response.raw_headers rescue => e Log.logger.warn "#{e.message} => Qiniu::HTTP.post('#{url}')" + if e.respond_to?(:response) && e.response.respond_to?(:code) then + return e.response.code, e.response.body, e.response.raw_headers + end return nil, nil, nil end # post diff --git a/lib/qiniu/pfop.rb b/lib/qiniu/pfop.rb new file mode 100755 index 0000000..29e571f --- /dev/null +++ b/lib/qiniu/pfop.rb @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +# vim: sw=2 ts=2 + +require 'qiniu/adt' +require 'qiniu/http' + +module Qiniu + module Fop + module Persistance + + class PfopPolicy + include ADT::Policy + + private + def initialize(bucket, + key, + fops, + notify_url) + @bucket = bucket + @key = key + @notify_url = notify_url + + self.fops!(fops) + end # initialize + + public + PARAMS = { + # 字符串类型参数 + :bucket => "bucket", + :key => "key", + :fops => "fops", + :notify_url => "notifyURL", + + # 数值类型参数 + :force => "force" + } # PARAMS + + PARAMS.each_pair do |key, fld| + attr_accessor key + end + + def params + return PARAMS + end # params + + def fops! (fops) + if fops.is_a?(Hash) then + fops = fops.values + end + + if fops.is_a?(Array) then + new_fops = [] + fops.each do |v| + if v.is_a?(ApiSpecification) then + new_fops.push(v.to_s) + end + end + + @fops = new_fops.join(";") + else + @fops = fops.to_s + end + end # fops! + + def force! + @force = 1 + end # force! + + alias :to_s :to_json + end # class PfopPolicy + + class << self + + API_HOST = 'http://api.qiniu.com' + + PFOP_URL = API_HOST + '/pfop/' + + def pfop (args) + ### 生成fop指令串 + if args.is_a?(PfopPolicy) then + # PfopPolicy的各个字段按固定顺序组织 + body = args.to_query_string() + elsif args.is_a?(Hash) then + # 无法保证固定字段顺序 + body = HTTP.generate_query_string(args) + else + # 由调用者保证固定字段顺序 + body = args.to_s + end + + ### 发送请求 + return HTTP.management_post(PFOP_URL, body) + end # pfop + + PREFOP_URL = API_HOST + '/status/get/prefop?id=' + + def prefop (persistent_id) + ### 抽取persistentId + if persistent_id.is_a?(Hash) then + pid = persistent_id['persistentId'] + else + pid = persistent_id.to_s + end + + ### 发送请求 + url = PREFOP_URL + pid + return HTTP.api_get(url) + end # prefop + + def generate_p1_url (url, fop) + # 如果fop是ApiSpecification，则各字段按固定顺序组织，保证一致性 + # 否则由调用者保证固定字段顺序 + fop = CGI.escape(fop.to_s).gsub('+', '%20') + + ### 生成url + return url + '?p/1/' + fop + end # generate_pl_url + + end # class << self + + end # module Persistance + end # module Fop +end # module Qiniu diff --git a/spec/qiniu/pfop_spec.rb b/spec/qiniu/pfop_spec.rb new file mode 100644 index 0000000..fb509a7 --- /dev/null +++ b/spec/qiniu/pfop_spec.rb @@ -0,0 +1,89 @@ +# -*- encoding: utf-8 -*- +# vim: sw=2 ts=2 + +require 'spec_helper' +require 'qiniu/auth' +require 'qiniu' +require 'qiniu/fop' + +module Qiniu + module Fop + module Persistance + describe Persistance do + + before :all do + ### 复用RubySDK-Test-Storage空间 + @bucket = 'RubySDK-Test-Storage' + @bucket = make_unique_bucket(@bucket) + + ### 尝试创建空间 + code, data, headers = Qiniu::Storage.make_a_public_bucket(@bucket) + puts code.inspect + puts data.inspect + puts headers.inspect + + pic_fname = "image_logo_for_test.png" + @key = make_unique_key_in_bucket(pic_fname) + + local_file = File.expand_path('../' + pic_fname, __FILE__) + + ### 检查测试文件存在性 + code, body, headers = Qiniu::Storage.stat(@bucket, @key) + if code == 404 || code == 612 then + # 文件不存在，尝试上传 + pp = Qiniu::Auth.PutPolicy.new(@bucket, @key) + code, body, headers = Qiniu::Storage.upload_with_put_policy( + pp, + local_file + ) + puts "Put a test file for Persistance cases" + puts code.inspect + puts body.inspect + puts header.inspect + end + end + + after :all do + ### 不删除Bucket以备下次使用 + end + + context ".pfop" do + it "should works" do + pp = Persistance::PfopPolicy.new( + @bucket, + @key, + 'imageView2/1/w/80/h/80', # fops + 'www.baidu.com' # notify_url + ) + + code, data, headers = Qiniu::Fop::Persistance.pfop(pp) + code.should == 200 + puts data.inspect + end + end + + context ".prefop" do + it "should works" do + code, data, headers = Qiniu::Fop::Persistance.prefop('fakePersistentId') + code.should == 612 + puts code.inspect + puts data.inspect + puts headers.inspect + end + end + + context ".p1" do + it "should works" do + url = 'http://fake.qiniudn.com/fake.jpg' + fop = 'imageView2/1/w/80/h/80' + target_url = "#{url}?p/1/#{CGI.escape(fop).gsub('+', '%20')}" + + p1_url = Qiniu::Fop::Persistance.generate_p1_url(url, fop) + p1_url.should == target_url + puts p1_url.inspect + end + end + end + end # module Persistance + end # module Fop +end # module Qiniu From 5718b720dfececbe714d09a5ebe248ac6d4754b2 Mon Sep 17 00:00:00 2001 From: Liang Tao Date: Tue, 8 Apr 2014 12:50:28 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9B=B4=E5=A4=9ADebug?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=BB=A5=E4=BE=BF=E8=BF=BD=E8=B8=AA=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/qiniu/resumable_upload.rb | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/qiniu/resumable_upload.rb b/lib/qiniu/resumable_upload.rb index 1c20946..f533f88 100755 --- a/lib/qiniu/resumable_upload.rb +++ b/lib/qiniu/resumable_upload.rb @@ -106,14 +106,14 @@ module Qiniu options[:headers][:content_type] = content_type end - code, data = HTTP.api_post(url, data, options) + code, data, raw_headers = HTTP.api_post(url, data, options) unless HTTP.is_response_ok?(code) retry_times += 1 if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times) end end - [code, data] + return code, data, raw_headers end # _call_binary_with_token def _mkblock(uptoken, block_size, body) @@ -150,7 +150,10 @@ module Qiniu if result_length != body_length raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end - code, data = _mkblock(uptoken, block_size, body) + + code, data, raw_headers = _mkblock(uptoken, block_size, body) + Utils.debug "Mkblk : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" + body_crc32 = Zlib.crc32(body) if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32 progress[:ctx] = data["ctx"] @@ -164,6 +167,7 @@ module Qiniu break elsif i == retry_times && data["crc32"] != body_crc32 Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + return code, data, raw_headers end end elsif progress[:offset] + progress[:restsize] != block_size @@ -181,7 +185,10 @@ module Qiniu if result_length != body_length raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end - code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body) + + code, data, raw_headers = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body) + Utils.debug "Bput : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" + body_crc32 = Zlib.crc32(body) if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32 progress[:ctx] = data["ctx"] @@ -195,11 +202,12 @@ module Qiniu break elsif i == retry_times && data["crc32"] != body_crc32 Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) + return code, data, raw_headers end end end # return - return [code, data] + return code, data, raw_headers end # _resumable_put_block def _block_count(fsize) @@ -291,19 +299,20 @@ module Qiniu checksums = [] block_count.times{checksums << ''} - code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) + code, data, raw_headers = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) if HTTP.is_response_ok?(code) uphost = data["host"] entry_uri = bucket + ':' + key - code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate) + code, data, raw_headers = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate) + Utils.debug "Mkfile : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" end if HTTP.is_response_ok?(code) Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded." end - [code, data] + return code, data, raw_headers end # _resumable_upload end # self class end # module Storage From ef0d79eb17685f10dc3c4e39a37d17eb180dce82 Mon Sep 17 00:00:00 2001 From: Liang Tao Date: Tue, 8 Apr 2014 14:05:00 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E4=B8=8D=E5=B8=B8?= =?UTF-8?q?=E8=A7=81=E7=9A=84Ruby=E7=89=88=E6=9C=AC=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E5=BF=85=E8=A6=81=E7=9A=84=E5=87=A0=E4=B8=AA?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b11d42..7a22ba1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,9 @@ language: ruby rvm: - - 1.8.7 - - 1.9.2 - 1.9.3 - 2.0.0 - 2.1.1 - - jruby-18mode - jruby-19mode - - ree before_script: - export QINIU_ACCESS_KEY=LGRqJsjX7LAvtdtX6hhyxs861lS57HwnOQmJsOuX - export QINIU_SECRET_KEY=g_OxhuN5iJ1O7BoA6YdXyF6zSszk34UZFKDfdIIW