require "erb" class ProcessPostWorker include Sidekiq::Worker include ERB::Util sidekiq_options queue: :post, retry: 3 def perform url post = Post.find_by url: url begin page = MetaInspector.new post.url post.title = page.title.try(:strip) post.summary = page.description post.processed = true if page.images.best.present? post.create_thumbnail file: URI.parse(page.images.best) end post.save rescue SocketError => e Rails.logger.error "出错了: #{e}" ensure if post.url.include?('//mp.weixin.qq.com/') get_content_by_nokogirl post else readability_extract_content(post) end end end private def readability_extract_content post token = 'f0365ca0cf184783625c676e3373b329d7b7f4f4' begin response = RestClient.get "https://readability.com/api/content/v1/parser?token=#{token}&url=#{url_encode(post.url)}" if response.code == 200 post_json = JSON.parse response.body post.content = HTMLEntities.new.decode post_json['content'] post.good_format = true if post.type != 'WebPage' post.title = post_json['title'] unless post.title post.create_thumbnail(file: URI.parse(post_json['lead_image_url'])) if post_json['lead_image_url'].present? post.save end rescue => e Sidekiq.logger.error "ReadAbility 获取文章失败:#{e}" end end def get_content_by_nokogirl post page = Nokogiri::HTML RestClient.get(url_encode(post.url)) content_html = page.css('div#js_content')[0].try(:to_html) post.content = content_html post.good_format = true if post.type != 'WebPage' post.save! end end