diff --git a/lib/wombat/crawler.rb b/lib/wombat/crawler.rb index 4d154d5..5a16f0c 100644 --- a/lib/wombat/crawler.rb +++ b/lib/wombat/crawler.rb @@ -17,7 +17,7 @@ class << self self.metadata = DSL::Metadata.new end - def crawl(url = nil, &block) + def crawl(url = nil, options = {}, &block) if block @metadata_dup = self.class.send(:metadata).clone instance_eval do @@ -27,14 +27,14 @@ def method_missing method, *args, &block end end self.instance_eval &block - parsed = parse(@metadata_dup, url) + parsed = parse(@metadata_dup, url, options) instance_eval do alias :method_missing :old_method_missing remove_instance_variable :@metadata_dup end parsed else - parse(self.class.send(:metadata), url) + parse(self.class.send(:metadata), url, options) end end diff --git a/lib/wombat/processing/parser.rb b/lib/wombat/processing/parser.rb index b80f23f..03611d8 100644 --- a/lib/wombat/processing/parser.rb +++ b/lib/wombat/processing/parser.rb @@ -3,6 +3,7 @@ require 'wombat/processing/node_selector' require 'mechanize' require 'restclient' +require 'uri' module Nokogiri module XML @@ -40,30 +41,39 @@ def initialize @mechanize.user_agent_alias = Wombat.user_agent_alias if Wombat.user_agent_alias end - def parse(metadata, url=nil) - @context = parser_for(metadata, url) - + def parse(metadata, url = nil, options = {}) + unless options.empty? + options = options.each_with_object({}) do |(k, v), memo| + memo[k.to_sym] = v.to_s unless v.nil? + end + end + @context = parser_for(metadata, url, options) + return nil if @context.nil? Wombat::Property::Locators::Factory.locator_for(metadata).locate(@context, @mechanize) end private - def parser_for(metadata, url) + + def parser_for(metadata, url, options = {}) url ||= "#{metadata[:base_url]}#{metadata[:path]}" - page = nil - parser = nil - _method = method_from(metadata[:http_method]) + return if url.nil? + method = method_from(metadata[:http_method]) data = metadata[:data] args = [url, data].compact begin @page = metadata[:page] if metadata[:document_format] == :html - @page = @mechanize.public_send(_method, *args) unless @page + @mechanize.set_proxy(*options[:proxy_args]) if options[:proxy_args] + @mechanize.user_agent = options[:user_agent] if options[:user_agent] + @mechanize.user_agent_alias = options[:user_agent_alias] if options[:user_agent_alias] + update_cookies(url, options[:cookies]) if options[:cookies] + @page = @mechanize.public_send(method, *args) unless @page parser = @page.parser # Nokogiri::HTML::Document parser.mechanize_page = @page # Mechanize::Page parser.headers = @page.header else - @page = RestClient.public_send(_method, *args) unless @page + @page = RestClient.public_send(method, *args) unless @page parser = Nokogiri::XML @page parser.headers = @page.headers end @@ -79,9 +89,19 @@ def parser_for(metadata, url) end end - def method_from(_method) - return :get if _method.nil? - HTTP_METHODS.detect(->{:get}){ |i| i == _method.downcase.to_sym } + def update_cookies(url, cookies) + domain = URI.parse(url).host + cookies.each do |k, v| + cookie = Mechanize::Cookie.new(k.to_s, v.to_s) + cookie.domain = domain + cookie.path = '/' + @mechanize.cookie_jar << cookie + end + end + + def method_from(method) + return :get if method.nil? + HTTP_METHODS.detect(-> {:get}){ |i| i == method.downcase.to_sym } end end end diff --git a/spec/crawler_spec.rb b/spec/crawler_spec.rb index 7d6d6ee..14a1bfe 100644 --- a/spec/crawler_spec.rb +++ b/spec/crawler_spec.rb @@ -134,14 +134,15 @@ it 'should crawl with url and block' do url = 'http://danielinc.com/itens' - expect(@crawler_instance).to receive(:parse).with(anything, url) - @crawler_instance.crawl(url) do + opts = {cookies: :cookies} + expect(@crawler_instance).to receive(:parse).with(anything, url, opts) + @crawler_instance.crawl(url, opts) do end another_instance = @crawler.new - expect(another_instance).to receive(:parse).with(anything, url) + expect(another_instance).to receive(:parse).with(anything, url, opts) - another_instance.crawl(url) + another_instance.crawl(url, opts) end it 'should remove created method missing' do