refactors HomeDepot connector

Author: dimanyc

app/connectors/home_depot.rb

@@ -1,50 +1,19 @@
 # Encapsulates instructions for
 # navigating Home Depot
 class HomeDepot < BaseConnector
-  attr_reader :listing_attrs
 
   BASE_URL = 'http://www.homedepot.com/p/'
   ABBREV   = 'hd'
 
   def abbrev; ABBREV end
 
   def process_listing(node)
-    id = node.id
-    visit_product_page(id)
+    visit_product_page(node.id)
     return if page_not_found?
-    fetch_product_attributes(id)
+    fetch_product_attributes(node.id)
     store_product_attributes(@listing_attrs)
   end
 
-  def visit_product_page(product_id)
-
-  end
-
-
-  def process_listings(nodes)
-    nodes.each_with_index do |node, index|
-      id = node[0]
-      sku = node[1]
-      # next if Listing.record_exists?(sku)
-      next if Listing.data_present?(sku, 'hd')
-      puts '>>> listing exists' if Listing.data_present?(sku, 'hd')
-      puts "=== starting id: #{id} \
-      | sku: #{sku} \
-      | iteration: #{index}"
-      visit_product_page(id)
-      next if page_not_found?
-      fetch_product_attributes(id)
-      vendor_sku = @listing_attrs[:vendor_sku]
-      vendor_url = @listing_attrs[:vendor_url]
-      Listing.append_hd_url(
-        vendor_sku,
-        vendor_url)
-      Listing.append_vendor_attrs(
-        vendor_sku,
-        @listing_attrs)
-    end
-  end
-
   def visit_product_page(hd_id)
     driver.visit("#{BASE_URL}#{hd_id}") unless hd_id.nil?
   end
@@ -55,14 +24,14 @@ def fetch_product_attributes(hd_id)
         vendor_url:   driver.current_url,
         vendor_id:    driver.doc.at('#product_internet_number').text,
         vendor_sku:   driver.doc.at('.modelNo').text.split(' ').last,
-        vendor_title: driver.doc.at('h1.product-title__title').text,
+        vendor_title: driver.doc.at('.product-title').text,
         vendor_price: driver.doc.at('#ajaxPrice').text.strip }
   end
 
   private
 
   def page_not_found?
-    driver.current_url == BASE_URL ||
+    driver.current_url != BASE_URL ||
       driver.doc.at('.no-results').present? ||
       driver.doc.at('.content_margin_404page').present?
   end

app/connectors/overstock.rb

@@ -16,7 +16,7 @@ def process_listing(node)
   # Visits the home page
   # return if driver is already on home page
   def visit_home_page
-    return if already_on_site?#driver.current_url == BASE_URL
+    return if already_on_site?
     close_popups
     driver.visit(BASE_URL)
   end

public/screenshot_test.jpg

@@ -1,9 +1,6 @@
-require 'rake'
-rake = Rake.application
-rake.init
-rake.load_rakefile
-
-file_path = 'lib/hd_skus.csv'
-nodes = CSVFeedParser.fetch_nodes(file_path)
+csv = 'lib/hd_skus.csv'
 connector = HomeDepot.new(PoltergeistCrawler.new)
-connector.process_listings(nodes)
+crawler = DirectPageAccessCrawler.new(connector)
+crawler.fetch_product_nodes(csv)
+crawler.process_listings
+

run_hd.rb

@@ -52,13 +52,49 @@
         connector.visit_product_page('204766736')
       end
 
-      it 'should be able to product attributes' do
+      it 'should be able to fetch vendor' do
         expect {
           connector.fetch_product_attributes('204766736')
         }.to change {
-          connector.listing_attrs
+          connector.listing_attrs[:vendor]
         } .from(nil)
-          .to(a_kind_of(Hash))
+          .to('hd')
+      end
+
+      it 'should be able to fetch vendor' do
+        expect {
+          connector.fetch_product_attributes('204766736')
+        }.to change {
+          connector.listing_attrs[:vendor_url]
+        } .from(nil)
+          .to('http://www.homedepot.com/p/204766736')
+      end
+
+      it 'should be able to fetch vendor' do
+        expect {
+          connector.fetch_product_attributes('204766736')
+        }.to change {
+          connector.listing_attrs[:vendor_id]
+        } .from(nil)
+          .to('204766736')
+      end
+
+      it 'should be able to fetch vendor' do
+        expect {
+          connector.fetch_product_attributes('204766736')
+        }.to change {
+          connector.listing_attrs[:vendor_sku]
+        } .from(nil)
+          .to('HD4872CS')
+      end
+
+      it 'should be able to fetch vendor' do
+        expect {
+          connector.fetch_product_attributes('204766736')
+        }.to change {
+          connector.listing_attrs[:vendor_title]
+        } .from(nil)
+          .to('Studio 5 ft. Reversible Drain Bathtub in Biscuit')
       end
 
       context 'specific attributes' do
@@ -90,7 +126,7 @@
         it 'should store product\'s vendor title' do
           connector.fetch_product_attributes('204766736')
           expect(connector.listing_attrs[:vendor_title])
-            .to eq('Peridot 6 ft. Acrylic Reversible Drain Rectangular Bathtub in White')
+            .to eq('Studio 5 ft. Reversible Drain Bathtub in Biscuit')
         end
 
         it 'should store product\'s vendor price' do
@@ -121,18 +157,4 @@
 
   end # crawling
 
-  context 'storing product attributes' do
-
-    it 'should store product attributes' do
-      expect_any_instance_of(BaseConnector)
-        .to receive(:store_attrs)
-        .with('123', a_kind_of(Hash))
-      connector.store_attrs('123', {})
-    end
-
-    it 'should append product attrs to listing vendor' do
-      Listing.append_menards_url( '123', 'foo.com' )
-    end
-
-  end # storing attributes
 end

spec/connectors/home_depot_spec.rb

@@ -1,7 +1,7 @@
 Billy.configure do |c|
   c.cache         = true
   c.cache_request_headers = true
-  c.persist_cache = true
+  # c.persist_cache = true
   c.cache_path    = 'spec/fixtures/billy_cache'
   c.non_whitelisted_requests_disabled = true
   c.ignore_params = ['http://www.homedepot.com/',