adds bulk skus processing for Menards

Author: dimanyc

app/connectors/menards.rb

@@ -9,37 +9,43 @@ def process_listings(nodes)
       id = node[0]
       sku = node[1]
       next if Listing.record_exists?(sku)
-      sleep(4) if index % 20 == 0
       puts "=== starting id: #{id} | iteration: #{index}"
-      sleep(rand(0.1...3))
       visit_product_page(id)
+      next if page_not_found?
       fetch_product_attributes(id)
-      Listing.append_menards_url(@listing_attrs[:vendor_sku],
-                                 @listing_attrs[:vendor_url])
+      vendor_sku = @listing_attrs[:vendor_sku]
+      vendor_url = @listing_attrs[:vendor_url]
+      Listing.append_menards_url(
+        vendor_sku,
+        vendor_url)
+      Listing.append_vendor_attrs(
+        vendor_sku,
+        @listing_attrs)
     end
   end
 
+
   def visit_product_page(menards_id)
     driver.visit("#{BASE_URL}p-#{menards_id}.html") unless menards_id.nil?
-    raise PageNotFoundError if page_not_found?
   end
 
   def fetch_product_attributes(menards_id)
-    return if page_not_found?
     @listing_attrs =
       { vendor:      'menards',
-        vendor_url:         driver.current_url,
-        vendor_id:   menards_id,
-        vendor_sku:  driver.doc.at('p.itemModelSku').children.last.text.strip,
-        vendor_title:       driver.doc.at('.itemCenterContent span h2').text,
-        vendor_price:       driver.doc.at('#totalItemPrice').children.first.text.strip }
+        vendor_url:   driver.current_url,
+        vendor_id:    menards_id,
+        vendor_sku:   driver.doc.at('p.itemModelSku').children.last.text.strip,
+        vendor_title: driver.doc.at('.itemCenterContent span h2').text,
+        vendor_price: driver.doc.at('#totalItemPrice').children.first.text.strip }
   end
 
   private
 
   def page_not_found?
     driver.current_url == BASE_URL ||
-      driver.doc.at('h5.error').present?
+      driver.doc.at('h5.error').present? ||
+      driver.doc.at('h3.resettitle').present? &&
+      driver.doc.at('h3.resettitle').text.include?('404 error')
   end
 
 end

app/models/listing.rb

@@ -10,6 +10,7 @@ class Listing < ActiveRecord::Base
 
   def self.append_vendor_attrs(sku, attrs)
     listing = find_by(sku: sku) || new(sku: sku)
+    listing.vendors_will_change!
     listing.fetch_vendor_data(attrs)
     listing.save!
   end
@@ -19,12 +20,12 @@ def self.record_exists?(sku)
   end
 
   def fetch_vendor_data(attrs)
-    vendors[attrs[:vendor]].merge!({
+    vendors[attrs[:vendor]] = ({
       id:    attrs[:vendor_id],
       sku:   attrs[:vendor_sku],
       url:   attrs[:vendor_url],
       title: attrs[:vendor_title],
-      price: attrs[:vendor_price] }).to_json
+      price: attrs[:vendor_price] })
   end
 
   VENDORS.each do |vendor|

app/workers/menards_crawler.rb

@@ -2,9 +2,17 @@ class MenardsCrawler
 
   include Sidekiq::Worker
 
-  def perform(ids)
-    connector = Menards.new(PoltergeistCrawler.new)
-    connector.process_listings(ids)
+  # def perform(ids)
+  #   connector = Menards.new(PoltergeistCrawler.new)
+  #   connector.process_listings(ids)
+  # end
+  #
+  def perform(id)
+    c = Menards.new(PoltergeistCrawler.new)
+    c.visit_product_page(id)
+    c.fetch_menards_skus(id)
+    Listing.append_menards_url(c.listing_attrs[:vendor_sku],
+                               c.listing_attrs[:vendor_url])
   end
 
 end

config/sidekiq.yml

@@ -0,0 +1,10 @@
+---
+:concurrency: 5
+:pidfile: tmp/pids/sidekiq.pid
+staging:
+  :concurrency: 10
+production:
+  :concurrency: 20
+:queues:
+  - default
+  - [myqueue, 2]