refactors Menards driver

Author: dimanyc

3

@@ -8,9 +8,8 @@ def initialize(driver)
     @driver = driver
   end
 
-  def append_url_to_listing(sku, url, abbrev: '', force_update: false)
-    return unless valid_options?(abbrev, force_update)
-    Listing.send("append_#{abbrev}_url", sku, url, force_update)
+  def store_attrs(sku, attrs)
+    Listing.append_vendor_attrs(sku, attrs)
   end
 
   private
@@ -25,11 +24,4 @@ def validate_driver(driver)
     end
   end
 
-  # Validates options keyword args
-  def valid_options?(abbrev, force_update)
-    abbrev.present? &&
-      Listing::CHANNELS.include?(abbrev) &&
-      [true, false].include?(force_update)
-  end
-
 end

app/connectors/base_connector.rb

@@ -4,21 +4,35 @@ class Menards < BaseConnector
 
   BASE_URL = 'http://www.menards.com/main/'
 
+  def process_listings(nodes)
+    nodes.each_with_index do |node, index|
+      id = node[0]
+      sku = node[1]
+      next if Listing.record_exists?(sku)
+      sleep(4) if index % 20 == 0
+      puts "=== starting id: #{id} | iteration: #{index}"
+      sleep(rand(0.1...3))
+      visit_product_page(id)
+      fetch_product_attributes(id)
+      Listing.append_menards_url(@listing_attrs[:vendor_sku],
+                                 @listing_attrs[:vendor_url])
+    end
+  end
+
   def visit_product_page(menards_id)
     driver.visit("#{BASE_URL}p-#{menards_id}.html") unless menards_id.nil?
+    raise PageNotFoundError if page_not_found?
   end
 
   def fetch_product_attributes(menards_id)
     return if page_not_found?
-    @listing_attrs = { url:         driver.current_url,
-                       vendor_id:   menards_id,
-                       vendor_sku:  driver.doc.at('p.itemModelSku').children.last.text.strip,
-                       title:       driver.doc.at('.itemCenterContent span h2').text,
-                       price:       driver.doc.at('#totalItemPrice').children.first.text.strip }
-  end
-
-  def store_product_attributes(listing_attrs)
-    Listing.append_attrs(listing_attrs)
+    @listing_attrs =
+      { vendor:      'menards',
+        vendor_url:         driver.current_url,
+        vendor_id:   menards_id,
+        vendor_sku:  driver.doc.at('p.itemModelSku').children.last.text.strip,
+        vendor_title:       driver.doc.at('.itemCenterContent span h2').text,
+        vendor_price:       driver.doc.at('#totalItemPrice').children.first.text.strip }
   end
 
   private

app/connectors/menards.rb

@@ -2,46 +2,36 @@
 # pertaining to a retailing channel
 class Listing < ActiveRecord::Base
 
-  validates_presence_of :sku
-  validate :channel_urls_format
+  VENDORS = %W( hd menards houzz overstock lowes build hmb wayfair )
 
-  CHANNELS = %w( hd menards overstock lowes build hmb wayfair houzz )
+  validates_presence_of :sku
+  serialize :vendors, HashSerializer
+  store_accessor :vendors, :hd, :wayfair, :lowes, :menards, :hmb, :houzz, :overstock, :build
 
-  def available_channel_urls
-    channel_names = CHANNELS.map { |channel| "#{channel}_url" }
-    channel_names.map do |name|
-      attributes.fetch_values(name)
-    end.flatten.compact
-  end
-
-  CHANNELS.each do |channel_name|
-    define_singleton_method("append_#{channel_name}_url") do |sku, url, force_update|
-      listing = find_by(sku: sku)
-      puts "=== SKU #{sku}"
-      puts "--- url #{url}"
-      puts "--- force_update  #{force_update}"
-      if force_update
-        listing.update_attributes!("#{channel_name}_url": url)
-      elsif listing
-        puts '--- listing already exists'
-        return
-      else
-        puts '--- updating listing'
-        create!(sku: sku, "#{channel_name}_url":url)
-      end
-    end
+  def self.append_vendor_attrs(sku, attrs)
+    listing = find_by(sku: sku) || new(sku: sku)
+    listing.fetch_vendor_data(attrs)
+    listing.save!
   end
 
   def self.record_exists?(sku)
     find_by(sku: sku).present?
   end
 
+  def fetch_vendor_data(attrs)
+    vendors[attrs[:vendor]].merge!({
+      id:    attrs[:vendor_id],
+      sku:   attrs[:vendor_sku],
+      url:   attrs[:vendor_url],
+      title: attrs[:vendor_title],
+      price: attrs[:vendor_price] }).to_json
+  end
 
-  private
-
-  def channel_urls_format
-    if available_channel_urls.any? { |url| url.scan(URI.regexp).empty? }
-      errors.add(:base, message: 'malformed url')
+  VENDORS.each do |vendor|
+    define_singleton_method("append_#{vendor}_url") do |sku, url|
+      listing = find_by(sku: sku) || new(sku: sku)
+      listing["#{vendor}_url"] = url
+      listing.save!
     end
   end
 

app/models/listing.rb

@@ -17,4 +17,12 @@ def self.stringify_skus(skus)
     skus.join(' ')
   end
 
+  def self.fetch_menards_skus(file_path)
+    nodes = {}
+    CSV.foreach(file_path) do |row|
+      nodes[row[0]] = row[1]
+    end
+    nodes
+  end
+
 end

app/parsers/csv_feed_parser.rb

@@ -0,0 +1,11 @@
+class HashSerializer
+
+  def self.dump(hash)
+    hash.to_json
+  end
+
+  def self.load(hash)
+    (hash || {}).with_indifferent_access
+  end
+
+end