extends direct page crawler spec

Author: dimanyc

Diff for: app/connectors/menards.rb

@@ -7,7 +7,6 @@ class Menards < BaseConnector
 
   def process_listing(node)
     id  = node[0]
-    # sku = node[1]
     visit_product_page(id)
     fetch_product_attributes(id)
     store_product_attributes(@listing_attrs)
@@ -53,6 +52,9 @@ def fetch_product_attributes(menards_id)
         vendor_price: driver.doc.at('#totalItemPrice').children.first.text.strip }
   end
 
+  def store_product_attributes(listing_attrs)
+  end
+
   private
 
   def page_not_found?

Diff for: app/crawlers/direct_page_access_crawler.rb

@@ -18,14 +18,16 @@ def fetch_product_nodes(file_path)
   end
 
   # Creates listings from
-  # fetched @nodes
-  def process_listings(nodes)
-    @nodes.each_with_index do |node, index|
+  # injected nodes (default: @nodes)
+  def process_listings(nodes = @nodes)
+    nodes.each_with_index do |node, index|
+      binding.pry
+      next if invalid_node?(node)
       next if data_exists?(node)
-      next if nil_id?(node)
       output_process_info(node, index)
-      validate_listing(id, sku, index)
-      index % 20 == 0 ? connector.restart : connector.process_listing(node, index)
+      # validate_listing(id, sku, index)
+      dispatch_action(node, index)
+      # (index != 0 && index % 20 == 0) ? connector.restart : connector.process_listing(node)
     end
   end
 
@@ -44,25 +46,41 @@ def validate_connector_type(obj)
   # returns 'true' if listing's vendor
   # data is present
   def data_exists?(node)
-    if Listing.data_present?(node[1],
-        connector.abbrev)
+    if Listing.data_present?(node.values.first, connector.abbrev)
       Notifier.raise_data_exists
       return true
     end
   end
 
   # returns 'true' if node's
   # 'id' or 'sku' are missing
+  # or if 'id' is '#N/A'
   def invalid_node?(node)
-    if node[0].nil? || node[1].nil?
+    if node.keys.first.nil? ||
+        node.keys.first.empty? ||
+        node.values.first.nil? ||
+        node.values.first.empty? ||
+        node.keys.first.to_s == '#N/A'
       Notifier.raise_invalid_node
       return true
     end
   end
 
   # Outputs process info to STDOUT
   def output_process_info(node, index)
-    Notifier.output_process_info(id, sku, index)
+    Notifier.output_process_info(node, index)
+  end
+
+  # Dispatches action based on current index param
+  # - when reaches % 20 restarts driver
+  # - else proceeds with data storing
+  def dispatch_action(node, index)
+    case
+    when index > 0 && index % 20 == 0
+      connector.restart
+    else
+      connector.process_listing(node)
+    end
   end
 
 end

Diff for: app/models/listing.rb

@@ -18,7 +18,13 @@ def self.append_vendor_attrs(sku, attrs)
 
   def self.record_exists?(sku)
     find_by(sku: sku).present?
+  end
 
+  # Constructs hash of
+  # '{ vendor_id: sku }'
+  # based on vendor injected
+  def to_node(vendor)
+    Hash[vendors["#{vendor}"][:vendor_id], sku]
   end
 
   def self.get_price(sku)

Diff for: app/utilities/notifier.rb

@@ -0,0 +1,14 @@
+class Notifier
+  class << self
+
+    def output_process_info(node, index)
+    end
+
+    def raise_invalid_node
+    end
+
+    def raise_data_exists
+    end
+
+  end
+end

Diff for: spec/connectors/menards_spec.rb

@@ -115,6 +115,10 @@
 
   context 'storing product attributes' do
 
+    it 'should stor product attributes' do
+      pending
+  end
+
     it 'should store product attributes' do
       expect_any_instance_of(BaseConnector)
         .to receive(:store_attrs)

Diff for: spec/crawlers/direct_page_access_crawler_spec.rb

@@ -29,9 +29,9 @@
         DirectPageAccessCrawler.new(fake_connector)
       end
 
-    end
+    end # driver validation
 
-  end
+  end  # init
 
   describe 'fetching product nodes' do
 
@@ -48,18 +48,168 @@
       expect(crawler.nodes).to_not be_nil
     end
 
-  end
+  end # fetching nodes
 
   describe 'processing' do
 
+    before do
+
+        allow(connector).to receive(:fetch_product_attributes)
+        allow(connector).to receive(:store_product_attributes)
+    end
+
+    it 'should iterate through stored nodes' do
+      csv = "#{Rails.root}/spec/fixtures/CSVs/sample_nodes.csv"
+      crawler.fetch_product_nodes(csv)
+      expect(crawler.process_listings).to be_valid
+    end
+
     it 'should iterate through listing nodes' do
-      listings = create_list(:listing, 3)
+      listings = create_list(:listing, 1, :with_menards_attrs)
+      nodes = [listings.first.to_node('menards')]
       expect(connector)
         .to receive(:process_listing)
+        .with(nodes.first)
+      crawler.process_listings(nodes)
+    end
+
+    it 'should output process details' do
+      listings = create_list(:listing, 1, :with_menards_attrs)
+      nodes = [listings.first.to_node('menards')]
+      expect(crawler)
+        .to receive(:output_process_info)
         .with(nodes.first, 0)
-      subject.process_listings(nodes)
+      crawler.process_listings(nodes)
     end
 
-  end
+    describe 'node validation' do
+
+      it 'should check if node has "sku" and "id" values' do
+        listings = create_list(:listing, 1, :with_menards_attrs)
+        node = listings.first.to_node('menards')
+        allow(connector).to receive(:fetch_product_attributes)
+        allow(connector).to receive(:store_product_attributes)
+        expect(crawler)
+          .to receive(:invalid_node?)
+          .with(node)
+          .once
+        crawler.process_listings([node])
+      end
+
+      context 'when node is invalid' do
+
+        describe 'beacuse of an empty id' do
+
+          let(:node) { {'': '234' } }
+
+          it 'should identify the invalid node' do
+            expect(crawler.send(:invalid_node?, node))
+              .to be_truthy
+          end
+
+          it 'should skip invalid node' do
+            nodes = { 'a': 1, '': 2 }
+            expect(crawler)
+              .to receive(:output_process_info)
+              .once
+            crawler.process_listings(nodes)
+
+          end
+
+          it 'should not process invalid node' do
+            expect(connector)
+              .to_not receive(:process_listing)
+            crawler.process_listings([node])
+          end
+
+          it 'should notify of invalid node' do
+            expect(Notifier)
+              .to receive(:raise_invalid_node)
+            crawler.process_listings([node])
+          end
+
+
+        end
+
+        describe 'because of an invalid id' do
+
+          let(:node) { { '#N/A': '123' } }
+
+          it 'should identify the invalid node' do
+            expect(crawler.send(:invalid_node?, node))
+              .to be_truthy
+          end
+
+          it 'should skip listing processing for that node' do
+            expect(connector)
+              .to_not receive(:process_listing)
+            crawler.process_listings([node])
+          end
+
+          it 'should notify of invalid node' do
+            expect(Notifier)
+              .to receive(:raise_invalid_node)
+            crawler.process_listings([node])
+          end
+
+        end
+
+        describe 'beacuse of an empty sku' do
+
+          let(:node) { {'123': '' } }
+
+          it 'should identify the invalid node' do
+            expect(crawler.send(:invalid_node?, node))
+              .to be_truthy
+          end
+
+          it 'should skip listing processing for that node' do
+            expect(connector)
+              .to_not receive(:process_listing)
+            crawler.process_listings([node])
+          end
+
+          it 'should notify of invalid node' do
+            expect(Notifier)
+              .to receive(:raise_invalid_node)
+            crawler.process_listings([node])
+          end
+
+        end # empty sku
+
+      end # invalid node
+
+    end # node validation
+
+    describe 'listing data validation' do
+
+      it 'should check if listing vendor data already exists' do
+        listings = create_list(:listing, 1, :with_menards_attrs)
+        node = listings.first.to_node('menards')
+        expect(crawler)
+          .to receive(:data_exists?)
+          .with(node)
+          .once
+        crawler.process_listings([node])
+      end
+
+      context 'when listing vendor data already exists' do
+
+        let(:listing) { create(:listing,
+                               :with_menards_attrs,
+                               :with_menards_url) }
+
+        it 'should notify about existing data' do
+          nodes = [listing.to_node('menards')]
+          expect(Notifier)
+            .to receive(:raise_data_exists)
+          crawler.process_listings(nodes)
+        end
+
+      end # when listing.vendors data exists
+
+    end # listing data validation
+
+  end # listing processing
 
 end

Diff for: spec/factories/listings.rb

@@ -6,15 +6,18 @@
         menards: {
           vendor_url:   Faker::Internet.url,
           vendor_id:    Faker::Number.number(10),
-          vendor_sku:   Faker::Code.isbn,
+          vendor_sku:   sku,
           vendor_title: Faker::Commerce.product_name,
           vendor_price: Faker::Commerce.price
         },
         hd: {}
       } }
     end
+
     trait :with_menards_url do
       menards_url Faker::Internet.url
     end
+
   end
+
 end

Diff for: spec/fixtures/CSVs/.~lock.sample_nodes.csv#

@@ -77,4 +77,15 @@
 
   end
 
+  describe 'helper methods' do
+
+    it 'should construct a node hash' do
+      listing = create(:listing, :with_menards_attrs)
+      expect(listing.to_node('menards')).to eq(
+        Hash[listing.sku, listing.vendors[:menards][:vendor_id]]
+      )
+    end
+
+  end
+
 end