Merge pull request #1185 from nexcess/devel

Devel
nexcess · May 31, 2016 · 1236f29 · 1236f29
2 parents bf24144 + 000362f
commit 1236f29
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 32 deletions.
diff --git a/app/code/community/Nexcessnet/Turpentine/Helper/Cron.php b/app/code/community/Nexcessnet/Turpentine/Helper/Cron.php
@@ -145,6 +145,24 @@ public function getCrawlerDebugEnabled() {
         return Mage::getStoreConfig('turpentine_varnish/general/crawler_debug');
     }
 
+    /**
+     * Get number of urls to crawl per batch
+     *
+     * @return int
+     */
+    public function getCrawlerBatchSize() {
+        return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchsize');
+    }
+
+    /**
+     * Get time in seconds to wait between url batches
+     *
+     * @return int
+     */
+    public function getCrawlerWaitPeriod() {
+        return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchwait');
+    }
+
     /**
      * Get the list of all URLs
      *

diff --git a/app/code/community/Nexcessnet/Turpentine/Model/Core/Session.php b/app/code/community/Nexcessnet/Turpentine/Model/Core/Session.php
@@ -50,4 +50,23 @@ public function real_getFormKey()
         }
         return $this->getData('_form_key');
     }
+
+    /**
+     * Creates new Form key
+     */
+    public function renewFormKey()
+    {
+     	$this->setData('_form_key', Mage::helper('core')->getRandomString(16));
+    }
+
+    /**
+     * Validates Form key
+     *
+     * @param string|null $formKey
+     * @return bool
+     */
+    public function validateFormKey($formKey)
+    {
+        return ($formKey === $this->getFormKey());
+    }
 }
diff --git a/app/code/community/Nexcessnet/Turpentine/Model/Observer/Ban.php b/app/code/community/Nexcessnet/Turpentine/Model/Observer/Ban.php
@@ -298,37 +298,39 @@ public function banCacheType($eventObject) {
      * @return bool
      */
     public function banProductReview($eventObject) {
-        $patterns = array();
-        /* @var $review \Mage_Review_Model_Review*/
-        $review = $eventObject->getObject();
-
-        /* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
-        $productCollection = $review->getProductCollection();
-
-        $products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();
+        if (Mage::helper('turpentine/varnish')->getVarnishEnabled()) {
+            $patterns = array();
+            /* @var $review \Mage_Review_Model_Review*/
+            $review = $eventObject->getObject();
+
+            /* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
+            $productCollection = $review->getProductCollection();
+
+            $products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();
 
-        $productIds = array_unique(array_map(
-            create_function('$p', 'return $p->getEntityId();'),
-            $products ));
-        $patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
-            implode('|', array_unique($productIds)));
-        $patterns[] = sprintf('/review/product/view/id/%d/',
-            $review->getEntityId());
-        $productPatterns = array();
-        foreach ($products as $p) {
-            $urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
-            if ($urlKey) {
-                $productPatterns[] = $urlKey;
+            $productIds = array_unique(array_map(
+                create_function('$p', 'return $p->getEntityId();'),
+                $products ));
+            $patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
+                implode('|', array_unique($productIds)));
+            $patterns[] = sprintf('/review/product/view/id/%d/',
+                $review->getEntityId());
+            $productPatterns = array();
+            foreach ($products as $p) {
+                $urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
+                if ($urlKey) {
+                    $productPatterns[] = $urlKey;
+                }
             }
-        }
-        if ( ! empty($productPatterns)) {
-            $productPatterns = array_unique($productPatterns);
-            $patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
-        }
-        $urlPattern = implode('|', $patterns);
+            if ( ! empty($productPatterns)) {
+                $productPatterns = array_unique($productPatterns);
+                $patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
+            }
+            $urlPattern = implode('|', $patterns);
 
-        $result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
-        return $this->_checkResult($result);
+            $result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
+            return $this->_checkResult($result);
+        }
     }
 
     /**

diff --git a/app/code/community/Nexcessnet/Turpentine/Model/Observer/Cron.php b/app/code/community/Nexcessnet/Turpentine/Model/Observer/Cron.php
@@ -49,6 +49,11 @@ public function crawlUrls($eventObject) {
             if ($maxRunTime === 0) {
                 $maxRunTime = self::MAX_CRAWL_TIME;
             }
+
+            $batchSize = $helper->getCrawlerBatchSize();
+            $timeout = $helper->getCrawlerWaitPeriod();
+            $crawlCount = 0;
+
             // just in case we have a silly short max_execution_time
             $maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER);
             while (($helper->getRunTime() < $maxRunTime) &&
@@ -57,6 +62,16 @@ public function crawlUrls($eventObject) {
                     Mage::helper('turpentine/debug')->logWarn(
                         'Failed to crawl URL: %s', $url );
                 }
+
+                if ($crawlCount > 0
+                    && $timeout > 0
+                    && $batchSize > 0
+                    && $crawlCount % $batchSize == 0
+                ) {
+                    Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds');
+                    sleep($timeout);
+                }
+                $crawlCount++;
             }
         }
     }

diff --git a/app/code/community/Nexcessnet/Turpentine/etc/config.xml b/app/code/community/Nexcessnet/Turpentine/etc/config.xml
@@ -35,6 +35,8 @@
                 <fix_product_toolbar>0</fix_product_toolbar>
                 <crawler_enable>0</crawler_enable>
                 <crawler_debug>0</crawler_debug>
+                <crawler_batchsize>0</crawler_batchsize>
+                <crawler_batchwait>0</crawler_batchwait>
             </general>
             <logging>
                 <use_custom_log_file>0</use_custom_log_file>
@@ -59,7 +61,7 @@
                 <frontend_timeout>300</frontend_timeout>
                 <admin_timeout>21600</admin_timeout>
                 <crawlers>127.0.0.1</crawlers>
-                <crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*]]></crawler_user_agents>
+                <crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*,.*PTST.*]]></crawler_user_agents>
             </backend>
             <normalization>
                 <encoding>1</encoding>

diff --git a/app/code/community/Nexcessnet/Turpentine/etc/system.xml b/app/code/community/Nexcessnet/Turpentine/etc/system.xml
@@ -132,6 +132,30 @@
                             <show_in_website>0</show_in_website>
                             <show_in_store>0</show_in_store>
                         </crawler_debug>
+                        <crawler_batchsize translate="label" module="turpentine">
+                            <label>Crawler Batch Size</label>
+                            <comment>Number of URLs to crawl per batch, when 0 requests will not be batched</comment>
+                            <frontend_type>text</frontend_type>
+                            <sort_order>90</sort_order>
+                            <show_in_default>1</show_in_default>
+                            <show_in_website>0</show_in_website>
+                            <show_in_store>0</show_in_store>
+                            <depends>
+                                <crawler_enable>1</crawler_enable>
+                            </depends>
+                        </crawler_batchsize>
+                        <crawler_batchwait translate="label" module="turpentine">
+                            <label>Crawler Batch Wait</label>
+                            <comment>Time in seconds to wait between batches</comment>
+                            <frontend_type>text</frontend_type>
+                            <sort_order>100</sort_order>
+                            <show_in_default>1</show_in_default>
+                            <show_in_website>0</show_in_website>
+                            <show_in_store>0</show_in_store>
+                            <depends>
+                                <crawler_enable>1</crawler_enable>
+                            </depends>
+                        </crawler_batchwait>
                     </fields>
                 </general>
                 <logging translate="label" module="turpentine">

diff --git a/app/code/community/Nexcessnet/Turpentine/misc/version-4.vcl b/app/code/community/Nexcessnet/Turpentine/misc/version-4.vcl
@@ -116,7 +116,7 @@ sub vcl_recv {
     if (!{{enable_caching}} || req.http.Authorization ||
         req.method !~ "^(GET|HEAD|OPTIONS)$" ||
         req.http.Cookie ~ "varnish_bypass={{secret_handshake}}") {
-        return (pipe);
+        return (pass);
     }
 
     if({{send_unmodified_url}}) {

diff --git a/build/build_package.py b/build/build_package.py
@@ -280,7 +280,7 @@ def _build_dependencies_tag(self, dependencies_tag, connect_dom):
         min_tag.text = connect_dom.findtext('depends_php_min')
         max_tag = ElementTree.SubElement(php_tag, 'max')
         max_tag.text = connect_dom.findtext('depends_php_max')
-        self._logger.debug('Finished adding dependancies')
+        self._logger.debug('Finished adding dependencies')
         return dependencies_tag
 
     def _get_module_dom(self, ext_name):

diff --git a/build/mage-package.xml b/build/mage-package.xml
@@ -26,7 +26,7 @@
       </email>
    </authors>
    <depends_php_min>5.2.13</depends_php_min>
-   <depends_php_max>6.0.0</depends_php_max>
+   <depends_php_max>7.1.0</depends_php_max>
    <depends>
       <package>
          <name>