Allow 127.0.0.1 as valid URL for scraping (#2560)

* allow 127.0.0.1 as valid url for scraping * update comments and lint --------- Co-authored-by: timothycarambat <[email protected]>
Mintplex-Labs · Oct 31, 2024 · 0bb4761 · 0bb4761
1 parent e719d05
commit 0bb4761
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 3 deletions.
diff --git a/collector/extensions/index.js b/collector/extensions/index.js
@@ -118,8 +118,7 @@ function extensions(app) {
       try {
         const websiteDepth = require("../utils/extensions/WebsiteDepth");
         const { url, depth = 1, maxLinks = 20 } = reqBody(request);
-        if (!validURL(url)) return { success: false, reason: "Not a valid URL." };
-
+        if (!validURL(url)) throw new Error("Not a valid URL.");
         const scrapedData = await websiteDepth(url, depth, maxLinks);
         response.status(200).json({ success: true, data: scrapedData });
       } catch (e) {

diff --git a/collector/utils/url/index.js b/collector/utils/url/index.js
@@ -1,7 +1,7 @@
 /**  ATTN: SECURITY RESEARCHERS
  * To Security researchers about to submit an SSRF report CVE - please don't.
  * We are aware that the code below is does not defend against any of the thousands of ways
- * you can map a hostname to another IP. The code below does not have intention of blocking this
+ * you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this
  * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects
  * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles.
  * If an attacker has those roles then the system is already vulnerable and this is not a primary concern.
@@ -14,15 +14,29 @@
 const VALID_PROTOCOLS = ["https:", "http:"];
 const INVALID_OCTETS = [192, 172, 10, 127];
 
+/**
+ * If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP.
+ * This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses.
+ * @param {URL} param0
+ * @param {URL['hostname']} param0.hostname
+ * @returns {boolean}
+ */
 function isInvalidIp({ hostname }) {
   const IPRegex = new RegExp(
     /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi
   );
+
+  // Not an IP address at all - passthrough
   if (!IPRegex.test(hostname)) return false;
   const [octetOne, ..._rest] = hostname.split(".");
 
   // If fails to validate to number - abort and return as invalid.
   if (isNaN(Number(octetOne))) return true;
+
+  // Allow localhost loopback and 0.0.0.0 for scraping convenience
+  // for locally hosted services or websites
+  if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false;
+
   return INVALID_OCTETS.includes(Number(octetOne));
 }