From 0bb47619dcdbbfc6af8a1c0e9a8b4e3d1b6f99bb Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Thu, 31 Oct 2024 09:57:28 -0700 Subject: [PATCH] Allow 127.0.0.1 as valid URL for scraping (#2560) * allow 127.0.0.1 as valid url for scraping * update comments and lint --------- Co-authored-by: timothycarambat --- collector/extensions/index.js | 3 +-- collector/utils/url/index.js | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/collector/extensions/index.js b/collector/extensions/index.js index 47989d5d5c..81a3a3dd79 100644 --- a/collector/extensions/index.js +++ b/collector/extensions/index.js @@ -118,8 +118,7 @@ function extensions(app) { try { const websiteDepth = require("../utils/extensions/WebsiteDepth"); const { url, depth = 1, maxLinks = 20 } = reqBody(request); - if (!validURL(url)) return { success: false, reason: "Not a valid URL." }; - + if (!validURL(url)) throw new Error("Not a valid URL."); const scrapedData = await websiteDepth(url, depth, maxLinks); response.status(200).json({ success: true, data: scrapedData }); } catch (e) { diff --git a/collector/utils/url/index.js b/collector/utils/url/index.js index 8a58dbd7aa..c9d87b295f 100644 --- a/collector/utils/url/index.js +++ b/collector/utils/url/index.js @@ -1,7 +1,7 @@ /** ATTN: SECURITY RESEARCHERS * To Security researchers about to submit an SSRF report CVE - please don't. * We are aware that the code below is does not defend against any of the thousands of ways - * you can map a hostname to another IP. The code below does not have intention of blocking this + * you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles. * If an attacker has those roles then the system is already vulnerable and this is not a primary concern. @@ -14,15 +14,29 @@ const VALID_PROTOCOLS = ["https:", "http:"]; const INVALID_OCTETS = [192, 172, 10, 127]; +/** + * If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP. + * This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses. + * @param {URL} param0 + * @param {URL['hostname']} param0.hostname + * @returns {boolean} + */ function isInvalidIp({ hostname }) { const IPRegex = new RegExp( /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi ); + + // Not an IP address at all - passthrough if (!IPRegex.test(hostname)) return false; const [octetOne, ..._rest] = hostname.split("."); // If fails to validate to number - abort and return as invalid. if (isNaN(Number(octetOne))) return true; + + // Allow localhost loopback and 0.0.0.0 for scraping convenience + // for locally hosted services or websites + if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false; + return INVALID_OCTETS.includes(Number(octetOne)); }