fix(docs): link checking with retries and user-agent header

Author: alicanerdurmaz

.github/workflows/scripts/check-existing-doc-links.js

@@ -4,11 +4,30 @@ const getUrlToCheck = (currentUrl, deploymentUrl) => {
   return url.toString();
 };
 
-const checkUrl = async (url) => {
-  const { status } = await fetch(url, {
-    method: "GET",
-  });
-  return [status, new URL(url).pathname];
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const checkUrl = async (url, retries = 3) => {
+  for (let i = 0; i < retries; i++) {
+    try {
+      const { status } = await fetch(url, {
+        method: "GET",
+        headers: {
+          "User-Agent": "refine-link-checker",
+        },
+      });
+      return [status, new URL(url).pathname];
+    } catch (error) {
+      if (i === retries - 1) {
+        console.error(
+          `Failed to fetch ${new URL(url).pathname} after ${retries} attempts:`,
+          error.message,
+        );
+        return [500, new URL(url).pathname];
+      }
+      // Wait before retrying
+      await sleep(1000 * (i + 1));
+    }
+  }
 };
 
 const toChunks = (array, chunkSize) => {
@@ -19,23 +38,19 @@ const toChunks = (array, chunkSize) => {
   return chunks;
 };
 
-const checkChunk = (chunk, deploymentUrl, success, fail) => {
-  return new Promise((resolve) => {
-    const promises = chunk.map((url) => {
-      return checkUrl(getUrlToCheck(url, deploymentUrl));
-    });
-
-    Promise.all(promises).then((results) => {
-      results.forEach(([status, url]) => {
-        if (status === 200) {
-          success.push(url);
-        } else {
-          fail.push(url);
-        }
-      });
-      resolve();
-    });
-  });
+const checkChunk = async (chunk, deploymentUrl, success, fail) => {
+  for (const url of chunk) {
+    const [status, pathname] = await checkUrl(
+      getUrlToCheck(url, deploymentUrl),
+    );
+    if (status === 200) {
+      success.push(pathname);
+    } else {
+      fail.push(pathname);
+    }
+    // Small delay between requests to avoid overwhelming the server
+    await sleep(100);
+  }
 };
 
 const checkExistingLinks = async (sitemapUrl, deploymentUrl) => {
@@ -50,17 +65,24 @@ const checkExistingLinks = async (sitemapUrl, deploymentUrl) => {
 
   console.log("Checking for existing urls in:", sitemapUrl);
   console.log("Deployment url:", deploymentUrl);
+  console.log(`Total URLs to check: ${urls.length}`);
 
-  const chunks = toChunks(urls, 10);
+  const chunks = toChunks(urls, 50);
 
   let done = 0;
 
   for (const chunk of chunks) {
-    console.log(`Checking chunk ${done + 1}/${chunks.length}`);
     done++;
+    console.log(
+      `Checking chunk ${done}/${chunks.length} (${
+        success.length + fail.length
+      }/${urls.length} URLs processed)`,
+    );
     await checkChunk(chunk, deploymentUrl, success, fail);
   }
 
+  console.log(`\nResults: ${success.length} successful, ${fail.length} failed`);
+
   if (fail.length > 0) {
     console.log("Broken links:");
     fail.forEach((link) => {