Update GitHub Actions workflow for scraping Codex links

Author: jessewashburn

.github/workflows/scrape_codex.yml

@@ -1,7 +1,47 @@
+name: Scrape Codex Links
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - "prompts/**/*.md"
+    branches:
+      - main
+
+jobs:
+  scrape:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y xvfb jq
+          npm install [email protected]
+
+      - name: Find new Codex links
+        id: find
+        run: |
+          links=$(grep -Eroh 'https://chatgpt.com/s/[A-Za-z0-9_]+' prompts || true)
+          if [ -z "$links" ]; then
+            echo "No Codex links found."
+            exit 0
+          fi
+          echo "links<<EOF" >> $GITHUB_OUTPUT
+          echo "$links" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
       - name: Scrape and append prompt text
         if: steps.find.outputs.links != ''
         run: |
-          # Inline Puppeteer scraper
           cat > scrape.js <<'EOF'
           import puppeteer from 'puppeteer';
           const [,, url] = process.argv;
@@ -12,15 +52,11 @@
             });
             const page = await browser.newPage();
             await page.goto(url, { waitUntil: 'networkidle2', timeout: 120000 });
-
-            // Delay for hydration (waitForTimeout removed in newer Puppeteer)
-            await new Promise(r => setTimeout(r, 5000));
-
+            await new Promise(r => setTimeout(r, 5000)); // delay for hydration
             const text = await page.evaluate(() => {
               const container = document.querySelector('main') || document.body;
               return container.innerText.trim();
             });
-
             console.log(text.slice(0, 3000));
             await browser.close();
           })();
@@ -50,5 +86,12 @@
             done
           done <<< "$(echo "${{ steps.find.outputs.links }}")"
 
-          # Clean up temp files to avoid 'untracked files' noise
           rm -rf node_modules package-lock.json package.json scrape.js
+
+      - name: Commit and push results
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add prompts
+          git commit -m "Auto-scraped Codex prompts" || echo "No new content to commit."
+          git push