Refactor to matrix-based parallel workflow with automated issue management

.github/workflows/flat.yml

@@ -1,104 +1,231 @@
 name: data
 on:
   schedule:
-    - cron: "*/5 * * * *"
-  workflow_dispatch: {}
+    - cron: "*/10 * * * *"
+  workflow_dispatch:
+    inputs:
+      banks:
+        description: 'Specific banks to fetch (comma-separated, leave empty for all)'
+        required: false
+        default: ''
   push:
     paths:
       - .github/workflows/flat.yml
-      - akbank.js
+      - scripts/fetch-bank.ts
+
+# Cancel redundant runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  scheduled:
+  fetch:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        bank:
+          - akbank
+          - ing
+          - garanti
+          - enpara
+          - teb
+          - hsbc
+          - kuveytturk
+          - isbank
+          - ziraat
+          - yapikredi
+    # Filter banks if specific ones are requested
+    if: ${{ github.event.inputs.banks == '' || contains(fromJSON(format('["{0}"]', replace(github.event.inputs.banks, ',', '","'))), matrix.bank) }}
     steps:
-      - name: Setup deno
-        uses: denoland/setup-deno@main
-        with:
-          deno-version: v1.x
       - name: Check out repo
         uses: actions/checkout@v3
         with:
           fetch-depth: 1
-      - name: Fetch akbank data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.akbank.com/_layouts/15/Akbank/CalcTools/Ajax.aspx/GetDovizKurlari
-          axios_config: akbank.json
-          downloaded_filename: akbank-current.json
-          postprocess: akbank.js
-      - name: Fetch ing data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.ing.com.tr/ProxyManagement/SiteManagerService_Script.aspx/GetCurrencyRates
-          downloaded_filename: ing-current.json
-          postprocess: ing.js
-      - name: Fetch garanti data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.ing.com.tr/ProxyManagement/SiteManagerService_Script.aspx/GetCurrencyRates
-          downloaded_filename: garanti-current.json
-          postprocess: garanti.js
-      - name: Fetch enpara data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.qnbfinansbank.enpara.com/hesaplar/doviz-ve-altin-kurlari
-          downloaded_filename: enpara-current.html
-          postprocess: enpara.js
-      - name: Fetch teb data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.cepteteb.com.tr/services/GetGunlukDovizKur
-          downloaded_filename: teb-current.json
-          postprocess: teb.js
-      - name: Fetch hsbc data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
-        with:
-          http_url: https://www.hsbcyatirim.com.tr/api/hsbcdata/getForeignCurrencies
-          downloaded_filename: hsbc-current.json
-          postprocess: hsbc.js
-      - name: Fetch kuveytturk data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
+
+      - name: Setup deno
+        uses: denoland/setup-deno@main
         with:
-          http_url: https://www.kuveytturk.com.tr/ck0d84?B83A1EF44DD940F2FEC85646BDB25EA0
-          downloaded_filename: kuveytturk-current.json
-          postprocess: kuveytturk.js
-      - name: Fetch isbank data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
+          deno-version: v1.x
+
+      - name: Fetch ${{ matrix.bank }} data
+        id: fetch
+        continue-on-error: true
+        run: |
+          deno run --allow-all scripts/fetch-bank.ts ${{ matrix.bank }}
+          echo "status=success" >> $GITHUB_OUTPUT
+        env:
+          BANK_NAME: ${{ matrix.bank }}
+
+      - name: Mark failure
+        if: steps.fetch.outcome == 'failure'
+        run: echo "status=failure" >> $GITHUB_OUTPUT
+
+      - name: Upload failure logs
+        if: steps.fetch.outcome == 'failure'
+        uses: actions/upload-artifact@v3
         with:
-          http_url: https://www.isbank.com.tr/_vti_bin/DV.Isbank/PriceAndRate/PriceAndRateService.svc/GetFxRates
-          downloaded_filename: isbank-current.json
-          postprocess: isbank.js
-      - name: Fetch ziraat data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
+          name: failure-log-${{ matrix.bank }}
+          path: logs/failures.jsonl
+          retention-days: 7
+          if-no-files-found: ignore
+
+      - name: Commit and push changes
+        if: steps.fetch.outcome == 'success'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add ${{ matrix.bank }}.csv
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+            exit 0
+          fi
+          git commit -m "Update ${{ matrix.bank }} data"
+          # Retry logic for push conflicts
+          for i in {1..5}; do
+            if git pull --rebase && git push; then
+              echo "Successfully pushed changes"
+              exit 0
+            fi
+            echo "Push failed, retrying in $i seconds..."
+            sleep $i
+          done
+          echo "Failed to push after 5 attempts"
+          exit 1
+
+  summary:
+    runs-on: ubuntu-latest
+    needs: fetch
+    if: always()
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Download all failure logs
+        uses: actions/download-artifact@v3
         with:
-          http_url: https://www.ing.com.tr/ProxyManagement/SiteManagerService_Script.aspx/GetCurrencyRates
-          downloaded_filename: ziraat-current.json
-          postprocess: ziraat.js
-      - name: Fetch yapikredi data
-        uses: githubocto/flat@v3
-        if: "!cancelled()"
+          path: failure-logs
+        continue-on-error: true
+
+      - name: Create summary and manage issues
+        uses: actions/github-script@v7
         with:
-          http_url: https://www.yapikredi.com.tr/_ajaxproxy/general.aspx/LoadMainCurrencies
-          axios_config: yapikredi.json
-          downloaded_filename: yapikredi-current.json
-          postprocess: yapikredi.js
-      #- name: Set up proxy environment variables
-      #  run: |
-      #    echo "http_proxy=http://176.88.175.190:8080" >> $GITHUB_ENV
-      #    echo "https_proxy=http://176.88.175.190:8080" >> $GITHUB_ENV
-      #- name: Fetch vakif data
-      #  uses: githubocto/flat@v3
-      #  if: "!cancelled()"
-      #  with:
-      #    http_url: https://www.ing.com.tr/ProxyManagement/SiteManagerService_Script.aspx/GetCurrencyRates
-      #    downloaded_filename: vakif-current.json
-      #    postprocess: vakif.js
+          script: |
+            const banks = ['akbank', 'ing', 'garanti', 'enpara', 'teb', 'hsbc', 'kuveytturk', 'isbank', 'ziraat', 'yapikredi'];
+            const fs = require('fs');
+            const path = require('path');
+
+            // Get job results
+            const jobs = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId
+            });
+
+            const results = {};
+            for (const job of jobs.data.jobs) {
+              const match = job.name.match(/fetch \((.*?)\)/);
+              if (match) {
+                const bank = match[1];
+                results[bank] = job.conclusion === 'success';
+              }
+            }
+
+            // Count successes and failures
+            const succeeded = Object.entries(results).filter(([_, success]) => success).map(([bank, _]) => bank);
+            const failed = Object.entries(results).filter(([_, success]) => !success).map(([bank, _]) => bank);
+
+            // Create job summary
+            let summary = '## Bank Fetch Results\n\n';
+            summary += `✅ **Successful**: ${succeeded.length}\n`;
+            summary += `❌ **Failed**: ${failed.length}\n\n`;
+
+            if (succeeded.length > 0) {
+              summary += '### Successful Banks\n';
+              summary += succeeded.map(b => `- ${b}`).join('\n') + '\n\n';
+            }
+
+            if (failed.length > 0) {
+              summary += '### Failed Banks\n';
+              summary += failed.map(b => `- ${b}`).join('\n') + '\n\n';
+            }
+
+            await core.summary.addRaw(summary).write();
+
+            // Manage GitHub issues for failures
+            for (const bank of banks) {
+              const isSuccess = results[bank];
+              const issueTitle = `Data fetch failure: ${bank}`;
+
+              // Check for existing open issues
+              const existingIssues = await github.rest.issues.listForRepo({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                state: 'open',
+                labels: 'automated,data-fetch',
+                per_page: 100
+              });
+
+              const existingIssue = existingIssues.data.find(issue =>
+                issue.title === issueTitle
+              );
+
+              if (!isSuccess) {
+                // Read error details from logs if available
+                let errorDetails = 'No detailed error log available.';
+                const logPath = path.join('failure-logs', `failure-log-${bank}`, 'failures.jsonl');
+
+                try {
+                  if (fs.existsSync(logPath)) {
+                    const logs = fs.readFileSync(logPath, 'utf8').trim().split('\n');
+                    const lastLog = JSON.parse(logs[logs.length - 1]);
+                    errorDetails = `**Error**: ${lastLog.error}\n\n**Timestamp**: ${lastLog.timestamp}`;
+                    if (lastLog.stack) {
+                      errorDetails += `\n\n**Stack Trace**:\n\`\`\`\n${lastLog.stack}\n\`\`\``;
+                    }
+                  }
+                } catch (e) {
+                  console.log(`Could not read error log for ${bank}:`, e.message);
+                }
+
+                // Create issue if none exists
+                if (!existingIssue) {
+                  await github.rest.issues.create({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    title: issueTitle,
+                    body: `The automated data fetch for **${bank}** has failed.\n\n${errorDetails}\n\n**Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+                    labels: ['automated', 'data-fetch']
+                  });
+                  console.log(`Created issue for ${bank}`);
+                } else {
+                  // Update existing issue with new failure
+                  await github.rest.issues.createComment({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    issue_number: existingIssue.number,
+                    body: `Still failing as of ${new Date().toISOString()}\n\n${errorDetails}\n\n**Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+                  });
+                  console.log(`Updated issue for ${bank}`);
+                }
+              } else if (existingIssue) {
+                // Close issue if bank has recovered
+                await github.rest.issues.update({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: existingIssue.number,
+                  state: 'closed'
+                });
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: existingIssue.number,
+                  body: `✅ Bank has recovered and is now fetching successfully. Closing issue.`
+                });
+                console.log(`Closed issue for ${bank} - recovered`);
+              }
+            }

.gitignore

@@ -0,0 +1,9 @@
+# Temporary files
+*-current.json
+*-current.html
+
+# Logs
+logs/
+
+# Deno
+.deno/