Merge pull request #22 from selfagency/dev

Author: selfagency

.github/workflows/feedbot.yml

@@ -7,24 +7,29 @@ jobs:
   rss-to-slack:
     runs-on: ubuntu-latest
     steps:
+      - name: Retrieve cache
+        uses: actions/cache@v2
+        with:
+          path: ~/slackfeedbot
+          key: slackfeedbot-cache
       - name: NYT
         uses: 'selfagency/slackfeedbot@dev'
         with:
           rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
-          interval: 15
+          cache_dir: ~/slackfeedbot
           unfurl: false
       - name: LAT
         uses: 'selfagency/slackfeedbot@dev'
         with:
           rss: 'https://www.latimes.com/rss2.0.xml'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
-          interval: 15
+          cache_dir: ~/slackfeedbot
           unfurl: false
       - name: WaPo
         uses: 'selfagency/slackfeedbot@dev'
         with:
           rss: 'https://feeds.washingtonpost.com/rss/homepage'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
-          interval: 15
+          cache_dir: ~/slackfeedbot
           unfurl: false

README.md

@@ -17,12 +17,19 @@ Push RSS feed updates to Slack via GitHub Actions
    - `rss` is an RSS feed URL.
    - `slack_webhook` is the URL of your Slack webhook (this can and probably
      should be a repository or organization secret).
+   - `cache_dir` is the folder in which you want to cache RSS data to prevent
+     publishing duplicates (e.g., `~/slackfeedbot`), or alternately...
    - `interval` is the number of minutes between runs of the parent workflow, as
-     specified in the `cron` section of the `schedule` workflow trigger.
+     specified in the `cron` section of the `schedule` workflow trigger (may
+     publish duplicates due to post pinning).
    - `unfurl` tells Slack to show the [Open Graph](https://ogp.me/) preview. If
      set to `false` the title, date, short description, and a link to the feed item
      will be posted. Defaults to `false` because it's kind of flaky.
 
+## Examples
+
+### With cache folder
+
 ```
 name: SlackFeedBot
 on:
@@ -32,23 +39,89 @@ jobs:
   rss-to-slack:
     runs-on: ubuntu-latest
     steps:
+      - name: Retrieve cache
+        uses: actions/cache@v2
+        with:
+          path: ~/slackfeedbot
+          key: slackfeedbot-cache
       - name: NYT
-        uses: 'selfagency/slackfeedbot@v1'
+        uses: 'selfagency/[email protected]'
+        with:
+          rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
+          slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
+          cache_dir: '~/slackfeedbot'
+```
+
+### With interval
+
+```
+name: SlackFeedBot
+on:
+  schedule:
+    - cron: '*/15 * * * *'
+jobs:
+  rss-to-slack:
+    runs-on: ubuntu-latest
+    steps:
+      - name: NYT
+        uses: 'selfagency/[email protected]'
         with:
           rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
           interval: 15
+```
+
+### Unfurl URLs
+
+```
+name: FeedBot
+on:
+  schedule:
+    - cron: '*/15 * * * *'
+jobs:
+  rss-to-slack:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Retrieve cache
+        uses: actions/cache@v2
+        with:
+          path: ~/slackfeedbot
+          key: slackfeedbot-cache
+      - name: NYT
+        uses: 'selfagency/[email protected]'
+        with:
+          rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
+          slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
+          cache_dir: '~/slackfeedbot'
           unfurl: true
+```
+
+### Multiple feeds
+
+```
+name: FeedBot
+on:
+  schedule:
+    - cron: '*/15 * * * *'
+jobs:
+  rss-to-slack:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Retrieve cache
+        uses: actions/cache@v2
+        with:
+          path: ~/slackfeedbot
+          key: slackfeedbot-cache
       - name: LAT
-        uses: 'selfagency/slackfeedbot@v1'
+        uses: 'selfagency/feedbot@v1.2.0'
         with:
           rss: 'https://www.latimes.com/rss2.0.xml'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
-          interval: 15
+          cache_dir: '~/slackfeedbot'
       - name: WaPo
-        uses: 'selfagency/slackfeedbot@v1'
+        uses: 'selfagency/feedbot@v1.2.0'
         with:
           rss: 'https://feeds.washingtonpost.com/rss/homepage'
           slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
-          interval: 15
+          cache_dir: '~/slackfeedbot'
 ```

action.js

@@ -1,14 +1,24 @@
 import core from '@actions/core';
+import { createHash } from 'crypto';
 import dayjs from 'dayjs';
+import { readFile, writeFile } from 'fs';
 import { compile } from 'html-to-text';
 import fetch from 'node-fetch';
 import { parse } from 'rss-to-json';
+import TurndownService from 'turndown';
+import { promisify } from 'util';
 
+const read = promisify(readFile);
+const write = promisify(writeFile);
+const { debug, setFailed, getInput } = core;
+const turndownService = new TurndownService();
 const html2txt = compile({
   wordwrap: 120
 });
 
-const { debug, setFailed, getInput } = core;
+function hash(string) {
+  return createHash('sha256').update(string).digest('hex');
+}
 
 const validate = () => {
   if (!getInput('rss') || !getInput('rss').startsWith('http')) {
@@ -19,8 +29,12 @@ const validate = () => {
     throw new Error('No Slack webhook or invalid webhook specified');
   }
 
-  if (!getInput('interval') || parseInt(getInput('interval')).toString() === 'NaN') {
-    throw new Error('No interval or invalid interval specified');
+  if (!getInput('interval') && !getInput('cache_dir')) {
+    throw new Error('No interval or cache folder specified');
+  }
+
+  if (getInput('interval') && parseInt(getInput('interval')).toString() === 'NaN') {
+    throw new Error('Invalid interval specified');
   }
 };
 
@@ -54,9 +68,12 @@ const run = async () => {
     validate();
 
     const rssFeed = getInput('rss');
+    const rssFeedUrl = new URL(rssFeed);
     const slackWebhook = getInput('slack_webhook');
     const interval = parseInt(getInput('interval'));
     const unfurl = getInput('unfurl').toString() === 'true';
+    const cacheDir = getInput('cache_dir');
+    const cachePath = `${cacheDir}/${rssFeedUrl.hostname.replace(/\./g, '_')}.json`;
 
     debug(`Retrieving ${rssFeed}`);
     const rss = await parse(rssFeed);
@@ -65,15 +82,32 @@ const run = async () => {
     debug('Checking for feed items');
     if (rss?.items?.length) {
       debug(`Selecting items posted in the last ${interval} minutes`);
-      const toSend = rss.items.filter(item => dayjs(item.created).isAfter(dayjs().subtract(interval, 'minute')));
+
+      let toSend = [];
+      let published = [];
+      if (cacheDir) {
+        debug(`Retrieving previously published entries`);
+        published = JSON.stringify(await read(cachePath, 'utf8'));
+
+        toSend = rss.items.filter(item => {
+          return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.description)));
+        });
+      } else {
+        toSend = rss.items.filter(item => {
+          return dayjs(item.created).isAfter(dayjs().subtract(interval, 'minute'));
+        });
+      }
 
       const blocks = toSend.map(item => {
         let text = '';
 
         if (!unfurl) {
           if (item.title) text += `*${html2txt(item.title)}*\n`;
           if (item.description) {
-            const description = item.description.replace(/[Rr]ead more/g, '…').replace(/\n/g, ' ');
+            const description = turndownService
+              .turndown(item.description)
+              .replace(/[Rr]ead more/g, '…')
+              .replace(/\n/g, ' ');
             text += `${description}\n`;
           }
           if (item.link) text += `<${item.link}|Read more>`;
@@ -112,6 +146,14 @@ const run = async () => {
           }
         });
         debug(res);
+
+        if (cacheDir) {
+          debug(`Writing cache to ${cachePath}`);
+          await write(
+            cachePath,
+            JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title + item.description)))])
+          );
+        }
       }
     } else {
       throw new Error('No feed items found');

action.yml

@@ -11,9 +11,12 @@ inputs:
   slack_webhook:
     description: 'Slack Webhook URL'
     required: true
+  cache_dir:
+    description: 'Cache folder'
+    required: true
   interval:
     description: 'Minutes between workflow runs'
-    required: true
+    required: false
   unfurl:
     description: 'Unfurl links'
     required: false