Skip to content

Commit cc103e3

Browse files
selfagencyselfagency
authored
Merge pull request #25 from selfagency/dev
fix caching
2 parents b63da04 + 682177d commit cc103e3

File tree

7 files changed

+96
-47
lines changed

7 files changed

+96
-47
lines changed

‎.github/workflows/feedbot.yml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,29 +7,36 @@ jobs:
77
rss-to-slack:
88
runs-on: ubuntu-latest
99
steps:
10+
- name: Generate cache key
11+
uses: actions/github-script@v6
12+
id: generate-key
13+
with:
14+
script: |
15+
core.setOutput('cache-key', new Date().valueOf())
1016
- name: Retrieve cache
1117
uses: actions/cache@v2
1218
with:
13-
path: ~/slackfeedbot
14-
key: slackfeedbot-cache
19+
path: ./slackfeedbot-cache
20+
key: feed-cache-${{ steps.generate-key.outputs.cache-key }}
21+
restore-keys: feed-cache-
1522
- name: NYT
1623
uses: 'selfagency/slackfeedbot@dev'
1724
with:
1825
rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
1926
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
20-
cache_dir: ~/slackfeedbot
27+
cache_dir: ./slackfeedbot-cache
2128
unfurl: false
2229
- name: LAT
2330
uses: 'selfagency/slackfeedbot@dev'
2431
with:
2532
rss: 'https://www.latimes.com/rss2.0.xml'
2633
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
27-
cache_dir: ~/slackfeedbot
34+
cache_dir: ./slackfeedbot-cache
2835
unfurl: false
2936
- name: WaPo
3037
uses: 'selfagency/slackfeedbot@dev'
3138
with:
3239
rss: 'https://feeds.washingtonpost.com/rss/homepage'
3340
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
34-
cache_dir: ~/slackfeedbot
41+
cache_dir: ./slackfeedbot-cache
3542
unfurl: false

‎.husky/post-receive‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
. "$(dirname "$0")/_/husky.sh"
3+
4+
npm i

‎README.md‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Push RSS feed updates to Slack via GitHub Actions
3030

3131
### With cache folder
3232

33+
Hashes and caches the post title + creation date to ensure no duplicates are posted.
34+
3335
```
3436
name: SlackFeedBot
3537
on:
@@ -39,13 +41,19 @@ jobs:
3941
rss-to-slack:
4042
runs-on: ubuntu-latest
4143
steps:
44+
- name: Generate cache key
45+
uses: actions/github-script@v6
46+
id: generate-key
47+
with:
48+
script: |
49+
core.setOutput('cache-key', new Date().valueOf())
4250
- name: Retrieve cache
4351
uses: actions/cache@v2
4452
with:
45-
path: ~/slackfeedbot
46-
key: slackfeedbot-cache
53+
path: ./slackfeedbot-cache
54+
key: feed-cache-${{ steps.generate-key.outputs.cache-key }}
4755
- name: NYT
48-
uses: 'selfagency/[email protected].2'
56+
uses: 'selfagency/[email protected].3'
4957
with:
5058
rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
5159
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
@@ -54,6 +62,8 @@ jobs:
5462

5563
### With interval
5664

65+
No cache, but maybe duplicates.
66+
5767
```
5868
name: SlackFeedBot
5969
on:
@@ -64,7 +74,7 @@ jobs:
6474
runs-on: ubuntu-latest
6575
steps:
6676
- name: NYT
67-
uses: 'selfagency/[email protected].2'
77+
uses: 'selfagency/[email protected].3'
6878
with:
6979
rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
7080
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
@@ -82,13 +92,19 @@ jobs:
8292
rss-to-slack:
8393
runs-on: ubuntu-latest
8494
steps:
95+
- name: Generate cache key
96+
uses: actions/github-script@v6
97+
id: generate-key
98+
with:
99+
script: |
100+
core.setOutput('cache-key', new Date().valueOf())
85101
- name: Retrieve cache
86102
uses: actions/cache@v2
87103
with:
88-
path: ~/slackfeedbot
89-
key: slackfeedbot-cache
104+
path: ./slackfeedbot-cache
105+
key: feed-cache-${{ steps.generate-key.outputs.cache-key }}
90106
- name: NYT
91-
uses: 'selfagency/[email protected].2'
107+
uses: 'selfagency/[email protected].3'
92108
with:
93109
rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'
94110
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
@@ -107,19 +123,25 @@ jobs:
107123
rss-to-slack:
108124
runs-on: ubuntu-latest
109125
steps:
126+
- name: Generate cache key
127+
uses: actions/github-script@v6
128+
id: generate-key
129+
with:
130+
script: |
131+
core.setOutput('cache-key', new Date().valueOf())
110132
- name: Retrieve cache
111133
uses: actions/cache@v2
112134
with:
113-
path: ~/slackfeedbot
114-
key: slackfeedbot-cache
135+
path: ./slackfeedbot-cache
136+
key: feed-cache-${{ steps.generate-key.outputs.cache-key }}
115137
- name: LAT
116-
uses: 'selfagency/[email protected].2'
138+
uses: 'selfagency/[email protected].3'
117139
with:
118140
rss: 'https://www.latimes.com/rss2.0.xml'
119141
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}
120142
cache_dir: '~/slackfeedbot'
121143
- name: WaPo
122-
uses: 'selfagency/[email protected].2'
144+
uses: 'selfagency/[email protected].3'
123145
with:
124146
rss: 'https://feeds.washingtonpost.com/rss/homepage'
125147
slack_webhook: ${{ secrets.SLACK_WEBHOOK }}

‎action.js‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
1+
import { mkdir, readFile, writeFile } from 'fs';
2+
3+
import { compile } from 'html-to-text';
14
import core from '@actions/core';
25
import { createHash } from 'crypto';
36
import dayjs from 'dayjs';
4-
import { readFile, writeFile } from 'fs';
5-
import html2md from 'html-to-md';
6-
import { compile } from 'html-to-text';
77
import fetch from 'node-fetch';
8+
import html2md from 'html-to-md';
89
import { parse } from 'rss-to-json';
910
import { promisify } from 'util';
1011

1112
const read = promisify(readFile);
1213
const write = promisify(writeFile);
13-
const { debug, setFailed, getInput } = core;
14+
const md = promisify(mkdir);
15+
const { debug, setFailed, getInput, getBooleanInput } = core;
1416
const html2txt = compile({
1517
wordwrap: 120
1618
});
@@ -70,7 +72,7 @@ const run = async () => {
7072
const rssFeedUrl = new URL(rssFeed);
7173
const slackWebhook = getInput('slack_webhook');
7274
const interval = parseInt(getInput('interval'));
73-
const unfurl = getInput('unfurl').toString() === 'true';
75+
const unfurl = getBooleanInput('unfurl');
7476
const cacheDir = getInput('cache_dir');
7577
const cachePath = `${cacheDir}/${rssFeedUrl.hostname.replace(/\./g, '_')}.json`;
7678

@@ -80,17 +82,16 @@ const run = async () => {
8082

8183
debug('Checking for feed items');
8284
if (rss?.items?.length) {
83-
debug(`Selecting items posted in the last ${interval} minutes`);
84-
8585
let toSend = [];
8686
let published = [];
8787
if (cacheDir) {
8888
debug(`Retrieving previously published entries`);
8989
try {
90-
published = JSON.stringify(await read(cachePath, 'utf8'));
90+
published = JSON.parse(await read(cachePath, 'utf8'));
91+
debug(published);
9192

9293
toSend = rss.items.filter(item => {
93-
return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.description)));
94+
return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.created)));
9495
});
9596
} catch (err) {
9697
debug(err.message);
@@ -99,6 +100,7 @@ const run = async () => {
99100
});
100101
}
101102
} else {
103+
debug(`Selecting items posted in the last ${interval} minutes`);
102104
toSend = rss.items.filter(item => {
103105
return dayjs(item.created).isAfter(dayjs().subtract(interval, 'minute'));
104106
});
@@ -117,7 +119,7 @@ const run = async () => {
117119
}
118120
if (item.link) text += `<${item.link}|Read more>`;
119121
} else {
120-
if (item.title) text += `<${item.link}|${html2txt(item.title)}>`;
122+
if (item.title) text += `<${item.link}|${html2txt(item.title + item.created)}>`;
121123
}
122124

123125
return {
@@ -154,9 +156,15 @@ const run = async () => {
154156

155157
if (cacheDir) {
156158
debug(`Writing cache to ${cachePath}`);
159+
try {
160+
await md(cacheDir, { recursive: true });
161+
} catch (err) {
162+
debug(err.message);
163+
}
164+
157165
await write(
158166
cachePath,
159-
JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title + item.description)))])
167+
JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title)))])
160168
);
161169
}
162170
}

‎dist/index.js‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21347,18 +21347,16 @@ var __webpack_exports__ = {};
2134721347
// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk.
2134821348
(() => {
2134921349

21350+
// EXTERNAL MODULE: external "fs"
21351+
var external_fs_ = __nccwpck_require__(7147);
21352+
// EXTERNAL MODULE: ./node_modules/html-to-text/index.js
21353+
var html_to_text = __nccwpck_require__(7015);
2135021354
// EXTERNAL MODULE: ./node_modules/@actions/core/lib/core.js
2135121355
var core = __nccwpck_require__(2186);
2135221356
;// CONCATENATED MODULE: external "crypto"
2135321357
const external_crypto_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("crypto");
2135421358
// EXTERNAL MODULE: ./node_modules/dayjs/dayjs.min.js
2135521359
var dayjs_min = __nccwpck_require__(7401);
21356-
// EXTERNAL MODULE: external "fs"
21357-
var external_fs_ = __nccwpck_require__(7147);
21358-
// EXTERNAL MODULE: ./node_modules/html-to-md/dist/index.js
21359-
var dist = __nccwpck_require__(7192);
21360-
// EXTERNAL MODULE: ./node_modules/html-to-text/index.js
21361-
var html_to_text = __nccwpck_require__(7015);
2136221360
;// CONCATENATED MODULE: external "node:http"
2136321361
const external_node_http_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:http");
2136421362
;// CONCATENATED MODULE: external "node:https"
@@ -21421,7 +21419,7 @@ function dataUriToBuffer(uri) {
2142121419
buffer.charset = charset;
2142221420
return buffer;
2142321421
}
21424-
/* harmony default export */ const data_uri_to_buffer_dist = (dataUriToBuffer);
21422+
/* harmony default export */ const dist = (dataUriToBuffer);
2142521423
//# sourceMappingURL=index.js.map
2142621424
;// CONCATENATED MODULE: external "node:util"
2142721425
const external_node_util_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:util");
@@ -23112,7 +23110,7 @@ async function fetch(url, options_) {
2311223110
}
2311323111

2311423112
if (parsedURL.protocol === 'data:') {
23115-
const data = data_uri_to_buffer_dist(request.url);
23113+
const data = dist(request.url);
2311623114
const response = new Response(data, {headers: {'Content-Type': data.typeFull}});
2311723115
resolve(response);
2311823116
return;
@@ -23466,6 +23464,8 @@ function fixResponseChunkedTransferBadEnding(request, errorCallback) {
2346623464
});
2346723465
}
2346823466

23467+
// EXTERNAL MODULE: ./node_modules/html-to-md/dist/index.js
23468+
var html_to_md_dist = __nccwpck_require__(7192);
2346923469
// EXTERNAL MODULE: ./node_modules/rss-to-json/dist/index.js
2347023470
var rss_to_json_dist = __nccwpck_require__(7235);
2347123471
// EXTERNAL MODULE: external "util"
@@ -23481,9 +23481,11 @@ var external_util_ = __nccwpck_require__(3837);
2348123481

2348223482

2348323483

23484+
2348423485
const read = (0,external_util_.promisify)(external_fs_.readFile);
2348523486
const write = (0,external_util_.promisify)(external_fs_.writeFile);
23486-
const { debug, setFailed, getInput } = core;
23487+
const md = (0,external_util_.promisify)(external_fs_.mkdir);
23488+
const { debug, setFailed, getInput, getBooleanInput } = core;
2348723489
const html2txt = (0,html_to_text.compile)({
2348823490
wordwrap: 120
2348923491
});
@@ -23543,7 +23545,7 @@ const run = async () => {
2354323545
const rssFeedUrl = new URL(rssFeed);
2354423546
const slackWebhook = getInput('slack_webhook');
2354523547
const interval = parseInt(getInput('interval'));
23546-
const unfurl = getInput('unfurl').toString() === 'true';
23548+
const unfurl = getBooleanInput('unfurl');
2354723549
const cacheDir = getInput('cache_dir');
2354823550
const cachePath = `${cacheDir}/${rssFeedUrl.hostname.replace(/\./g, '_')}.json`;
2354923551

@@ -23553,17 +23555,16 @@ const run = async () => {
2355323555

2355423556
debug('Checking for feed items');
2355523557
if (rss?.items?.length) {
23556-
debug(`Selecting items posted in the last ${interval} minutes`);
23557-
2355823558
let toSend = [];
2355923559
let published = [];
2356023560
if (cacheDir) {
2356123561
debug(`Retrieving previously published entries`);
2356223562
try {
23563-
published = JSON.stringify(await read(cachePath, 'utf8'));
23563+
published = JSON.parse(await read(cachePath, 'utf8'));
23564+
debug(published);
2356423565

2356523566
toSend = rss.items.filter(item => {
23566-
return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.description)));
23567+
return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.created)));
2356723568
});
2356823569
} catch (err) {
2356923570
debug(err.message);
@@ -23572,6 +23573,7 @@ const run = async () => {
2357223573
});
2357323574
}
2357423575
} else {
23576+
debug(`Selecting items posted in the last ${interval} minutes`);
2357523577
toSend = rss.items.filter(item => {
2357623578
return dayjs_min(item.created).isAfter(dayjs_min().subtract(interval, 'minute'));
2357723579
});
@@ -23583,14 +23585,14 @@ const run = async () => {
2358323585
if (!unfurl) {
2358423586
if (item.title) text += `*${html2txt(item.title)}*\n`;
2358523587
if (item.description) {
23586-
const description = dist(item.description)
23588+
const description = html_to_md_dist(item.description)
2358723589
.replace(/[Rr]ead more/g, '…')
2358823590
.replace(/\n/g, ' ');
2358923591
text += `${description}\n`;
2359023592
}
2359123593
if (item.link) text += `<${item.link}|Read more>`;
2359223594
} else {
23593-
if (item.title) text += `<${item.link}|${html2txt(item.title)}>`;
23595+
if (item.title) text += `<${item.link}|${html2txt(item.title + item.created)}>`;
2359423596
}
2359523597

2359623598
return {
@@ -23627,9 +23629,15 @@ const run = async () => {
2362723629

2362823630
if (cacheDir) {
2362923631
debug(`Writing cache to ${cachePath}`);
23632+
try {
23633+
await md(cacheDir, { recursive: true });
23634+
} catch (err) {
23635+
debug(err.message);
23636+
}
23637+
2363023638
await write(
2363123639
cachePath,
23632-
JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title + item.description)))])
23640+
JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title)))])
2363323641
);
2363423642
}
2363523643
}

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "slackfeedbot",
3-
"version": "1.2.2",
3+
"version": "1.2.3",
44
"license": "MIT",
55
"author": "Daniel Sieradski <[email protected]>",
66
"description": "Push RSS feed updates to Slack via GitHub Actions",

0 commit comments

Comments
 (0)