Skip to content

Commit

Permalink
Merge pull request #2486 from DIYgod/master
Browse files Browse the repository at this point in the history
[pull] master from diygod:master
  • Loading branch information
pull[bot] authored Apr 22, 2024
2 parents 81a0ce2 + 4e4afaa commit 3f29650
Show file tree
Hide file tree
Showing 29 changed files with 1,118 additions and 251 deletions.
16 changes: 11 additions & 5 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ jobs:
chromium:
- name: bundled Chromium
dependency: ''
environment: '{}'
environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "0" }'
- name: Chromium from Ubuntu
dependency: chromium-browser
environment: '{ "CHROMIUM_EXECUTABLE_PATH": "chromium-browser" }'
environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "1" }'
- name: Chrome from Google
dependency: google-chrome-stable
environment: '{ "CHROMIUM_EXECUTABLE_PATH": "google-chrome-stable" }'
environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "1" }'
name: Vitest puppeteer on Node ${{ matrix.node-version }} with ${{ matrix.chromium.name }}
steps:
- uses: actions/checkout@v4
Expand All @@ -83,26 +83,32 @@ jobs:
cache: 'pnpm'
- name: Install dependencies (pnpm)
run: pnpm i
env: ${{ fromJSON(matrix.chromium.environment) }}
- name: Run postinstall script for dependencies
run: pnpm rb
env: ${{ fromJSON(matrix.chromium.environment) }}
- name: Build routes
run: pnpm build
env: ${{ fromJSON(matrix.chromium.environment) }}
- name: Install Chromium
if: ${{ matrix.chromium.dependency != '' }}
# 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means
# nothing since it calls Snap (disgusting!) to install Chromium, which should be up-to-date.
# That's not really a problem since the Chromium-bundled Docker image is based on Debian bookworm,
# which provides up-to-date native packages.
run: |
set -ex
set -eux
curl -s "https://dl.google.com/linux/linux_signing_key.pub" | gpg --dearmor |
sudo tee /etc/apt/trusted.gpg.d/google-chrome.gpg > /dev/null
echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" |
sudo tee /etc/apt/sources.list.d/google-chrome.list > /dev/null
sudo apt-get update
sudo apt-get install -yq --no-install-recommends ${{ matrix.chromium.dependency }}
- name: Test puppeteer
run: pnpm run vitest puppeteer
run: |
set -eux
export CHROMIUM_EXECUTABLE_PATH="$(which ${{ matrix.chromium.dependency }})"
pnpm run vitest puppeteer
env: ${{ fromJSON(matrix.chromium.environment) }}

all:
Expand Down
8 changes: 3 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -146,20 +146,18 @@ RUN \
apt-get install -yq --no-install-recommends \
chromium \
&& \
echo 'CHROMIUM_EXECUTABLE_PATH=chromium' | tee /app/.env ; \
echo "CHROMIUM_EXECUTABLE_PATH=$(which chromium)" | tee /app/.env ; \
fi; \
fi; \
rm -rf /var/lib/apt/lists/*

COPY --from=chromium-downloader /app/node_modules/.cache/puppeteer /app/node_modules/.cache/puppeteer

# if grep matches nothing then it will exit with 1, thus, we cannot `set -e` here
RUN \
set -x && \
set -ex && \
if [ "$PUPPETEER_SKIP_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
echo 'Verifying Chromium installation...' && \
ldd $(find /app/node_modules/.cache/puppeteer/ -name chrome -type f) | grep "not found" ; \
if [ "$?" = 0 ]; then \
if ldd $(find /app/node_modules/.cache/puppeteer/ -name chrome -type f) | grep "not found"; then \
echo "!!! Chromium has unmet shared libs !!!" && \
exit 1 ; \
else \
Expand Down
6 changes: 6 additions & 0 deletions lib/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,9 @@ export type Config = {
zodgame: {
cookie?: string;
};
zsxq: {
accessToken?: string;
};
};

const value: Config | Record<string, any> = {};
Expand Down Expand Up @@ -651,6 +654,9 @@ const calculateValue = () => {
zodgame: {
cookie: envs.ZODGAME_COOKIE,
},
zsxq: {
accessToken: envs.ZSXQ_ACCESS_TOKEN,
},
};

for (const name in _value) {
Expand Down
31 changes: 17 additions & 14 deletions lib/routes/aljazeera/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ const __dirname = getCurrentPath(import.meta.url);

import { getSubPath } from '@/utils/common-utils';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import { art } from '@/utils/render';
import path from 'node:path';
import { ofetch } from 'ofetch';

const languages = {
arabic: {
Expand Down Expand Up @@ -45,12 +44,8 @@ async function handler(ctx) {
const rootUrl = languages[language].rootUrl;
const currentUrl = `${rootUrl}/${isRSS ? languages[language].rssUrl : params.join('/')}`;

const response = await got({
method: 'get',
url: currentUrl,
});

const $ = load(response.data);
const response = await ofetch(currentUrl);
const $ = load(response);

let items = isRSS
? response.data.match(new RegExp('<link>' + rootUrl + '/(.*?)</link>', 'g')).map((item) => ({
Expand All @@ -69,19 +64,27 @@ async function handler(ctx) {
items = await Promise.all(
items.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50).map((item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const detailResponse = await ofetch(item.link);

const content = load(detailResponse.data);
const content = load(detailResponse);

content('.more-on').parent().remove();
content('.responsive-image img').removeAttr('srcset');
let pubDate;

const datePublished = detailResponse.match(/"datePublished": ?"(.*?)",/);
if (datePublished && datePublished.length > 1) {
pubDate = detailResponse.match(/"datePublished": ?"(.*?)",/)[1];
} else {
// uploadDate replaces datePublished for video articles
const uploadDate = detailResponse.match(/"uploadDate": ?"(.*?)",/)[1];

pubDate = uploadDate && uploadDate.length > 1 ? uploadDate : content('div.date-simple > span:nth-child(2)').text();
}

item.title = content('h1').first().text();
item.author = content('.author').text();
item.pubDate = parseDate(detailResponse.data.match(/"datePublished": ?"(.*?)",/)[1]);
item.pubDate = pubDate;
item.description = art(path.join(__dirname, 'templates/description.art'), {
image: content('.article-featured-image').html(),
description: content('.wysiwyg').html(),
Expand Down
202 changes: 202 additions & 0 deletions lib/routes/bnu/mba.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
import { Route } from '@/types';

import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';

export const handler = async (ctx) => {
const { category = 'xwdt' } = ctx.req.param();
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20;

const rootUrl = 'https://mba.bnu.edu.cn';
const currentUrl = new URL(`${category.replace(/\/$/, '')}/`, rootUrl).href;

const { data: response } = await got(currentUrl);

const $ = load(response);

const language = $('html').prop('lang');

let items = $('ul.concrcc li')
.slice(0, limit)
.toArray()
.map((item) => {
item = $(item);

const a = item.find('a.listlj');
const title = a.text();

return {
title,
pubDate: parseDate(item.find('div.crq').text()),
link: new URL(a.prop('href'), currentUrl).href,
language,
};
});

items = await Promise.all(
items.map((item) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);

const $$ = load(detailResponse);

const title = $$('div.connewst').text();
const description = $$('div.concrczw').html();
const image = $$('div.concrczw img').first().prop('src');

item.title = title;
item.description = description;
item.pubDate = parseDate($$('div.connewstis-time').text().split(//).pop());
item.content = {
html: description,
text: $$('div.concrczw').text(),
};
item.image = image;
item.banner = image;
item.language = language;

return item;
})
)
);

const author = $('title').text();
const image = new URL('images/logo5.png', rootUrl).href;

return {
title: `${author} - ${$('div.concrchbt').text()}`,
link: currentUrl,
item: items,
allowEmpty: true,
image,
author,
language,
};
};

export const route: Route = {
path: '/mba/:category{.+}?',
name: '经济与工商管理学院MBA',
url: 'mba.bnu.edu.cn',
maintainers: ['nczitzk'],
handler,
example: '/bnu/mba/xwdt',
parameters: { category: '分类,默认为 xwdt,即新闻聚焦' },
description: `:::tip
若订阅 [新闻聚焦](https://mba.bnu.edu.cn/xwdt/index.html),网址为 \`https://mba.bnu.edu.cn/xwdt/index.html\`。截取 \`https://mba.bnu.edu.cn/\` 到末尾 \`/index.html\` 的部分 \`xwdt\` 作为参数填入,此时路由为 [\`/bnu/mba/xwdt\`](https://rsshub.app/bnu/mba/xwdt)。
:::
#### [主页](https://mba.bnu.edu.cn)
| [新闻聚焦](https://mba.bnu.edu.cn/xwdt/index.html) | [通知公告](https://mba.bnu.edu.cn/tzgg/index.html) | [MBA 系列讲座](https://mba.bnu.edu.cn/mbaxljz/index.html) |
| -------------------------------------------------- | -------------------------------------------------- | --------------------------------------------------------- |
| [xwdt](https://rsshub.app/bnu/mba/xwdt) | [tzgg](https://rsshub.app/bnu/mba/tzgg) | [mbaxljz](https://rsshub.app/bnu/mba/mbaxljz) |
#### [招生动态](https://mba.bnu.edu.cn/zsdt/zsjz/index.html)
| [下载专区](https://mba.bnu.edu.cn/zsdt/cjwt/index.html) |
| ------------------------------------------------------- |
| [zsdt/cjwt](https://rsshub.app/bnu/mba/zsdt/cjwt) |
#### [国际视野](https://mba.bnu.edu.cn/gjhz/hwjd/index.html)
| [海外基地](https://mba.bnu.edu.cn/gjhz/hwjd/index.html) | [学位合作](https://mba.bnu.edu.cn/gjhz/xwhz/index.html) | [长期交换](https://mba.bnu.edu.cn/gjhz/zqjh/index.html) | [短期项目](https://mba.bnu.edu.cn/gjhz/dqxm/index.html) |
| ------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------- |
| [gjhz/hwjd](https://rsshub.app/bnu/mba/gjhz/hwjd) | [gjhz/xwhz](https://rsshub.app/bnu/mba/gjhz/xwhz) | [gjhz/zqjh](https://rsshub.app/bnu/mba/gjhz/zqjh) | [gjhz/dqxm](https://rsshub.app/bnu/mba/gjhz/dqxm) |
#### [校园生活](https://mba.bnu.edu.cn/xysh/xszz/index.html)
| [学生组织](https://mba.bnu.edu.cn/xysh/xszz/index.html) |
| ------------------------------------------------------- |
| [xysh/xszz](https://rsshub.app/bnu/mba/xysh/xszz) |
#### [职业发展](https://mba.bnu.edu.cn/zyfz/xwds/index.html)
| [校外导师](https://mba.bnu.edu.cn/zyfz/xwds/index.html) | [企业实践](https://mba.bnu.edu.cn/zyfz/zycp/index.html) | [就业创业](https://mba.bnu.edu.cn/zyfz/jycy/index.html) |
| ------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------- |
| [zyfz/xwds](https://rsshub.app/bnu/mba/zyfz/xwds) | [zyfz/zycp](https://rsshub.app/bnu/mba/zyfz/zycp) | [zyfz/jycy](https://rsshub.app/bnu/mba/zyfz/jycy) |
`,
categories: ['university'],

features: {
requireConfig: false,
requirePuppeteer: false,
antiCrawler: false,
supportRadar: true,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
radar: [
{
source: ['mba.bnu.edu.cn/:category?'],
target: (params) => {
const category = params.category;

return category ? `/${category.replace(/\/index\.html$/, '')}` : '';
},
},
{
title: '新闻聚焦',
source: ['mba.bnu.edu.cn/xwdt/index.html'],
target: '/mba/xwdt',
},
{
title: '通知公告',
source: ['mba.bnu.edu.cn/tzgg/index.html'],
target: '/mba/tzgg',
},
{
title: 'MBA系列讲座',
source: ['mba.bnu.edu.cn/mbaxljz/index.html'],
target: '/mba/mbaxljz',
},
{
title: '招生动态 - 下载专区',
source: ['mba.bnu.edu.cn/zsdt/cjwt/index.html'],
target: '/mba/zsdt/cjwt',
},
{
title: '国际视野 - 海外基地',
source: ['mba.bnu.edu.cn/gjhz/hwjd/index.html'],
target: '/mba/gjhz/hwjd',
},
{
title: '国际视野 - 学位合作',
source: ['mba.bnu.edu.cn/gjhz/xwhz/index.html'],
target: '/mba/gjhz/xwhz',
},
{
title: '国际视野 - 长期交换',
source: ['mba.bnu.edu.cn/gjhz/zqjh/index.html'],
target: '/mba/gjhz/zqjh',
},
{
title: '国际视野 - 短期项目',
source: ['mba.bnu.edu.cn/gjhz/dqxm/index.html'],
target: '/mba/gjhz/dqxm',
},
{
title: '校园生活 - 学生组织',
source: ['mba.bnu.edu.cn/xysh/xszz/index.html'],
target: '/mba/xysh/xszz',
},
{
title: '职业发展 - 校外导师',
source: ['mba.bnu.edu.cn/zyfz/xwds/index.html'],
target: '/mba/zyfz/xwds',
},
{
title: '职业发展 - 企业实践',
source: ['mba.bnu.edu.cn/zyfz/zycp/index.html'],
target: '/mba/zyfz/zycp',
},
{
title: '职业发展 - 就业创业',
source: ['mba.bnu.edu.cn/zyfz/jycy/index.html'],
target: '/mba/zyfz/jycy',
},
],
};
6 changes: 6 additions & 0 deletions lib/routes/byau/namespace.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import type { Namespace } from '@/types';

export const namespace: Namespace = {
name: '黑龙江八一农垦大学',
url: 'byau.edu.cn',
};
Loading

0 comments on commit 3f29650

Please sign in to comment.