Skip to content

Commit 13a01dd

Browse files
trim21trim21
committedJul 2, 2023
init commit
0 parents  commit 13a01dd

14 files changed

+1498
-0
lines changed
 

‎.github/codecov.yml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
coverage:
2+
precision: 1
3+
round: down
4+
range: 90...95 # custom range of coverage colors from red -> yellow -> green
5+
status:
6+
project:
7+
default:
8+
# basic
9+
target: 99%
10+
threshold: 1%
11+
12+
codecov:
13+
require_ci_to_pass: true

‎.github/renovate.json

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
3+
"extends": [
4+
"local>Trim21/renovate-config",
5+
"local>Trim21/renovate-config:monthly",
6+
"local>Trim21/renovate-config:poetry"
7+
],
8+
"separateMajorMinor": true,
9+
"separateMinorPatch": false,
10+
"rangeStrategy": "pin",
11+
"pre-commit": {
12+
"enabled": true
13+
},
14+
"semanticCommitType": "build",
15+
"packageRules": [
16+
{
17+
"matchManagers": ["pre-commit"],
18+
"semanticCommitType": "chore",
19+
"groupName": "pre-commit",
20+
"semanticCommitScope": ""
21+
},
22+
{
23+
"matchManagers": ["dockerfile"],
24+
"rangeStrategy": "pin",
25+
"minor": {
26+
"enabled": false
27+
},
28+
"major": {
29+
"enabled": false
30+
}
31+
}
32+
]
33+
}

‎.github/workflows/ci.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: ci
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- "master"
7+
push:
8+
branches:
9+
- "master"
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- uses: actions/checkout@v3
17+
- run: docker build -t base-image -f etc/base.dockerfile .
18+
- run: docker build -t tmp -f etc/final.dockerfile .
19+
- run: docker run tmp --help

‎.github/workflows/lint.yaml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: lint
2+
3+
on:
4+
push:
5+
branches-ignore:
6+
- renovate/**
7+
paths-ignore:
8+
- "**.md"
9+
pull_request:
10+
branches:
11+
- master
12+
paths-ignore:
13+
- "**.md"
14+
15+
jobs:
16+
mypy:
17+
runs-on: ubuntu-latest
18+
steps:
19+
- uses: actions/checkout@v3
20+
21+
- uses: actions/setup-python@v4
22+
with:
23+
python-version: "3.11"
24+
25+
- uses: Trim21/setup-poetry@dist/v1
26+
27+
- uses: Trim21/install-poetry-project@dist/v1
28+
29+
- uses: liskin/gh-problem-matcher-wrap@v2
30+
with:
31+
action: add
32+
linters: mypy
33+
34+
- name: mypy
35+
run: mypy --show-column-numbers app
36+
37+
lint:
38+
runs-on: ubuntu-latest
39+
steps:
40+
- uses: actions/checkout@v3
41+
42+
- uses: actions/setup-python@v4
43+
with:
44+
python-version: "3.11"
45+
46+
- uses: trim21/setup-poetry@dist/v1
47+
- uses: trim21/install-poetry-project@dist/v1
48+
- uses: trim21/actions/pre-commit@master

‎.github/workflows/release.yaml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: release
2+
3+
on:
4+
push:
5+
tags:
6+
- v*
7+
branches:
8+
- master
9+
10+
jobs:
11+
docker:
12+
runs-on: ubuntu-latest
13+
env:
14+
IMAGE: "ghcr.io/${{ github.repository_owner }}/clean-old-site-cache"
15+
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.sha }}
18+
cancel-in-progress: false
19+
20+
steps:
21+
- uses: actions/checkout@v3
22+
23+
- uses: docker/login-action@v2
24+
with:
25+
registry: ghcr.io
26+
username: ${{ github.actor }}
27+
password: ${{ github.token }}
28+
29+
- name: build base docker image
30+
env:
31+
FILES_HASH: "${{ hashFiles('poetry.lock', 'etc/*.dockerfile') }}"
32+
run: |
33+
export "TAG_HASH=base-${FILES_HASH}"
34+
echo "TAG_HASH=${TAG_HASH}" >>$GITHUB_ENV
35+
36+
if ! docker pull "${IMAGE}:${TAG_HASH}"; then
37+
echo "NEED_BUILD_BASE=true" >>$GITHUB_ENV
38+
else
39+
echo "NEED_BUILD_BASE=false" >>$GITHUB_ENV
40+
fi
41+
42+
- name: Build Base Docker Image (if needed)
43+
uses: docker/build-push-action@v4
44+
if: ${{ env.NEED_BUILD_BASE }}
45+
with:
46+
context: ./
47+
file: ./etc/base.dockerfile
48+
provenance: false
49+
push: true
50+
tags: ${{ env.IMAGE }}:${{ env.TAG_HASH }}
51+
52+
- run: docker tag "${IMAGE}:${TAG_HASH}" base-image
53+
54+
- run: echo "SHA=$(git show --no-patch --no-notes --date=short-local --pretty='%as-%h')" >> $GITHUB_ENV
55+
env:
56+
TZ: UTC
57+
58+
- name: Docker metadata
59+
id: meta
60+
uses: docker/metadata-action@v4
61+
with:
62+
images: ${{ env.IMAGE }}
63+
tags: |
64+
type=semver,pattern=v{{version}}
65+
66+
type=ref,event=branch
67+
type=sha,prefix={{branch}}-
68+
type=ref,event=branch,suffix=-${{ env.SHA }}
69+
70+
type=raw,value=${{ env.SHA }}
71+
72+
- name: Build Final Docker Image
73+
uses: docker/build-push-action@v4
74+
with:
75+
context: ./
76+
provenance: false
77+
file: ./etc/final.dockerfile
78+
push: true
79+
tags: ${{ steps.meta.outputs.tags }}
80+
labels: ${{ steps.meta.outputs.labels }}

‎.gitignore

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
.idea
3+
.vscode
4+
ics
5+
env
6+
.env
7+
.envrc
8+
source.sh
9+
.scrapy
10+
a.html
11+
*.py[cod]
12+
.mypy_cache
13+
.pytest_cache
14+
docs/build
15+
docs/source/openapi.json
16+
*.egg-info
17+
.venv/
18+
dev-env/
19+
.coverage
20+
*.pyc.*
21+
/data/
22+
tmp/
23+
.task/

‎.pre-commit-config.yaml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
exclude: "api/.*"
2+
repos:
3+
- repo: https://github.com/pre-commit/pre-commit-hooks
4+
rev: v4.4.0
5+
hooks:
6+
- id: check-case-conflict
7+
- id: check-ast
8+
- id: check-builtin-literals
9+
- id: fix-byte-order-marker
10+
- id: check-toml
11+
- id: check-yaml
12+
- id: check-json
13+
- id: check-docstring-first
14+
- id: check-merge-conflict
15+
- id: check-added-large-files # check for file bigger than 500kb
16+
- id: debug-statements
17+
- id: trailing-whitespace
18+
- id: debug-statements
19+
- id: mixed-line-ending
20+
args: [--fix=lf]
21+
- id: end-of-file-fixer
22+
- id: fix-encoding-pragma
23+
args: [--remove]
24+
25+
- repo: https://github.com/python-poetry/poetry
26+
rev: "1.5.1"
27+
hooks:
28+
- id: poetry-check
29+
- id: poetry-lock
30+
name: poetry-lock-check
31+
args: [--check]
32+
33+
- repo: https://github.com/asottile/pyupgrade
34+
rev: v3.7.0
35+
hooks:
36+
- id: pyupgrade
37+
args: [--py38-plus]
38+
39+
- repo: https://github.com/myint/autoflake
40+
rev: v2.2.0
41+
hooks:
42+
- id: autoflake
43+
args:
44+
- --in-place
45+
- --remove-unused-variables
46+
- --remove-all-unused-imports
47+
- --expand-star-imports
48+
49+
- repo: https://github.com/astral-sh/ruff-pre-commit
50+
# Ruff version.
51+
rev: v0.0.275
52+
hooks:
53+
- id: ruff
54+
args: [ --fix, --exit-non-zero-on-fix ]
55+
56+
- repo: https://github.com/psf/black
57+
rev: 23.3.0
58+
hooks:
59+
- id: black

‎app/config.py

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from dotenv import load_dotenv
2+
from pydantic import AnyUrl, Field
3+
from pydantic_settings import BaseSettings
4+
5+
6+
class Settings(BaseSettings):
7+
broker: AnyUrl = Field(env="BROKER")
8+
memcached: str = Field(env="MEMCACHED")
9+
10+
COMMIT_REF: str = Field(env="COMMIT_REF", default="dev")
11+
12+
13+
load_dotenv()
14+
config = Settings()
15+
16+
if __name__ == "__main__":
17+
print(config.model_dump())

‎app/main.py

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
from collections.abc import Iterable
2+
3+
import msgspec
4+
import pymemcache
5+
from kafka import KafkaConsumer
6+
from kafka.consumer.fetcher import ConsumerRecord
7+
8+
from app.config import config
9+
10+
11+
class ChiiInterest(msgspec.Struct):
12+
interest_id: int
13+
interest_uid: int
14+
interest_subject_id: int
15+
interest_subject_type: int
16+
interest_rate: int
17+
interest_type: int
18+
interest_has_comment: int
19+
interest_comment: str
20+
interest_tag: str
21+
interest_ep_status: int
22+
interest_vol_status: int
23+
interest_wish_dateline: int
24+
interest_doing_dateline: int
25+
interest_collect_dateline: int
26+
interest_on_hold_dateline: int
27+
interest_dropped_dateline: int
28+
interest_create_ip: str
29+
interest_lasttouch_ip: str
30+
interest_lasttouch: int
31+
interest_private: int
32+
33+
34+
class ValuePayload(msgspec.Struct):
35+
before: ChiiInterest | None
36+
after: ChiiInterest | None
37+
op: str
38+
39+
40+
class Value(msgspec.Struct):
41+
payload: ValuePayload
42+
43+
44+
def kafka_events() -> Iterable[tuple[int, int]]:
45+
consumer = KafkaConsumer(
46+
"debezium.chii.bangumi.chii_subject_interests",
47+
group_id="py-cache-clean",
48+
bootstrap_servers=f"{config.broker.host}:{config.broker.port}",
49+
auto_offset_reset="earliest",
50+
)
51+
52+
msg: ConsumerRecord
53+
for msg in consumer:
54+
value = msgspec.json.decode(msg.value, type=Value)
55+
before = value.payload.before
56+
after = value.payload.after
57+
if after is not None:
58+
yield after.interest_uid, after.interest_subject_id
59+
elif before is not None:
60+
yield before.interest_uid, before.interest_subject_id
61+
62+
63+
def user_collection_change():
64+
if config.broker.scheme == "kafka":
65+
yield from kafka_events()
66+
else:
67+
raise ValueError(f"event broker not support {config.broker.scheme!r}")
68+
69+
70+
def main():
71+
client = pymemcache.Client(config.memcached)
72+
while True:
73+
for user_id, subject_id in kafka_events():
74+
print(user_id, subject_id)
75+
client.delete(f"{user_id}:{subject_id}", True)
76+
77+
78+
if __name__ == "__main__":
79+
main()

0 commit comments

Comments
 (0)