Skip to content

Commit

Permalink
feat: add ability to provide database code as full url
Browse files Browse the repository at this point in the history
  • Loading branch information
Phumrapee Limpianchop committed Nov 12, 2024
1 parent 4172cd0 commit 368f725
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 2 deletions.
1 change: 1 addition & 0 deletions commons/src/@types/DatabaseCode.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export type DatabaseCode =
| string
| number
| {
code: number
Expand Down
5 changes: 3 additions & 2 deletions scraper/src/commands/fetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { Kysely } from 'kysely'
import { SQLDatabase } from '../@types/SQLDatabase'
import { createDBConnection } from '../functions/createDBConnection'
import { getRemoteHentai } from '../functions/getRemoteHentai'
import {parseUrl} from "../functions/parseUrl";

const fetchQueue = new PQueue({
concurrency: 8,
Expand Down Expand Up @@ -42,7 +43,7 @@ export const fetch = async (entryPoint: string) => {
const chunkFile = path.join(prebuiltChunkDirectory, `chunk-${i + 1}.json`)
await fs.promises.writeFile(
chunkFile,
JSON.stringify(chunk.map(o => (typeof o === 'number' ? o : o.code)))
JSON.stringify(chunk.map(o => (typeof o === 'number' ? o : typeof o === 'string' ? parseUrl(o) : o.code)))
)
})
)
Expand Down Expand Up @@ -87,7 +88,7 @@ export const fetch = async (entryPoint: string) => {
const orderedHentai = codes
.map(code => {
try {
const targetCode = typeof code === 'number' ? code : code.code
const targetCode = typeof code === 'number' ? code : typeof code === 'string' ? parseUrl(code) : code.code
const targetHentai: Hentai = JSON.parse(
fs
.readFileSync(path.join(hentaiDirectory, `${targetCode}.json`))
Expand Down
7 changes: 7 additions & 0 deletions scraper/src/functions/parseUrl.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export const parseUrl = (input: string) => {
const matcher = input.match(/https?:\/\/nhentai.net\/g\/(\d+)\/?/)
if (matcher !== null && Number.isInteger(Number(matcher[1])))
return Number(matcher[1])
else
throw new Error("Cannot parse " + input)
}

0 comments on commit 368f725

Please sign in to comment.