feat: improved toc; fixes

Author: transitive-bullshit

package.json

@@ -1,7 +1,7 @@
 {
   "name": "kindle-ai-export",
   "private": true,
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Export any Kindle book you own as text, PDF, EPUB, or as a custom, AI-narrated audiobook.",
   "author": "Travis Fischer <[email protected]>",
   "license": "MIT",
@@ -25,10 +25,8 @@
     "@inquirer/prompts": "^7.0.0",
     "dotenv": "^17.2.3",
     "fluent-ffmpeg": "^2.1.3",
-    "globby": "^15.0.0",
     "hash-object": "^5.0.1",
     "hh-mm-ss": "^1.2.0",
-    "kindle-api-ky": "^1.0.1",
     "ky": "^1.12.0",
     "node-id3": "^0.2.6",
     "openai-fetch": "^3.4.2",

src/export-book-markdown.ts

@@ -41,7 +41,7 @@ async function main() {
 
   let output = `# ${title}
 
-By ${authors.join(', ')}
+> By ${authors.join(', ')}
 
 ---
 
@@ -53,7 +53,7 @@ ${metadata.toc
   )
   .map(
     (tocItem) =>
-      `- [${tocItem.label}](#${tocItem.label.toLowerCase().replaceAll(/[^\da-z]+/g, '-')})`
+      `${'  '.repeat(tocItem.depth)}- [${tocItem.label}](#${tocItem.label.toLowerCase().replaceAll(/[^\da-z]+/g, '-')})`
   )
   .join('\n')}
 
@@ -78,7 +78,7 @@ ${metadata.toc
 
     output += `
 
-## ${tocItem.label}
+${'#'.repeat(tocItem.depth + 2)} ${tocItem.label}
 
 ${text}`
 

src/export-book-pdf.ts

@@ -84,7 +84,7 @@ async function main() {
     const text = chunks.map((chunk) => chunk.text).join(' ')
 
     ;(doc as any).outline.addItem(tocItem.label)
-    doc.fontSize(20)
+    doc.fontSize(tocItem.depth === 1 ? 16 : 20)
     doc.text(tocItem.label, { align: 'center', lineGap: 16 })
 
     doc.fontSize(fontSize)

src/extract-kindle-book.ts

@@ -325,6 +325,7 @@ async function main() {
 
     const numTocItems = await page.locator('ion-list ion-item').count()
     const $tocTopLevelItems = await page
+      // TODO: this is pretty brittle
       .locator('ion-list > div > ion-item')
       .all()
     const tocItems: Array<TocItem> = []
@@ -356,9 +357,11 @@ async function main() {
 
       const currentTocItem: TocItem = {
         label,
+        depth: 0,
         ...pageNav
       }
       tocItems.push(currentTocItem)
+      console.warn(currentTocItem)
 
       // if (pageNav.page !== undefined) {
       //   // TODO: this assumes the toc items are in order and contiguous...
@@ -372,7 +375,6 @@ async function main() {
         .all()
 
       if (subTocItems.length > 0) {
-        currentTocItem.entries = []
         console.warn(`${label}: found ${subTocItems.length} sub-TOC items...`)
 
         for (const $subTocItem of subTocItems) {
@@ -385,8 +387,9 @@ async function main() {
           const pageNav = await getPageNav()
           assert(pageNav)
 
-          currentTocItem.entries!.push({
+          tocItems.push({
             label,
+            depth: 1,
             ...pageNav
           })
 
@@ -396,8 +399,6 @@ async function main() {
           })
         }
       }
-
-      console.warn(currentTocItem)
     }
 
     result.toc = tocItems

src/transcribe-book-content.ts

@@ -3,41 +3,58 @@ import 'dotenv/config'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 
-import { globby } from 'globby'
 import { OpenAIClient } from 'openai-fetch'
 import pMap from 'p-map'
 
-import type { ContentChunk } from './types'
+import type { BookMetadata, ContentChunk, TocItem } from './types'
 import { assert, getEnv } from './utils'
 
 async function main() {
   const asin = getEnv('ASIN')
   assert(asin, 'ASIN is required')
 
   const outDir = path.join('out', asin)
-  const pageScreenshotsDir = path.join(outDir, 'pages')
-  const pageScreenshots = await globby(`${pageScreenshotsDir}/*.png`)
-  assert(pageScreenshots.length, 'no page screenshots found')
+  const metadata = JSON.parse(
+    await fs.readFile(path.join(outDir, 'metadata.json'), 'utf8')
+  ) as BookMetadata
+  assert(metadata.pages?.length, 'no page screenshots found')
+  assert(metadata.toc?.length, 'invalid book metadata: missing toc')
+
+  // eslint-disable-next-line unicorn/no-array-reduce
+  const pageToTocItemMap = metadata.toc.reduce(
+    (acc, tocItem) => {
+      if (tocItem.page !== undefined) {
+        acc[tocItem.page] = tocItem
+      }
+      return acc
+    },
+    {} as Record<number, TocItem>
+  )
+
+  // const pageScreenshotsDir = path.join(outDir, 'pages')
+  // const pageScreenshots = await globby(`${pageScreenshotsDir}/*.png`)
+  // assert(pageScreenshots.length, 'no page screenshots found')
 
   const openai = new OpenAIClient()
 
   const content: ContentChunk[] = (
     await pMap(
-      pageScreenshots,
-      async (screenshot) => {
+      metadata.pages,
+      async (pageChunk, pageChunkIndex) => {
+        const { screenshot, index, page } = pageChunk
         const screenshotBuffer = await fs.readFile(screenshot)
         const screenshotBase64 = `data:image/png;base64,${screenshotBuffer.toString('base64')}`
-        const metadataMatch = screenshot.match(/0*(\d+)-\0*(\d+).png/)
-        assert(
-          metadataMatch?.[1] && metadataMatch?.[2],
-          `invalid screenshot filename: ${screenshot}`
-        )
-        const index = Number.parseInt(metadataMatch[1]!, 10)
-        const page = Number.parseInt(metadataMatch[2]!, 10)
-        assert(
-          !Number.isNaN(index) && !Number.isNaN(page),
-          `invalid screenshot filename: ${screenshot}`
-        )
+        // const metadataMatch = screenshot.match(/0*(\d+)-\0*(\d+).png/)
+        // assert(
+        //   metadataMatch?.[1] && metadataMatch?.[2],
+        //   `invalid screenshot filename: ${screenshot}`
+        // )
+        // const index = Number.parseInt(metadataMatch[1]!, 10)
+        // const page = Number.parseInt(metadataMatch[2]!, 10)
+        // assert(
+        //   !Number.isNaN(index) && !Number.isNaN(page),
+        //   `invalid screenshot filename: ${screenshot}`
+        // )
 
         try {
           const maxRetries = 20
@@ -69,7 +86,7 @@ Do not include any additional text, descriptions, or punctuation. Ignore any emb
             })
 
             const rawText = res.choices[0]!.message.content!
-            const text = rawText
+            let text = rawText
               .replace(/^\s*\d+\s*$\n+/m, '')
               // .replaceAll(/\n+/g, '\n')
               .replaceAll(/^\s*/gm, '')
@@ -95,6 +112,18 @@ Do not include any additional text, descriptions, or punctuation. Ignore any emb
               continue
             }
 
+            const prevPageChunk = metadata.pages[pageChunkIndex - 1]
+            if (prevPageChunk && prevPageChunk.page !== page) {
+              const tocItem = pageToTocItemMap[page]
+              if (tocItem) {
+                text = text.replace(
+                  // eslint-disable-next-line security/detect-non-literal-regexp
+                  new RegExp(`^${tocItem.label}\\s*`, 'i'),
+                  ''
+                )
+              }
+            }
+
             const result: ContentChunk = {
               index,
               page,

src/types.ts

@@ -30,8 +30,8 @@ export interface PageNav {
 export type TocItem = SimplifyDeep<
   PageNav & {
     label: string
-    tocPositionId?: number
-    entries?: Simplify<Omit<TocItem, 'entries'>>[]
+    // tocPositionId?: number
+    depth: number
   }
 >
 

src/utils.ts

@@ -7,13 +7,62 @@ import timeFormat from 'hh-mm-ss'
 import { extract } from 'tar'
 import { temporaryDirectory } from 'tempy'
 
-export {
-  assert,
-  getEnv,
-  normalizeAuthors,
-  parseJsonpResponse
-} from 'kindle-api-ky'
+export function assert(
+  value: unknown,
+  message?: string | Error
+): asserts value {
+  if (value) {
+    return
+  }
+
+  if (!message) {
+    throw new Error('Assertion failed')
+  }
+
+  throw typeof message === 'string' ? new Error(message) : message
+}
+
+export function getEnv(name: string): string | undefined {
+  try {
+    return typeof process !== 'undefined'
+      ? // eslint-disable-next-line no-process-env
+        process.env?.[name]
+      : undefined
+  } catch {
+    return undefined
+  }
+}
+
+export function normalizeAuthors(rawAuthors: string[]): string[] {
+  if (!rawAuthors?.length) {
+    return []
+  }
+
+  const rawAuthor = rawAuthors[0]!
+
+  return Array.from(new Set(rawAuthor.split(':').filter(Boolean)), (authors) =>
+    authors
+      .split(',')
+      .map((elems) => elems.trim())
+      .toReversed()
+      .join(' ')
+  )
+}
+
+const JSONP_REGEX = /\(({.*})\)/
 
+export function parseJsonpResponse<T = unknown>(body: string): T | undefined {
+  const content = body?.match(JSONP_REGEX)?.[1]
+  if (!content) {
+    return
+  }
+
+  try {
+    return JSON.parse(content) as T
+  } catch {
+    return
+  }
+}
 const numerals = { I: 1, V: 5, X: 10, L: 50, C: 100, D: 500, M: 1000 }
 
 export function deromanize(romanNumeral: string): number {