Push some old experiments

Signed-off-by: Marcus Crane <[email protected]>

Author: marcus-crane

v2/README.md

@@ -0,0 +1,19 @@
+# v2
+
+This folder contains some experiments with rewriting October. It may or may not amount to anything but I'd like to untangle various elements of the backend.
+
+It isn't intended to be usable in its current state, I'm approximating the upload process by pointing at files on disc instead of actually using my mounted Kobo.
+
+This version will also be attempting to incorporate better parsing by processing epubs and probably parallelised uploading using Goroutines
+
+## Prerequisites
+
+Quickest way to bootstrap a DB for experimenting:
+
+```bash
+git clone [email protected]:marcus-crane/kobodbgen
+cd kobodbgen
+./initdb.sh
+```
+
+This will create a mock Kobo DB for usage although I'm using a dump of my Kobo stored on my desktop at present

v2/go.mod

@@ -0,0 +1,10 @@
+module github.com/marcus-crane/october/v2
+
+go 1.19
+
+require (
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/pgaskin/koboutils/v2 v2.1.2-0.20220306004009-a07e72ebae42
+	github.com/taylorskalyo/goreader v0.0.0-20220528130152-945e7448ceb5
+)

v2/go.sum

@@ -0,0 +1,16 @@
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
+github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/pgaskin/koboutils/v2 v2.1.1 h1:Or5y+z8rXlip0Al8tiSj+Fb9NkuLhkcw1UPpzPPvKWY=
+github.com/pgaskin/koboutils/v2 v2.1.1/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM=
+github.com/pgaskin/koboutils/v2 v2.1.2-0.20220306004009-a07e72ebae42 h1:wwU2E+7+IN4HBv9v5p+TkoaaFSnF5BtguW5bTxopuvk=
+github.com/pgaskin/koboutils/v2 v2.1.2-0.20220306004009-a07e72ebae42/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/taylorskalyo/goreader v0.0.0-20220528130152-945e7448ceb5 h1:dW3HLfusjJuR5/7MCKKcBKzTmRjZnEAEO8AVrHIqqC8=
+github.com/taylorskalyo/goreader v0.0.0-20220528130152-945e7448ceb5/go.mod h1:06vTtAxpkyCBMlqDyYuvHgeQec6ne7NWXIEgJNhq2Ks=

v2/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"sync"
+
+	"github.com/marcus-crane/october/v2/pkg/epub"
+	"github.com/marcus-crane/october/v2/pkg/kobo"
+)
+
+func main() {
+	root, err := epub.LoadEpub("kobodbgen/epubs/source/james-hogg_the-private-memoirs-and-confessions-of-a-justified-sinner.epub")
+	if err != nil {
+		log.Fatalf("Failed to load epub: %+v", err)
+	}
+	// Spines are the correct order where manifests may be out of order
+	// May need to check the manifest for covers
+	for i, entry := range root.Spine.Itemrefs {
+		idx := i + 1 // We increment by 1 as we don't trust external systems to not drop zero padding
+		fmt.Printf("%d - %s\n", idx, entry.HREF)
+	}
+
+	connection := kobo.NewKobo("/Users/marcus/Desktop/Kobo", "/Users/marcus/Desktop/Kobo/.kobo/KoboReader.sqlite")
+	if err := connection.Connect(); err != nil {
+		log.Fatalf("Failed to connect to database: %+v", err)
+	}
+	defer connection.Disconnect()
+	numBookmarks, err := kobo.CountBookmarks(&connection)
+	if err != nil {
+		log.Fatalf("Failed to count bookmarks")
+	}
+	log.Printf("Your device has %d bookmarks", numBookmarks)
+	numContent, err := kobo.CountContent(&connection)
+	if err != nil {
+		log.Fatalf("Failed to count content")
+	}
+	log.Printf("Your device has %d non-unique pieces of content", numContent)
+	volumes, err := kobo.QueryDistinctVolumes(&connection)
+	if err != nil {
+		log.Fatalf("Failed to check volumes")
+	}
+	log.Printf("Detected %d unique books containing highlights and notes", len(volumes))
+	// VERY important to keep this buffered (ie; set length) or the goroutine will block
+	bookmarks := make(chan kobo.Bookmark, len(volumes))
+	var wg sync.WaitGroup
+	for _, volume := range volumes {
+		wg.Add(1)
+		go func(connection kobo.Kobo, volume string) {
+			defer wg.Done()
+			fmt.Printf("Saw %s and tried to ping DB with result %+v\n", volume, connection.Ping())
+			bookmarks <- kobo.Bookmark{
+				Text: fmt.Sprintf("Toot from %s", volume),
+			}
+		}(connection, volume)
+	}
+	wg.Wait()
+	close(bookmarks)
+	fmt.Println("Finished waiting")
+
+	for bookmark := range bookmarks {
+		fmt.Printf("Received bookmark with text: %s\n", bookmark.Text)
+	}
+}

v2/pkg/epub/epub.go

@@ -0,0 +1,36 @@
+package epub
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/taylorskalyo/goreader/epub"
+)
+
+// NOTE TO SELF: Cover images can be retrieved like so:
+// 1) Find meta name="cover" in OPF metadata and check the content
+// 2) Find item entry with id set to same value as metadata content
+// See https://github.com/kobolabs/epub-spec#cover-images
+// Also covers may technically be SVGs so check the types and
+// explicitly mention what is supported
+
+// See https://github.com/kobolabs/epub-spec#for-epub3 for a list of nav fallbacks
+// if OPF doesn't have the desired information
+
+// LoadEpub takes a relative path to an epub file (aka VolumeID) and returns the content of the epub
+// Further parsing is required to actually read files for example but it gives a full book spine
+// among other useful metadata that may or may not be present in the database, particular with
+// epubs (not kepubs) that have been loaded directly onto the device without using a tool like Calibre
+func LoadEpub(path string) (*epub.Rootfile, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, err
+	}
+	rc, err := epub.OpenReader(path)
+	if err != nil {
+		return nil, err
+	}
+	if len(rc.Rootfiles) == 0 {
+		return nil, fmt.Errorf("no root files found in epub")
+	}
+	return rc.Rootfiles[0], nil
+}

v2/pkg/kobo/bookmark.go

@@ -0,0 +1,51 @@
+package kobo
+
+type Bookmark struct {
+	BookmarkID               string  `db:"BookmarkID"`
+	VolumeID                 string  `db:"VolumeID"`
+	ContentID                string  `db:"ContentID"`
+	StartContainerPath       string  `db:"StartContainerPath"`
+	StartContainerChild      string  `db:"StartContainerChild"`
+	StartContainerChildIndex int     `db:"StartContainerChildIndex"`
+	StartOffset              int     `db:"StartOffset"`
+	EndContainerPath         string  `db:"EndContainerPath"`
+	EndContainerChildIndex   int     `db:"EndContainerChildIndex"`
+	EndOffset                int     `db:"EndOffset"`
+	Text                     string  `db:"Text"`
+	Annotation               string  `db:"Annotation"`
+	ExtraAnnotationData      string  `db:"ExtraAnnotationData"`
+	DateCreated              string  `db:"DateCreated"`
+	ChapterProgress          float64 `db:"ChapterProgress"`
+	Hidden                   bool    `db:"Hidden"`
+	Version                  string  `db:"Version"`
+	DateModified             string  `db:"DateModified"`
+	Creator                  string  `db:"Creator"`
+	UUID                     string  `db:"UUID"`
+	UserID                   string  `db:"UserID"`
+	SyncTime                 string  `db:"SyncTime"`
+	Published                bool    `db:"Published"`
+	ContextString            string  `db:"ContextString"`
+	Type                     string  `db:"Type"`
+}
+
+func CountBookmarks(kobo *Kobo) (int, error) {
+	var count int
+	if err := kobo.dbClient.Get(&count, "SELECT count(*) FROM Bookmark"); err != nil {
+		return count, err
+	}
+	return count, nil
+}
+
+// QueryDistinctVolumes retrieves all books that contains highlights and reside (or did at one point)
+// on the device itself. We exclude Kobo store books for simplicity where the path is a GUID which is
+// not useful in the context of retrieving the underlying epub for scanning.
+func QueryDistinctVolumes(kobo *Kobo) ([]string, error) {
+	var contentWithBookmarks []string
+	if err := kobo.dbClient.Select(
+		&contentWithBookmarks,
+		"SELECT DISTINCT VolumeID FROM Bookmark WHERE VolumeID LIKE '%file:///%' ORDER BY VolumeID;",
+	); err != nil {
+		return contentWithBookmarks, err
+	}
+	return contentWithBookmarks, nil
+}

v2/pkg/kobo/content.go

@@ -0,0 +1,115 @@
+package kobo
+
+type Content struct {
+	ContentID                    string  `db:"ContentID"`
+	ContentType                  string  `db:"ContentType"`
+	MimeType                     string  `db:"MimeType"`
+	BookID                       string  `db:"BookID"`
+	BookTitle                    string  `db:"BookTitle"`
+	ImageId                      string  `db:"ImageId"`
+	Title                        string  `db:"Title"`
+	Attribution                  string  `db:"Attribution"`
+	Description                  string  `db:"Description"`
+	DateCreated                  string  `db:"DateCreated"`
+	ShortCoverKey                string  `db:"ShortCoverKey"`
+	AdobeLocation                string  `db:"adobe_location"`
+	Publisher                    string  `db:"Publisher"`
+	IsEncrypted                  bool    `db:"IsEncrypted"`
+	DateLastRead                 string  `db:"DateLastRead"`
+	FirstTimeReading             bool    `db:"FirstTimeReading"`
+	ChapterIDBookmarked          string  `db:"ChapterIDBookmarked"`
+	ParagraphBookmarked          int     `db:"ParagraphBookmarked"`
+	BookmarkWordOffset           int     `db:"BookmarkWordOffset"`
+	NumShortcovers               int     `db:"NumShortcovers"`
+	VolumeIndex                  int     `db:"VolumeIndex"`
+	NumPages                     int     `db:"___NumPages"`
+	ReadStatus                   int     `db:"ReadStatus"`
+	SyncTime                     string  `db:"___SyncTime"`
+	UserID                       string  `db:"___UserID"`
+	PublicationId                string  `db:"PublicationId"`
+	FileOffset                   int     `db:"___FileOffset"`
+	FileSize                     int     `db:"___FileSize"`
+	PercentRead                  int     `db:"___PercentRead"`
+	ExpirationStatus             int     `db:"___ExpirationStatus"`
+	FavouritesIndex              int     `db:"FavouritesIndex"`
+	Accessibility                int     `db:"Accessibility"`
+	ContentURL                   string  `db:"ContentURL"`
+	Language                     string  `db:"Language"`
+	BookshelfTags                string  `db:"BookshelfTags"`
+	IsDownloaded                 bool    `db:"IsDownloaded"`
+	FeedbackType                 int     `db:"FeedbackType"`
+	AverageRating                int     `db:"AverageRating"`
+	Depth                        int     `db:"Depth"`
+	PageProgressDirection        string  `db:"PageProgressDirection"`
+	InWishlist                   bool    `db:"InWishlist"`
+	ISBN                         string  `db:"ISBN"`
+	WishlistedDate               string  `db:"WishlistedDate"`
+	FeedbackTypeSynced           int     `db:"FeedbackTypeSynced"`
+	IsSocialEnabled              bool    `db:"IsSocialEnabled"`
+	EpubType                     int     `db:"EpubType"`
+	Monetization                 int     `db:"Monetization"`
+	ExternalId                   string  `db:"ExternalId"`
+	Series                       string  `db:"Series"`
+	SeriesNumber                 string  `db:"SeriesNumber"`
+	Subtitle                     string  `db:"Subtitle"`
+	WordCount                    int     `db:"WordCount"`
+	Fallback                     string  `db:"Fallback"`
+	RestOfBookEstimate           int     `db:"RestOfBookEstimate"`
+	CurrentChapterEstimate       int     `db:"CurrentChapterEstimate"`
+	CurrentChapterProgress       float32 `db:"CurrentChapterProgress"`
+	PocketStatus                 int     `db:"PocketStatus"`
+	UnsyncedPocketChanges        string  `db:"UnsyncedPocketChanges"`
+	ImageUrl                     string  `db:"ImageUrl"`
+	DateAdded                    string  `db:"DateAdded"`
+	WorkId                       string  `db:"WorkId"`
+	Properties                   string  `db:"Properties"`
+	RenditionSpread              string  `db:"RenditionSpread"`
+	RatingCount                  int     `db:"RatingCount"`
+	ReviewsSyncDate              string  `db:"ReviewsSyncDate"`
+	MediaOverlay                 string  `db:"MediaOverlay"`
+	MediaOverlayType             string  `db:"MediaOverlayType"`
+	RedirectPreviewUrl           bool    `db:"RedirectPreviewUrl"`
+	PreviewFileSize              int     `db:"PreviewFileSize"`
+	EntitlementId                string  `db:"EntitlementId"`
+	CrossRevisionId              string  `db:"CrossRevisionId"`
+	DownloadUrl                  string  `db:"DownloadUrl"`
+	ReadStateSynced              bool    `db:"ReadStateSynced"`
+	TimesStartedReading          int     `db:"TimesStartedReading"`
+	TimeSpentReading             int     `db:"TimeSpentReading"`
+	LastTimeStartedReading       string  `db:"LastTimeStartedReading"`
+	LastTimeFinishedReading      string  `db:"LastTimeFinishedReading"`
+	ApplicableSubscriptions      string  `db:"ApplicableSubscriptions"`
+	ExternalIds                  string  `db:"ExternalIds"`
+	PurchaseRevisionId           string  `db:"PurchaseRevisionId"`
+	SeriesID                     string  `db:"SeriesID"`
+	SeriesNumberFloat            float64 `db:"SeriesNumberFloat"`
+	AdobeLoanExpiration          string  `db:"AdobeLoanExpiration"`
+	HideFromHomePage             bool    `db:"HideFromHomePage"`
+	IsInternetArchive            bool    `db:"IsInternetArchive"`
+	TitleKana                    string  `db:"titleKana"`
+	SubtitleKana                 string  `db:"subtitleKana"`
+	SeriesKana                   string  `db:"seriesKana"`
+	AttributionKana              string  `db:"attributionKana"`
+	PublisherKana                string  `db:"publisherKana"`
+	IsPurchaseable               bool    `db:"IsPurchaseable"`
+	IsSupported                  bool    `db:"IsSupported"`
+	AnnotationsSyncToken         string  `db:"AnnotationsSyncToken"`
+	DateModified                 string  `db:"DateModified"`
+	StorePages                   int     `db:"StorePages"`
+	StoreWordCount               int     `db:"StoreWordCount"`
+	StoreTimeToReadLowerEstimate int     `db:"StoreTimeToReadLowerEstimate"`
+	StoreTimeToReadUpperEstimate int     `db:"StoreTimeToReadUpperEstimate"`
+	Duration                     int     `db:"Duration"`
+	IsAbridged                   bool    `db:"IsAbridged"`
+}
+
+func CountContent(kobo *Kobo) (count int, err error) {
+	if err := kobo.dbClient.Get(
+		&count,
+		"SELECT count(*) FROM content WHERE ContentType = ? AND VolumeIndex = ? AND MimeType = ?",
+		6, -1, "application/x-kobo-epub+zip",
+	); err != nil {
+		return count, err
+	}
+	return count, nil
+}

v2/pkg/kobo/detection.go

@@ -0,0 +1,54 @@
+package kobo
+
+import (
+	"github.com/pgaskin/koboutils/v2/kobo"
+)
+
+// FindMountDevices runs a variety of detection methods for different operating systems (Linux, macOS, Windows)
+// returning any results it can find. Those results are then checked for the existence of a .kobo
+// folder which is used to determine whether a device is valid or not.
+func FindMountedDevices() ([]string, error) {
+	confirmedLocations := []string{}
+	locations, err := kobo.Find()
+	if err != nil {
+		return nil, err
+	}
+	for _, location := range locations {
+		if kobo.IsKobo(location) {
+			confirmedLocations = append(confirmedLocations, location)
+		}
+	}
+	return confirmedLocations, nil
+}
+
+// GetDeviceMetadata looks up the connected Kobo by checking the version string located at <kobo>/.kobo/version
+// An uninitialised KoboConnection will be returned populated with identifiers like device name, storage etc
+// If a device is unknown (ie; released before we support it), a minimally populated KoboConnection will be
+// returned which should still contain enough information for October to still connect regardless.
+func GetDeviceMetadata(path string) (Kobo, error) {
+	serial, version, deviceId, err := kobo.ParseKoboVersion(path)
+	if err != nil {
+		return Kobo{}, err
+	}
+	device, known := kobo.DeviceByID(deviceId)
+	if !known {
+		// We can handle unsupported Kobos that release in future, before support is added to koboutils
+		// but we should tell the user that no support is 100% guaranteed if there are DB changes etc
+		// We only read data anyway so there is very little risk of just trying out best
+		return Kobo{
+			Name:      "Unknown Device",
+			MountPath: path,
+			DbPath:    formatUsualDbPath(path),
+		}, nil
+	}
+	return Kobo{
+		Name:       device.Name(),
+		Storage:    device.StorageGB(),
+		DisplayPPI: device.DisplayPPI(),
+		MountPath:  path,
+		DbPath:     formatUsualDbPath(path),
+		Serial:     serial,
+		Version:    version,
+		DeviceId:   deviceId,
+	}, nil
+}