Refactor scraping to include related object fields

pkg/scraper/cache.go

@@ -16,7 +16,6 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/txn"
 )
 
@@ -262,19 +261,23 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
 		return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
 	}
 
-	ignoredRegex := c.compileExcludeTagPatterns()
-
-	var ignoredTags []string
-	for i, cc := range content {
-		var thisIgnoredTags []string
-		content[i], thisIgnoredTags, err = c.postScrape(ctx, cc, ignoredRegex)
-		if err != nil {
-			return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
+	pp := postScraper{
+		Cache:        c,
+		excludeTagRE: c.compileExcludeTagPatterns(),
+	}
+	if err := c.repository.WithReadTxn(ctx, func(ctx context.Context) error {
+		for i, cc := range content {
+			content[i], err = pp.postScrape(ctx, cc)
+			if err != nil {
+				return fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
+			}
 		}
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
+		return nil
+	}); err != nil {
+		return nil, err
 	}
 
-	LogIgnoredTags(ignoredTags)
+	LogIgnoredTags(pp.ignoredTags)
 
 	return content, nil
 }

pkg/scraper/image.go

@@ -37,88 +37,43 @@ func setPerformerImage(ctx context.Context, client *http.Client, p *models.Scrap
 	return nil
 }
 
-func setSceneImage(ctx context.Context, client *http.Client, s *models.ScrapedScene, globalConfig GlobalConfig) error {
-	// don't try to get the image if it doesn't appear to be a URL
-	if s.Image == nil || !strings.HasPrefix(*s.Image, "http") {
-		// nothing to do
-		return nil
-	}
-
-	img, err := getImage(ctx, *s.Image, client, globalConfig)
-	if err != nil {
-		return err
-	}
-
-	s.Image = img
-
-	return nil
-}
-
-func setMovieFrontImage(ctx context.Context, client *http.Client, m *models.ScrapedMovie, globalConfig GlobalConfig) error {
-	// don't try to get the image if it doesn't appear to be a URL
-	if m.FrontImage == nil || !strings.HasPrefix(*m.FrontImage, "http") {
-		// nothing to do
-		return nil
-	}
-
-	img, err := getImage(ctx, *m.FrontImage, client, globalConfig)
-	if err != nil {
-		return err
-	}
-
-	m.FrontImage = img
-
-	return nil
-}
-
-func setMovieBackImage(ctx context.Context, client *http.Client, m *models.ScrapedMovie, globalConfig GlobalConfig) error {
-	// don't try to get the image if it doesn't appear to be a URL
-	if m.BackImage == nil || !strings.HasPrefix(*m.BackImage, "http") {
+func setStudioImage(ctx context.Context, client *http.Client, p *models.ScrapedStudio, globalConfig GlobalConfig) error {
+	// backwards compatibility: we fetch the image if it's a URL and set it to the first image
+	// Image is deprecated, so only do this if Images is unset
+	if p.Image == nil || len(p.Images) > 0 {
 		// nothing to do
 		return nil
 	}
 
-	img, err := getImage(ctx, *m.BackImage, client, globalConfig)
-	if err != nil {
-		return err
-	}
-
-	m.BackImage = img
-
-	return nil
-}
-
-func setGroupFrontImage(ctx context.Context, client *http.Client, m *models.ScrapedGroup, globalConfig GlobalConfig) error {
 	// don't try to get the image if it doesn't appear to be a URL
-	if m.FrontImage == nil || !strings.HasPrefix(*m.FrontImage, "http") {
-		// nothing to do
+	if !strings.HasPrefix(*p.Image, "http") {
+		p.Images = []string{*p.Image}
 		return nil
 	}
 
-	img, err := getImage(ctx, *m.FrontImage, client, globalConfig)
+	img, err := getImage(ctx, *p.Image, client, globalConfig)
 	if err != nil {
 		return err
 	}
 
-	m.FrontImage = img
+	p.Image = img
+	// Image is deprecated. Use images instead
+	p.Images = []string{*img}
 
 	return nil
 }
 
-func setGroupBackImage(ctx context.Context, client *http.Client, m *models.ScrapedGroup, globalConfig GlobalConfig) error {
-	// don't try to get the image if it doesn't appear to be a URL
-	if m.BackImage == nil || !strings.HasPrefix(*m.BackImage, "http") {
-		// nothing to do
+func processImageField(ctx context.Context, imageField *string, client *http.Client, globalConfig GlobalConfig) error {
+	if imageField == nil {
 		return nil
 	}
 
-	img, err := getImage(ctx, *m.BackImage, client, globalConfig)
+	img, err := getImage(ctx, *imageField, client, globalConfig)
 	if err != nil {
 		return err
 	}
 
-	m.BackImage = img
-
+	*imageField = *img
 	return nil
 }