scraper.go

package main

import (
	"context"
	"database/sql"
	"encoding/xml"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"
	"github.com/jscmurph/blog_aggregator/internal/database"
)

func startScraping(db *database.Queries, concurrency int, timeBetweenRequest time.Duration) {
    log.Printf("Collecting feeds every %s on %v goroutines...", timeBetweenRequest, concurrency)
    ticker := time.NewTicker(timeBetweenRequest)

    for ; ; <- ticker.C {
        feeds, err := db.GetNextFeedsToFetch(context.Background(), int32(concurrency))
        if err != nil {
            log.Println("Couldn't get next feeds to fetch", err)
            continue
        }

        log.Printf("Found %v feeds to fetch", len(feeds))

        wg := &sync.WaitGroup{}
        for _, feed := range feeds {
            wg.Add(1)
            go scrapeFeed(db, wg, feed)
        }
        wg.Wait()
    }
}

func scrapeFeed(db *database.Queries, wg *sync.WaitGroup, feed database.RssFeed) {
    defer wg.Done()
    _, err := db.MarkFeedFetched(context.Background(), feed.ID)
    if err != nil {
        log.Printf("Couldn't mark feed %s fetched: %v", feed.Name, err)
        return
    }

    feedData, err := fetchFeed(feed.Url)
    if err != nil {
        log.Printf("Couldn't collect feed %s: %v", feed.Name, err)
        return
    }

    for _, item := range feedData.Channel.Item {
        publishedAt := sql.NullTime{}
        if t, err := time.Parse(time.RFC1123Z, item.PubDate); err == nil {
            publishedAt = sql.NullTime{
                Time: t,
                Valid: true,
            }
        }

        _, err = db.CreatePost(context.Background(), database.CreatePostParams{
            ID: uuid.New(),
            CreatedAt: time.Now().UTC(),
            UpdatedAt: time.Now().UTC(),
            FeedID: feed.ID,
            Title: item.Title,
            Description: sql.NullString{
                String: item.Description,
                Valid: true,
            },
            Url: item.Link,
            PublishedAt: publishedAt,
        })

        if err != nil {
            if strings.Contains(err.Error(), "duplicate key value violates unique constrant") {
                continue
            }
            log.Printf("Couldn't create post: %v", err)
            continue
        }
    }

    log.Printf("Feed %s collected, %v posts found", feed.Name, len(feedData.Channel.Item))
}

type RSSFeed struct {
	Channel struct {
		Title       string    `xml:"title"`
		Link        string    `xml:"link"`
		Description string    `xml:"description"`
		Language    string    `xml:"language"`
		Item        []RSSItem `xml:"item"`
	} `xml:"channel"`
}

type RSSItem struct {
	Title       string `xml:"title"`
	Link        string `xml:"link"`
	Description string `xml:"description"`
	PubDate     string `xml:"pubDate"`
}

func fetchFeed(feedURL string) (*RSSFeed, error) {
    httpClient := http.Client {
        Timeout: 10 * time.Second,
    }

    resp, err := httpClient.Get(feedURL)
    if err != nil {
        return nil, err
    }

    defer resp.Body.Close()

    dat, err := io.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }

    var rssFeed RSSFeed
    err = xml.Unmarshal(dat, &rssFeed)
    if err != nil {
        return nil, err
    }

    return &rssFeed, nil
}