feeds.sh (#2)

* feat(feeds): init

.env.example

@@ -92,3 +92,24 @@ IMGS_PROTOCOL=http
 IMGS_ALLOW_REGISTER=1
 IMGS_STORAGE_DIR=.storage
 IMGS_DEBUG=1
+
+SENDGRID_API_KEY=
+FEEDS_V4=
+FEEDS_V6=
+FEEDS_HTTP_V4=$FEEDS_V4:80
+FEEDS_HTTP_V6=[$FEEDS_V6]:80
+FEEDS_HTTPS_V4=$FEEDS_V4:443
+FEEDS_HTTPS_V6=[$FEEDS_V6]:443
+FEEDS_SSH_V4=$FEEDS_V4:22
+FEEDS_SSH_V6=[$FEEDS_V6]:22
+FEEDS_HOST=
+FEEDS_SSH_PORT=2222
+FEEDS_WEB_PORT=3000
+FEEDS_PROM_PORT=9222
+FEEDS_DOMAIN=feeds.dev.pico.sh:3004
+FEEDS_EMAIL=[email protected]
+FEEDS_SUBDOMAINS=1
+FEEDS_CUSTOMDOMAINS=1
+FEEDS_PROTOCOL=http
+FEEDS_ALLOW_REGISTER=1
+FEEDS_DEBUG=1

.github/workflows/build.yml

@@ -84,6 +84,12 @@ jobs:
         app: imgs
         platforms: ${{ env.PLATFORMS }}
         registry: ${{ env.REGISTRY }}
+    - name: Run docker build for imgs
+      uses: ./.github/actions/build
+      with:
+        app: feeds
+        platforms: ${{ env.PLATFORMS }}
+        registry: ${{ env.REGISTRY }}
   build-caddy:
     runs-on: ubuntu-22.04
     needs: test

Makefile

@@ -12,6 +12,7 @@ css:
 	cp ./smol.css prose/public/main.css
 	cp ./smol.css pastes/public/main.css
 	cp ./smol.css imgs/public/main.css
+	cp ./smol.css feeds/public/main.css
 
 	cp ./syntax.css pastes/public/syntax.css
 	cp ./syntax.css prose/public/syntax.css
@@ -35,15 +36,15 @@ bp-%: bp-setup
 	$(DOCKER_BUILDX_BUILD) --build-arg "APP=$*" -t "ghcr.io/picosh/pico/$*-web:$(DOCKER_TAG)" --target release-web .
 .PHONY: bp-%
 
-bp-all: bp-prose bp-lists bp-pastes bp-imgs
+bp-all: bp-prose bp-lists bp-pastes bp-imgs bp-feeds
 .PHONY: bp-all
 
 build-%:
 	go build -o "build/$*-web" "./cmd/$*/web"
 	go build -o "build/$*-ssh" "./cmd/$*/ssh"
 .PHONY: build-%
 
-build: build-prose build-lists build-pastes build-imgs
+build: build-prose build-lists build-pastes build-imgs build-feeds
 .PHONY: build
 
 format:

cmd/feeds/ssh/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/charmbracelet/promwish"
+	"github.com/charmbracelet/wish"
+	bm "github.com/charmbracelet/wish/bubbletea"
+	lm "github.com/charmbracelet/wish/logging"
+	"github.com/gliderlabs/ssh"
+	"github.com/picosh/pico/db/postgres"
+	"github.com/picosh/pico/feeds"
+	"github.com/picosh/pico/filehandlers"
+	"github.com/picosh/pico/imgs/storage"
+	"github.com/picosh/pico/shared"
+	"github.com/picosh/pico/wish/cms"
+	"github.com/picosh/pico/wish/list"
+	"github.com/picosh/pico/wish/pipe"
+	"github.com/picosh/pico/wish/proxy"
+	"github.com/picosh/pico/wish/send/auth"
+	wishrsync "github.com/picosh/pico/wish/send/rsync"
+	"github.com/picosh/pico/wish/send/scp"
+	"github.com/picosh/pico/wish/send/sftp"
+)
+
+type SSHServer struct{}
+
+func (me *SSHServer) authHandler(ctx ssh.Context, key ssh.PublicKey) bool {
+	return true
+}
+
+func createRouter(handler *filehandlers.ScpUploadHandler) proxy.Router {
+	return func(sh ssh.Handler, s ssh.Session) []wish.Middleware {
+		return []wish.Middleware{
+			pipe.Middleware(handler, ".txt"),
+			list.Middleware(handler),
+			scp.Middleware(handler),
+			wishrsync.Middleware(handler),
+			auth.Middleware(handler),
+			bm.Middleware(cms.Middleware(&handler.Cfg.ConfigCms, handler.Cfg)),
+			lm.Middleware(),
+		}
+	}
+}
+
+func withProxy(handler *filehandlers.ScpUploadHandler, otherMiddleware ...wish.Middleware) ssh.Option {
+	return func(server *ssh.Server) error {
+		err := sftp.SSHOption(handler)(server)
+		if err != nil {
+			return err
+		}
+
+		return proxy.WithProxy(createRouter(handler), otherMiddleware...)(server)
+	}
+}
+
+func main() {
+	host := shared.GetEnv("LISTS_HOST", "0.0.0.0")
+	port := shared.GetEnv("LISTS_SSH_PORT", "2222")
+	promPort := shared.GetEnv("LISTS_PROM_PORT", "9222")
+	cfg := feeds.NewConfigSite()
+	logger := cfg.Logger
+	dbh := postgres.NewDB(&cfg.ConfigCms)
+	defer dbh.Close()
+
+	hooks := &feeds.FeedHooks{
+		Cfg: cfg,
+		Db:  dbh,
+	}
+
+	var st storage.ObjectStorage
+	var err error
+	if cfg.MinioURL == "" {
+		st, err = storage.NewStorageFS(cfg.StorageDir)
+	} else {
+		st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
+	}
+
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	handler := filehandlers.NewScpPostHandler(dbh, cfg, hooks, st)
+
+	sshServer := &SSHServer{}
+	s, err := wish.NewServer(
+		wish.WithAddress(fmt.Sprintf("%s:%s", host, port)),
+		wish.WithHostKeyPath("ssh_data/term_info_ed25519"),
+		wish.WithPublicKeyAuth(sshServer.authHandler),
+		withProxy(
+			handler,
+			promwish.Middleware(fmt.Sprintf("%s:%s", host, promPort), "feeds-ssh"),
+		),
+	)
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	logger.Infof("Starting SSH server on %s:%s", host, port)
+	go func() {
+		if err = s.ListenAndServe(); err != nil {
+			logger.Fatal(err)
+		}
+	}()
+
+	<-done
+	logger.Info("Stopping SSH server")
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer func() { cancel() }()
+	if err := s.Shutdown(ctx); err != nil {
+		logger.Fatal(err)
+	}
+}

cmd/feeds/web/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/picosh/pico/feeds"
+
+func main() {
+	feeds.StartApiServer()
+}

cmd/scripts/dates/dates.go

@@ -10,7 +10,6 @@ import (
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/db/postgres"
 	"github.com/picosh/pico/imgs"
-	"github.com/picosh/pico/lists"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/wish/cms/config"
 	"go.uber.org/zap"
@@ -117,15 +116,20 @@ func main() {
 				}
 			}
 		} else if post.Space == "lists" {
-			parsed := lists.ParseText(post.Text, linkify)
+			linkify := imgs.NewImgsLinkify(post.Username)
+			parsed := shared.ListParseText(post.Text, linkify)
+			if err != nil {
+				logger.Error(err)
+				continue
+			}
 
-			if parsed.MetaData.PublishAt != nil && !parsed.MetaData.PublishAt.IsZero() {
-				err = updateDates(tx, post.ID, parsed.MetaData.PublishAt)
+			if parsed.PublishAt != nil && !parsed.PublishAt.IsZero() {
+				err = updateDates(tx, post.ID, parsed.PublishAt)
 				if err != nil {
 					logger.Error(err)
 					continue
 				}
-				if !parsed.MetaData.PublishAt.Equal(*post.PublishAt) {
+				if !parsed.PublishAt.Equal(*post.PublishAt) {
 					datesFixed = append(datesFixed, post.ID)
 				}
 			}

db/db.go

@@ -27,7 +27,8 @@ type User struct {
 }
 
 type PostData struct {
-	ImgPath string `json:"img_path"`
+	ImgPath    string     `json:"img_path"`
+	LastDigest *time.Time `json:"last_digest"`
 }
 
 // Make the Attrs struct implement the driver.Valuer interface. This method

docker-compose.override.yml

@@ -94,3 +94,23 @@ services:
       - ./data/imgs-ssh/data:/app/ssh_data
     ports:
       - "2223:2222"
+  feeds-web:
+    build:
+      args:
+        APP: feeds
+      target: release-web
+    env_file:
+      - .env.example
+    ports:
+      - "3004:3000"
+  feeds-ssh:
+    build:
+      args:
+        APP: feeds
+      target: release-ssh
+    env_file:
+      - .env.example
+    volumes:
+      - ./data/feeds-ssh/data:/app/ssh_data
+    ports:
+      - "2224:2222"

docker-compose.prod.yml

@@ -204,6 +204,48 @@ services:
     ports:
       - "${IMGS_SSH_V4:-22}:2222"
       - "${IMGS_SSH_V6:-[::1]:22}:2222"
+  feeds-caddy:
+    image: ghcr.io/picosh/pico/caddy:latest
+    restart: always
+    networks:
+      - feeds
+    env_file:
+      - .env.prod
+    environment:
+      APP_DOMAIN: ${FEEDS_DOMAIN:-feeds.sh}
+      APP_EMAIL: ${FEEDS_EMAIL:[email protected]}
+    volumes:
+      - ./caddy/Caddyfile:/etc/caddy/Caddyfile
+      - ./data/feeds-caddy/data:/data
+      - ./data/feeds-caddy/config:/config
+    ports:
+      - "${FEEDS_HTTPS_V4:-443}:443"
+      - "${FEEDS_HTTP_V4:-80}:80"
+      - "${FEEDS_HTTPS_V6:-[::1]:443}:443"
+      - "${FEEDS_HTTP_V6:-[::1]:80}:80"
+    profiles:
+      - feeds
+      - caddy
+      - all
+  feeds-web:
+    networks:
+      feeds:
+        aliases:
+          - web
+    env_file:
+      - .env.prod
+  feeds-ssh:
+    networks:
+      feeds:
+        aliases:
+          - ssh
+    env_file:
+      - .env.prod
+    volumes:
+      - ./data/feeds-ssh/data:/app/ssh_data
+    ports:
+      - "${FEEDS_SSH_V4:-22}:2222"
+      - "${FEEDS_SSH_V6:-[::1]:22}:2222"
 
 networks:
   default:
@@ -235,3 +277,9 @@ networks:
     ipam:
       config:
         - subnet: 172.21.0.0/16
+  feeds:
+    driver_opts:
+      com.docker.network.bridge.name: feeds
+    ipam:
+      config:
+        - subnet: 172.22.0.0/16

docker-compose.yml

@@ -69,3 +69,17 @@ services:
       - imgs
       - services
       - all
+  feeds-web:
+    image: ghcr.io/picosh/pico/feeds-web:latest
+    restart: always
+    profiles:
+      - feeds
+      - services
+      - all
+  feeds-ssh:
+    image: ghcr.io/picosh/pico/feeds-ssh:latest
+    restart: always
+    profiles:
+      - feeds
+      - services
+      - all