Skip to content

Commit 350d563

Browse files
neurosnapneurosnap
authored
feeds.sh (#2)
* feat(feeds): init
1 parent 32a6c64 commit 350d563

36 files changed

+1693
-535
lines changed

‎.env.example

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,24 @@ IMGS_PROTOCOL=http
9292
IMGS_ALLOW_REGISTER=1
9393
IMGS_STORAGE_DIR=.storage
9494
IMGS_DEBUG=1
95+
96+
SENDGRID_API_KEY=
97+
FEEDS_V4=
98+
FEEDS_V6=
99+
FEEDS_HTTP_V4=$FEEDS_V4:80
100+
FEEDS_HTTP_V6=[$FEEDS_V6]:80
101+
FEEDS_HTTPS_V4=$FEEDS_V4:443
102+
FEEDS_HTTPS_V6=[$FEEDS_V6]:443
103+
FEEDS_SSH_V4=$FEEDS_V4:22
104+
FEEDS_SSH_V6=[$FEEDS_V6]:22
105+
FEEDS_HOST=
106+
FEEDS_SSH_PORT=2222
107+
FEEDS_WEB_PORT=3000
108+
FEEDS_PROM_PORT=9222
109+
FEEDS_DOMAIN=feeds.dev.pico.sh:3004
110+
FEEDS_EMAIL=[email protected]
111+
FEEDS_SUBDOMAINS=1
112+
FEEDS_CUSTOMDOMAINS=1
113+
FEEDS_PROTOCOL=http
114+
FEEDS_ALLOW_REGISTER=1
115+
FEEDS_DEBUG=1

‎.github/workflows/build.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ jobs:
8484
app: imgs
8585
platforms: ${{ env.PLATFORMS }}
8686
registry: ${{ env.REGISTRY }}
87+
- name: Run docker build for imgs
88+
uses: ./.github/actions/build
89+
with:
90+
app: feeds
91+
platforms: ${{ env.PLATFORMS }}
92+
registry: ${{ env.REGISTRY }}
8793
build-caddy:
8894
runs-on: ubuntu-22.04
8995
needs: test

‎Makefile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ css:
1212
cp ./smol.css prose/public/main.css
1313
cp ./smol.css pastes/public/main.css
1414
cp ./smol.css imgs/public/main.css
15+
cp ./smol.css feeds/public/main.css
1516

1617
cp ./syntax.css pastes/public/syntax.css
1718
cp ./syntax.css prose/public/syntax.css
@@ -35,15 +36,15 @@ bp-%: bp-setup
3536
$(DOCKER_BUILDX_BUILD) --build-arg "APP=$*" -t "ghcr.io/picosh/pico/$*-web:$(DOCKER_TAG)" --target release-web .
3637
.PHONY: bp-%
3738

38-
bp-all: bp-prose bp-lists bp-pastes bp-imgs
39+
bp-all: bp-prose bp-lists bp-pastes bp-imgs bp-feeds
3940
.PHONY: bp-all
4041

4142
build-%:
4243
go build -o "build/$*-web" "./cmd/$*/web"
4344
go build -o "build/$*-ssh" "./cmd/$*/ssh"
4445
.PHONY: build-%
4546

46-
build: build-prose build-lists build-pastes build-imgs
47+
build: build-prose build-lists build-pastes build-imgs build-feeds
4748
.PHONY: build
4849

4950
format:

‎cmd/feeds/ssh/main.go

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"syscall"
9+
"time"
10+
11+
"github.com/charmbracelet/promwish"
12+
"github.com/charmbracelet/wish"
13+
bm "github.com/charmbracelet/wish/bubbletea"
14+
lm "github.com/charmbracelet/wish/logging"
15+
"github.com/gliderlabs/ssh"
16+
"github.com/picosh/pico/db/postgres"
17+
"github.com/picosh/pico/feeds"
18+
"github.com/picosh/pico/filehandlers"
19+
"github.com/picosh/pico/imgs/storage"
20+
"github.com/picosh/pico/shared"
21+
"github.com/picosh/pico/wish/cms"
22+
"github.com/picosh/pico/wish/list"
23+
"github.com/picosh/pico/wish/pipe"
24+
"github.com/picosh/pico/wish/proxy"
25+
"github.com/picosh/pico/wish/send/auth"
26+
wishrsync "github.com/picosh/pico/wish/send/rsync"
27+
"github.com/picosh/pico/wish/send/scp"
28+
"github.com/picosh/pico/wish/send/sftp"
29+
)
30+
31+
type SSHServer struct{}
32+
33+
func (me *SSHServer) authHandler(ctx ssh.Context, key ssh.PublicKey) bool {
34+
return true
35+
}
36+
37+
func createRouter(handler *filehandlers.ScpUploadHandler) proxy.Router {
38+
return func(sh ssh.Handler, s ssh.Session) []wish.Middleware {
39+
return []wish.Middleware{
40+
pipe.Middleware(handler, ".txt"),
41+
list.Middleware(handler),
42+
scp.Middleware(handler),
43+
wishrsync.Middleware(handler),
44+
auth.Middleware(handler),
45+
bm.Middleware(cms.Middleware(&handler.Cfg.ConfigCms, handler.Cfg)),
46+
lm.Middleware(),
47+
}
48+
}
49+
}
50+
51+
func withProxy(handler *filehandlers.ScpUploadHandler, otherMiddleware ...wish.Middleware) ssh.Option {
52+
return func(server *ssh.Server) error {
53+
err := sftp.SSHOption(handler)(server)
54+
if err != nil {
55+
return err
56+
}
57+
58+
return proxy.WithProxy(createRouter(handler), otherMiddleware...)(server)
59+
}
60+
}
61+
62+
func main() {
63+
host := shared.GetEnv("LISTS_HOST", "0.0.0.0")
64+
port := shared.GetEnv("LISTS_SSH_PORT", "2222")
65+
promPort := shared.GetEnv("LISTS_PROM_PORT", "9222")
66+
cfg := feeds.NewConfigSite()
67+
logger := cfg.Logger
68+
dbh := postgres.NewDB(&cfg.ConfigCms)
69+
defer dbh.Close()
70+
71+
hooks := &feeds.FeedHooks{
72+
Cfg: cfg,
73+
Db: dbh,
74+
}
75+
76+
var st storage.ObjectStorage
77+
var err error
78+
if cfg.MinioURL == "" {
79+
st, err = storage.NewStorageFS(cfg.StorageDir)
80+
} else {
81+
st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
82+
}
83+
84+
if err != nil {
85+
logger.Fatal(err)
86+
}
87+
88+
handler := filehandlers.NewScpPostHandler(dbh, cfg, hooks, st)
89+
90+
sshServer := &SSHServer{}
91+
s, err := wish.NewServer(
92+
wish.WithAddress(fmt.Sprintf("%s:%s", host, port)),
93+
wish.WithHostKeyPath("ssh_data/term_info_ed25519"),
94+
wish.WithPublicKeyAuth(sshServer.authHandler),
95+
withProxy(
96+
handler,
97+
promwish.Middleware(fmt.Sprintf("%s:%s", host, promPort), "feeds-ssh"),
98+
),
99+
)
100+
if err != nil {
101+
logger.Fatal(err)
102+
}
103+
104+
done := make(chan os.Signal, 1)
105+
signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
106+
logger.Infof("Starting SSH server on %s:%s", host, port)
107+
go func() {
108+
if err = s.ListenAndServe(); err != nil {
109+
logger.Fatal(err)
110+
}
111+
}()
112+
113+
<-done
114+
logger.Info("Stopping SSH server")
115+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
116+
defer func() { cancel() }()
117+
if err := s.Shutdown(ctx); err != nil {
118+
logger.Fatal(err)
119+
}
120+
}

‎cmd/feeds/web/main.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package main
2+
3+
import "github.com/picosh/pico/feeds"
4+
5+
func main() {
6+
feeds.StartApiServer()
7+
}

‎cmd/scripts/dates/dates.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"github.com/picosh/pico/db"
1111
"github.com/picosh/pico/db/postgres"
1212
"github.com/picosh/pico/imgs"
13-
"github.com/picosh/pico/lists"
1413
"github.com/picosh/pico/shared"
1514
"github.com/picosh/pico/wish/cms/config"
1615
"go.uber.org/zap"
@@ -117,15 +116,20 @@ func main() {
117116
}
118117
}
119118
} else if post.Space == "lists" {
120-
parsed := lists.ParseText(post.Text, linkify)
119+
linkify := imgs.NewImgsLinkify(post.Username)
120+
parsed := shared.ListParseText(post.Text, linkify)
121+
if err != nil {
122+
logger.Error(err)
123+
continue
124+
}
121125

122-
if parsed.MetaData.PublishAt != nil && !parsed.MetaData.PublishAt.IsZero() {
123-
err = updateDates(tx, post.ID, parsed.MetaData.PublishAt)
126+
if parsed.PublishAt != nil && !parsed.PublishAt.IsZero() {
127+
err = updateDates(tx, post.ID, parsed.PublishAt)
124128
if err != nil {
125129
logger.Error(err)
126130
continue
127131
}
128-
if !parsed.MetaData.PublishAt.Equal(*post.PublishAt) {
132+
if !parsed.PublishAt.Equal(*post.PublishAt) {
129133
datesFixed = append(datesFixed, post.ID)
130134
}
131135
}

‎db/db.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ type User struct {
2727
}
2828

2929
type PostData struct {
30-
ImgPath string `json:"img_path"`
30+
ImgPath string `json:"img_path"`
31+
LastDigest *time.Time `json:"last_digest"`
3132
}
3233

3334
// Make the Attrs struct implement the driver.Valuer interface. This method

‎docker-compose.override.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,23 @@ services:
9494
- ./data/imgs-ssh/data:/app/ssh_data
9595
ports:
9696
- "2223:2222"
97+
feeds-web:
98+
build:
99+
args:
100+
APP: feeds
101+
target: release-web
102+
env_file:
103+
- .env.example
104+
ports:
105+
- "3004:3000"
106+
feeds-ssh:
107+
build:
108+
args:
109+
APP: feeds
110+
target: release-ssh
111+
env_file:
112+
- .env.example
113+
volumes:
114+
- ./data/feeds-ssh/data:/app/ssh_data
115+
ports:
116+
- "2224:2222"

‎docker-compose.prod.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,48 @@ services:
204204
ports:
205205
- "${IMGS_SSH_V4:-22}:2222"
206206
- "${IMGS_SSH_V6:-[::1]:22}:2222"
207+
feeds-caddy:
208+
image: ghcr.io/picosh/pico/caddy:latest
209+
restart: always
210+
networks:
211+
- feeds
212+
env_file:
213+
- .env.prod
214+
environment:
215+
APP_DOMAIN: ${FEEDS_DOMAIN:-feeds.sh}
216+
APP_EMAIL: ${FEEDS_EMAIL:[email protected]}
217+
volumes:
218+
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
219+
- ./data/feeds-caddy/data:/data
220+
- ./data/feeds-caddy/config:/config
221+
ports:
222+
- "${FEEDS_HTTPS_V4:-443}:443"
223+
- "${FEEDS_HTTP_V4:-80}:80"
224+
- "${FEEDS_HTTPS_V6:-[::1]:443}:443"
225+
- "${FEEDS_HTTP_V6:-[::1]:80}:80"
226+
profiles:
227+
- feeds
228+
- caddy
229+
- all
230+
feeds-web:
231+
networks:
232+
feeds:
233+
aliases:
234+
- web
235+
env_file:
236+
- .env.prod
237+
feeds-ssh:
238+
networks:
239+
feeds:
240+
aliases:
241+
- ssh
242+
env_file:
243+
- .env.prod
244+
volumes:
245+
- ./data/feeds-ssh/data:/app/ssh_data
246+
ports:
247+
- "${FEEDS_SSH_V4:-22}:2222"
248+
- "${FEEDS_SSH_V6:-[::1]:22}:2222"
207249

208250
networks:
209251
default:
@@ -235,3 +277,9 @@ networks:
235277
ipam:
236278
config:
237279
- subnet: 172.21.0.0/16
280+
feeds:
281+
driver_opts:
282+
com.docker.network.bridge.name: feeds
283+
ipam:
284+
config:
285+
- subnet: 172.22.0.0/16

‎docker-compose.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,17 @@ services:
6969
- imgs
7070
- services
7171
- all
72+
feeds-web:
73+
image: ghcr.io/picosh/pico/feeds-web:latest
74+
restart: always
75+
profiles:
76+
- feeds
77+
- services
78+
- all
79+
feeds-ssh:
80+
image: ghcr.io/picosh/pico/feeds-ssh:latest
81+
restart: always
82+
profiles:
83+
- feeds
84+
- services
85+
- all

0 commit comments

Comments
 (0)