file upload api

Makefile

@@ -16,6 +16,10 @@ build:
 	@mkdir -p ./build
 	go build -trimpath -ldflags "-X github.com/flashbots/system-api/common.Version=${VERSION}" -v -o ./build/system-api cmd/system-api/*.go
 
+.PHONY: run
+run:
+	SHELL_TO_USE=/bin/bash go run cmd/system-api/main.go --config systemapi-config.toml
+
 # .PHONY: build-httpserver
 # build-httpserver:
 # 	@mkdir -p ./build

README.md

@@ -8,17 +8,45 @@ It currently does the following things:
  used to record and query events. Useful to record service startup/shutdown, errors, progress updates,
  hashes, etc.
 - **Actions**: Ability to execute shell commands via API
+- **Configuration** through file uploads
 
 Future features:
 
-- Operator can set a password for http-basic-auth (persisted, for all future requests)
-- Operator-provided configuration (i.e. config values, secrets, etc.)
+- Set a password for http-basic-auth (persisted, for all future requests)
 
 ---
 
- ## Event log
+## Getting started
 
- Events can be added via local named pipe (i.e. file `pipe.fifo`) or through HTTP API:
+```bash
+# start the server
+make run
+
+# add events
+echo "hello world" > pipe.fifo
+curl localhost:3535/api/v1/new_event?message=this+is+a+test
+
+# execute actions
+curl -v localhost:3535/api/v1/actions/echo_test
+
+# upload files
+curl -v -X POST -d "@README.md" localhost:3535/api/v1/file-upload/testfile
+
+# get event log
+curl localhost:3535/api/v1/events?format=text
+2024-11-05T22:03:23Z     hello world
+2024-11-05T22:03:26Z     this is a test
+2024-11-05T22:03:29Z     [system-api] executing action: echo_test = echo test
+2024-11-05T22:03:29Z     [system-api] executing action success: echo_test = echo test
+2024-11-05T22:03:31Z     [system-api] file upload: testfile = /tmp/testfile.txt
+2024-11-05T22:03:31Z     [system-api] file upload success: testfile = /tmp/testfile.txt - content: 1991 bytes
+```
+
+---
+
+## Event log
+
+Events can be added via local named pipe (i.e. file `pipe.fifo`) or through HTTP API:
 
 ```bash
 # Start the server
@@ -39,10 +67,26 @@ $ curl localhost:3535/api/v1/events?format=text
  Actions are shell commands that can be executed via API. The commands are defined in the config file,
  see [systemapi-config.toml](./systemapi-config.toml) for examples.
 
+Actions are recorded in the event log.
+
 ```bash
 # Start the server
 $ go run cmd/system-api/main.go --config systemapi-config.toml
 
 # Execute the example action
 $ curl -v localhost:3535/api/v1/actions/echo_test
-```
+```
+
+## File Uploads
+
+Upload destinations are defined in the config file (see [systemapi-config.toml](./systemapi-config.toml)).
+
+File uploads are recorded in the event log.
+
+```bash
+# Start the server
+$ go run cmd/system-api/main.go --config systemapi-config.toml
+
+# Execute the example action
+$ curl -v -X POST -d "@README.md" localhost:3535/api/v1/file-upload/testfile
+```

common/utils.go

@@ -1,7 +1,9 @@
 package common
 
 import (
+	"bytes"
 	"os"
+	"os/exec"
 	"strconv"
 )
 
@@ -24,3 +26,13 @@ func GetEnv(key, defaultValue string) string {
 	}
 	return defaultValue
 }
+
+func Shellout(command string) (string, string, error) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd := exec.Command(ShellToUse, "-c", command)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	return stdout.String(), stderr.String(), err
+}

common/vars.go

@@ -1,3 +1,6 @@
 package common
 
-var Version = "dev"
+var (
+	Version    = "dev"
+	ShellToUse = GetEnv("SHELL_TO_USE", "/bin/ash")
+)

systemapi-config.toml

@@ -3,3 +3,6 @@
 # rbuilder_restart = "/etc/init.d/rbuilder restart"
 # rbuilder_stop = "/etc/init.d/rbuilder stop"
 echo_test = "echo test"
+
+[file_uploads]
+testfile = "/tmp/testfile.txt"

systemapi/config.go

@@ -7,7 +7,8 @@ import (
 )
 
 type SystemAPIConfig struct {
-	Actions map[string]string
+	Actions     map[string]string
+	FileUploads map[string]string `toml:"file_uploads"`
 }
 
 func LoadConfigFromFile(path string) (*SystemAPIConfig, error) {

systemapi/server.go

@@ -3,18 +3,19 @@ package systemapi
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
 	"net/http"
 	"os"
-	"os/exec"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/flashbots/system-api/common"
 	chi "github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/httplog/v2"
@@ -88,6 +89,7 @@ func (s *Server) getRouter() http.Handler {
 	mux.Get("/api/v1/new_event", s.handleNewEvent)
 	mux.Get("/api/v1/events", s.handleGetEvents)
 	mux.Get("/api/v1/actions/{action}", s.handleAction)
+	mux.Post("/api/v1/file-upload/{file}", s.handleFileUpload)
 
 	if s.cfg.EnablePprof {
 		s.log.Info("pprof API enabled")
@@ -155,6 +157,13 @@ func (s *Server) addEvent(event Event) {
 	s.eventsLock.Unlock()
 }
 
+func (s *Server) addInternalEvent(msg string) {
+	s.addEvent(Event{
+		ReceivedAt: time.Now().UTC(),
+		Message:    "[system-api] " + msg,
+	})
+}
+
 func (s *Server) handleNewEvent(w http.ResponseWriter, r *http.Request) {
 	msg := r.URL.Query().Get("message")
 	s.log.Info("Received new event", "message", msg)
@@ -205,28 +214,66 @@ func (s *Server) handleAction(w http.ResponseWriter, r *http.Request) {
 
 	cmd, ok := s.cfg.Config.Actions[action]
 	if !ok {
-		w.WriteHeader(http.StatusNotImplemented)
+		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
 
 	s.log.Info("Executing action", "action", action, "cmd", cmd)
-	stdout, stderr, err := Shellout(cmd)
+	s.addInternalEvent("executing action: " + action + " = " + cmd)
+
+	stdout, stderr, err := common.Shellout(cmd)
 	if err != nil {
 		s.log.Error("Failed to execute action", "action", action, "cmd", cmd, "err", err, "stderr", stderr)
+		s.addInternalEvent("error executing action: " + action + " - error: " + err.Error() + " (stderr: " + stderr + ")")
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
 
 	s.log.Info("Action executed", "action", action, "cmd", cmd, "stdout", stdout, "stderr", stderr)
+	s.addInternalEvent("executing action success: " + action + " = " + cmd)
 	w.WriteHeader(http.StatusOK)
 }
 
-func Shellout(command string) (string, string, error) {
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-	cmd := exec.Command(ShellToUse, "-c", command)
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	return stdout.String(), stderr.String(), err
+func (s *Server) handleFileUpload(w http.ResponseWriter, r *http.Request) {
+	fileArg := chi.URLParam(r, "file")
+	log := s.log.With("file", fileArg)
+	log.Info("Receiving file upload")
+
+	if s.cfg.Config == nil {
+		w.WriteHeader(http.StatusNotImplemented)
+		return
+	}
+
+	filename, ok := s.cfg.Config.FileUploads[fileArg]
+	if !ok {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	log = log.With("filename", filename)
+	s.addInternalEvent("file upload: " + fileArg + " = " + filename)
+
+	// 1. read content from payload r.Body
+	content, err := io.ReadAll(r.Body)
+	if err != nil {
+		log.Error("Failed to read content from payload", "err", err)
+		s.addInternalEvent("file upload error (failed to read): " + fileArg + " = " + filename + " - error: " + err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	log.Debug("Content read from payload", "content", string(content))
+
+	// 2. write content to file
+	err = os.WriteFile(filename, content, 0o600)
+	if err != nil {
+		log.Error("Failed to write content to file", "err", err)
+		s.addInternalEvent("file upload error (failed to write): " + fileArg + " = " + filename + " - error: " + err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	log.Info("File uploaded")
+	s.addInternalEvent(fmt.Sprintf("file upload success: %s = %s - content: %d bytes", fileArg, filename, len(content)))
+	w.WriteHeader(http.StatusOK)
 }

systemapi/vars.go

@@ -2,7 +2,4 @@ package systemapi
 
 import "github.com/flashbots/system-api/common"
 
-var (
-	MaxEvents  = common.GetEnvInt("MAX_EVENTS", 1000)
-	ShellToUse = common.GetEnv("SHELL_TO_USE", "/bin/ash")
-)
+var MaxEvents = common.GetEnvInt("MAX_EVENTS", 1000)