diff --git a/Makefile b/Makefile index 9bce923..b96ca1a 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ build: @mkdir -p ./build go build -trimpath -ldflags "-X github.com/flashbots/system-api/common.Version=${VERSION}" -v -o ./build/system-api cmd/system-api/*.go +.PHONY: run +run: + SHELL_TO_USE=/bin/bash go run cmd/system-api/main.go --config systemapi-config.toml + # .PHONY: build-httpserver # build-httpserver: # @mkdir -p ./build diff --git a/README.md b/README.md index 33c02d6..beebad3 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,45 @@ It currently does the following things: used to record and query events. Useful to record service startup/shutdown, errors, progress updates, hashes, etc. - **Actions**: Ability to execute shell commands via API +- **Configuration** through file uploads Future features: -- Operator can set a password for http-basic-auth (persisted, for all future requests) -- Operator-provided configuration (i.e. config values, secrets, etc.) +- Set a password for http-basic-auth (persisted, for all future requests) --- - ## Event log +## Getting started - Events can be added via local named pipe (i.e. file `pipe.fifo`) or through HTTP API: +```bash +# start the server +make run + +# add events +echo "hello world" > pipe.fifo +curl localhost:3535/api/v1/new_event?message=this+is+a+test + +# execute actions +curl -v localhost:3535/api/v1/actions/echo_test + +# upload files +curl -v -X POST -d "@README.md" localhost:3535/api/v1/file-upload/testfile + +# get event log +curl localhost:3535/api/v1/events?format=text +2024-11-05T22:03:23Z hello world +2024-11-05T22:03:26Z this is a test +2024-11-05T22:03:29Z [system-api] executing action: echo_test = echo test +2024-11-05T22:03:29Z [system-api] executing action success: echo_test = echo test +2024-11-05T22:03:31Z [system-api] file upload: testfile = /tmp/testfile.txt +2024-11-05T22:03:31Z [system-api] file upload success: testfile = /tmp/testfile.txt - content: 1991 bytes +``` + +--- + +## Event log + +Events can be added via local named pipe (i.e. file `pipe.fifo`) or through HTTP API: ```bash # Start the server @@ -39,10 +67,26 @@ $ curl localhost:3535/api/v1/events?format=text Actions are shell commands that can be executed via API. The commands are defined in the config file, see [systemapi-config.toml](./systemapi-config.toml) for examples. +Actions are recorded in the event log. + ```bash # Start the server $ go run cmd/system-api/main.go --config systemapi-config.toml # Execute the example action $ curl -v localhost:3535/api/v1/actions/echo_test -``` \ No newline at end of file +``` + +## File Uploads + +Upload destinations are defined in the config file (see [systemapi-config.toml](./systemapi-config.toml)). + +File uploads are recorded in the event log. + +```bash +# Start the server +$ go run cmd/system-api/main.go --config systemapi-config.toml + +# Execute the example action +$ curl -v -X POST -d "@README.md" localhost:3535/api/v1/file-upload/testfile +``` diff --git a/common/utils.go b/common/utils.go index 89beb52..3d522cb 100644 --- a/common/utils.go +++ b/common/utils.go @@ -1,7 +1,9 @@ package common import ( + "bytes" "os" + "os/exec" "strconv" ) @@ -24,3 +26,13 @@ func GetEnv(key, defaultValue string) string { } return defaultValue } + +func Shellout(command string) (string, string, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd := exec.Command(ShellToUse, "-c", command) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} diff --git a/common/vars.go b/common/vars.go index f6c5d47..70fbaa8 100644 --- a/common/vars.go +++ b/common/vars.go @@ -1,3 +1,6 @@ package common -var Version = "dev" +var ( + Version = "dev" + ShellToUse = GetEnv("SHELL_TO_USE", "/bin/ash") +) diff --git a/systemapi-config.toml b/systemapi-config.toml index 7880d8f..20952f0 100644 --- a/systemapi-config.toml +++ b/systemapi-config.toml @@ -3,3 +3,6 @@ # rbuilder_restart = "/etc/init.d/rbuilder restart" # rbuilder_stop = "/etc/init.d/rbuilder stop" echo_test = "echo test" + +[file_uploads] +testfile = "/tmp/testfile.txt" diff --git a/systemapi/config.go b/systemapi/config.go index 35aee39..175a75e 100644 --- a/systemapi/config.go +++ b/systemapi/config.go @@ -7,7 +7,8 @@ import ( ) type SystemAPIConfig struct { - Actions map[string]string + Actions map[string]string + FileUploads map[string]string `toml:"file_uploads"` } func LoadConfigFromFile(path string) (*SystemAPIConfig, error) { diff --git a/systemapi/server.go b/systemapi/server.go index a976857..b62b215 100644 --- a/systemapi/server.go +++ b/systemapi/server.go @@ -3,18 +3,19 @@ package systemapi import ( "bufio" - "bytes" "context" "encoding/json" "errors" + "fmt" + "io" "net/http" "os" - "os/exec" "strings" "sync" "syscall" "time" + "github.com/flashbots/system-api/common" chi "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httplog/v2" @@ -88,6 +89,7 @@ func (s *Server) getRouter() http.Handler { mux.Get("/api/v1/new_event", s.handleNewEvent) mux.Get("/api/v1/events", s.handleGetEvents) mux.Get("/api/v1/actions/{action}", s.handleAction) + mux.Post("/api/v1/file-upload/{file}", s.handleFileUpload) if s.cfg.EnablePprof { s.log.Info("pprof API enabled") @@ -155,6 +157,13 @@ func (s *Server) addEvent(event Event) { s.eventsLock.Unlock() } +func (s *Server) addInternalEvent(msg string) { + s.addEvent(Event{ + ReceivedAt: time.Now().UTC(), + Message: "[system-api] " + msg, + }) +} + func (s *Server) handleNewEvent(w http.ResponseWriter, r *http.Request) { msg := r.URL.Query().Get("message") s.log.Info("Received new event", "message", msg) @@ -205,28 +214,66 @@ func (s *Server) handleAction(w http.ResponseWriter, r *http.Request) { cmd, ok := s.cfg.Config.Actions[action] if !ok { - w.WriteHeader(http.StatusNotImplemented) + w.WriteHeader(http.StatusBadRequest) return } s.log.Info("Executing action", "action", action, "cmd", cmd) - stdout, stderr, err := Shellout(cmd) + s.addInternalEvent("executing action: " + action + " = " + cmd) + + stdout, stderr, err := common.Shellout(cmd) if err != nil { s.log.Error("Failed to execute action", "action", action, "cmd", cmd, "err", err, "stderr", stderr) + s.addInternalEvent("error executing action: " + action + " - error: " + err.Error() + " (stderr: " + stderr + ")") w.WriteHeader(http.StatusInternalServerError) return } s.log.Info("Action executed", "action", action, "cmd", cmd, "stdout", stdout, "stderr", stderr) + s.addInternalEvent("executing action success: " + action + " = " + cmd) w.WriteHeader(http.StatusOK) } -func Shellout(command string) (string, string, error) { - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd := exec.Command(ShellToUse, "-c", command) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - return stdout.String(), stderr.String(), err +func (s *Server) handleFileUpload(w http.ResponseWriter, r *http.Request) { + fileArg := chi.URLParam(r, "file") + log := s.log.With("file", fileArg) + log.Info("Receiving file upload") + + if s.cfg.Config == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + + filename, ok := s.cfg.Config.FileUploads[fileArg] + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + + log = log.With("filename", filename) + s.addInternalEvent("file upload: " + fileArg + " = " + filename) + + // 1. read content from payload r.Body + content, err := io.ReadAll(r.Body) + if err != nil { + log.Error("Failed to read content from payload", "err", err) + s.addInternalEvent("file upload error (failed to read): " + fileArg + " = " + filename + " - error: " + err.Error()) + w.WriteHeader(http.StatusInternalServerError) + return + } + + log.Debug("Content read from payload", "content", string(content)) + + // 2. write content to file + err = os.WriteFile(filename, content, 0o600) + if err != nil { + log.Error("Failed to write content to file", "err", err) + s.addInternalEvent("file upload error (failed to write): " + fileArg + " = " + filename + " - error: " + err.Error()) + w.WriteHeader(http.StatusInternalServerError) + return + } + + log.Info("File uploaded") + s.addInternalEvent(fmt.Sprintf("file upload success: %s = %s - content: %d bytes", fileArg, filename, len(content))) + w.WriteHeader(http.StatusOK) } diff --git a/systemapi/vars.go b/systemapi/vars.go index 35674b1..2439cb4 100644 --- a/systemapi/vars.go +++ b/systemapi/vars.go @@ -2,7 +2,4 @@ package systemapi import "github.com/flashbots/system-api/common" -var ( - MaxEvents = common.GetEnvInt("MAX_EVENTS", 1000) - ShellToUse = common.GetEnv("SHELL_TO_USE", "/bin/ash") -) +var MaxEvents = common.GetEnvInt("MAX_EVENTS", 1000)