Merge branch 'master' into with-environment

Author: mahadzaryab1

.github/workflows/audit.yml

@@ -1,5 +1,6 @@
 name: Security audit
 on:
+  pull_request:
   workflow_dispatch:
   schedule:
     - cron: '0 0 * * *'

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
       - uses: actions/checkout@v3
-      - run: go test ./...
+      - run: go test -race ./...
 
   gocritic:
     runs-on: ubuntu-latest

README.md

@@ -34,6 +34,7 @@ If you're already familiar with shell scripting and the Unix toolset, here is a
 | `>`                | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) |
 | `>>`               | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) |
 | `$*`               | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) |
+| `base64`           | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) |
 | `basename`         | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) |
 | `cat`              | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) |
 | `curl`             | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) / [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) / [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) |
@@ -290,9 +291,11 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to
 | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | removes leading path components from each line, leaving only the filename |
 | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | Nth column of input |
 | [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | contents of multiple files |
+| [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) | input decoded from base64 |
 | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | removes filename from each line, leaving only leading path components |
 | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) | response to supplied HTTP request |
 | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Echo) | all input replaced by given string |
+| [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | input encoded to base64 |
 | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Exec) | filtered through external command |
 | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | execute given command template for each line of input |
 | [`Filter`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Filter) | user-supplied function filtering a reader to a writer |
@@ -330,13 +333,14 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext
 | [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Slice) | | data as `[]string`, error  |
 | [`Stdout`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Stdout) | standard output | bytes written, error  |
 | [`String`](https://pkg.go.dev/github.com/bitfield/script#Pipe.String) | | data as `string`, error  |
-| [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) | | none  |
+| [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) | | error  |
 | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | specified file, truncating if it exists | bytes written, error  |
 
 # What's new
 
 | Version | New |
 | ----------- | ------- |
+| _next_  | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) |
 | v0.22.0 | [`Tee`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Tee), [`WithStderr`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithStderr) |
 | v0.21.0 | HTTP support: [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do), [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get), [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) |
 | v0.20.0 | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) |
@@ -347,7 +351,7 @@ See the [contributor's guide](CONTRIBUTING.md) for some helpful tips if you'd li
 
 # Links
 
-- [Scripting with Go](https://bitfieldconsulting.com/golang/scripting)
+- [Scripting with Go](https://bitfieldconsulting.com/posts/scripting)
 - [Code Club: Script](https://www.youtube.com/watch?v=6S5EqzVwpEg)
 - [Bitfield Consulting](https://bitfieldconsulting.com/)
 - [Go books by John Arundel](https://bitfieldconsulting.com/books)

script.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"container/ring"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -27,9 +28,9 @@ import (
 // Pipe represents a pipe object with an associated [ReadAutoCloser].
 type Pipe struct {
 	// Reader is the underlying reader.
-	Reader         ReadAutoCloser
-	stdout, stderr io.Writer
-	httpClient     *http.Client
+	Reader     ReadAutoCloser
+	stdout     io.Writer
+	httpClient *http.Client
 
 	// because pipe stages are concurrent, protect 'err'
 	mu  *sync.Mutex
@@ -40,6 +41,11 @@ type Pipe struct {
 	// If env is not nil, it will replace the default environment variables
 	// when executing commands.
 	env []string
+
+	// because pipe stages are concurrent, protect 'err' and 'stderr'
+	mu     *sync.Mutex
+	err    error
+	stderr io.Writer
 }
 
 // Args creates a pipe containing the program's command-line arguments from
@@ -282,6 +288,18 @@ func (p *Pipe) CountLines() (lines int, err error) {
 	return lines, p.Error()
 }
 
+// DecodeBase64 produces the string represented by the base64 encoded input.
+func (p *Pipe) DecodeBase64() *Pipe {
+	return p.Filter(func(r io.Reader, w io.Writer) error {
+		decoder := base64.NewDecoder(base64.StdEncoding, r)
+		_, err := io.Copy(w, decoder)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 // Dirname reads paths from the pipe, one per line, and produces only the
 // parent directories of each path. For example, /usr/local/bin/foo would
 // become just /usr/local/bin. This is the complementary operation to
@@ -354,7 +372,23 @@ func (p *Pipe) Echo(s string) *Pipe {
 	return p.WithReader(strings.NewReader(s))
 }
 
+// EncodeBase64 produces the base64 encoding of the input.
+func (p *Pipe) EncodeBase64() *Pipe {
+	return p.Filter(func(r io.Reader, w io.Writer) error {
+		encoder := base64.NewEncoder(base64.StdEncoding, w)
+		defer encoder.Close()
+		_, err := io.Copy(encoder, r)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 // Error returns any error present on the pipe, or nil otherwise.
+// Error is not a sink and does not wait until the pipe reaches
+// completion. To wait for completion before returning the error,
+// see [Pipe.Wait].
 func (p *Pipe) Error() error {
 	if p.mu == nil { // uninitialised pipe
 		return nil
@@ -392,8 +426,9 @@ func (p *Pipe) Exec(cmdLine string) *Pipe {
 		cmd.Stdin = r
 		cmd.Stdout = w
 		cmd.Stderr = w
-		if p.stderr != nil {
-			cmd.Stderr = p.stderr
+		pipeStderr := p.stdErr()
+		if pipeStderr != nil {
+			cmd.Stderr = pipeStderr
 		}
 		if p.env != nil {
 			cmd.Env = p.env
@@ -435,8 +470,9 @@ func (p *Pipe) ExecForEach(cmdLine string) *Pipe {
 			cmd := exec.Command(args[0], args[1:]...)
 			cmd.Stdout = w
 			cmd.Stderr = w
-			if p.stderr != nil {
-				cmd.Stderr = p.stderr
+			pipeStderr := p.stdErr()
+			if pipeStderr != nil {
+				cmd.Stderr = pipeStderr
 			}
 			if p.env != nil {
 				cmd.Env = p.env
@@ -823,6 +859,18 @@ func (p *Pipe) Slice() ([]string, error) {
 	return result, p.Error()
 }
 
+// stdErr returns the pipe's configured standard error writer for commands run
+// via [Pipe.Exec] and [Pipe.ExecForEach]. The default is nil, which means that
+// error output will go to the pipe.
+func (p *Pipe) stdErr() io.Writer {
+	if p.mu == nil { // uninitialised pipe
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.stderr
+}
+
 // Stdout copies the pipe's contents to its configured standard output (using
 // [Pipe.WithStdout]), or to [os.Stdout] otherwise, and returns the number of
 // bytes successfully written, together with any error.
@@ -861,14 +909,15 @@ func (p *Pipe) Tee(writers ...io.Writer) *Pipe {
 	return p.WithReader(io.TeeReader(p.Reader, teeWriter))
 }
 
-// Wait reads the pipe to completion and discards the result. This is mostly
-// useful for waiting until concurrent filters have completed (see
-// [Pipe.Filter]).
-func (p *Pipe) Wait() {
+// Wait reads the pipe to completion and returns any error present on
+// the pipe, or nil otherwise. This is mostly useful for waiting until
+// concurrent filters have completed (see [Pipe.Filter]).
+func (p *Pipe) Wait() error {
 	_, err := io.Copy(io.Discard, p)
 	if err != nil {
 		p.SetError(err)
 	}
+	return p.Error()
 }
 
 // WithEnv sets the environment for subsequent [Pipe.Exec] and [Pipe.ExecForEach] commands
@@ -905,10 +954,11 @@ func (p *Pipe) WithReader(r io.Reader) *Pipe {
 	return p
 }
 
-// WithStderr redirects the standard error output for commands run via
-// [Pipe.Exec] or [Pipe.ExecForEach] to the writer w, instead of going to the
-// pipe as it normally would.
+// WithStderr sets the standard error output for [Pipe.Exec] or
+// [Pipe.ExecForEach] commands to w, instead of the pipe.
 func (p *Pipe) WithStderr(w io.Writer) *Pipe {
+	p.mu.Lock()
+	defer p.mu.Unlock()
 	p.stderr = w
 	return p
 }

script_test.go

@@ -1896,6 +1896,135 @@ func TestReadReturnsErrorGivenReadErrorOnPipe(t *testing.T) {
 	}
 }
 
+func TestWait_ReturnsErrorPresentOnPipe(t *testing.T) {
+	t.Parallel()
+	p := script.Echo("a\nb\nc\n").ExecForEach("{{invalid template syntax}}")
+	if p.Wait() == nil {
+		t.Error("want error, got nil")
+	}
+}
+
+func TestWait_DoesNotReturnErrorForValidExecution(t *testing.T) {
+	t.Parallel()
+	p := script.Echo("a\nb\nc\n").ExecForEach("echo \"{{.}}\"")
+	if err := p.Wait(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+var base64Cases = []struct {
+	name    string
+	decoded string
+	encoded string
+}{
+	{
+		name:    "empty string",
+		decoded: "",
+		encoded: "",
+	},
+	{
+		name:    "single line string",
+		decoded: "hello world",
+		encoded: "aGVsbG8gd29ybGQ=",
+	},
+	{
+		name:    "multi line string",
+		decoded: "hello\nthere\nworld\n",
+		encoded: "aGVsbG8KdGhlcmUKd29ybGQK",
+	},
+}
+
+func TestEncodeBase64_CorrectlyEncodes(t *testing.T) {
+	t.Parallel()
+	for _, tc := range base64Cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := script.Echo(tc.decoded).EncodeBase64().String()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tc.encoded {
+				t.Logf("input %q incorrectly encoded:", tc.decoded)
+				t.Error(cmp.Diff(tc.encoded, got))
+			}
+		})
+	}
+}
+
+func TestDecodeBase64_CorrectlyDecodes(t *testing.T) {
+	t.Parallel()
+	for _, tc := range base64Cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := script.Echo(tc.encoded).DecodeBase64().String()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tc.decoded {
+				t.Logf("input %q incorrectly decoded:", tc.encoded)
+				t.Error(cmp.Diff(tc.decoded, got))
+			}
+		})
+	}
+}
+
+func TestEncodeBase64_FollowedByDecodeRecoversOriginal(t *testing.T) {
+	t.Parallel()
+	for _, tc := range base64Cases {
+		t.Run(tc.name, func(t *testing.T) {
+			decoded, err := script.Echo(tc.decoded).EncodeBase64().DecodeBase64().String()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if decoded != tc.decoded {
+				t.Error("encode-decode round trip failed:", cmp.Diff(tc.decoded, decoded))
+			}
+			encoded, err := script.Echo(tc.encoded).DecodeBase64().EncodeBase64().String()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if encoded != tc.encoded {
+				t.Error("decode-encode round trip failed:", cmp.Diff(tc.encoded, encoded))
+			}
+		})
+	}
+}
+
+func TestDecodeBase64_CorrectlyDecodesInputToBytes(t *testing.T) {
+	t.Parallel()
+	input := "CAAAEA=="
+	got, err := script.Echo(input).DecodeBase64().Bytes()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []byte{8, 0, 0, 16}
+	if !bytes.Equal(want, got) {
+		t.Logf("input %#v incorrectly decoded:", input)
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
+func TestEncodeBase64_CorrectlyEncodesInputBytes(t *testing.T) {
+	t.Parallel()
+	input := []byte{8, 0, 0, 16}
+	reader := bytes.NewReader(input)
+	want := "CAAAEA=="
+	got, err := script.NewPipe().WithReader(reader).EncodeBase64().String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Logf("input %#v incorrectly encoded:", input)
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
+func TestWithStdErr_IsConcurrencySafeAfterExec(t *testing.T) {
+	t.Parallel()
+	err := script.Exec("echo").WithStderr(nil).Wait()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func ExampleArgs() {
 	script.Args().Stdout()
 	// prints command-line arguments
@@ -2015,6 +2144,12 @@ func ExamplePipe_CountLines() {
 	// 3
 }
 
+func ExamplePipe_DecodeBase64() {
+	script.Echo("SGVsbG8sIHdvcmxkIQ==").DecodeBase64().Stdout()
+	// Output:
+	// Hello, world!
+}
+
 func ExamplePipe_Do() {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		data, err := io.ReadAll(r.Body)
@@ -2050,6 +2185,12 @@ func ExamplePipe_Echo() {
 	// Hello, world!
 }
 
+func ExamplePipe_EncodeBase64() {
+	script.Echo("Hello, world!").EncodeBase64().Stdout()
+	// Output:
+	// SGVsbG8sIHdvcmxkIQ==
+}
+
 func ExamplePipe_ExitStatus() {
 	p := script.Exec("echo")
 	fmt.Println(p.ExitStatus())