Switch back to original conftest test --update behavior (#1138)

* fix: Switch back to original --update behavior

In conftest v0.61.0 the behavior of --update changed in order to fetch
the policy URLs and stored them in a temporary directory.

It broke the previous behavior that updated current policy directory
and instead aggregated the current policy (under `policy`) with the
one(s) used in `--update`.

It also broke the behavior of a project that had no existing `policy`.

This reverts commit ed0ff0a.

Fixes #1136.

Signed-off-by: Leonardo Taccari <[email protected]>

* test: Test conftest test --update behavior

Add an acceptance test in order to ensure that:

- conftest test --update create/populate policy directory
- conftest test --update update the policy directory with the now
  policies that are fetched remotely

Fixes #1136.

---------

Signed-off-by: Leonardo Taccari <[email protected]>

Author: iamleot

runner/test.go

@@ -55,21 +55,12 @@ func (t *TestRunner) Run(ctx context.Context, fileList []string) (output.CheckRe
 		return nil, fmt.Errorf("parse configurations: %w", err)
 	}
 
-	// When there are policies to download, they are placed in temporary
-	// directory which is passed to the policy engine.
-	// Downloaded policies are removed after the Run to keep the system intact.
+	// When there are policies to download, they are currently placed in the first
+	// directory that appears in the list of policies.
 	if len(t.Update) > 0 {
-		policyDir, err := os.MkdirTemp(os.TempDir(), "remote-policy-")
-		if err != nil {
-			return nil, fmt.Errorf("create temp dir: %w", err)
-		}
-		defer os.RemoveAll(policyDir)
-
-		if err := downloader.Download(ctx, policyDir, t.Update); err != nil {
+		if err := downloader.Download(ctx, t.Policy[0], t.Update); err != nil {
 			return nil, fmt.Errorf("update policies: %w", err)
 		}
-
-		t.Policy = append(t.Policy, policyDir)
 	}
 
 	capabilities, err := policy.LoadCapabilities(t.Capabilities)

tests/multiple-runs-with-update/file.json

@@ -0,0 +1 @@
+{ "a": "a", "b": "b" }

tests/multiple-runs-with-update/remote-policy/policy.rego

@@ -3,6 +3,6 @@ package main
 import rego.v1
 
 deny contains msg if {
-    input.bar == "baz"
-    msg := "local-policy"
+	input.a
+	msg := "a should not be present"
 }

tests/multiple-runs-with-update/test.bats

@@ -0,0 +1,13 @@
+package main
+
+import rego.v1
+
+deny contains msg if {
+	input.a
+	msg := "a should not be present"
+}
+
+deny contains msg if {
+	input.b
+	msg := "b should not be present"
+}

tests/pull-update/file.json

@@ -0,0 +1,43 @@
+#!/usr/bin/env bats
+
+setup_file() {
+	# Create a temporary directory shared by all the tests
+	export TEMP_DIR=$(mktemp -d)
+
+	# Copy all the files there
+	cp -r . "${TEMP_DIR}"
+}
+
+teardown_file() {
+	# Cleanup temporary directory
+	rm -rf "${TEMP_DIR}"
+}
+
+@test "Ensure that policy do not exists" {
+	run test -e "${TEMP_DIR}/policy"
+
+	[ "$status" -eq 1 ]
+}
+
+@test "Pull and update first version policy" {
+	run $CONFTEST test --policy "${TEMP_DIR}/policy" --update "file://${TEMP_DIR}/remote-policy/a" file.json
+
+	[ "$status" -eq 1 ]
+	[[ "$output" =~ "a should not be present" ]]
+	[[ "$output" =~ "1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions" ]]
+}
+
+@test "Ensure that policy directory exists" {
+	run test -d "${TEMP_DIR}/policy"
+
+	[ "$status" -eq 0 ]
+}
+
+@test "Pull and update second version policy" {
+	run $CONFTEST test --policy "${TEMP_DIR}/policy" --update "file://${TEMP_DIR}/remote-policy/b" file.json
+
+	[ "$status" -eq 1 ]
+	[[ "$output" =~ "a should not be present" ]]
+	[[ "$output" =~ "b should not be present" ]]
+	[[ "$output" =~ "2 tests, 0 passed, 0 warnings, 2 failures, 0 exceptions" ]]
+}