fix: synchronize tf init to avoid plugin race condition

Author: rafatio

Makefile

@@ -115,7 +115,7 @@ GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v3.8.7
 CONTROLLER_TOOLS_VERSION ?= v0.16.4
-GOLANGCI_LINT_VERSION ?= v1.60.3
+GOLANGCI_LINT_VERSION ?= v2.5.0
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
@@ -135,7 +135,7 @@ $(ENVTEST): $(LOCALBIN)
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary
 $(GOLANGCI_LINT): $(LOCALBIN)
-	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 
 # Development environment targets
 setup-dev-env: create-kind init-tilt

controllers/controlplane/kopscontrolplane_controller.go

@@ -184,7 +184,9 @@ func (r *KopsControlPlaneReconciler) PrepareCustomCloudResources(ctx context.Con
 		if err != nil {
 			return err
 		}
-		defer karpenterResourcesContent.Close()
+		defer func() {
+			_ = karpenterResourcesContent.Close()
+		}()
 
 		// This is needed because the apply will fail if the file is empty
 		placeholder := corev1.ConfigMap{

pkg/utils/kops_utils.go

@@ -367,7 +367,9 @@ func GetUserDataFromTerraformFile(clusterName, igName, terraformOutputDir string
 	if err != nil {
 		return "", err
 	}
-	defer userDataFile.Close()
+	defer func() {
+		_ = userDataFile.Close()
+	}()
 	userData, err := io.ReadAll(userDataFile)
 	if err != nil {
 		return "", err

pkg/utils/kops_utils_test.go

@@ -125,10 +125,10 @@ func TestParseSpotinstFeatureflags(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			os.Unsetenv("SPOTINST_TOKEN")
-			os.Unsetenv("SPOTINST_ACCOUNT")
+			_ = os.Unsetenv("SPOTINST_TOKEN")
+			_ = os.Unsetenv("SPOTINST_ACCOUNT")
 			for key, value := range tc.environmentVariables {
-				os.Setenv(key, value)
+				_ = os.Setenv(key, value)
 			}
 
 			err := ParseSpotinstFeatureflags(tc.input)

pkg/utils/terraform_utils.go

@@ -8,7 +8,10 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"syscall"
 	"text/template"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -25,6 +28,40 @@ type Template struct {
 //go:embed templates/*.tpl
 var templates embed.FS
 
+var tfPluginMux sync.Mutex
+
+func lockPluginCache(pluginCacheDir string) (*os.File, error) {
+	if err := os.MkdirAll(pluginCacheDir, 0755); err != nil {
+		return nil, err
+	}
+
+	lockPath := filepath.Join(pluginCacheDir, ".terraform-plugin.lock")
+	lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open lock file: %w", err)
+	}
+
+	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil {
+		_ = lockFile.Close()
+		return nil, fmt.Errorf("failed to acquire exclusive lock: %w", err)
+	}
+
+	return lockFile, nil
+}
+
+func unlockPluginCache(lockFile *os.File) error {
+	if lockFile == nil {
+		return nil
+	}
+
+	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN); err != nil {
+		_ = lockFile.Close()
+		return fmt.Errorf("failed to unlock plugin cache: %w", err)
+	}
+
+	return lockFile.Close()
+}
+
 // CreateTerraformFileFromTemplate populates a Terraform template and create files in the state
 func CreateTerraformFilesFromTemplate(terraformTemplateFilePath string, TerraformOutputFileName string, terraformOutputDir string, templateData any) error {
 	template := Template{
@@ -43,7 +80,9 @@ func CreateAdditionalTerraformFiles(tfFiles ...Template) error {
 		if err != nil {
 			return err
 		}
-		defer file.Close()
+		defer func() {
+			_ = file.Close()
+		}()
 
 		t := template.New(filepath.Base(tfFile.TemplateFilename)).Funcs(template.FuncMap{
 			"stringReplace": strings.Replace,
@@ -96,23 +135,52 @@ func initTerraform(ctx context.Context, workingDir, terraformExecPath string, cr
 		return nil, err
 	}
 
+	pluginCacheDir := fmt.Sprintf("%s/plugin-cache", filepath.Dir(terraformExecPath))
+
 	env := map[string]string{
 		"AWS_ACCESS_KEY_ID":     credentials.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY": credentials.SecretAccessKey,
 		"SPOTINST_TOKEN":        os.Getenv("SPOTINST_TOKEN"),
 		"SPOTINST_ACCOUNT":      os.Getenv("SPOTINST_ACCOUNT"),
-		"TF_PLUGIN_CACHE_DIR":   fmt.Sprintf("%s/plugin-cache", filepath.Dir(terraformExecPath)),
+		"TF_PLUGIN_CACHE_DIR":   pluginCacheDir,
 	}
 
-	// this overrides all ENVVARs that are passed to Terraform
 	err = tf.SetEnv(env)
 	if err != nil {
 		return nil, err
 	}
 
-	err = tf.Init(ctx, tfexec.Upgrade(true))
+	tfPluginMux.Lock()
+	defer tfPluginMux.Unlock()
+
+	lockFile, err := lockPluginCache(pluginCacheDir)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to acquire plugin cache lock: %w", err)
+	}
+	defer func() {
+		time.Sleep(500 * time.Millisecond)
+		_ = unlockPluginCache(lockFile)
+	}()
+
+	var initErr error
+	maxRetries := 3
+	for i := 0; i < maxRetries; i++ {
+		initErr = tf.Init(ctx, tfexec.Upgrade(true))
+		if initErr == nil {
+			break
+		}
+
+		if strings.Contains(initErr.Error(), "text file busy") && i < maxRetries-1 {
+			waitTime := time.Duration(i+1) * 2 * time.Second
+			time.Sleep(waitTime)
+			continue
+		}
+
+		break
+	}
+
+	if initErr != nil {
+		return nil, initErr
 	}
 
 	return tf, nil
@@ -127,12 +195,24 @@ func ApplyTerraform(ctx context.Context, workingDir, terraformExecPath string, c
 		return err
 	}
 
-	err = tf.Apply(ctx)
-	if err != nil {
-		return err
+	var applyErr error
+	maxRetries := 5
+	for i := 0; i < maxRetries; i++ {
+		applyErr = tf.Apply(ctx)
+		if applyErr == nil {
+			return nil
+		}
+
+		if strings.Contains(applyErr.Error(), "text file busy") && i < maxRetries-1 {
+			waitTime := time.Duration(i+1) * time.Second
+			time.Sleep(waitTime)
+			continue
+		}
+
+		break
 	}
 
-	return nil
+	return applyErr
 }
 
 // PlanTerraform just applies the already created terraform files
@@ -150,12 +230,24 @@ func PlanTerraform(ctx context.Context, workingDir, terraformExecPath string, cr
 		return err
 	}
 
-	_, err = tf.Plan(ctx, tfexec.Out(workingDir+"/plan.out"))
-	if err != nil {
-		return err
+	var planErr error
+	maxRetries := 5
+	for i := 0; i < maxRetries; i++ {
+		_, planErr = tf.Plan(ctx, tfexec.Out(workingDir+"/plan.out"))
+		if planErr == nil {
+			return nil
+		}
+
+		if strings.Contains(planErr.Error(), "text file busy") && i < maxRetries-1 {
+			waitTime := time.Duration(i+1) * time.Second
+			time.Sleep(waitTime)
+			continue
+		}
+
+		break
 	}
 
-	return nil
+	return planErr
 }
 
 func DestroyTerraform(ctx context.Context, workingDir, terraformExecPath string, credentials aws.Credentials) error {
@@ -164,20 +256,34 @@ func DestroyTerraform(ctx context.Context, workingDir, terraformExecPath string,
 		return err
 	}
 
-	err = tf.Destroy(ctx)
-	if err != nil {
-		return err
+	var destroyErr error
+	maxRetries := 5
+	for i := 0; i < maxRetries; i++ {
+		destroyErr = tf.Destroy(ctx)
+		if destroyErr == nil {
+			return nil
+		}
+
+		if strings.Contains(destroyErr.Error(), "text file busy") && i < maxRetries-1 {
+			waitTime := time.Duration(i+1) * time.Second
+			time.Sleep(waitTime)
+			continue
+		}
+
+		break
 	}
 
-	return nil
+	return destroyErr
 }
 
 func CleanupTerraformDirectory(dir string) error {
 	d, err := os.Open(dir)
 	if err != nil {
 		return err
 	}
-	defer d.Close()
+	defer func() {
+		_ = d.Close()
+	}()
 	names, err := d.Readdirnames(-1)
 	if err != nil {
 		return err

pkg/utils/terraform_utils_test.go

@@ -222,7 +222,9 @@ func TestModifyTerraformProviderVersion(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			tmpDir, err := os.MkdirTemp("", "test_terraform_provider")
 			g.Expect(err).NotTo(HaveOccurred())
-			defer os.RemoveAll(tmpDir)
+			defer func() {
+				_ = os.RemoveAll(tmpDir)
+			}()
 
 			kubernetesFile := fmt.Sprintf("%s/kubernetes.tf", tmpDir)