FileContent Detector should account for allowed patterns in talismanRC

tinamthomas · svishwanath-tw · commit fe13d1ada25f · 2022-11-01T21:54:43.000+05:30
diff --git a/.talismanrc b/.talismanrc
@@ -11,7 +11,7 @@ fileignoreconfig:
   ignore_detectors:
   - filecontent
 - filename: detector/filecontent/filecontent_detector_test.go
-  checksum: affb25839a87476dcef4f4169ccb9b54b2d2f2437cef3aca24f4d3b69d5886c5
+  checksum: 05d00bb99452d37ab45de28c9e074357e5be22a80d95dad1b2cdd01074b25b2a
 - filename: detector/filename/filename_detector.go
   checksum: 5404565683a7e812fa98ff2d14237c4d1ba7dc5b4aca2dd3ba663b33dc8ddae7
 - filename: detector/filename/filename_detector_test.go
@@ -32,6 +32,8 @@ fileignoreconfig:
   checksum: bf0da9b8b6f779f502564166323a903a49b56b9d8df2597729a5b96c8f066074
 - filename: install.sh
   checksum: c909f6a1caefba3f196d489f9262608044be596a44793c2173ec55b98ecec649
+- filename: talismanrc/talismanrc_test.go
+  checksum: eab40d0745dc215267da86ef7f926be77c4d46e9a248dadbc7e52aa186e82853
 scopeconfig:
 - scope: go
 version: "1.0"
diff --git a/detector/filecontent/filecontent_detector.go b/detector/filecontent/filecontent_detector.go
@@ -119,6 +119,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
 				data := []byte(content)
 				addition.Data = data
 			}
+			addition.Data = []byte(talismanRC.FilterAllowedPatternsFromAddition(addition))
 			for _, ct := range contentTypes {
 				contents <- content{
 					name:        addition.Name,
diff --git a/detector/filecontent/filecontent_detector_test.go b/detector/filecontent/filecontent_detector_test.go
@@ -2,6 +2,7 @@ package filecontent
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 	"talisman/detector/helpers"
 	"talisman/detector/severity"
@@ -20,10 +21,10 @@ var emptyTalismanRC = &talismanrc.TalismanRC{IgnoreConfigs: []talismanrc.IgnoreC
 var defaultChecksumCompareUtility = helpers.
 	NewChecksumCompare(nil, utility.MakeHasher("default", "."), emptyTalismanRC)
 var dummyCallback = func() {}
+var filename = "filename"
 
 func TestShouldNotFlagSafeText(t *testing.T) {
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))}
 
 	NewFileContentDetector(emptyTalismanRC).
@@ -33,7 +34,6 @@ func TestShouldNotFlagSafeText(t *testing.T) {
 
 func TestShouldIgnoreFileIfNeeded(t *testing.T) {
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))}
 	talismanRCIWithFilenameIgnore := &talismanrc.TalismanRC{
 		IgnoreConfigs: []talismanrc.IgnoreConfig{
@@ -59,7 +59,6 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) {
 		 the encoded value of i· rather just a plain abcd input see
 	stackoverflow.com/questions/8571501/how-to-check-whether-the-string-is-base64-encoded-or-not#comment23919648_8571649*/
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("abcd"))}
 
 	NewFileContentDetector(emptyTalismanRC).
@@ -71,7 +70,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
 	const lowEntropyString string = "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWEK"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(lowEntropyString)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
 	NewFileContentDetector(emptyTalismanRC).
@@ -82,7 +80,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
 func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
 	const awsSecretAccessKey string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsSecretAccessKey))}
 	filePath := additions[0].Path
 
@@ -99,7 +96,6 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
 func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(t *testing.T) {
 	const secret string = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9asdfa"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(secret))}
 	filePath := additions[0].Path
 
@@ -117,7 +113,6 @@ func TestShouldFlagPotentialJWT(t *testing.T) {
 		"OiJDaHJpcyBTZXZpbGxlamEiLCJhZG1pbiI6dHJ1ZX0.03f329983b86f7d9a9f5fef85305880101d5e302afafa20154d094b229f757"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(jwt)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
@@ -141,7 +136,6 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) {
 		"}"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(dangerousJavaCode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
@@ -162,7 +156,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
 		"	}\r\n\r\n" +
 		"}"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeJavaCode))}
 
 	NewFileContentDetector(emptyTalismanRC).
@@ -173,7 +166,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
 func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
 	safeLongMethodName := "TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeLongMethodName))}
 
 	NewFileContentDetector(emptyTalismanRC).
@@ -184,7 +176,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
 func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
 	const hex string = "68656C6C6F20776F726C6421"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))}
 	filePath := additions[0].Path
 
@@ -198,7 +189,6 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
 func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) {
 	const creditCardNumber string = "340000000000009"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
 	talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High}
 	checksumCompareWithThreshold := helpers.
@@ -210,12 +200,44 @@ func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) {
 	assert.False(t, results.HasFailures(), "Expected no base64 detection when threshold is higher")
 }
 
+func TestShouldNotFlagPotentialSecretsIfIgnored(t *testing.T) {
+	const hex string = "68656C6C6F20776F726C6421"
+	talismanRCWithIgnores := &talismanrc.TalismanRC{
+		AllowedPatterns: []*regexp.Regexp{regexp.MustCompile("[0-9a-fA-F]*")}}
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
+	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))}
+
+	NewFileContentDetector(emptyTalismanRC).
+		Test(defaultChecksumCompareUtility, additions, talismanRCWithIgnores, results, dummyCallback)
+
+	assert.False(t, results.HasFailures(), "Expected file ignore allowed pattern for hex text")
+}
+
+func TestResultsShouldNotFlagCreditCardNumberIfSpecifiedInFileIgnores(t *testing.T) {
+	const creditCardNumber string = "340000000000009"
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
+	fileIgnoreConfig := &talismanrc.FileIgnoreConfig{
+		FileName: filename, Checksum: "",
+		AllowedPatterns: []string{creditCardNumber},
+	}
+	talismanRCWithFileIgnore := &talismanrc.TalismanRC{
+		IgnoreConfigs:   []talismanrc.IgnoreConfig{fileIgnoreConfig},
+	}
+	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
+
+	NewFileContentDetector(emptyTalismanRC).
+		Test(defaultChecksumCompareUtility, additions, talismanRCWithFileIgnore, results, dummyCallback)
+	
+	assert.False(t, results.HasFailures(), "Expected the creditcard number to be ignored based on talisman RC")
+
+}
+
+
 func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
 	const hex string = "68656C6C6F20776F726C6421"
 	const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	const hexAndBase64 = hex + "\n" + base64
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))}
 	filePath := additions[0].Path
 
@@ -232,7 +254,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
 	const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	const hexAndBase64 = hex + "\n" + base64
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))}
 	filePath := additions[0].Path
 
@@ -248,7 +269,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
 func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *testing.T) {
 	const creditCardNumber string = "340000000000009"
 	results := helpers.NewDetectionResults(talismanrc.HookMode)
-	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
 	filePath := additions[0].Path
 
diff --git a/detector/pattern/pattern_detector.go b/detector/pattern/pattern_detector.go
@@ -50,7 +50,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current
 				ignoredFilePaths <- addition.Path
 				return
 			}
-			detections := detector.secretsPattern.check(processAllowedPatterns(addition, ignoreConfig), ignoreConfig.Threshold)
+			detections := detector.secretsPattern.check(ignoreConfig.FilterAllowedPatternsFromAddition(addition), ignoreConfig.Threshold)
 			matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits}
 		}(addition)
 	}
@@ -77,24 +77,6 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current
 	}
 }
 
-func processAllowedPatterns(addition gitrepo.Addition, tRC *talismanrc.TalismanRC) string {
-	additionPathAsString := string(addition.Path)
-	// Processing global allowed patterns
-	for _, pattern := range tRC.AllowedPatterns {
-		addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
-	}
-
-	// Processing allowed patterns based on file path
-	for _, ignoreConfig := range tRC.IgnoreConfigs {
-		if ignoreConfig.GetFileName() == additionPathAsString {
-			for _, pattern := range ignoreConfig.GetAllowedPatterns() {
-				addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
-			}
-		}
-	}
-	return string(addition.Data)
-}
-
 func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result *helpers.DetectionResults) {
 	log.WithFields(log.Fields{
 		"filePath": ignoredFilePath,
diff --git a/talismanrc/talismanrc.go b/talismanrc/talismanrc.go
@@ -164,6 +164,25 @@ func (tRC *TalismanRC) Deny(addition gitrepo.Addition, detectorName string) bool
 	return false
 }
 
+//Strip git addition 
+func(tRC *TalismanRC) FilterAllowedPatternsFromAddition(addition gitrepo.Addition) string {
+	additionPathAsString := string(addition.Path)
+	// Processing global allowed patterns
+	for _, pattern := range tRC.AllowedPatterns {
+		addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
+	}
+
+	// Processing allowed patterns based on file path
+	for _, ignoreConfig := range tRC.IgnoreConfigs {
+		if ignoreConfig.GetFileName() == additionPathAsString {
+			for _, pattern := range ignoreConfig.GetAllowedPatterns() {
+				addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
+			}
+		}
+	}
+	return string(addition.Data)
+}
+
 func (tRC *TalismanRC) effectiveRules(detectorName string) []string {
 	var result []string
 	for _, ignore := range tRC.IgnoreConfigs {
diff --git a/talismanrc/talismanrc_test.go b/talismanrc/talismanrc_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"testing"
 
 	"talisman/detector/severity"
@@ -47,6 +48,31 @@ func TestShouldIgnoreUnformattedFiles(t *testing.T) {
 	setRepoFileReader(defaultRepoFileReader)
 }
 
+func TestShouldFilterAllowedPatternsFromAddition(t *testing.T) {
+	const hex string = "68656C6C6F20776F726C6421"
+	const fileContent string = "Prefix content" + hex
+	gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent))
+	talismanrc := &TalismanRC{AllowedPatterns: []*regexp.Regexp{regexp.MustCompile(hex)}}
+
+	fileContentFiltered := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1)
+
+	assert.Equal(t, fileContentFiltered, "Prefix content")
+}
+
+func TestShouldFilterAllowedPatternsFromAdditionBasedOnFileConfig(t *testing.T) {
+	const hexContent string = "68656C6C6F20776F726C6421"
+	const fileContent string = "Prefix content" + hexContent
+	gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent))
+	gitRepoAddition2 := testAdditionWithData("file2", []byte(fileContent))
+	talismanrc := createTalismanRCWithFileIgnores("file1", "somedetector", []string{hexContent})
+
+	fileContentFiltered1 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1)
+	fileContentFiltered2 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition2)
+
+	assert.Equal(t, fileContentFiltered1, "Prefix content")
+	assert.Equal(t, fileContentFiltered2, fileContent)
+}
+
 func TestShouldConvertThresholdToValue(t *testing.T) {
 	talismanRCContents := []byte("threshold: high")
 	assert.Equal(t, newPersistedRC(talismanRCContents).Threshold, severity.High)
@@ -142,27 +168,34 @@ func assertDenies(line, ignoreDetector string, path string, t *testing.T) {
 }
 
 func assertDeniesDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) {
-	assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path)
+	assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path)
 }
 
 func assertAccepts(line, ignoreDetector string, path string, t *testing.T, detectorNames ...string) {
 	assertAcceptsDetector(line, ignoreDetector, path, "someDetector", t)
 }
 
 func assertAcceptsDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) {
-	assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path)
+	assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path)
 }
 
 func testAddition(path string) gitrepo.Addition {
 	return gitrepo.NewAddition(path, make([]byte, 0))
 }
 
-func createTalismanRCWithFileIgnores(filename string, detector string) *TalismanRC {
+func testAdditionWithData(path string, content []byte) gitrepo.Addition {
+	return gitrepo.NewAddition(path, content)
+}
+
+func createTalismanRCWithFileIgnores(filename string, detector string, allowedPatterns []string) *TalismanRC {
 	fileIgnoreConfig := &FileIgnoreConfig{}
 	fileIgnoreConfig.FileName = filename
 	if detector != "" {
 		fileIgnoreConfig.IgnoreDetectors = []string{detector}
 	}
+	if len(allowedPatterns) != 0 {
+		fileIgnoreConfig.AllowedPatterns = allowedPatterns
+	}
 
 	return &TalismanRC{IgnoreConfigs: []IgnoreConfig{fileIgnoreConfig}}
 }

Original file line number	Diff line number	Diff line change
`@@ -119,6 +119,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA`
`119`	`119`	`data := []byte(content)`
`120`	`120`	`addition.Data = data`
`121`	`121`	`}`
	`122`	`+ addition.Data = []byte(talismanRC.FilterAllowedPatternsFromAddition(addition))`
`122`	`123`	`for _, ct := range contentTypes {`
`123`	`124`	`contents <- content{`
`124`	`125`	`name: addition.Name,`