From fe13d1ada25f9cb05e20b21f19e16787c4d3f00e Mon Sep 17 00:00:00 2001 From: Tina Thomas Date: Mon, 31 Oct 2022 11:35:54 -0400 Subject: [PATCH] FileContent Detector should account for allowed patterns in talismanRC --- .talismanrc | 4 +- detector/filecontent/filecontent_detector.go | 1 + .../filecontent/filecontent_detector_test.go | 50 +++++++++++++------ detector/pattern/pattern_detector.go | 20 +------- talismanrc/talismanrc.go | 19 +++++++ talismanrc/talismanrc_test.go | 39 +++++++++++++-- 6 files changed, 95 insertions(+), 38 deletions(-) diff --git a/.talismanrc b/.talismanrc index 7b14263b..afac5cf3 100644 --- a/.talismanrc +++ b/.talismanrc @@ -11,7 +11,7 @@ fileignoreconfig: ignore_detectors: - filecontent - filename: detector/filecontent/filecontent_detector_test.go - checksum: affb25839a87476dcef4f4169ccb9b54b2d2f2437cef3aca24f4d3b69d5886c5 + checksum: 05d00bb99452d37ab45de28c9e074357e5be22a80d95dad1b2cdd01074b25b2a - filename: detector/filename/filename_detector.go checksum: 5404565683a7e812fa98ff2d14237c4d1ba7dc5b4aca2dd3ba663b33dc8ddae7 - filename: detector/filename/filename_detector_test.go @@ -32,6 +32,8 @@ fileignoreconfig: checksum: bf0da9b8b6f779f502564166323a903a49b56b9d8df2597729a5b96c8f066074 - filename: install.sh checksum: c909f6a1caefba3f196d489f9262608044be596a44793c2173ec55b98ecec649 +- filename: talismanrc/talismanrc_test.go + checksum: eab40d0745dc215267da86ef7f926be77c4d46e9a248dadbc7e52aa186e82853 scopeconfig: - scope: go version: "1.0" diff --git a/detector/filecontent/filecontent_detector.go b/detector/filecontent/filecontent_detector.go index 79407bfe..e01fadbf 100644 --- a/detector/filecontent/filecontent_detector.go +++ b/detector/filecontent/filecontent_detector.go @@ -119,6 +119,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA data := []byte(content) addition.Data = data } + addition.Data = []byte(talismanRC.FilterAllowedPatternsFromAddition(addition)) for _, ct := range contentTypes { contents <- content{ name: addition.Name, diff --git a/detector/filecontent/filecontent_detector_test.go b/detector/filecontent/filecontent_detector_test.go index 43d81ede..055ba4c9 100644 --- a/detector/filecontent/filecontent_detector_test.go +++ b/detector/filecontent/filecontent_detector_test.go @@ -2,6 +2,7 @@ package filecontent import ( "fmt" + "regexp" "strings" "talisman/detector/helpers" "talisman/detector/severity" @@ -20,10 +21,10 @@ var emptyTalismanRC = &talismanrc.TalismanRC{IgnoreConfigs: []talismanrc.IgnoreC var defaultChecksumCompareUtility = helpers. NewChecksumCompare(nil, utility.MakeHasher("default", "."), emptyTalismanRC) var dummyCallback = func() {} +var filename = "filename" func TestShouldNotFlagSafeText(t *testing.T) { results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))} NewFileContentDetector(emptyTalismanRC). @@ -33,7 +34,6 @@ func TestShouldNotFlagSafeText(t *testing.T) { func TestShouldIgnoreFileIfNeeded(t *testing.T) { results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))} talismanRCIWithFilenameIgnore := &talismanrc.TalismanRC{ IgnoreConfigs: []talismanrc.IgnoreConfig{ @@ -59,7 +59,6 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) { the encoded value of i· rather just a plain abcd input see stackoverflow.com/questions/8571501/how-to-check-whether-the-string-is-base64-encoded-or-not#comment23919648_8571649*/ results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("abcd"))} NewFileContentDetector(emptyTalismanRC). @@ -71,7 +70,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) { const lowEntropyString string = "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWEK" results := helpers.NewDetectionResults(talismanrc.HookMode) content := []byte(lowEntropyString) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} NewFileContentDetector(emptyTalismanRC). @@ -82,7 +80,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) { func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) { const awsSecretAccessKey string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsSecretAccessKey))} filePath := additions[0].Path @@ -99,7 +96,6 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) { func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(t *testing.T) { const secret string = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9asdfa" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(secret))} filePath := additions[0].Path @@ -117,7 +113,6 @@ func TestShouldFlagPotentialJWT(t *testing.T) { "OiJDaHJpcyBTZXZpbGxlamEiLCJhZG1pbiI6dHJ1ZX0.03f329983b86f7d9a9f5fef85305880101d5e302afafa20154d094b229f757" results := helpers.NewDetectionResults(talismanrc.HookMode) content := []byte(jwt) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path @@ -141,7 +136,6 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) { "}" results := helpers.NewDetectionResults(talismanrc.HookMode) content := []byte(dangerousJavaCode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path @@ -162,7 +156,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) { " }\r\n\r\n" + "}" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeJavaCode))} NewFileContentDetector(emptyTalismanRC). @@ -173,7 +166,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) { func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) { safeLongMethodName := "TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeLongMethodName))} NewFileContentDetector(emptyTalismanRC). @@ -184,7 +176,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) { func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) { const hex string = "68656C6C6F20776F726C6421" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))} filePath := additions[0].Path @@ -198,7 +189,6 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) { func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) { const creditCardNumber string = "340000000000009" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))} talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High} checksumCompareWithThreshold := helpers. @@ -210,12 +200,44 @@ func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) { assert.False(t, results.HasFailures(), "Expected no base64 detection when threshold is higher") } +func TestShouldNotFlagPotentialSecretsIfIgnored(t *testing.T) { + const hex string = "68656C6C6F20776F726C6421" + talismanRCWithIgnores := &talismanrc.TalismanRC{ + AllowedPatterns: []*regexp.Regexp{regexp.MustCompile("[0-9a-fA-F]*")}} + results := helpers.NewDetectionResults(talismanrc.HookMode) + additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))} + + NewFileContentDetector(emptyTalismanRC). + Test(defaultChecksumCompareUtility, additions, talismanRCWithIgnores, results, dummyCallback) + + assert.False(t, results.HasFailures(), "Expected file ignore allowed pattern for hex text") +} + +func TestResultsShouldNotFlagCreditCardNumberIfSpecifiedInFileIgnores(t *testing.T) { + const creditCardNumber string = "340000000000009" + results := helpers.NewDetectionResults(talismanrc.HookMode) + fileIgnoreConfig := &talismanrc.FileIgnoreConfig{ + FileName: filename, Checksum: "", + AllowedPatterns: []string{creditCardNumber}, + } + talismanRCWithFileIgnore := &talismanrc.TalismanRC{ + IgnoreConfigs: []talismanrc.IgnoreConfig{fileIgnoreConfig}, + } + additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))} + + NewFileContentDetector(emptyTalismanRC). + Test(defaultChecksumCompareUtility, additions, talismanRCWithFileIgnore, results, dummyCallback) + + assert.False(t, results.HasFailures(), "Expected the creditcard number to be ignored based on talisman RC") + +} + + func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) { const hex string = "68656C6C6F20776F726C6421" const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" const hexAndBase64 = hex + "\n" + base64 results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))} filePath := additions[0].Path @@ -232,7 +254,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T) const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" const hexAndBase64 = hex + "\n" + base64 results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))} filePath := additions[0].Path @@ -248,7 +269,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T) func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *testing.T) { const creditCardNumber string = "340000000000009" results := helpers.NewDetectionResults(talismanrc.HookMode) - filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))} filePath := additions[0].Path diff --git a/detector/pattern/pattern_detector.go b/detector/pattern/pattern_detector.go index 1b475026..3a428b0a 100644 --- a/detector/pattern/pattern_detector.go +++ b/detector/pattern/pattern_detector.go @@ -50,7 +50,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current ignoredFilePaths <- addition.Path return } - detections := detector.secretsPattern.check(processAllowedPatterns(addition, ignoreConfig), ignoreConfig.Threshold) + detections := detector.secretsPattern.check(ignoreConfig.FilterAllowedPatternsFromAddition(addition), ignoreConfig.Threshold) matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits} }(addition) } @@ -77,24 +77,6 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current } } -func processAllowedPatterns(addition gitrepo.Addition, tRC *talismanrc.TalismanRC) string { - additionPathAsString := string(addition.Path) - // Processing global allowed patterns - for _, pattern := range tRC.AllowedPatterns { - addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) - } - - // Processing allowed patterns based on file path - for _, ignoreConfig := range tRC.IgnoreConfigs { - if ignoreConfig.GetFileName() == additionPathAsString { - for _, pattern := range ignoreConfig.GetAllowedPatterns() { - addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) - } - } - } - return string(addition.Data) -} - func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result *helpers.DetectionResults) { log.WithFields(log.Fields{ "filePath": ignoredFilePath, diff --git a/talismanrc/talismanrc.go b/talismanrc/talismanrc.go index 4bc00d91..90bdd588 100644 --- a/talismanrc/talismanrc.go +++ b/talismanrc/talismanrc.go @@ -164,6 +164,25 @@ func (tRC *TalismanRC) Deny(addition gitrepo.Addition, detectorName string) bool return false } +//Strip git addition +func(tRC *TalismanRC) FilterAllowedPatternsFromAddition(addition gitrepo.Addition) string { + additionPathAsString := string(addition.Path) + // Processing global allowed patterns + for _, pattern := range tRC.AllowedPatterns { + addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) + } + + // Processing allowed patterns based on file path + for _, ignoreConfig := range tRC.IgnoreConfigs { + if ignoreConfig.GetFileName() == additionPathAsString { + for _, pattern := range ignoreConfig.GetAllowedPatterns() { + addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) + } + } + } + return string(addition.Data) +} + func (tRC *TalismanRC) effectiveRules(detectorName string) []string { var result []string for _, ignore := range tRC.IgnoreConfigs { diff --git a/talismanrc/talismanrc_test.go b/talismanrc/talismanrc_test.go index 99792a43..1da13561 100644 --- a/talismanrc/talismanrc_test.go +++ b/talismanrc/talismanrc_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "regexp" "testing" "talisman/detector/severity" @@ -47,6 +48,31 @@ func TestShouldIgnoreUnformattedFiles(t *testing.T) { setRepoFileReader(defaultRepoFileReader) } +func TestShouldFilterAllowedPatternsFromAddition(t *testing.T) { + const hex string = "68656C6C6F20776F726C6421" + const fileContent string = "Prefix content" + hex + gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent)) + talismanrc := &TalismanRC{AllowedPatterns: []*regexp.Regexp{regexp.MustCompile(hex)}} + + fileContentFiltered := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1) + + assert.Equal(t, fileContentFiltered, "Prefix content") +} + +func TestShouldFilterAllowedPatternsFromAdditionBasedOnFileConfig(t *testing.T) { + const hexContent string = "68656C6C6F20776F726C6421" + const fileContent string = "Prefix content" + hexContent + gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent)) + gitRepoAddition2 := testAdditionWithData("file2", []byte(fileContent)) + talismanrc := createTalismanRCWithFileIgnores("file1", "somedetector", []string{hexContent}) + + fileContentFiltered1 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1) + fileContentFiltered2 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition2) + + assert.Equal(t, fileContentFiltered1, "Prefix content") + assert.Equal(t, fileContentFiltered2, fileContent) +} + func TestShouldConvertThresholdToValue(t *testing.T) { talismanRCContents := []byte("threshold: high") assert.Equal(t, newPersistedRC(talismanRCContents).Threshold, severity.High) @@ -142,7 +168,7 @@ func assertDenies(line, ignoreDetector string, path string, t *testing.T) { } func assertDeniesDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) { - assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path) + assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path) } func assertAccepts(line, ignoreDetector string, path string, t *testing.T, detectorNames ...string) { @@ -150,19 +176,26 @@ func assertAccepts(line, ignoreDetector string, path string, t *testing.T, detec } func assertAcceptsDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) { - assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path) + assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path) } func testAddition(path string) gitrepo.Addition { return gitrepo.NewAddition(path, make([]byte, 0)) } -func createTalismanRCWithFileIgnores(filename string, detector string) *TalismanRC { +func testAdditionWithData(path string, content []byte) gitrepo.Addition { + return gitrepo.NewAddition(path, content) +} + +func createTalismanRCWithFileIgnores(filename string, detector string, allowedPatterns []string) *TalismanRC { fileIgnoreConfig := &FileIgnoreConfig{} fileIgnoreConfig.FileName = filename if detector != "" { fileIgnoreConfig.IgnoreDetectors = []string{detector} } + if len(allowedPatterns) != 0 { + fileIgnoreConfig.AllowedPatterns = allowedPatterns + } return &TalismanRC{IgnoreConfigs: []IgnoreConfig{fileIgnoreConfig}} }