From 91f71e0ef616b01a723efe891d8a7258a2b6875a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 9 May 2025 22:48:56 +0400 Subject: [PATCH] feat(cli): upgrade secret scanner --- cli/config/allowlist_test.go | 115 - cli/config/config.go | 279 -- cli/config/config_test.go | 170 - cli/config/example-infisical-relay.yaml | 8 - cli/config/infisical-relay.yaml | 8 - cli/config/infisical-scan.toml | 2803 --------------- cli/config/rule.go | 43 - cli/config/utils.go | 24 - cli/detect/baseline.go | 82 +- cli/detect/baseline_test.go | 160 - cli/detect/cmd/scm/scm.go | 66 + cli/{ => detect}/config/allowlist.go | 108 +- cli/detect/config/config.go | 426 +++ cli/detect/config/gitleaks.toml | 3130 +++++++++++++++++ cli/detect/config/rule.go | 114 + .../report.go => detect/config/utils.go} | 44 +- cli/detect/decoder.go | 328 ++ cli/detect/detect.go | 889 ++--- cli/detect/detect_test.go | 754 ---- cli/detect/directory.go | 225 ++ cli/detect/git.go | 214 ++ cli/detect/git/git.go | 147 - cli/detect/git/git_test.go | 158 - cli/detect/location.go | 1 + cli/detect/location_test.go | 82 - cli/detect/logging/log.go | 72 + cli/detect/reader.go | 149 + cli/detect/regexp/stdlib_regex.go | 37 + cli/detect/regexp/wasilibs_regex.go | 37 + cli/{ => detect}/report/constants.go | 1 + cli/{ => detect}/report/csv.go | 43 +- cli/{ => detect}/report/finding.go | 36 +- cli/{ => detect}/report/json.go | 10 +- cli/detect/report/junit.go | 129 + .../report/report.go} | 36 +- cli/{ => detect}/report/sarif.go | 54 +- cli/detect/report/template.go | 68 + cli/detect/sources/directory.go | 127 + cli/detect/sources/git.go | 211 ++ cli/detect/utils.go | 100 +- cli/go.mod | 12 +- cli/go.sum | 20 + cli/packages/cmd/scan.go | 144 +- cli/report/csv_test.go | 108 - cli/report/json_test.go | 111 - cli/report/report_test.go | 133 - cli/report/sarif_test.go | 122 - 47 files changed, 6276 insertions(+), 5862 deletions(-) delete mode 100644 cli/config/allowlist_test.go delete mode 100644 cli/config/config.go delete mode 100644 cli/config/config_test.go delete mode 100644 cli/config/example-infisical-relay.yaml delete mode 100644 cli/config/infisical-relay.yaml delete mode 100644 cli/config/infisical-scan.toml delete mode 100644 cli/config/rule.go delete mode 100644 cli/config/utils.go delete mode 100644 cli/detect/baseline_test.go create mode 100644 cli/detect/cmd/scm/scm.go rename cli/{ => detect}/config/allowlist.go (53%) create mode 100644 cli/detect/config/config.go create mode 100644 cli/detect/config/gitleaks.toml create mode 100644 cli/detect/config/rule.go rename cli/{report/report.go => detect/config/utils.go} (65%) create mode 100644 cli/detect/decoder.go delete mode 100644 cli/detect/detect_test.go create mode 100644 cli/detect/directory.go create mode 100644 cli/detect/git.go delete mode 100644 cli/detect/git/git.go delete mode 100644 cli/detect/git/git_test.go delete mode 100644 cli/detect/location_test.go create mode 100644 cli/detect/logging/log.go create mode 100644 cli/detect/reader.go create mode 100644 cli/detect/regexp/stdlib_regex.go create mode 100644 cli/detect/regexp/wasilibs_regex.go rename cli/{ => detect}/report/constants.go (99%) rename cli/{ => detect}/report/csv.go (72%) rename cli/{ => detect}/report/finding.go (75%) rename cli/{ => detect}/report/json.go (89%) create mode 100644 cli/detect/report/junit.go rename cli/{report/finding_test.go => detect/report/report.go} (73%) rename cli/{ => detect}/report/sarif.go (84%) create mode 100644 cli/detect/report/template.go create mode 100644 cli/detect/sources/directory.go create mode 100644 cli/detect/sources/git.go delete mode 100644 cli/report/csv_test.go delete mode 100644 cli/report/json_test.go delete mode 100644 cli/report/report_test.go delete mode 100644 cli/report/sarif_test.go diff --git a/cli/config/allowlist_test.go b/cli/config/allowlist_test.go deleted file mode 100644 index 52766e3cd..000000000 --- a/cli/config/allowlist_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - "regexp" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCommitAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - commit string - commitAllowed bool - }{ - { - allowlist: Allowlist{ - Commits: []string{"commitA"}, - }, - commit: "commitA", - commitAllowed: true, - }, - { - allowlist: Allowlist{ - Commits: []string{"commitB"}, - }, - commit: "commitA", - commitAllowed: false, - }, - { - allowlist: Allowlist{ - Commits: []string{"commitB"}, - }, - commit: "", - commitAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.commitAllowed, tt.allowlist.CommitAllowed(tt.commit)) - } -} - -func TestRegexAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - secret string - regexAllowed bool - }{ - { - allowlist: Allowlist{ - Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")}, - }, - secret: "a secret: matchthis, done", - regexAllowed: true, - }, - { - allowlist: Allowlist{ - Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")}, - }, - secret: "a secret", - regexAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.regexAllowed, tt.allowlist.RegexAllowed(tt.secret)) - } -} - -func TestPathAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - path string - pathAllowed bool - }{ - { - allowlist: Allowlist{ - Paths: []*regexp.Regexp{regexp.MustCompile("path")}, - }, - path: "a path", - pathAllowed: true, - }, - { - allowlist: Allowlist{ - Paths: []*regexp.Regexp{regexp.MustCompile("path")}, - }, - path: "a ???", - pathAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.pathAllowed, tt.allowlist.PathAllowed(tt.path)) - } -} diff --git a/cli/config/config.go b/cli/config/config.go deleted file mode 100644 index b1ce08e2b..000000000 --- a/cli/config/config.go +++ /dev/null @@ -1,279 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - _ "embed" - "fmt" - "regexp" - "strings" - - "github.com/rs/zerolog/log" - "github.com/spf13/viper" -) - -//go:embed infisical-scan.toml -var DefaultConfig string - -// use to keep track of how many configs we can extend -// yea I know, globals bad -var extendDepth int - -const maxExtendDepth = 2 - -const DefaultScanConfigFileName = ".infisical-scan.toml" -const DefaultScanConfigEnvName = "INFISICAL_SCAN_CONFIG" -const DefaultInfisicalIgnoreFineName = ".infisicalignore" - -// ViperConfig is the config struct used by the Viper config package -// to parse the config file. This struct does not include regular expressions. -// It is used as an intermediary to convert the Viper config to the Config struct. -type ViperConfig struct { - Description string - Extend Extend - Rules []struct { - ID string - Description string - Entropy float64 - SecretGroup int - Regex string - Keywords []string - Path string - Tags []string - - Allowlist struct { - RegexTarget string - Regexes []string - Paths []string - Commits []string - StopWords []string - } - } - Allowlist struct { - RegexTarget string - Regexes []string - Paths []string - Commits []string - StopWords []string - } -} - -// Config is a configuration struct that contains rules and an allowlist if present. -type Config struct { - Extend Extend - Path string - Description string - Rules map[string]Rule - Allowlist Allowlist - Keywords []string - - // used to keep sarif results consistent - orderedRules []string -} - -// Extend is a struct that allows users to define how they want their -// configuration extended by other configuration files. -type Extend struct { - Path string - URL string - UseDefault bool -} - -func (vc *ViperConfig) Translate() (Config, error) { - var ( - keywords []string - orderedRules []string - ) - rulesMap := make(map[string]Rule) - - for _, r := range vc.Rules { - var allowlistRegexes []*regexp.Regexp - for _, a := range r.Allowlist.Regexes { - allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) - } - var allowlistPaths []*regexp.Regexp - for _, a := range r.Allowlist.Paths { - allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) - } - - if r.Keywords == nil { - r.Keywords = []string{} - } else { - for _, k := range r.Keywords { - keywords = append(keywords, strings.ToLower(k)) - } - } - - if r.Tags == nil { - r.Tags = []string{} - } - - var configRegex *regexp.Regexp - var configPathRegex *regexp.Regexp - if r.Regex == "" { - configRegex = nil - } else { - configRegex = regexp.MustCompile(r.Regex) - } - if r.Path == "" { - configPathRegex = nil - } else { - configPathRegex = regexp.MustCompile(r.Path) - } - r := Rule{ - Description: r.Description, - RuleID: r.ID, - Regex: configRegex, - Path: configPathRegex, - SecretGroup: r.SecretGroup, - Entropy: r.Entropy, - Tags: r.Tags, - Keywords: r.Keywords, - Allowlist: Allowlist{ - RegexTarget: r.Allowlist.RegexTarget, - Regexes: allowlistRegexes, - Paths: allowlistPaths, - Commits: r.Allowlist.Commits, - StopWords: r.Allowlist.StopWords, - }, - } - orderedRules = append(orderedRules, r.RuleID) - - if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() { - return Config{}, fmt.Errorf("%s invalid regex secret group %d, max regex secret group %d", r.Description, r.SecretGroup, r.Regex.NumSubexp()) - } - rulesMap[r.RuleID] = r - } - var allowlistRegexes []*regexp.Regexp - for _, a := range vc.Allowlist.Regexes { - allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) - } - var allowlistPaths []*regexp.Regexp - for _, a := range vc.Allowlist.Paths { - allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) - } - c := Config{ - Description: vc.Description, - Extend: vc.Extend, - Rules: rulesMap, - Allowlist: Allowlist{ - RegexTarget: vc.Allowlist.RegexTarget, - Regexes: allowlistRegexes, - Paths: allowlistPaths, - Commits: vc.Allowlist.Commits, - StopWords: vc.Allowlist.StopWords, - }, - Keywords: keywords, - orderedRules: orderedRules, - } - - if maxExtendDepth != extendDepth { - // disallow both usedefault and path from being set - if c.Extend.Path != "" && c.Extend.UseDefault { - log.Fatal().Msg("unable to load config due to extend.path and extend.useDefault being set") - } - if c.Extend.UseDefault { - c.extendDefault() - } else if c.Extend.Path != "" { - c.extendPath() - } - - } - - return c, nil -} - -func (c *Config) OrderedRules() []Rule { - var orderedRules []Rule - for _, id := range c.orderedRules { - if _, ok := c.Rules[id]; ok { - orderedRules = append(orderedRules, c.Rules[id]) - } - } - return orderedRules -} - -func (c *Config) extendDefault() { - extendDepth++ - viper.SetConfigType("toml") - if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - defaultViperConfig := ViperConfig{} - if err := viper.Unmarshal(&defaultViperConfig); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - cfg, err := defaultViperConfig.Translate() - if err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - log.Debug().Msg("extending config with default config") - c.extend(cfg) - -} - -func (c *Config) extendPath() { - extendDepth++ - viper.SetConfigFile(c.Extend.Path) - if err := viper.ReadInConfig(); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - extensionViperConfig := ViperConfig{} - if err := viper.Unmarshal(&extensionViperConfig); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - cfg, err := extensionViperConfig.Translate() - if err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - log.Debug().Msgf("extending config with %s", c.Extend.Path) - c.extend(cfg) -} - -func (c *Config) extendURL() { - // TODO -} - -func (c *Config) extend(extensionConfig Config) { - for ruleID, rule := range extensionConfig.Rules { - if _, ok := c.Rules[ruleID]; !ok { - log.Trace().Msgf("adding %s to base config", ruleID) - c.Rules[ruleID] = rule - c.Keywords = append(c.Keywords, rule.Keywords...) - } - } - - // append allowlists, not attempting to merge - c.Allowlist.Commits = append(c.Allowlist.Commits, - extensionConfig.Allowlist.Commits...) - c.Allowlist.Paths = append(c.Allowlist.Paths, - extensionConfig.Allowlist.Paths...) - c.Allowlist.Regexes = append(c.Allowlist.Regexes, - extensionConfig.Allowlist.Regexes...) -} diff --git a/cli/config/config_test.go b/cli/config/config_test.go deleted file mode 100644 index e8f4d47b1..000000000 --- a/cli/config/config_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - "fmt" - "regexp" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" -) - -const configPath = "../testdata/config/" - -func TestTranslate(t *testing.T) { - tests := []struct { - cfgName string - cfg Config - wantError error - }{ - { - cfgName: "allow_aws_re", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Regexes: []*regexp.Regexp{ - regexp.MustCompile("AKIALALEMEL33243OLIA"), - }, - }, - }, - }, - }, - }, - { - cfgName: "allow_commit", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Commits: []string{"allowthiscommit"}, - }, - }, - }, - }, - }, - { - cfgName: "allow_path", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Paths: []*regexp.Regexp{ - regexp.MustCompile(".go"), - }, - }, - }, - }, - }, - }, - { - cfgName: "entropy_group", - cfg: Config{ - Rules: map[string]Rule{"discord-api-key": { - Description: "Discord API key", - Regex: regexp.MustCompile(`(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]`), - RuleID: "discord-api-key", - Allowlist: Allowlist{}, - Entropy: 3.5, - SecretGroup: 3, - Tags: []string{}, - Keywords: []string{}, - }, - }, - }, - }, - { - cfgName: "bad_entropy_group", - cfg: Config{}, - wantError: fmt.Errorf("Discord API key invalid regex secret group 5, max regex secret group 3"), - }, - { - cfgName: "base", - cfg: Config{ - Rules: map[string]Rule{ - "aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - }, - "aws-secret-key": { - Description: "AWS Secret Key", - Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-secret-key", - }, - "aws-secret-key-again": { - Description: "AWS Secret Key", - Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-secret-key-again", - }, - }, - }, - }, - } - - for _, tt := range tests { - viper.Reset() - viper.AddConfigPath(configPath) - viper.SetConfigName(tt.cfgName) - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if tt.wantError != nil { - if err == nil { - t.Errorf("expected error") - } - assert.Equal(t, tt.wantError, err) - } - - assert.Equal(t, cfg.Rules, tt.cfg.Rules) - } -} diff --git a/cli/config/example-infisical-relay.yaml b/cli/config/example-infisical-relay.yaml deleted file mode 100644 index c913ed757..000000000 --- a/cli/config/example-infisical-relay.yaml +++ /dev/null @@ -1,8 +0,0 @@ -public_ip: 127.0.0.1 -auth_secret: example-auth-secret -realm: infisical.org -# set port 5349 for tls -# port: 5349 -# tls_private_key_path: /full-path -# tls_ca_path: /full-path -# tls_cert_path: /full-path diff --git a/cli/config/infisical-relay.yaml b/cli/config/infisical-relay.yaml deleted file mode 100644 index 89c6b5e45..000000000 --- a/cli/config/infisical-relay.yaml +++ /dev/null @@ -1,8 +0,0 @@ -public_ip: 127.0.0.1 -auth_secret: changeThisOnProduction -realm: infisical.org -# set port 5349 for tls -# port: 5349 -# tls_private_key_path: /full-path -# tls_ca_path: /full-path -# tls_cert_path: /full-path diff --git a/cli/config/infisical-scan.toml b/cli/config/infisical-scan.toml deleted file mode 100644 index 193883444..000000000 --- a/cli/config/infisical-scan.toml +++ /dev/null @@ -1,2803 +0,0 @@ - -# This file has been auto-generated. Do not edit manually. -# If you would like to contribute new rules, please use -# cmd/generate/config/main.go and follow the contributing guidelines -# at https://github.com/zricethezav/gitleaks/blob/master/CONTRIBUTING.md - -# This is the default gitleaks configuration file. -# Rules and allowlists are defined within this file. -# Rules instruct gitleaks on what should be considered a secret. -# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. - -title = "gitleaks config" - -[allowlist] -description = "global allow lists" -paths = [ - '''infisical-scan.toml''', - '''(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket)$''', - '''(go.mod|go.sum)$''', - '''gradle.lockfile''', - '''node_modules''', - '''package-lock.json''', - '''pnpm-lock.yaml''', - '''Database.refactorlog''', - '''vendor''', -] - -[[rules]] -description = "Adafruit API Key" -id = "adafruit-api-key" -regex = '''(?i)(?:adafruit)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "adafruit", -] - -[[rules]] -description = "Adobe Client ID (OAuth Web)" -id = "adobe-client-id" -regex = '''(?i)(?:adobe)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "adobe", -] - -[[rules]] -description = "Adobe Client Secret" -id = "adobe-client-secret" -regex = '''(?i)\b((p8e-)(?i)[a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "p8e-", -] - -[[rules]] -description = "Age secret key" -id = "age secret key" -regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' -keywords = [ - "age-secret-key-1", -] - -[[rules]] -description = "Airtable API Key" -id = "airtable-api-key" -regex = '''(?i)(?:airtable)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{17})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "airtable", -] - -[[rules]] -description = "Algolia API Key" -id = "algolia-api-key" -regex = '''(?i)(?:algolia)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "algolia", -] - -[[rules]] -description = "Alibaba AccessKey ID" -id = "alibaba-access-key-id" -regex = '''(?i)\b((LTAI)(?i)[a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "ltai", -] - -[[rules]] -description = "Alibaba Secret Key" -id = "alibaba-secret-key" -regex = '''(?i)(?:alibaba)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "alibaba", -] - -[[rules]] -description = "Asana Client ID" -id = "asana-client-id" -regex = '''(?i)(?:asana)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "asana", -] - -[[rules]] -description = "Asana Client Secret" -id = "asana-client-secret" -regex = '''(?i)(?:asana)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "asana", -] - -[[rules]] -description = "Atlassian API token" -id = "atlassian-api-token" -regex = '''(?i)(?:atlassian|confluence|jira)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "atlassian","confluence","jira", -] - -[[rules]] -description = "Authress Service Client Access Key" -id = "authress-service-client-access-key" -regex = '''(?i)\b((?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc_[a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sc_","ext_","scauth_","authress_", -] - -[[rules]] -description = "AWS" -id = "aws-access-token" -regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' -keywords = [ - "akia","agpa","aida","aroa","aipa","anpa","anva","asia", -] - -[[rules]] -description = "Beamer API token" -id = "beamer-api-token" -regex = '''(?i)(?:beamer)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(b_[a-z0-9=_\-]{44})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "beamer", -] - -[[rules]] -description = "Bitbucket Client ID" -id = "bitbucket-client-id" -regex = '''(?i)(?:bitbucket)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bitbucket", -] - -[[rules]] -description = "Bitbucket Client Secret" -id = "bitbucket-client-secret" -regex = '''(?i)(?:bitbucket)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bitbucket", -] - -[[rules]] -description = "Bittrex Access Key" -id = "bittrex-access-key" -regex = '''(?i)(?:bittrex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bittrex", -] - -[[rules]] -description = "Bittrex Secret Key" -id = "bittrex-secret-key" -regex = '''(?i)(?:bittrex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bittrex", -] - -[[rules]] -description = "Clojars API token" -id = "clojars-api-token" -regex = '''(?i)(CLOJARS_)[a-z0-9]{60}''' -keywords = [ - "clojars", -] - -[[rules]] -description = "Codecov Access Token" -id = "codecov-access-token" -regex = '''(?i)(?:codecov)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "codecov", -] - -[[rules]] -description = "Coinbase Access Token" -id = "coinbase-access-token" -regex = '''(?i)(?:coinbase)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "coinbase", -] - -[[rules]] -description = "Confluent Access Token" -id = "confluent-access-token" -regex = '''(?i)(?:confluent)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "confluent", -] - -[[rules]] -description = "Confluent Secret Key" -id = "confluent-secret-key" -regex = '''(?i)(?:confluent)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "confluent", -] - -[[rules]] -description = "Contentful delivery API token" -id = "contentful-delivery-api-token" -regex = '''(?i)(?:contentful)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{43})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "contentful", -] - -[[rules]] -description = "Databricks API token" -id = "databricks-api-token" -regex = '''(?i)\b(dapi[a-h0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dapi", -] - -[[rules]] -description = "Datadog Access Token" -id = "datadog-access-token" -regex = '''(?i)(?:datadog)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "datadog", -] - -[[rules]] -description = "Defined Networking API token" -id = "defined-networking-api-token" -regex = '''(?i)(?:dnkey)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dnkey", -] - -[[rules]] -description = "DigitalOcean OAuth Access Token" -id = "digitalocean-access-token" -regex = '''(?i)\b(doo_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "doo_v1_", -] - -[[rules]] -description = "DigitalOcean Personal Access Token" -id = "digitalocean-pat" -regex = '''(?i)\b(dop_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dop_v1_", -] - -[[rules]] -description = "DigitalOcean OAuth Refresh Token" -id = "digitalocean-refresh-token" -regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dor_v1_", -] - -[[rules]] -description = "Discord API key" -id = "discord-api-token" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Discord client ID" -id = "discord-client-id" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{18})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Discord client secret" -id = "discord-client-secret" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Doppler API token" -id = "doppler-api-token" -regex = '''(dp\.pt\.)(?i)[a-z0-9]{43}''' -keywords = [ - "doppler", -] - -[[rules]] -description = "Droneci Access Token" -id = "droneci-access-token" -regex = '''(?i)(?:droneci)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "droneci", -] - -[[rules]] -description = "Dropbox API secret" -id = "dropbox-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{15})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dropbox", -] - -[[rules]] -description = "Dropbox long lived API token" -id = "dropbox-long-lived-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dropbox", -] - -[[rules]] -description = "Dropbox short lived API token" -id = "dropbox-short-lived-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(sl\.[a-z0-9\-=_]{135})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dropbox", -] - -[[rules]] -description = "Duffel API token" -id = "duffel-api-token" -regex = '''duffel_(test|live)_(?i)[a-z0-9_\-=]{43}''' -keywords = [ - "duffel", -] - -[[rules]] -description = "Dynatrace API token" -id = "dynatrace-api-token" -regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' -keywords = [ - "dynatrace", -] - -[[rules]] -description = "EasyPost API token" -id = "easypost-api-token" -regex = '''\bEZAK(?i)[a-z0-9]{54}''' -keywords = [ - "ezak", -] - -[[rules]] -description = "EasyPost test API token" -id = "easypost-test-api-token" -regex = '''\bEZTK(?i)[a-z0-9]{54}''' -keywords = [ - "eztk", -] - -[[rules]] -description = "Etsy Access Token" -id = "etsy-access-token" -regex = '''(?i)(?:etsy)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "etsy", -] - -[[rules]] -description = "Facebook Access Token" -id = "facebook" -regex = '''(?i)(?:facebook)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "facebook", -] - -[[rules]] -description = "Fastly API key" -id = "fastly-api-token" -regex = '''(?i)(?:fastly)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "fastly", -] - -[[rules]] -description = "Finicity API token" -id = "finicity-api-token" -regex = '''(?i)(?:finicity)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finicity", -] - -[[rules]] -description = "Finicity Client Secret" -id = "finicity-client-secret" -regex = '''(?i)(?:finicity)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finicity", -] - -[[rules]] -description = "Finnhub Access Token" -id = "finnhub-access-token" -regex = '''(?i)(?:finnhub)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finnhub", -] - -[[rules]] -description = "Flickr Access Token" -id = "flickr-access-token" -regex = '''(?i)(?:flickr)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "flickr", -] - -[[rules]] -description = "Flutterwave Encryption Key" -id = "flutterwave-encryption-key" -regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' -keywords = [ - "flwseck_test", -] - -[[rules]] -description = "Finicity Public Key" -id = "flutterwave-public-key" -regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' -keywords = [ - "flwpubk_test", -] - -[[rules]] -description = "Flutterwave Secret Key" -id = "flutterwave-secret-key" -regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' -keywords = [ - "flwseck_test", -] - -[[rules]] -description = "Frame.io API token" -id = "frameio-api-token" -regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' -keywords = [ - "fio-u-", -] - -[[rules]] -description = "Freshbooks Access Token" -id = "freshbooks-access-token" -regex = '''(?i)(?:freshbooks)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "freshbooks", -] - -[[rules]] -description = "GCP API key" -id = "gcp-api-key" -regex = '''(?i)\b(AIza[0-9A-Za-z\\-_]{35})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "aiza", -] - -[[rules]] -description = "Generic API Key" -id = "generic-api-key" -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -entropy = 3.5 -keywords = [ - "key","api","token","secret","client","passwd","password","auth","access", -] -[rules.allowlist] -stopwords= [ - "client", - "endpoint", - "vpn", - "_ec2_", - "aws_", - "authorize", - "author", - "define", - "config", - "credential", - "setting", - "sample", - "xxxxxx", - "000000", - "buffer", - "delete", - "aaaaaa", - "fewfwef", - "getenv", - "env_", - "system", - "example", - "ecdsa", - "sha256", - "sha1", - "sha2", - "md5", - "alert", - "wizard", - "target", - "onboard", - "welcome", - "page", - "exploit", - "experiment", - "expire", - "rabbitmq", - "scraper", - "widget", - "music", - "dns_", - "dns-", - "yahoo", - "want", - "json", - "action", - "script", - "fix_", - "fix-", - "develop", - "compas", - "stripe", - "service", - "master", - "metric", - "tech", - "gitignore", - "rich", - "open", - "stack", - "irc_", - "irc-", - "sublime", - "kohana", - "has_", - "has-", - "fabric", - "wordpres", - "role", - "osx_", - "osx-", - "boost", - "addres", - "queue", - "working", - "sandbox", - "internet", - "print", - "vision", - "tracking", - "being", - "generator", - "traffic", - "world", - "pull", - "rust", - "watcher", - "small", - "auth", - "full", - "hash", - "more", - "install", - "auto", - "complete", - "learn", - "paper", - "installer", - "research", - "acces", - "last", - "binding", - "spine", - "into", - "chat", - "algorithm", - "resource", - "uploader", - "video", - "maker", - "next", - "proc", - "lock", - "robot", - "snake", - "patch", - "matrix", - "drill", - "terminal", - "term", - "stuff", - "genetic", - "generic", - "identity", - "audit", - "pattern", - "audio", - "web_", - "web-", - "crud", - "problem", - "statu", - "cms-", - "cms_", - "arch", - "coffee", - "workflow", - "changelog", - "another", - "uiview", - "content", - "kitchen", - "gnu_", - "gnu-", - "gnu.", - "conf", - "couchdb", - "client", - "opencv", - "rendering", - "update", - "concept", - "varnish", - "gui_", - "gui-", - "gui.", - "version", - "shared", - "extra", - "product", - "still", - "not_", - "not-", - "not.", - "drop", - "ring", - "png_", - "png-", - "png.", - "actively", - "import", - "output", - "backup", - "start", - "embedded", - "registry", - "pool", - "semantic", - "instagram", - "bash", - "system", - "ninja", - "drupal", - "jquery", - "polyfill", - "physic", - "league", - "guide", - "pack", - "synopsi", - "sketch", - "injection", - "svg_", - "svg-", - "svg.", - "friendly", - "wave", - "convert", - "manage", - "camera", - "link", - "slide", - "timer", - "wrapper", - "gallery", - "url_", - "url-", - "url.", - "todomvc", - "requirej", - "party", - "http", - "payment", - "async", - "library", - "home", - "coco", - "gaia", - "display", - "universal", - "func", - "metadata", - "hipchat", - "under", - "room", - "config", - "personal", - "realtime", - "resume", - "database", - "testing", - "tiny", - "basic", - "forum", - "meetup", - "yet_", - "yet-", - "yet.", - "cento", - "dead", - "fluentd", - "editor", - "utilitie", - "run_", - "run-", - "run.", - "box_", - "box-", - "box.", - "bot_", - "bot-", - "bot.", - "making", - "sample", - "group", - "monitor", - "ajax", - "parallel", - "cassandra", - "ultimate", - "site", - "get_", - "get-", - "get.", - "gen_", - "gen-", - "gen.", - "gem_", - "gem-", - "gem.", - "extended", - "image", - "knife", - "asset", - "nested", - "zero", - "plugin", - "bracket", - "mule", - "mozilla", - "number", - "act_", - "act-", - "act.", - "map_", - "map-", - "map.", - "micro", - "debug", - "openshift", - "chart", - "expres", - "backend", - "task", - "source", - "translate", - "jbos", - "composer", - "sqlite", - "profile", - "mustache", - "mqtt", - "yeoman", - "have", - "builder", - "smart", - "like", - "oauth", - "school", - "guideline", - "captcha", - "filter", - "bitcoin", - "bridge", - "color", - "toolbox", - "discovery", - "new_", - "new-", - "new.", - "dashboard", - "when", - "setting", - "level", - "post", - "standard", - "port", - "platform", - "yui_", - "yui-", - "yui.", - "grunt", - "animation", - "haskell", - "icon", - "latex", - "cheat", - "lua_", - "lua-", - "lua.", - "gulp", - "case", - "author", - "without", - "simulator", - "wifi", - "directory", - "lisp", - "list", - "flat", - "adventure", - "story", - "storm", - "gpu_", - "gpu-", - "gpu.", - "store", - "caching", - "attention", - "solr", - "logger", - "demo", - "shortener", - "hadoop", - "finder", - "phone", - "pipeline", - "range", - "textmate", - "showcase", - "app_", - "app-", - "app.", - "idiomatic", - "edit", - "our_", - "our-", - "our.", - "out_", - "out-", - "out.", - "sentiment", - "linked", - "why_", - "why-", - "why.", - "local", - "cube", - "gmail", - "job_", - "job-", - "job.", - "rpc_", - "rpc-", - "rpc.", - "contest", - "tcp_", - "tcp-", - "tcp.", - "usage", - "buildout", - "weather", - "transfer", - "automated", - "sphinx", - "issue", - "sas_", - "sas-", - "sas.", - "parallax", - "jasmine", - "addon", - "machine", - "solution", - "dsl_", - "dsl-", - "dsl.", - "episode", - "menu", - "theme", - "best", - "adapter", - "debugger", - "chrome", - "tutorial", - "life", - "step", - "people", - "joomla", - "paypal", - "developer", - "solver", - "team", - "current", - "love", - "visual", - "date", - "data", - "canva", - "container", - "future", - "xml_", - "xml-", - "xml.", - "twig", - "nagio", - "spatial", - "original", - "sync", - "archived", - "refinery", - "science", - "mapping", - "gitlab", - "play", - "ext_", - "ext-", - "ext.", - "session", - "impact", - "set_", - "set-", - "set.", - "see_", - "see-", - "see.", - "migration", - "commit", - "community", - "shopify", - "what'", - "cucumber", - "statamic", - "mysql", - "location", - "tower", - "line", - "code", - "amqp", - "hello", - "send", - "index", - "high", - "notebook", - "alloy", - "python", - "field", - "document", - "soap", - "edition", - "email", - "php_", - "php-", - "php.", - "command", - "transport", - "official", - "upload", - "study", - "secure", - "angularj", - "akka", - "scalable", - "package", - "request", - "con_", - "con-", - "con.", - "flexible", - "security", - "comment", - "module", - "flask", - "graph", - "flash", - "apache", - "change", - "window", - "space", - "lambda", - "sheet", - "bookmark", - "carousel", - "friend", - "objective", - "jekyll", - "bootstrap", - "first", - "article", - "gwt_", - "gwt-", - "gwt.", - "classic", - "media", - "websocket", - "touch", - "desktop", - "real", - "read", - "recorder", - "moved", - "storage", - "validator", - "add-on", - "pusher", - "scs_", - "scs-", - "scs.", - "inline", - "asp_", - "asp-", - "asp.", - "timeline", - "base", - "encoding", - "ffmpeg", - "kindle", - "tinymce", - "pretty", - "jpa_", - "jpa-", - "jpa.", - "used", - "user", - "required", - "webhook", - "download", - "resque", - "espresso", - "cloud", - "mongo", - "benchmark", - "pure", - "cakephp", - "modx", - "mode", - "reactive", - "fuel", - "written", - "flickr", - "mail", - "brunch", - "meteor", - "dynamic", - "neo_", - "neo-", - "neo.", - "new_", - "new-", - "new.", - "net_", - "net-", - "net.", - "typo", - "type", - "keyboard", - "erlang", - "adobe", - "logging", - "ckeditor", - "message", - "iso_", - "iso-", - "iso.", - "hook", - "ldap", - "folder", - "reference", - "railscast", - "www_", - "www-", - "www.", - "tracker", - "azure", - "fork", - "form", - "digital", - "exporter", - "skin", - "string", - "template", - "designer", - "gollum", - "fluent", - "entity", - "language", - "alfred", - "summary", - "wiki", - "kernel", - "calendar", - "plupload", - "symfony", - "foundry", - "remote", - "talk", - "search", - "dev_", - "dev-", - "dev.", - "del_", - "del-", - "del.", - "token", - "idea", - "sencha", - "selector", - "interface", - "create", - "fun_", - "fun-", - "fun.", - "groovy", - "query", - "grail", - "red_", - "red-", - "red.", - "laravel", - "monkey", - "slack", - "supported", - "instant", - "value", - "center", - "latest", - "work", - "but_", - "but-", - "but.", - "bug_", - "bug-", - "bug.", - "virtual", - "tweet", - "statsd", - "studio", - "path", - "real-time", - "frontend", - "notifier", - "coding", - "tool", - "firmware", - "flow", - "random", - "mediawiki", - "bosh", - "been", - "beer", - "lightbox", - "theory", - "origin", - "redmine", - "hub_", - "hub-", - "hub.", - "require", - "pro_", - "pro-", - "pro.", - "ant_", - "ant-", - "ant.", - "any_", - "any-", - "any.", - "recipe", - "closure", - "mapper", - "event", - "todo", - "model", - "redi", - "provider", - "rvm_", - "rvm-", - "rvm.", - "program", - "memcached", - "rail", - "silex", - "foreman", - "activity", - "license", - "strategy", - "batch", - "streaming", - "fast", - "use_", - "use-", - "use.", - "usb_", - "usb-", - "usb.", - "impres", - "academy", - "slider", - "please", - "layer", - "cros", - "now_", - "now-", - "now.", - "miner", - "extension", - "own_", - "own-", - "own.", - "app_", - "app-", - "app.", - "debian", - "symphony", - "example", - "feature", - "serie", - "tree", - "project", - "runner", - "entry", - "leetcode", - "layout", - "webrtc", - "logic", - "login", - "worker", - "toolkit", - "mocha", - "support", - "back", - "inside", - "device", - "jenkin", - "contact", - "fake", - "awesome", - "ocaml", - "bit_", - "bit-", - "bit.", - "drive", - "screen", - "prototype", - "gist", - "binary", - "nosql", - "rest", - "overview", - "dart", - "dark", - "emac", - "mongoid", - "solarized", - "homepage", - "emulator", - "commander", - "django", - "yandex", - "gradle", - "xcode", - "writer", - "crm_", - "crm-", - "crm.", - "jade", - "startup", - "error", - "using", - "format", - "name", - "spring", - "parser", - "scratch", - "magic", - "try_", - "try-", - "try.", - "rack", - "directive", - "challenge", - "slim", - "counter", - "element", - "chosen", - "doc_", - "doc-", - "doc.", - "meta", - "should", - "button", - "packet", - "stream", - "hardware", - "android", - "infinite", - "password", - "software", - "ghost", - "xamarin", - "spec", - "chef", - "interview", - "hubot", - "mvc_", - "mvc-", - "mvc.", - "exercise", - "leaflet", - "launcher", - "air_", - "air-", - "air.", - "photo", - "board", - "boxen", - "way_", - "way-", - "way.", - "computing", - "welcome", - "notepad", - "portfolio", - "cat_", - "cat-", - "cat.", - "can_", - "can-", - "can.", - "magento", - "yaml", - "domain", - "card", - "yii_", - "yii-", - "yii.", - "checker", - "browser", - "upgrade", - "only", - "progres", - "aura", - "ruby_", - "ruby-", - "ruby.", - "polymer", - "util", - "lite", - "hackathon", - "rule", - "log_", - "log-", - "log.", - "opengl", - "stanford", - "skeleton", - "history", - "inspector", - "help", - "soon", - "selenium", - "lab_", - "lab-", - "lab.", - "scheme", - "schema", - "look", - "ready", - "leveldb", - "docker", - "game", - "minimal", - "logstash", - "messaging", - "within", - "heroku", - "mongodb", - "kata", - "suite", - "picker", - "win_", - "win-", - "win.", - "wip_", - "wip-", - "wip.", - "panel", - "started", - "starter", - "front-end", - "detector", - "deploy", - "editing", - "based", - "admin", - "capture", - "spree", - "page", - "bundle", - "goal", - "rpg_", - "rpg-", - "rpg.", - "setup", - "side", - "mean", - "reader", - "cookbook", - "mini", - "modern", - "seed", - "dom_", - "dom-", - "dom.", - "doc_", - "doc-", - "doc.", - "dot_", - "dot-", - "dot.", - "syntax", - "sugar", - "loader", - "website", - "make", - "kit_", - "kit-", - "kit.", - "protocol", - "human", - "daemon", - "golang", - "manager", - "countdown", - "connector", - "swagger", - "map_", - "map-", - "map.", - "mac_", - "mac-", - "mac.", - "man_", - "man-", - "man.", - "orm_", - "orm-", - "orm.", - "org_", - "org-", - "org.", - "little", - "zsh_", - "zsh-", - "zsh.", - "shop", - "show", - "workshop", - "money", - "grid", - "server", - "octopres", - "svn_", - "svn-", - "svn.", - "ember", - "embed", - "general", - "file", - "important", - "dropbox", - "portable", - "public", - "docpad", - "fish", - "sbt_", - "sbt-", - "sbt.", - "done", - "para", - "network", - "common", - "readme", - "popup", - "simple", - "purpose", - "mirror", - "single", - "cordova", - "exchange", - "object", - "design", - "gateway", - "account", - "lamp", - "intellij", - "math", - "mit_", - "mit-", - "mit.", - "control", - "enhanced", - "emitter", - "multi", - "add_", - "add-", - "add.", - "about", - "socket", - "preview", - "vagrant", - "cli_", - "cli-", - "cli.", - "powerful", - "top_", - "top-", - "top.", - "radio", - "watch", - "fluid", - "amazon", - "report", - "couchbase", - "automatic", - "detection", - "sprite", - "pyramid", - "portal", - "advanced", - "plu_", - "plu-", - "plu.", - "runtime", - "git_", - "git-", - "git.", - "uri_", - "uri-", - "uri.", - "haml", - "node", - "sql_", - "sql-", - "sql.", - "cool", - "core", - "obsolete", - "handler", - "iphone", - "extractor", - "array", - "copy", - "nlp_", - "nlp-", - "nlp.", - "reveal", - "pop_", - "pop-", - "pop.", - "engine", - "parse", - "check", - "html", - "nest", - "all_", - "all-", - "all.", - "chinese", - "buildpack", - "what", - "tag_", - "tag-", - "tag.", - "proxy", - "style", - "cookie", - "feed", - "restful", - "compiler", - "creating", - "prelude", - "context", - "java", - "rspec", - "mock", - "backbone", - "light", - "spotify", - "flex", - "related", - "shell", - "which", - "clas", - "webapp", - "swift", - "ansible", - "unity", - "console", - "tumblr", - "export", - "campfire", - "conway'", - "made", - "riak", - "hero", - "here", - "unix", - "unit", - "glas", - "smtp", - "how_", - "how-", - "how.", - "hot_", - "hot-", - "hot.", - "debug", - "release", - "diff", - "player", - "easy", - "right", - "old_", - "old-", - "old.", - "animate", - "time", - "push", - "explorer", - "course", - "training", - "nette", - "router", - "draft", - "structure", - "note", - "salt", - "where", - "spark", - "trello", - "power", - "method", - "social", - "via_", - "via-", - "via.", - "vim_", - "vim-", - "vim.", - "select", - "webkit", - "github", - "ftp_", - "ftp-", - "ftp.", - "creator", - "mongoose", - "led_", - "led-", - "led.", - "movie", - "currently", - "pdf_", - "pdf-", - "pdf.", - "load", - "markdown", - "phalcon", - "input", - "custom", - "atom", - "oracle", - "phonegap", - "ubuntu", - "great", - "rdf_", - "rdf-", - "rdf.", - "popcorn", - "firefox", - "zip_", - "zip-", - "zip.", - "cuda", - "dotfile", - "static", - "openwrt", - "viewer", - "powered", - "graphic", - "les_", - "les-", - "les.", - "doe_", - "doe-", - "doe.", - "maven", - "word", - "eclipse", - "lab_", - "lab-", - "lab.", - "hacking", - "steam", - "analytic", - "option", - "abstract", - "archive", - "reality", - "switcher", - "club", - "write", - "kafka", - "arduino", - "angular", - "online", - "title", - "don't", - "contao", - "notice", - "analyzer", - "learning", - "zend", - "external", - "staging", - "busines", - "tdd_", - "tdd-", - "tdd.", - "scanner", - "building", - "snippet", - "modular", - "bower", - "stm_", - "stm-", - "stm.", - "lib_", - "lib-", - "lib.", - "alpha", - "mobile", - "clean", - "linux", - "nginx", - "manifest", - "some", - "raspberry", - "gnome", - "ide_", - "ide-", - "ide.", - "block", - "statistic", - "info", - "drag", - "youtube", - "koan", - "facebook", - "paperclip", - "art_", - "art-", - "art.", - "quality", - "tab_", - "tab-", - "tab.", - "need", - "dojo", - "shield", - "computer", - "stat", - "state", - "twitter", - "utility", - "converter", - "hosting", - "devise", - "liferay", - "updated", - "force", - "tip_", - "tip-", - "tip.", - "behavior", - "active", - "call", - "answer", - "deck", - "better", - "principle", - "ches", - "bar_", - "bar-", - "bar.", - "reddit", - "three", - "haxe", - "just", - "plug-in", - "agile", - "manual", - "tetri", - "super", - "beta", - "parsing", - "doctrine", - "minecraft", - "useful", - "perl", - "sharing", - "agent", - "switch", - "view", - "dash", - "channel", - "repo", - "pebble", - "profiler", - "warning", - "cluster", - "running", - "markup", - "evented", - "mod_", - "mod-", - "mod.", - "share", - "csv_", - "csv-", - "csv.", - "response", - "good", - "house", - "connect", - "built", - "build", - "find", - "ipython", - "webgl", - "big_", - "big-", - "big.", - "google", - "scala", - "sdl_", - "sdl-", - "sdl.", - "sdk_", - "sdk-", - "sdk.", - "native", - "day_", - "day-", - "day.", - "puppet", - "text", - "routing", - "helper", - "linkedin", - "crawler", - "host", - "guard", - "merchant", - "poker", - "over", - "writing", - "free", - "classe", - "component", - "craft", - "nodej", - "phoenix", - "longer", - "quick", - "lazy", - "memory", - "clone", - "hacker", - "middleman", - "factory", - "motion", - "multiple", - "tornado", - "hack", - "ssh_", - "ssh-", - "ssh.", - "review", - "vimrc", - "driver", - "driven", - "blog", - "particle", - "table", - "intro", - "importer", - "thrift", - "xmpp", - "framework", - "refresh", - "react", - "font", - "librarie", - "variou", - "formatter", - "analysi", - "karma", - "scroll", - "tut_", - "tut-", - "tut.", - "apple", - "tag_", - "tag-", - "tag.", - "tab_", - "tab-", - "tab.", - "category", - "ionic", - "cache", - "homebrew", - "reverse", - "english", - "getting", - "shipping", - "clojure", - "boot", - "book", - "branch", - "combination", - "combo", -] -[[rules]] -description = "GitHub App Token" -id = "github-app-token" -regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' -keywords = [ - "ghu_","ghs_", -] - -[[rules]] -description = "GitHub Fine-Grained Personal Access Token" -id = "github-fine-grained-pat" -regex = '''github_pat_[0-9a-zA-Z_]{82}''' -keywords = [ - "github_pat_", -] - -[[rules]] -description = "GitHub OAuth Access Token" -id = "github-oauth" -regex = '''gho_[0-9a-zA-Z]{36}''' -keywords = [ - "gho_", -] - -[[rules]] -description = "GitHub Personal Access Token" -id = "github-pat" -regex = '''ghp_[0-9a-zA-Z]{36}''' -keywords = [ - "ghp_", -] - -[[rules]] -description = "GitHub Refresh Token" -id = "github-refresh-token" -regex = '''ghr_[0-9a-zA-Z]{36}''' -keywords = [ - "ghr_", -] - -[[rules]] -description = "GitLab Personal Access Token" -id = "gitlab-pat" -regex = '''glpat-[0-9a-zA-Z\-\_]{20}''' -keywords = [ - "glpat-", -] - -[[rules]] -description = "GitLab Pipeline Trigger Token" -id = "gitlab-ptt" -regex = '''glptt-[0-9a-f]{40}''' -keywords = [ - "glptt-", -] - -[[rules]] -description = "GitLab Runner Registration Token" -id = "gitlab-rrt" -regex = '''GR1348941[0-9a-zA-Z\-\_]{20}''' -keywords = [ - "gr1348941", -] - -[[rules]] -description = "Gitter Access Token" -id = "gitter-access-token" -regex = '''(?i)(?:gitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "gitter", -] - -[[rules]] -description = "GoCardless API token" -id = "gocardless-api-token" -regex = '''(?i)(?:gocardless)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(live_(?i)[a-z0-9\-_=]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "live_","gocardless", -] - -[[rules]] -description = "Grafana api key (or Grafana cloud api key)" -id = "grafana-api-key" -regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "eyjrijoi", -] - -[[rules]] -description = "Grafana cloud api token" -id = "grafana-cloud-api-token" -regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "glc_", -] - -[[rules]] -description = "Grafana service account token" -id = "grafana-service-account-token" -regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "glsa_", -] - -[[rules]] -description = "HashiCorp Terraform user/org API token" -id = "hashicorp-tf-api-token" -regex = '''(?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}''' -keywords = [ - "atlasv1", -] - -[[rules]] -description = "Heroku API Key" -id = "heroku-api-key" -regex = '''(?i)(?:heroku)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "heroku", -] - -[[rules]] -description = "HubSpot API Token" -id = "hubspot-api-key" -regex = '''(?i)(?:hubspot)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "hubspot", -] - -[[rules]] -description = "Intercom API Token" -id = "intercom-api-key" -regex = '''(?i)(?:intercom)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{60})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "intercom", -] - -[[rules]] -description = "JSON Web Token" -id = "jwt" -regex = '''(?i)\b(ey[0-9a-z]{30,34}\.ey[0-9a-z-\/_]{30,500}\.[0-9a-zA-Z-\/_]{10,200}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "ey", -] - -[[rules]] -description = "Kraken Access Token" -id = "kraken-access-token" -regex = '''(?i)(?:kraken)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9\/=_\+\-]{80,90})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kraken", -] - -[[rules]] -description = "Kucoin Access Token" -id = "kucoin-access-token" -regex = '''(?i)(?:kucoin)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kucoin", -] - -[[rules]] -description = "Kucoin Secret Key" -id = "kucoin-secret-key" -regex = '''(?i)(?:kucoin)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kucoin", -] - -[[rules]] -description = "Launchdarkly Access Token" -id = "launchdarkly-access-token" -regex = '''(?i)(?:launchdarkly)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "launchdarkly", -] - -[[rules]] -description = "Linear API Token" -id = "linear-api-key" -regex = '''lin_api_(?i)[a-z0-9]{40}''' -keywords = [ - "lin_api_", -] - -[[rules]] -description = "Linear Client Secret" -id = "linear-client-secret" -regex = '''(?i)(?:linear)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linear", -] - -[[rules]] -description = "LinkedIn Client ID" -id = "linkedin-client-id" -regex = '''(?i)(?:linkedin|linked-in)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{14})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linkedin","linked-in", -] - -[[rules]] -description = "LinkedIn Client secret" -id = "linkedin-client-secret" -regex = '''(?i)(?:linkedin|linked-in)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linkedin","linked-in", -] - -[[rules]] -description = "Lob API Key" -id = "lob-api-key" -regex = '''(?i)(?:lob)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}((live|test)_[a-f0-9]{35})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "test_","live_", -] - -[[rules]] -description = "Lob Publishable API Key" -id = "lob-pub-api-key" -regex = '''(?i)(?:lob)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}((test|live)_pub_[a-f0-9]{31})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "test_pub","live_pub","_pub", -] - -[[rules]] -description = "Mailchimp API key" -id = "mailchimp-api-key" -regex = '''(?i)(?:mailchimp)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32}-us20)(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailchimp", -] - -[[rules]] -description = "Mailgun private API token" -id = "mailgun-private-api-token" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(key-[a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "Mailgun public validation key" -id = "mailgun-pub-key" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(pubkey-[a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "Mailgun webhook signing key" -id = "mailgun-signing-key" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "MapBox API token" -id = "mapbox-api-token" -regex = '''(?i)(?:mapbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mapbox", -] - -[[rules]] -description = "Mattermost Access Token" -id = "mattermost-access-token" -regex = '''(?i)(?:mattermost)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{26})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mattermost", -] - -[[rules]] -description = "MessageBird API token" -id = "messagebird-api-token" -regex = '''(?i)(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{25})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "messagebird","message-bird","message_bird", -] - -[[rules]] -description = "MessageBird client ID" -id = "messagebird-client-id" -regex = '''(?i)(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "messagebird","message-bird","message_bird", -] - -[[rules]] -description = "Microsoft Teams Webhook" -id = "microsoft-teams-webhook" -regex = '''https:\/\/[a-z0-9]+\.webhook\.office\.com\/webhookb2\/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\/IncomingWebhook\/[a-z0-9]{32}\/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' -keywords = [ - "webhook.office.com","webhookb2","incomingwebhook", -] - -[[rules]] -description = "Netlify Access Token" -id = "netlify-access-token" -regex = '''(?i)(?:netlify)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{40,46})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "netlify", -] - -[[rules]] -description = "New Relic ingest browser API token" -id = "new-relic-browser-api-token" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(NRJS-[a-f0-9]{19})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nrjs-", -] - -[[rules]] -description = "New Relic user API ID" -id = "new-relic-user-api-id" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "new-relic","newrelic","new_relic", -] - -[[rules]] -description = "New Relic user API Key" -id = "new-relic-user-api-key" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(NRAK-[a-z0-9]{27})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nrak", -] - -[[rules]] -description = "npm access token" -id = "npm-access-token" -regex = '''(?i)\b(npm_[a-z0-9]{36})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "npm_", -] - -[[rules]] -description = "Nytimes Access Token" -id = "nytimes-access-token" -regex = '''(?i)(?:nytimes|new-york-times,|newyorktimes)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nytimes","new-york-times","newyorktimes", -] - -[[rules]] -description = "Okta Access Token" -id = "okta-access-token" -regex = '''(?i)(?:okta)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{42})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "okta", -] - -[[rules]] -description = "Plaid API Token" -id = "plaid-api-token" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "Plaid Client ID" -id = "plaid-client-id" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "Plaid Secret key" -id = "plaid-secret-key" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "PlanetScale API token" -id = "planetscale-api-token" -regex = '''(?i)\b(pscale_tkn_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_tkn_", -] - -[[rules]] -description = "PlanetScale OAuth token" -id = "planetscale-oauth-token" -regex = '''(?i)\b(pscale_oauth_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_oauth_", -] - -[[rules]] -description = "PlanetScale password" -id = "planetscale-password" -regex = '''(?i)\b(pscale_pw_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_pw_", -] - -[[rules]] -description = "Postman API token" -id = "postman-api-token" -regex = '''(?i)\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pmak-", -] - -[[rules]] -description = "Prefect API token" -id = "prefect-api-token" -regex = '''(?i)\b(pnu_[a-z0-9]{36})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pnu_", -] - -[[rules]] -description = "Private Key" -id = "private-key" -regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY( BLOCK)?-----[\s\S-]*KEY( BLOCK)?----''' -keywords = [ - "-----begin", -] - -[[rules]] -description = "Pulumi API token" -id = "pulumi-api-token" -regex = '''(?i)\b(pul-[a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pul-", -] - -[[rules]] -description = "PyPI upload token" -id = "pypi-upload-token" -regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}''' -keywords = [ - "pypi-ageichlwas5vcmc", -] - -[[rules]] -description = "RapidAPI Access Token" -id = "rapidapi-access-token" -regex = '''(?i)(?:rapidapi)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{50})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rapidapi", -] - -[[rules]] -description = "Readme API token" -id = "readme-api-token" -regex = '''(?i)\b(rdme_[a-z0-9]{70})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rdme_", -] - -[[rules]] -description = "Rubygem API token" -id = "rubygems-api-token" -regex = '''(?i)\b(rubygems_[a-f0-9]{48})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rubygems_", -] - -[[rules]] -description = "Sendbird Access ID" -id = "sendbird-access-id" -regex = '''(?i)(?:sendbird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sendbird", -] - -[[rules]] -description = "Sendbird Access Token" -id = "sendbird-access-token" -regex = '''(?i)(?:sendbird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sendbird", -] - -[[rules]] -description = "SendGrid API token" -id = "sendgrid-api-token" -regex = '''(?i)\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sg.", -] - -[[rules]] -description = "Sendinblue API token" -id = "sendinblue-api-token" -regex = '''(?i)\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "xkeysib-", -] - -[[rules]] -description = "Sentry Access Token" -id = "sentry-access-token" -regex = '''(?i)(?:sentry)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sentry", -] - -[[rules]] -description = "Shippo API token" -id = "shippo-api-token" -regex = '''(?i)\b(shippo_(live|test)_[a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "shippo_", -] - -[[rules]] -description = "Shopify access token" -id = "shopify-access-token" -regex = '''shpat_[a-fA-F0-9]{32}''' -keywords = [ - "shpat_", -] - -[[rules]] -description = "Shopify custom access token" -id = "shopify-custom-access-token" -regex = '''shpca_[a-fA-F0-9]{32}''' -keywords = [ - "shpca_", -] - -[[rules]] -description = "Shopify private app access token" -id = "shopify-private-app-access-token" -regex = '''shppa_[a-fA-F0-9]{32}''' -keywords = [ - "shppa_", -] - -[[rules]] -description = "Shopify shared secret" -id = "shopify-shared-secret" -regex = '''shpss_[a-fA-F0-9]{32}''' -keywords = [ - "shpss_", -] - -[[rules]] -description = "Sidekiq Secret" -id = "sidekiq-secret" -regex = '''(?i)(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bundle_enterprise__contribsys__com","bundle_gems__contribsys__com", -] - -[[rules]] -description = "Sidekiq Sensitive URL" -id = "sidekiq-sensitive-url" -regex = '''(?i)\b(http(?:s??):\/\/)([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' -secretGroup = 2 -keywords = [ - "gems.contribsys.com","enterprise.contribsys.com", -] - -[[rules]] -description = "Slack token" -id = "slack-access-token" -regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})''' -keywords = [ - "xoxb","xoxa","xoxp","xoxr","xoxs", -] - -[[rules]] -description = "Slack Webhook" -id = "slack-web-hook" -regex = '''https:\/\/hooks.slack.com\/(services|workflows)\/[A-Za-z0-9+\/]{44,46}''' -keywords = [ - "hooks.slack.com", -] - -[[rules]] -description = "Square Access Token" -id = "square-access-token" -regex = '''(?i)\b(sq0atp-[0-9A-Za-z\-_]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "sq0atp-", -] - -[[rules]] -description = "Squarespace Access Token" -id = "squarespace-access-token" -regex = '''(?i)(?:squarespace)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "squarespace", -] - -[[rules]] -description = "Stripe Access Token" -id = "stripe-access-token" -regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}''' -keywords = [ - "sk_test","pk_test","sk_live","pk_live", -] - -[[rules]] -description = "SumoLogic Access ID" -id = "sumologic-access-id" -regex = '''(?i)(?:sumo)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{14})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sumo", -] - -[[rules]] -description = "SumoLogic Access Token" -id = "sumologic-access-token" -regex = '''(?i)(?:sumo)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sumo", -] - -[[rules]] -description = "Telegram Bot API Token" -id = "telegram-bot-api-token" -regex = '''(?i)(?:^|[^0-9])([0-9]{5,16}:A[a-zA-Z0-9_\-]{34})(?:$|[^a-zA-Z0-9_\-])''' -secretGroup = 1 -keywords = [ - "telegram","api","bot","token","url", -] - -[[rules]] -description = "Travis CI Access Token" -id = "travisci-access-token" -regex = '''(?i)(?:travis)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "travis", -] - -[[rules]] -description = "Twilio API Key" -id = "twilio-api-key" -regex = '''SK[0-9a-fA-F]{32}''' -keywords = [ - "twilio", -] - -[[rules]] -description = "Twitch API token" -id = "twitch-api-token" -regex = '''(?i)(?:twitch)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitch", -] - -[[rules]] -description = "Twitter Access Secret" -id = "twitter-access-secret" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{45})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter Access Token" -id = "twitter-access-token" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter API Key" -id = "twitter-api-key" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{25})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter API Secret" -id = "twitter-api-secret" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{50})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter Bearer Token" -id = "twitter-bearer-token" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Typeform API token" -id = "typeform-api-token" -regex = '''(?i)(?:typeform)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(tfp_[a-z0-9\-_\.=]{59})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "tfp_", -] - -[[rules]] -description = "Vault Batch Token" -id = "vault-batch-token" -regex = '''(?i)\b(hvb\.[a-z0-9_-]{138,212})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "hvb", -] - -[[rules]] -description = "Vault Service Token" -id = "vault-service-token" -regex = '''(?i)\b(hvs\.[a-z0-9_-]{90,100})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "hvs", -] - -[[rules]] -description = "Yandex Access Token" -id = "yandex-access-token" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Yandex API Key" -id = "yandex-api-key" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Yandex AWS Access Token" -id = "yandex-aws-access-token" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(YC[a-zA-Z0-9_\-]{38})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Zendesk Secret Key" -id = "zendesk-secret-key" -regex = '''(?i)(?:zendesk)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "zendesk", -] - - diff --git a/cli/config/rule.go b/cli/config/rule.go deleted file mode 100644 index b7c8c1518..000000000 --- a/cli/config/rule.go +++ /dev/null @@ -1,43 +0,0 @@ -package config - -import ( - "regexp" -) - -// Rules contain information that define details on how to detect secrets -type Rule struct { - // Description is the description of the rule. - Description string - - // RuleID is a unique identifier for this rule - RuleID string - - // Entropy is a float representing the minimum shannon - // entropy a regex group must have to be considered a secret. - Entropy float64 - - // SecretGroup is an int used to extract secret from regex - // match and used as the group that will have its entropy - // checked if `entropy` is set. - SecretGroup int - - // Regex is a golang regular expression used to detect secrets. - Regex *regexp.Regexp - - // Path is a golang regular expression used to - // filter secrets by path - Path *regexp.Regexp - - // Tags is an array of strings used for metadata - // and reporting purposes. - Tags []string - - // Keywords are used for pre-regex check filtering. Rules that contain - // keywords will perform a quick string compare check to make sure the - // keyword(s) are in the content being scanned. - Keywords []string - - // Allowlist allows a rule to be ignored for specific - // regexes, paths, and/or commits - Allowlist Allowlist -} diff --git a/cli/config/utils.go b/cli/config/utils.go deleted file mode 100644 index ada6ff0fe..000000000 --- a/cli/config/utils.go +++ /dev/null @@ -1,24 +0,0 @@ -package config - -import ( - "regexp" -) - -func anyRegexMatch(f string, res []*regexp.Regexp) bool { - for _, re := range res { - if regexMatched(f, re) { - return true - } - } - return false -} - -func regexMatched(f string, re *regexp.Regexp) bool { - if re == nil { - return false - } - if re.FindString(f) != "" { - return true - } - return false -} diff --git a/cli/detect/baseline.go b/cli/detect/baseline.go index bd4c25665..eeaa2a73a 100644 --- a/cli/detect/baseline.go +++ b/cli/detect/baseline.go @@ -25,35 +25,31 @@ package detect import ( "encoding/json" "fmt" - "io" "os" + "path/filepath" - "github.com/rs/zerolog/log" - - "github.com/Infisical/infisical-merge/report" + "github.com/Infisical/infisical-merge/detect/report" ) -func IsNew(finding report.Finding, baseline []report.Finding) bool { +func IsNew(finding report.Finding, redact uint, baseline []report.Finding) bool { // Explicitly testing each property as it gives significantly better performance in comparison to cmp.Equal(). Drawback is that - // the code requires maintanance if/when the Finding struct changes + // the code requires maintenance if/when the Finding struct changes for _, b := range baseline { - - if finding.Author == b.Author && - finding.Commit == b.Commit && - finding.Date == b.Date && + if finding.RuleID == b.RuleID && finding.Description == b.Description && - finding.Email == b.Email && - finding.EndColumn == b.EndColumn && + finding.StartLine == b.StartLine && finding.EndLine == b.EndLine && - finding.Entropy == b.Entropy && - finding.File == b.File && - // Omit checking finding.Fingerprint - if the format of the fingerprint changes, the users will see unexpected behaviour - finding.Match == b.Match && - finding.Message == b.Message && - finding.RuleID == b.RuleID && - finding.Secret == b.Secret && finding.StartColumn == b.StartColumn && - finding.StartLine == b.StartLine { + finding.EndColumn == b.EndColumn && + (redact > 0 || (finding.Match == b.Match && finding.Secret == b.Secret)) && + finding.File == b.File && + finding.Commit == b.Commit && + finding.Author == b.Author && + finding.Email == b.Email && + finding.Date == b.Date && + finding.Message == b.Message && + // Omit checking finding.Fingerprint - if the format of the fingerprint changes, the users will see unexpected behaviour + finding.Entropy == b.Entropy { return false } } @@ -61,23 +57,12 @@ func IsNew(finding report.Finding, baseline []report.Finding) bool { } func LoadBaseline(baselinePath string) ([]report.Finding, error) { - var previousFindings []report.Finding - jsonFile, err := os.Open(baselinePath) + bytes, err := os.ReadFile(baselinePath) if err != nil { return nil, fmt.Errorf("could not open %s", baselinePath) } - defer func() { - if cerr := jsonFile.Close(); cerr != nil { - log.Warn().Err(cerr).Msg("problem closing jsonFile handle") - } - }() - - bytes, err := io.ReadAll(jsonFile) - if err != nil { - return nil, fmt.Errorf("could not read data from the file %s", baselinePath) - } - + var previousFindings []report.Finding err = json.Unmarshal(bytes, &previousFindings) if err != nil { return nil, fmt.Errorf("the format of the file %s is not supported", baselinePath) @@ -85,3 +70,34 @@ func LoadBaseline(baselinePath string) ([]report.Finding, error) { return previousFindings, nil } + +func (d *Detector) AddBaseline(baselinePath string, source string) error { + if baselinePath != "" { + absoluteSource, err := filepath.Abs(source) + if err != nil { + return err + } + + absoluteBaseline, err := filepath.Abs(baselinePath) + if err != nil { + return err + } + + relativeBaseline, err := filepath.Rel(absoluteSource, absoluteBaseline) + if err != nil { + return err + } + + baseline, err := LoadBaseline(baselinePath) + if err != nil { + return err + } + + d.baseline = baseline + baselinePath = relativeBaseline + + } + + d.baselinePath = baselinePath + return nil +} diff --git a/cli/detect/baseline_test.go b/cli/detect/baseline_test.go deleted file mode 100644 index 91d2eb72e..000000000 --- a/cli/detect/baseline_test.go +++ /dev/null @@ -1,160 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/Infisical/infisical-merge/report" -) - -func TestIsNew(t *testing.T) { - tests := []struct { - findings report.Finding - baseline []report.Finding - expect bool - }{ - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0000", - }, - }, - expect: false, - }, - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0002", - }, - }, - expect: true, - }, - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - Tags: []string{"a", "b"}, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0000", - Tags: []string{"a", "c"}, - }, - }, - expect: false, // Updated tags doesn't make it a new finding - }, - } - for _, test := range tests { - assert.Equal(t, test.expect, IsNew(test.findings, test.baseline)) - } -} - -func TestFileLoadBaseline(t *testing.T) { - tests := []struct { - Filename string - ExpectedError error - }{ - { - Filename: "../testdata/baseline/baseline.csv", - ExpectedError: errors.New("the format of the file ../testdata/baseline/baseline.csv is not supported"), - }, - { - Filename: "../testdata/baseline/baseline.sarif", - ExpectedError: errors.New("the format of the file ../testdata/baseline/baseline.sarif is not supported"), - }, - { - Filename: "../testdata/baseline/notfound.json", - ExpectedError: errors.New("could not open ../testdata/baseline/notfound.json"), - }, - } - - for _, test := range tests { - _, err := LoadBaseline(test.Filename) - assert.Equal(t, test.ExpectedError.Error(), err.Error()) - } -} - -func TestIgnoreIssuesInBaseline(t *testing.T) { - tests := []struct { - findings []report.Finding - baseline []report.Finding - expectCount int - }{ - { - findings: []report.Finding{ - { - Author: "a", - Commit: "5", - }, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "5", - }, - }, - expectCount: 0, - }, - { - findings: []report.Finding{ - { - Author: "a", - Commit: "5", - Fingerprint: "a", - }, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "5", - Fingerprint: "b", - }, - }, - expectCount: 0, - }, - } - - for _, test := range tests { - d, _ := NewDetectorDefaultConfig() - d.baseline = test.baseline - for _, finding := range test.findings { - d.addFinding(finding) - } - assert.Equal(t, test.expectCount, len(d.findings)) - } -} diff --git a/cli/detect/cmd/scm/scm.go b/cli/detect/cmd/scm/scm.go new file mode 100644 index 000000000..66868aadc --- /dev/null +++ b/cli/detect/cmd/scm/scm.go @@ -0,0 +1,66 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package scm + +import ( + "fmt" + "strings" +) + +type Platform int + +const ( + UnknownPlatform Platform = iota + NoPlatform // Explicitly disable the feature + GitHubPlatform + GitLabPlatform + AzureDevOpsPlatform + // TODO: Add others. +) + +func (p Platform) String() string { + return [...]string{ + "unknown", + "none", + "github", + "gitlab", + "azuredevops", + }[p] +} + +func PlatformFromString(s string) (Platform, error) { + switch strings.ToLower(s) { + case "", "unknown": + return UnknownPlatform, nil + case "none": + return NoPlatform, nil + case "github": + return GitHubPlatform, nil + case "gitlab": + return GitLabPlatform, nil + case "azuredevops": + return AzureDevOpsPlatform, nil + default: + return UnknownPlatform, fmt.Errorf("invalid scm platform value: %s", s) + } +} diff --git a/cli/config/allowlist.go b/cli/detect/config/allowlist.go similarity index 53% rename from cli/config/allowlist.go rename to cli/detect/config/allowlist.go index 373325758..d91188f68 100644 --- a/cli/config/allowlist.go +++ b/cli/detect/config/allowlist.go @@ -23,63 +23,137 @@ package config import ( - "regexp" + "fmt" "strings" + + "golang.org/x/exp/maps" + + "github.com/Infisical/infisical-merge/detect/regexp" ) +type AllowlistMatchCondition int + +const ( + AllowlistMatchOr AllowlistMatchCondition = iota + AllowlistMatchAnd +) + +func (a AllowlistMatchCondition) String() string { + return [...]string{ + "OR", + "AND", + }[a] +} + // Allowlist allows a rule to be ignored for specific // regexes, paths, and/or commits type Allowlist struct { // Short human readable description of the allowlist. Description string - // Regexes is slice of content regular expressions that are allowed to be ignored. - Regexes []*regexp.Regexp + // MatchCondition determines whether all criteria must match. + MatchCondition AllowlistMatchCondition - // RegexTarget - RegexTarget string + // Commits is a slice of commit SHAs that are allowed to be ignored. Defaults to "OR". + Commits []string // Paths is a slice of path regular expressions that are allowed to be ignored. Paths []*regexp.Regexp - // Commits is a slice of commit SHAs that are allowed to be ignored. - Commits []string + // Can be `match` or `line`. + // + // If `match` the _Regexes_ will be tested against the match of the _Rule.Regex_. + // + // If `line` the _Regexes_ will be tested against the entire line. + // + // If RegexTarget is empty, it will be tested against the found secret. + RegexTarget string + + // Regexes is slice of content regular expressions that are allowed to be ignored. + Regexes []*regexp.Regexp // StopWords is a slice of stop words that are allowed to be ignored. // This targets the _secret_, not the content of the regex match like the // Regexes slice. StopWords []string + + // validated is an internal flag to track whether `Validate()` has been called. + validated bool +} + +func (a *Allowlist) Validate() error { + if a.validated { + return nil + } + + // Disallow empty allowlists. + if len(a.Commits) == 0 && + len(a.Paths) == 0 && + len(a.Regexes) == 0 && + len(a.StopWords) == 0 { + return fmt.Errorf("must contain at least one check for: commits, paths, regexes, or stopwords") + } + + // Deduplicate commits and stopwords. + if len(a.Commits) > 0 { + uniqueCommits := make(map[string]struct{}) + for _, commit := range a.Commits { + uniqueCommits[commit] = struct{}{} + } + a.Commits = maps.Keys(uniqueCommits) + } + if len(a.StopWords) > 0 { + uniqueStopwords := make(map[string]struct{}) + for _, stopWord := range a.StopWords { + uniqueStopwords[stopWord] = struct{}{} + } + a.StopWords = maps.Keys(uniqueStopwords) + } + + a.validated = true + return nil } // CommitAllowed returns true if the commit is allowed to be ignored. -func (a *Allowlist) CommitAllowed(c string) bool { - if c == "" { - return false +func (a *Allowlist) CommitAllowed(c string) (bool, string) { + if a == nil || c == "" { + return false, "" } + for _, commit := range a.Commits { if commit == c { - return true + return true, c } } - return false + return false, "" } // PathAllowed returns true if the path is allowed to be ignored. func (a *Allowlist) PathAllowed(path string) bool { + if a == nil || path == "" { + return false + } return anyRegexMatch(path, a.Paths) } // RegexAllowed returns true if the regex is allowed to be ignored. -func (a *Allowlist) RegexAllowed(s string) bool { - return anyRegexMatch(s, a.Regexes) +func (a *Allowlist) RegexAllowed(secret string) bool { + if a == nil || secret == "" { + return false + } + return anyRegexMatch(secret, a.Regexes) } -func (a *Allowlist) ContainsStopWord(s string) bool { +func (a *Allowlist) ContainsStopWord(s string) (bool, string) { + if a == nil || s == "" { + return false, "" + } + s = strings.ToLower(s) for _, stopWord := range a.StopWords { if strings.Contains(s, strings.ToLower(stopWord)) { - return true + return true, stopWord } } - return false + return false, "" } diff --git a/cli/detect/config/config.go b/cli/detect/config/config.go new file mode 100644 index 000000000..10c6db7e0 --- /dev/null +++ b/cli/detect/config/config.go @@ -0,0 +1,426 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package config + +import ( + _ "embed" + "errors" + "fmt" + "sort" + "strings" + + "github.com/spf13/viper" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/regexp" +) + +const DefaultScanConfigFileName = ".infisical-scan.toml" +const DefaultScanConfigEnvName = "INFISICAL_SCAN_CONFIG" +const DefaultInfisicalIgnoreFineName = ".infisicalignore" + +var ( + //go:embed gitleaks.toml + DefaultConfig string + + // use to keep track of how many configs we can extend + // yea I know, globals bad + extendDepth int +) + +const maxExtendDepth = 2 + +// ViperConfig is the config struct used by the Viper config package +// to parse the config file. This struct does not include regular expressions. +// It is used as an intermediary to convert the Viper config to the Config struct. +type ViperConfig struct { + Title string + Description string + Extend Extend + Rules []struct { + ID string + Description string + Path string + Regex string + SecretGroup int + Entropy float64 + Keywords []string + Tags []string + + // Deprecated: this is a shim for backwards-compatibility. + // TODO: Remove this in 9.x. + AllowList *viperRuleAllowlist + Allowlists []*viperRuleAllowlist + } + // Deprecated: this is a shim for backwards-compatibility. + // TODO: Remove this in 9.x. + AllowList *viperGlobalAllowlist + Allowlists []*viperGlobalAllowlist +} + +type viperRuleAllowlist struct { + Description string + Condition string + Commits []string + Paths []string + RegexTarget string + Regexes []string + StopWords []string +} + +type viperGlobalAllowlist struct { + TargetRules []string + viperRuleAllowlist `mapstructure:",squash"` +} + +// Config is a configuration struct that contains rules and an allowlist if present. +type Config struct { + Title string + Extend Extend + Path string + Description string + Rules map[string]Rule + Keywords map[string]struct{} + // used to keep sarif results consistent + OrderedRules []string + Allowlists []*Allowlist +} + +// Extend is a struct that allows users to define how they want their +// configuration extended by other configuration files. +type Extend struct { + Path string + URL string + UseDefault bool + DisabledRules []string +} + +func (vc *ViperConfig) Translate() (Config, error) { + var ( + keywords = make(map[string]struct{}) + orderedRules []string + rulesMap = make(map[string]Rule) + ruleAllowlists = make(map[string][]*Allowlist) + ) + + // Validate individual rules. + for _, vr := range vc.Rules { + var ( + pathPat *regexp.Regexp + regexPat *regexp.Regexp + ) + if vr.Path != "" { + pathPat = regexp.MustCompile(vr.Path) + } + if vr.Regex != "" { + regexPat = regexp.MustCompile(vr.Regex) + } + if vr.Keywords == nil { + vr.Keywords = []string{} + } else { + for i, k := range vr.Keywords { + keyword := strings.ToLower(k) + keywords[keyword] = struct{}{} + vr.Keywords[i] = keyword + } + } + if vr.Tags == nil { + vr.Tags = []string{} + } + cr := Rule{ + RuleID: vr.ID, + Description: vr.Description, + Regex: regexPat, + SecretGroup: vr.SecretGroup, + Entropy: vr.Entropy, + Path: pathPat, + Keywords: vr.Keywords, + Tags: vr.Tags, + } + + // Parse the rule allowlists, including the older format for backwards compatibility. + if vr.AllowList != nil { + // TODO: Remove this in v9. + if len(vr.Allowlists) > 0 { + return Config{}, fmt.Errorf("%s: [rules.allowlist] is deprecated, it cannot be used alongside [[rules.allowlist]]", cr.RuleID) + } + vr.Allowlists = append(vr.Allowlists, vr.AllowList) + } + for _, a := range vr.Allowlists { + allowlist, err := parseAllowlist(a) + if err != nil { + return Config{}, fmt.Errorf("%s: [[rules.allowlists]] %w", cr.RuleID, err) + } + cr.Allowlists = append(cr.Allowlists, allowlist) + } + orderedRules = append(orderedRules, cr.RuleID) + rulesMap[cr.RuleID] = cr + } + + // Assemble the config. + c := Config{ + Title: vc.Title, + Description: vc.Description, + Extend: vc.Extend, + Rules: rulesMap, + Keywords: keywords, + OrderedRules: orderedRules, + } + // Parse the config allowlists, including the older format for backwards compatibility. + if vc.AllowList != nil { + // TODO: Remove this in v9. + if len(vc.Allowlists) > 0 { + return Config{}, errors.New("[allowlist] is deprecated, it cannot be used alongside [[allowlists]]") + } + vc.Allowlists = append(vc.Allowlists, vc.AllowList) + } + for _, a := range vc.Allowlists { + allowlist, err := parseAllowlist(&a.viperRuleAllowlist) + if err != nil { + return Config{}, fmt.Errorf("[[allowlists]] %w", err) + } + // Allowlists with |targetRules| aren't added to the global list. + if len(a.TargetRules) > 0 { + for _, ruleID := range a.TargetRules { + // It's not possible to validate |ruleID| until after extend. + ruleAllowlists[ruleID] = append(ruleAllowlists[ruleID], allowlist) + } + } else { + c.Allowlists = append(c.Allowlists, allowlist) + } + } + + if maxExtendDepth != extendDepth { + // disallow both usedefault and path from being set + if c.Extend.Path != "" && c.Extend.UseDefault { + return Config{}, errors.New("unable to load config due to extend.path and extend.useDefault being set") + } + if c.Extend.UseDefault { + if err := c.extendDefault(); err != nil { + return Config{}, err + } + } else if c.Extend.Path != "" { + if err := c.extendPath(); err != nil { + return Config{}, err + } + } + } + + // Validate the rules after everything has been assembled (including extended configs). + if extendDepth == 0 { + for _, rule := range c.Rules { + if err := rule.Validate(); err != nil { + return Config{}, err + } + } + + // Populate targeted configs. + for ruleID, allowlists := range ruleAllowlists { + rule, ok := c.Rules[ruleID] + if !ok { + return Config{}, fmt.Errorf("[[allowlists]] target rule ID '%s' does not exist", ruleID) + } + rule.Allowlists = append(rule.Allowlists, allowlists...) + c.Rules[ruleID] = rule + } + } + + return c, nil +} + +func parseAllowlist(a *viperRuleAllowlist) (*Allowlist, error) { + var matchCondition AllowlistMatchCondition + switch strings.ToUpper(a.Condition) { + case "AND", "&&": + matchCondition = AllowlistMatchAnd + case "", "OR", "||": + matchCondition = AllowlistMatchOr + default: + return nil, fmt.Errorf("unknown allowlist |condition| '%s' (expected 'and', 'or')", a.Condition) + } + + // Validate the target. + regexTarget := a.RegexTarget + if regexTarget != "" { + switch regexTarget { + case "secret": + regexTarget = "" + case "match", "line": + // do nothing + default: + return nil, fmt.Errorf("unknown allowlist |regexTarget| '%s' (expected 'match', 'line')", regexTarget) + } + } + var allowlistRegexes []*regexp.Regexp + for _, a := range a.Regexes { + allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) + } + var allowlistPaths []*regexp.Regexp + for _, a := range a.Paths { + allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) + } + + allowlist := &Allowlist{ + Description: a.Description, + MatchCondition: matchCondition, + Commits: a.Commits, + Paths: allowlistPaths, + RegexTarget: regexTarget, + Regexes: allowlistRegexes, + StopWords: a.StopWords, + } + if err := allowlist.Validate(); err != nil { + return nil, err + } + return allowlist, nil +} + +func (c *Config) GetOrderedRules() []Rule { + var orderedRules []Rule + for _, id := range c.OrderedRules { + if _, ok := c.Rules[id]; ok { + orderedRules = append(orderedRules, c.Rules[id]) + } + } + return orderedRules +} + +func (c *Config) extendDefault() error { + extendDepth++ + viper.SetConfigType("toml") + if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + } + defaultViperConfig := ViperConfig{} + if err := viper.Unmarshal(&defaultViperConfig); err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + } + cfg, err := defaultViperConfig.Translate() + if err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + + } + logging.Debug().Msg("extending config with default config") + c.extend(cfg) + return nil +} + +func (c *Config) extendPath() error { + extendDepth++ + viper.SetConfigFile(c.Extend.Path) + if err := viper.ReadInConfig(); err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + extensionViperConfig := ViperConfig{} + if err := viper.Unmarshal(&extensionViperConfig); err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + cfg, err := extensionViperConfig.Translate() + if err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + logging.Debug().Msgf("extending config with %s", c.Extend.Path) + c.extend(cfg) + return nil +} + +func (c *Config) extendURL() { + // TODO +} + +func (c *Config) extend(extensionConfig Config) { + // Get config name for helpful log messages. + var configName string + if c.Extend.Path != "" { + configName = c.Extend.Path + } else { + configName = "default" + } + // Convert |Config.DisabledRules| into a map for ease of access. + disabledRuleIDs := map[string]struct{}{} + for _, id := range c.Extend.DisabledRules { + if _, ok := extensionConfig.Rules[id]; !ok { + logging.Warn(). + Str("rule-id", id). + Str("config", configName). + Msg("Disabled rule doesn't exist in extended config.") + } + disabledRuleIDs[id] = struct{}{} + } + + for ruleID, baseRule := range extensionConfig.Rules { + // Skip the rule. + if _, ok := disabledRuleIDs[ruleID]; ok { + logging.Debug(). + Str("rule-id", ruleID). + Str("config", configName). + Msg("Ignoring rule from extended config.") + continue + } + + currentRule, ok := c.Rules[ruleID] + if !ok { + // Rule doesn't exist, add it to the config. + c.Rules[ruleID] = baseRule + for _, k := range baseRule.Keywords { + c.Keywords[k] = struct{}{} + } + c.OrderedRules = append(c.OrderedRules, ruleID) + } else { + // Rule exists, merge our changes into the base. + if currentRule.Description != "" { + baseRule.Description = currentRule.Description + } + if currentRule.Entropy != 0 { + baseRule.Entropy = currentRule.Entropy + } + if currentRule.SecretGroup != 0 { + baseRule.SecretGroup = currentRule.SecretGroup + } + if currentRule.Regex != nil { + baseRule.Regex = currentRule.Regex + } + if currentRule.Path != nil { + baseRule.Path = currentRule.Path + } + baseRule.Tags = append(baseRule.Tags, currentRule.Tags...) + baseRule.Keywords = append(baseRule.Keywords, currentRule.Keywords...) + for _, a := range currentRule.Allowlists { + baseRule.Allowlists = append(baseRule.Allowlists, a) + } + // The keywords from the base rule and the extended rule must be merged into the global keywords list + for _, k := range baseRule.Keywords { + c.Keywords[k] = struct{}{} + } + c.Rules[ruleID] = baseRule + } + } + + // append allowlists, not attempting to merge + for _, a := range extensionConfig.Allowlists { + c.Allowlists = append(c.Allowlists, a) + } + + // sort to keep extended rules in order + sort.Strings(c.OrderedRules) +} diff --git a/cli/detect/config/gitleaks.toml b/cli/detect/config/gitleaks.toml new file mode 100644 index 000000000..92a06a319 --- /dev/null +++ b/cli/detect/config/gitleaks.toml @@ -0,0 +1,3130 @@ +# This file has been auto-generated. Do not edit manually. +# If you would like to contribute new rules, please use +# cmd/generate/config/main.go and follow the contributing guidelines +# at https://github.com/gitleaks/gitleaks/blob/master/CONTRIBUTING.md +# +# How the hell does secret scanning work? Read this: +# https://lookingatcomputer.substack.com/p/regex-is-almost-all-you-need +# +# This is the default gitleaks configuration file. +# Rules and allowlists are defined within this file. +# Rules instruct gitleaks on what should be considered a secret. +# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. + +title = "gitleaks config" + +# TODO: change to [[allowlists]] +[allowlist] +description = "global allow lists" +paths = [ + '''gitleaks\.toml''', + '''(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$''', + '''(?i)\.(?:eot|[ot]tf|woff2?)$''', + '''(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf|zip)$''', + '''go\.(?:mod|sum|work(?:\.sum)?)$''', + '''(?:^|/)vendor/modules\.txt$''', + '''(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$''', + '''(?:^|/)gradlew(?:\.bat)?$''', + '''(?:^|/)gradle\.lockfile$''', + '''(?:^|/)mvnw(?:\.cmd)?$''', + '''(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''', + '''(?:^|/)node_modules(?:/.*)?$''', + '''(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''', + '''(?:^|/)bower_components(?:/.*)?$''', + '''(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$''', + '''(?:^|/)javascript\.json$''', + '''(?:^|/)(?:Pipfile|poetry)\.lock$''', + '''(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$''', + '''(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$''', + '''(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$''', + '''(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$''', + '''\.gem$''', + '''verification-metadata\.xml''', + '''Database.refactorlog''', +] +regexes = [ + '''(?i)^true|false|null$''', + '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''', + '''^\$(?:\d+|{\d+})$''', + '''^\$(?:[A-Z_]+|[a-z_]+)$''', + '''^\${(?:[A-Z_]+|[a-z_]+)}$''', + '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''', + '''^\$\{\{[ \t]*(?:(?:env|github|secrets|vars)(?:\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''', + '''^%(?:[A-Z_]+|[a-z_]+)%$''', + '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''', + '''^\{\d{0,2}}$''', + '''^@(?:[A-Z_]+|[a-z_]+)@$''', + '''^/Users/(?i)[a-z0-9]+/[\w .-/]+$''', + '''^/(?:bin|etc|home|opt|tmp|usr|var)/[\w ./-]+$''', +] +stopwords = [ + "abcdefghijklmnopqrstuvwxyz", + "014df517-39d1-4453-b7b3-9930c563627c", +] + +[[rules]] +id = "1password-secret-key" +description = "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults." +regex = '''\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b''' +entropy = 3.8 +keywords = ["a3-"] + +[[rules]] +id = "1password-service-account-token" +description = "Uncovered a possible 1Password service account token, potentially compromising access to secrets in vaults." +regex = '''ops_eyJ[a-zA-Z0-9+/]{250,}={0,3}''' +entropy = 4 +keywords = ["ops_"] + +[[rules]] +id = "adafruit-api-key" +description = "Identified a potential Adafruit API Key, which could lead to unauthorized access to Adafruit services and sensitive data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:adafruit)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["adafruit"] + +[[rules]] +id = "adobe-client-id" +description = "Detected a pattern that resembles an Adobe OAuth Web Client ID, posing a risk of compromised Adobe integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:adobe)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["adobe"] + +[[rules]] +id = "adobe-client-secret" +description = "Discovered a potential Adobe Client Secret, which, if exposed, could allow unauthorized Adobe service access and data manipulation." +regex = '''\b(p8e-(?i)[a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["p8e-"] + +[[rules]] +id = "age-secret-key" +description = "Discovered a potential Age encryption tool secret key, risking data decryption and unauthorized access to sensitive information." +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' +keywords = ["age-secret-key-1"] + +[[rules]] +id = "airtable-api-key" +description = "Uncovered a possible Airtable API Key, potentially compromising database access and leading to data leakage or alteration." +regex = '''(?i)[\w.-]{0,50}?(?:airtable)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{17})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["airtable"] + +[[rules]] +id = "algolia-api-key" +description = "Identified an Algolia API Key, which could result in unauthorized search operations and data exposure on Algolia-managed platforms." +regex = '''(?i)[\w.-]{0,50}?(?:algolia)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["algolia"] + +[[rules]] +id = "alibaba-access-key-id" +description = "Detected an Alibaba Cloud AccessKey ID, posing a risk of unauthorized cloud resource access and potential data compromise." +regex = '''\b(LTAI(?i)[a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["ltai"] + +[[rules]] +id = "alibaba-secret-key" +description = "Discovered a potential Alibaba Cloud Secret Key, potentially allowing unauthorized operations and data access within Alibaba Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:alibaba)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["alibaba"] + +[[rules]] +id = "asana-client-id" +description = "Discovered a potential Asana Client ID, risking unauthorized access to Asana projects and sensitive task information." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "asana-client-secret" +description = "Identified an Asana Client Secret, which could lead to compromised project management integrity and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "atlassian-api-token" +description = "Detected an Atlassian API token, posing a threat to project management and collaboration tool security and data confidentiality." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:atlassian|confluence|jira)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)|\b(ATATT3[A-Za-z0-9_\-=]{186})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "atlassian", + "confluence", + "jira", + "atatt3", +] + +[[rules]] +id = "authress-service-client-access-key" +description = "Uncovered a possible Authress Service Client Access Key, which may compromise access control services and sensitive data." +regex = '''\b((?:sc|ext|scauth|authress)_(?i)[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.(?-i:acc)[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sc_", + "ext_", + "scauth_", + "authress_", +] + +[[rules]] +id = "aws-access-token" +description = "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms." +regex = '''\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16})\b''' +entropy = 3 +keywords = [ + "a3t", + "akia", + "asia", + "abia", + "acca", +] +[[rules.allowlists]] +regexes = [ + '''.+EXAMPLE$''', +] + +[[rules]] +id = "azure-ad-client-secret" +description = "Azure AD Client Secret" +regex = '''(?:^|[\\'"\x60\s>=:(,)])([a-zA-Z0-9_~.]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:$|[\\'"\x60\s<),])''' +entropy = 3 +keywords = ["q~"] + +[[rules]] +id = "beamer-api-token" +description = "Detected a Beamer API token, potentially compromising content management and exposing sensitive notifications and updates." +regex = '''(?i)[\w.-]{0,50}?(?:beamer)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(b_[a-z0-9=_\-]{44})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["beamer"] + +[[rules]] +id = "bitbucket-client-id" +description = "Discovered a potential Bitbucket Client ID, risking unauthorized repository access and potential codebase exposure." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bitbucket-client-secret" +description = "Discovered a potential Bitbucket Client Secret, posing a risk of compromised code repositories and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bittrex-access-key" +description = "Identified a Bittrex Access Key, which could lead to unauthorized access to cryptocurrency trading accounts and financial loss." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "bittrex-secret-key" +description = "Detected a Bittrex Secret Key, potentially compromising cryptocurrency transactions and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "cisco-meraki-api-key" +description = "Cisco Meraki is a cloud-managed IT solution that provides networking, security, and device management through an easy-to-use interface." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Mm]eraki|MERAKI))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["meraki"] + +[[rules]] +id = "clickhouse-cloud-api-secret-key" +description = "Identified a pattern that may indicate clickhouse cloud API secret key, risking unauthorized clickhouse cloud api access and data breaches on ClickHouse Cloud platforms." +regex = '''\b(4b1d[A-Za-z0-9]{38})\b''' +entropy = 3 +keywords = ["4b1d"] + +[[rules]] +id = "clojars-api-token" +description = "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation." +regex = '''(?i)CLOJARS_[a-z0-9]{60}''' +entropy = 2 +keywords = ["clojars_"] + +[[rules]] +id = "cloudflare-api-key" +description = "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-global-api-key" +description = "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{37})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-origin-ca-key" +description = "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security." +regex = '''\b(v1\.0-[a-f0-9]{24}-[a-f0-9]{146})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "cloudflare", + "v1.0-", +] + +[[rules]] +id = "codecov-access-token" +description = "Found a pattern resembling a Codecov Access Token, posing a risk of unauthorized access to code coverage reports and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:codecov)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["codecov"] + +[[rules]] +id = "cohere-api-token" +description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:cohere|CO_API_KEY)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "cohere", + "co_api_key", +] + +[[rules]] +id = "coinbase-access-token" +description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions." +regex = '''(?i)[\w.-]{0,50}?(?:coinbase)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["coinbase"] + +[[rules]] +id = "confluent-access-token" +description = "Identified a Confluent Access Token, which could compromise access to streaming data platforms and sensitive data flow." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "confluent-secret-key" +description = "Found a Confluent Secret Key, potentially risking unauthorized operations and data access within Confluent services." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "contentful-delivery-api-token" +description = "Discovered a Contentful delivery API token, posing a risk to content management systems and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:contentful)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["contentful"] + +[[rules]] +id = "curl-auth-header" +description = "Discovered a potential authorization token provided in a curl command header, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*?|.*?(?:[\r\n]{1,2}.*?){1,5})[ \t\n\r](?:-H|--header)(?:=|[ \t]{0,5})(?:"(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))"|'(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))')(?:\B|\s|\z)''' +entropy = 2.75 +keywords = ["curl"] + +[[rules]] +id = "curl-auth-user" +description = "Discovered a potential basic authorization token provided in a curl command, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*|.*(?:[\r\n]{1,2}.*){1,5})[ \t\n\r](?:-u|--user)(?:=|[ \t]{0,5})("(:[^"]{3,}|[^:"]{3,}:|[^:"]{3,}:[^"]{3,})"|'([^:']{3,}:[^']{3,})'|((?:"[^"]{3,}"|'[^']{3,}'|[\w$@.-]+):(?:"[^"]{3,}"|'[^']{3,}'|[\w${}@.-]+)))(?:\s|\z)''' +entropy = 2 +keywords = ["curl"] +[[rules.allowlists]] +regexes = [ + '''[^:]+:(?:change(?:it|me)|pass(?:word)?|pwd|test|token|\*+|x+)''', + '''['"]?<[^>]+>['"]?:['"]?<[^>]+>|<[^:]+:[^>]+>['"]?''', + '''[^:]+:\[[^]]+]''', + '''['"]?[^:]+['"]?:['"]?\$(?:\d|\w+|\{(?:\d|\w+)})['"]?''', + '''\$\([^)]+\):\$\([^)]+\)''', + '''['"]?\$?{{[^}]+}}['"]?:['"]?\$?{{[^}]+}}['"]?''', +] + +[[rules]] +id = "databricks-api-token" +description = "Uncovered a Databricks API token, which may compromise big data analytics platforms and sensitive data processing." +regex = '''\b(dapi[a-f0-9]{32}(?:-\d)?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dapi"] + +[[rules]] +id = "datadog-access-token" +description = "Detected a Datadog Access Token, potentially risking monitoring and analytics data exposure and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:datadog)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["datadog"] + +[[rules]] +id = "defined-networking-api-token" +description = "Identified a Defined Networking API token, which could lead to unauthorized network operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:dnkey)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dnkey"] + +[[rules]] +id = "digitalocean-access-token" +description = "Found a DigitalOcean OAuth Access Token, risking unauthorized cloud resource access and data compromise." +regex = '''\b(doo_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["doo_v1_"] + +[[rules]] +id = "digitalocean-pat" +description = "Discovered a DigitalOcean Personal Access Token, posing a threat to cloud infrastructure security and data privacy." +regex = '''\b(dop_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dop_v1_"] + +[[rules]] +id = "digitalocean-refresh-token" +description = "Uncovered a DigitalOcean OAuth Refresh Token, which could allow prolonged unauthorized access and resource manipulation." +regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dor_v1_"] + +[[rules]] +id = "discord-api-token" +description = "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["discord"] + +[[rules]] +id = "discord-client-id" +description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{18})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "discord-client-secret" +description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "doppler-api-token" +description = "Discovered a Doppler API token, posing a risk to environment and secrets management security." +regex = '''dp\.pt\.(?i)[a-z0-9]{43}''' +entropy = 2 +keywords = ["dp.pt."] + +[[rules]] +id = "droneci-access-token" +description = "Detected a Droneci Access Token, potentially compromising continuous integration and deployment workflows." +regex = '''(?i)[\w.-]{0,50}?(?:droneci)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["droneci"] + +[[rules]] +id = "dropbox-api-token" +description = "Identified a Dropbox API secret, which could lead to unauthorized file access and data breaches in Dropbox storage." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{15})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Found a Dropbox long-lived API token, risking prolonged unauthorized access to cloud storage and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Discovered a Dropbox short-lived API token, posing a risk of temporary but potentially harmful data access and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(sl\.[a-z0-9\-=_]{135})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "duffel-api-token" +description = "Uncovered a Duffel API token, which may compromise travel platform integrations and sensitive customer data." +regex = '''duffel_(?:test|live)_(?i)[a-z0-9_\-=]{43}''' +entropy = 2 +keywords = ["duffel_"] + +[[rules]] +id = "dynatrace-api-token" +description = "Detected a Dynatrace API token, potentially risking application performance monitoring and data exposure." +regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' +entropy = 4 +keywords = ["dt0c01."] + +[[rules]] +id = "easypost-api-token" +description = "Identified an EasyPost API token, which could lead to unauthorized postal and shipment service access and data exposure." +regex = '''\bEZAK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["ezak"] + +[[rules]] +id = "easypost-test-api-token" +description = "Detected an EasyPost test API token, risking exposure of test environments and potentially sensitive shipment data." +regex = '''\bEZTK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["eztk"] + +[[rules]] +id = "etsy-access-token" +description = "Found an Etsy Access Token, potentially compromising Etsy shop management and customer data." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ETSY|[Ee]tsy))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["etsy"] + +[[rules]] +id = "facebook-access-token" +description = "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)\b(\d{15,16}(\||%)[0-9a-z\-_]{27,40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "facebook-page-access-token" +description = "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''\b(EAA[MC](?i)[a-z0-9]{100,})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "eaam", + "eaac", +] + +[[rules]] +id = "facebook-secret" +description = "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:facebook)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "fastly-api-token" +description = "Uncovered a Fastly API key, which may compromise CDN and edge cloud services, leading to content delivery and security issues." +regex = '''(?i)[\w.-]{0,50}?(?:fastly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["fastly"] + +[[rules]] +id = "finicity-api-token" +description = "Detected a Finicity API token, potentially risking financial data access and unauthorized financial operations." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finicity-client-secret" +description = "Identified a Finicity Client Secret, which could lead to compromised financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finnhub-access-token" +description = "Found a Finnhub Access Token, risking unauthorized access to financial market data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:finnhub)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finnhub"] + +[[rules]] +id = "flickr-access-token" +description = "Discovered a Flickr Access Token, posing a risk of unauthorized photo management and potential data leakage." +regex = '''(?i)[\w.-]{0,50}?(?:flickr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["flickr"] + +[[rules]] +id = "flutterwave-encryption-key" +description = "Uncovered a Flutterwave Encryption Key, which may compromise payment processing and sensitive financial information." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flutterwave-public-key" +description = "Detected a Finicity Public Key, potentially exposing public cryptographic operations and integrations." +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwpubk_test"] + +[[rules]] +id = "flutterwave-secret-key" +description = "Identified a Flutterwave Secret Key, risking unauthorized financial transactions and data breaches." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flyio-access-token" +description = "Uncovered a Fly.io API key" +regex = '''\b((?:fo1_[\w-]{43}|fm1[ar]_[a-zA-Z0-9+\/]{100,}={0,3}|fm2_[a-zA-Z0-9+\/]{100,}={0,3}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "fo1_", + "fm1", + "fm2_", +] + +[[rules]] +id = "frameio-api-token" +description = "Found a Frame.io API token, potentially compromising video collaboration and project management." +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' +keywords = ["fio-u-"] + +[[rules]] +id = "freemius-secret-key" +description = "Detected a Freemius secret key, potentially exposing sensitive information." +regex = '''(?i)["']secret_key["']\s*=>\s*["'](sk_[\S]{29})["']''' +path = '''(?i)\.php$''' +keywords = ["secret_key"] + +[[rules]] +id = "freshbooks-access-token" +description = "Discovered a Freshbooks Access Token, posing a risk to accounting software access and sensitive financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:freshbooks)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["freshbooks"] + +[[rules]] +id = "gcp-api-key" +description = "Uncovered a GCP API key, which could lead to unauthorized access to Google Cloud services and data breaches." +regex = '''\b(AIza[\w-]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["aiza"] +[[rules.allowlists]] +regexes = [ + '''AIzaSyabcdefghijklmnopqrstuvwxyz1234567''', + '''AIzaSyAnLA7NfeLquW1tJFpx_eQCxoX-oo6YyIs''', + '''AIzaSyCkEhVjf3pduRDt6d1yKOMitrUEke8agEM''', + '''AIzaSyDMAScliyLx7F0NPDEJi1QmyCgHIAODrlU''', + '''AIzaSyD3asb-2pEZVqMkmL6M9N6nHZRR_znhrh0''', + '''AIzayDNSXIbFmlXbIE6mCzDLQAqITYefhixbX4A''', + '''AIzaSyAdOS2zB6NCsk1pCdZ4-P6GBdi_UUPwX7c''', + '''AIzaSyASWm6HmTMdYWpgMnjRBjxcQ9CKctWmLd4''', + '''AIzaSyANUvH9H9BsUccjsu2pCmEkOPjjaXeDQgY''', + '''AIzaSyA5_iVawFQ8ABuTZNUdcwERLJv_a_p4wtM''', + '''AIzaSyA4UrcGxgwQFTfaI3no3t7Lt1sjmdnP5sQ''', + '''AIzaSyDSb51JiIcB6OJpwwMicseKRhhrOq1cS7g''', + '''AIzaSyBF2RrAIm4a0mO64EShQfqfd2AFnzAvvuU''', + '''AIzaSyBcE-OOIbhjyR83gm4r2MFCu4MJmprNXsw''', + '''AIzaSyB8qGxt4ec15vitgn44duC5ucxaOi4FmqE''', + '''AIzaSyA8vmApnrHNFE0bApF4hoZ11srVL_n0nvY''', +] + +[[rules]] +id = "generic-api-key" +description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." +regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "access", + "api", + "auth", + "key", + "credential", + "creds", + "passwd", + "password", + "secret", + "token", +] +[[rules.allowlists]] +regexes = [ + '''^[a-zA-Z_.-]+$''', +] +[[rules.allowlists]] +description = "Allowlist for Generic API Keys" +regexTarget = "match" +regexes = [ + '''(?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))''', +] +stopwords = [ + "000000", + "6fe4476ee5a1832882e326b506d14126", + "_ec2_", + "aaaaaa", + "about", + "abstract", + "academy", + "acces", + "account", + "act-", + "act.", + "act_", + "action", + "active", + "actively", + "activity", + "adapter", + "add-", + "add-on", + "add.", + "add_", + "addon", + "addres", + "admin", + "adobe", + "advanced", + "adventure", + "agent", + "agile", + "air-", + "air.", + "air_", + "ajax", + "akka", + "alert", + "alfred", + "algorithm", + "all-", + "all.", + "all_", + "alloy", + "alpha", + "amazon", + "amqp", + "analysi", + "analytic", + "analyzer", + "android", + "angular", + "angularj", + "animate", + "animation", + "another", + "ansible", + "answer", + "ant-", + "ant.", + "ant_", + "any-", + "any.", + "any_", + "apache", + "app-", + "app.", + "app_", + "apple", + "arch", + "archive", + "archived", + "arduino", + "array", + "art-", + "art.", + "art_", + "article", + "asp-", + "asp.", + "asp_", + "asset", + "async", + "atom", + "attention", + "audio", + "audit", + "aura", + "auth", + "author", + "authorize", + "auto", + "automated", + "automatic", + "awesome", + "aws_", + "azure", + "back", + "backbone", + "backend", + "backup", + "bar-", + "bar.", + "bar_", + "base", + "based", + "bash", + "basic", + "batch", + "been", + "beer", + "behavior", + "being", + "benchmark", + "best", + "beta", + "better", + "big-", + "big.", + "big_", + "binary", + "binding", + "bit-", + "bit.", + "bit_", + "bitcoin", + "block", + "blog", + "board", + "book", + "bookmark", + "boost", + "boot", + "bootstrap", + "bosh", + "bot-", + "bot.", + "bot_", + "bower", + "box-", + "box.", + "box_", + "boxen", + "bracket", + "branch", + "bridge", + "browser", + "brunch", + "buffer", + "bug-", + "bug.", + "bug_", + "build", + "builder", + "building", + "buildout", + "buildpack", + "built", + "bundle", + "busines", + "but-", + "but.", + "but_", + "button", + "cache", + "caching", + "cakephp", + "calendar", + "call", + "camera", + "campfire", + "can-", + "can.", + "can_", + "canva", + "captcha", + "capture", + "card", + "carousel", + "case", + "cassandra", + "cat-", + "cat.", + "cat_", + "category", + "center", + "cento", + "challenge", + "change", + "changelog", + "channel", + "chart", + "chat", + "cheat", + "check", + "checker", + "chef", + "ches", + "chinese", + "chosen", + "chrome", + "ckeditor", + "clas", + "classe", + "classic", + "clean", + "cli-", + "cli.", + "cli_", + "client", + "clojure", + "clone", + "closure", + "cloud", + "club", + "cluster", + "cms-", + "cms_", + "coco", + "code", + "coding", + "coffee", + "color", + "combination", + "combo", + "command", + "commander", + "comment", + "commit", + "common", + "community", + "compas", + "compiler", + "complete", + "component", + "composer", + "computer", + "computing", + "con-", + "con.", + "con_", + "concept", + "conf", + "config", + "connect", + "connector", + "console", + "contact", + "container", + "contao", + "content", + "contest", + "context", + "control", + "convert", + "converter", + "conway'", + "cookbook", + "cookie", + "cool", + "copy", + "cordova", + "core", + "couchbase", + "couchdb", + "countdown", + "counter", + "course", + "craft", + "crawler", + "create", + "creating", + "creator", + "credential", + "crm-", + "crm.", + "crm_", + "cros", + "crud", + "csv-", + "csv.", + "csv_", + "cube", + "cucumber", + "cuda", + "current", + "currently", + "custom", + "daemon", + "dark", + "dart", + "dash", + "dashboard", + "data", + "database", + "date", + "day-", + "day.", + "day_", + "dead", + "debian", + "debug", + "debugger", + "deck", + "define", + "del-", + "del.", + "del_", + "delete", + "demo", + "deploy", + "design", + "designer", + "desktop", + "detection", + "detector", + "dev-", + "dev.", + "dev_", + "develop", + "developer", + "device", + "devise", + "diff", + "digital", + "directive", + "directory", + "discovery", + "display", + "django", + "dns-", + "dns_", + "doc-", + "doc.", + "doc_", + "docker", + "docpad", + "doctrine", + "document", + "doe-", + "doe.", + "doe_", + "dojo", + "dom-", + "dom.", + "dom_", + "domain", + "don't", + "done", + "dot-", + "dot.", + "dot_", + "dotfile", + "download", + "draft", + "drag", + "drill", + "drive", + "driven", + "driver", + "drop", + "dropbox", + "drupal", + "dsl-", + "dsl.", + "dsl_", + "dynamic", + "easy", + "ecdsa", + "eclipse", + "edit", + "editing", + "edition", + "editor", + "element", + "emac", + "email", + "embed", + "embedded", + "ember", + "emitter", + "emulator", + "encoding", + "endpoint", + "engine", + "english", + "enhanced", + "entity", + "entry", + "env_", + "episode", + "erlang", + "error", + "espresso", + "event", + "evented", + "example", + "exchange", + "exercise", + "experiment", + "expire", + "exploit", + "explorer", + "export", + "exporter", + "expres", + "ext-", + "ext.", + "ext_", + "extended", + "extension", + "external", + "extra", + "extractor", + "fabric", + "facebook", + "factory", + "fake", + "fast", + "feature", + "feed", + "fewfwef", + "ffmpeg", + "field", + "file", + "filter", + "find", + "finder", + "firefox", + "firmware", + "first", + "fish", + "fix-", + "fix_", + "flash", + "flask", + "flat", + "flex", + "flexible", + "flickr", + "flow", + "fluent", + "fluentd", + "fluid", + "folder", + "font", + "force", + "foreman", + "fork", + "form", + "format", + "formatter", + "forum", + "foundry", + "framework", + "free", + "friend", + "friendly", + "front-end", + "frontend", + "ftp-", + "ftp.", + "ftp_", + "fuel", + "full", + "fun-", + "fun.", + "fun_", + "func", + "future", + "gaia", + "gallery", + "game", + "gateway", + "gem-", + "gem.", + "gem_", + "gen-", + "gen.", + "gen_", + "general", + "generator", + "generic", + "genetic", + "get-", + "get.", + "get_", + "getenv", + "getting", + "ghost", + "gist", + "git-", + "git.", + "git_", + "github", + "gitignore", + "gitlab", + "glas", + "gmail", + "gnome", + "gnu-", + "gnu.", + "gnu_", + "goal", + "golang", + "gollum", + "good", + "google", + "gpu-", + "gpu.", + "gpu_", + "gradle", + "grail", + "graph", + "graphic", + "great", + "grid", + "groovy", + "group", + "grunt", + "guard", + "gui-", + "gui.", + "gui_", + "guide", + "guideline", + "gulp", + "gwt-", + "gwt.", + "gwt_", + "hack", + "hackathon", + "hacker", + "hacking", + "hadoop", + "haml", + "handler", + "hardware", + "has-", + "has_", + "hash", + "haskell", + "have", + "haxe", + "hello", + "help", + "helper", + "here", + "hero", + "heroku", + "high", + "hipchat", + "history", + "home", + "homebrew", + "homepage", + "hook", + "host", + "hosting", + "hot-", + "hot.", + "hot_", + "house", + "how-", + "how.", + "how_", + "html", + "http", + "hub-", + "hub.", + "hub_", + "hubot", + "human", + "icon", + "ide-", + "ide.", + "ide_", + "idea", + "identity", + "idiomatic", + "image", + "impact", + "import", + "important", + "importer", + "impres", + "index", + "infinite", + "info", + "injection", + "inline", + "input", + "inside", + "inspector", + "instagram", + "install", + "installer", + "instant", + "intellij", + "interface", + "internet", + "interview", + "into", + "intro", + "ionic", + "iphone", + "ipython", + "irc-", + "irc_", + "iso-", + "iso.", + "iso_", + "issue", + "jade", + "jasmine", + "java", + "jbos", + "jekyll", + "jenkin", + "jetbrains", + "job-", + "job.", + "job_", + "joomla", + "jpa-", + "jpa.", + "jpa_", + "jquery", + "json", + "just", + "kafka", + "karma", + "kata", + "kernel", + "keyboard", + "kindle", + "kit-", + "kit.", + "kit_", + "kitchen", + "knife", + "koan", + "kohana", + "lab-", + "lab.", + "lab_", + "lambda", + "lamp", + "language", + "laravel", + "last", + "latest", + "latex", + "launcher", + "layer", + "layout", + "lazy", + "ldap", + "leaflet", + "league", + "learn", + "learning", + "led-", + "led.", + "led_", + "leetcode", + "les-", + "les.", + "les_", + "level", + "leveldb", + "lib-", + "lib.", + "lib_", + "librarie", + "library", + "license", + "life", + "liferay", + "light", + "lightbox", + "like", + "line", + "link", + "linked", + "linkedin", + "linux", + "lisp", + "list", + "lite", + "little", + "load", + "loader", + "local", + "location", + "lock", + "log-", + "log.", + "log_", + "logger", + "logging", + "logic", + "login", + "logstash", + "longer", + "look", + "love", + "lua-", + "lua.", + "lua_", + "mac-", + "mac.", + "mac_", + "machine", + "made", + "magento", + "magic", + "mail", + "make", + "maker", + "making", + "man-", + "man.", + "man_", + "manage", + "manager", + "manifest", + "manual", + "map-", + "map.", + "map_", + "mapper", + "mapping", + "markdown", + "markup", + "master", + "math", + "matrix", + "maven", + "md5", + "mean", + "media", + "mediawiki", + "meetup", + "memcached", + "memory", + "menu", + "merchant", + "message", + "messaging", + "meta", + "metadata", + "meteor", + "method", + "metric", + "micro", + "middleman", + "migration", + "minecraft", + "miner", + "mini", + "minimal", + "mirror", + "mit-", + "mit.", + "mit_", + "mobile", + "mocha", + "mock", + "mod-", + "mod.", + "mod_", + "mode", + "model", + "modern", + "modular", + "module", + "modx", + "money", + "mongo", + "mongodb", + "mongoid", + "mongoose", + "monitor", + "monkey", + "more", + "motion", + "moved", + "movie", + "mozilla", + "mqtt", + "mule", + "multi", + "multiple", + "music", + "mustache", + "mvc-", + "mvc.", + "mvc_", + "mysql", + "nagio", + "name", + "native", + "need", + "neo-", + "neo.", + "neo_", + "nest", + "nested", + "net-", + "net.", + "net_", + "nette", + "network", + "new-", + "new.", + "new_", + "next", + "nginx", + "ninja", + "nlp-", + "nlp.", + "nlp_", + "node", + "nodej", + "nosql", + "not-", + "not.", + "not_", + "note", + "notebook", + "notepad", + "notice", + "notifier", + "now-", + "now.", + "now_", + "number", + "oauth", + "object", + "objective", + "obsolete", + "ocaml", + "octopres", + "official", + "old-", + "old.", + "old_", + "onboard", + "online", + "only", + "open", + "opencv", + "opengl", + "openshift", + "openwrt", + "option", + "oracle", + "org-", + "org.", + "org_", + "origin", + "original", + "orm-", + "orm.", + "orm_", + "osx-", + "osx_", + "our-", + "our.", + "our_", + "out-", + "out.", + "out_", + "output", + "over", + "overview", + "own-", + "own.", + "own_", + "pack", + "package", + "packet", + "page", + "panel", + "paper", + "paperclip", + "para", + "parallax", + "parallel", + "parse", + "parser", + "parsing", + "particle", + "party", + "password", + "patch", + "path", + "pattern", + "payment", + "paypal", + "pdf-", + "pdf.", + "pdf_", + "pebble", + "people", + "perl", + "personal", + "phalcon", + "phoenix", + "phone", + "phonegap", + "photo", + "php-", + "php.", + "php_", + "physic", + "picker", + "pipeline", + "platform", + "play", + "player", + "please", + "plu-", + "plu.", + "plu_", + "plug-in", + "plugin", + "plupload", + "png-", + "png.", + "png_", + "poker", + "polyfill", + "polymer", + "pool", + "pop-", + "pop.", + "pop_", + "popcorn", + "popup", + "port", + "portable", + "portal", + "portfolio", + "post", + "power", + "powered", + "powerful", + "prelude", + "pretty", + "preview", + "principle", + "print", + "pro-", + "pro.", + "pro_", + "problem", + "proc", + "product", + "profile", + "profiler", + "program", + "progres", + "project", + "protocol", + "prototype", + "provider", + "proxy", + "public", + "pull", + "puppet", + "pure", + "purpose", + "push", + "pusher", + "pyramid", + "python", + "quality", + "query", + "queue", + "quick", + "rabbitmq", + "rack", + "radio", + "rail", + "railscast", + "random", + "range", + "raspberry", + "rdf-", + "rdf.", + "rdf_", + "react", + "reactive", + "read", + "reader", + "readme", + "ready", + "real", + "real-time", + "reality", + "realtime", + "recipe", + "recorder", + "red-", + "red.", + "red_", + "reddit", + "redi", + "redmine", + "reference", + "refinery", + "refresh", + "registry", + "related", + "release", + "remote", + "rendering", + "repo", + "report", + "request", + "require", + "required", + "requirej", + "research", + "resource", + "response", + "resque", + "rest", + "restful", + "resume", + "reveal", + "reverse", + "review", + "riak", + "rich", + "right", + "ring", + "robot", + "role", + "room", + "router", + "routing", + "rpc-", + "rpc.", + "rpc_", + "rpg-", + "rpg.", + "rpg_", + "rspec", + "ruby-", + "ruby.", + "ruby_", + "rule", + "run-", + "run.", + "run_", + "runner", + "running", + "runtime", + "rust", + "rvm-", + "rvm.", + "rvm_", + "salt", + "sample", + "sandbox", + "sas-", + "sas.", + "sas_", + "sbt-", + "sbt.", + "sbt_", + "scala", + "scalable", + "scanner", + "schema", + "scheme", + "school", + "science", + "scraper", + "scratch", + "screen", + "script", + "scroll", + "scs-", + "scs.", + "scs_", + "sdk-", + "sdk.", + "sdk_", + "sdl-", + "sdl.", + "sdl_", + "search", + "secure", + "security", + "see-", + "see.", + "see_", + "seed", + "select", + "selector", + "selenium", + "semantic", + "sencha", + "send", + "sentiment", + "serie", + "server", + "service", + "session", + "set-", + "set.", + "set_", + "setting", + "setup", + "sha1", + "sha2", + "sha256", + "share", + "shared", + "sharing", + "sheet", + "shell", + "shield", + "shipping", + "shop", + "shopify", + "shortener", + "should", + "show", + "showcase", + "side", + "silex", + "simple", + "simulator", + "single", + "site", + "skeleton", + "sketch", + "skin", + "slack", + "slide", + "slider", + "slim", + "small", + "smart", + "smtp", + "snake", + "snapshot", + "snippet", + "soap", + "social", + "socket", + "software", + "solarized", + "solr", + "solution", + "solver", + "some", + "soon", + "source", + "space", + "spark", + "spatial", + "spec", + "sphinx", + "spine", + "spotify", + "spree", + "spring", + "sprite", + "sql-", + "sql.", + "sql_", + "sqlite", + "ssh-", + "ssh.", + "ssh_", + "stack", + "staging", + "standard", + "stanford", + "start", + "started", + "starter", + "startup", + "stat", + "statamic", + "state", + "static", + "statistic", + "statsd", + "statu", + "steam", + "step", + "still", + "stm-", + "stm.", + "stm_", + "storage", + "store", + "storm", + "story", + "strategy", + "stream", + "streaming", + "string", + "stripe", + "structure", + "studio", + "study", + "stuff", + "style", + "sublime", + "sugar", + "suite", + "summary", + "super", + "support", + "supported", + "svg-", + "svg.", + "svg_", + "svn-", + "svn.", + "svn_", + "swagger", + "swift", + "switch", + "switcher", + "symfony", + "symphony", + "sync", + "synopsi", + "syntax", + "system", + "tab-", + "tab.", + "tab_", + "table", + "tag-", + "tag.", + "tag_", + "talk", + "target", + "task", + "tcp-", + "tcp.", + "tcp_", + "tdd-", + "tdd.", + "tdd_", + "team", + "tech", + "template", + "term", + "terminal", + "testing", + "tetri", + "text", + "textmate", + "theme", + "theory", + "three", + "thrift", + "time", + "timeline", + "timer", + "tiny", + "tinymce", + "tip-", + "tip.", + "tip_", + "title", + "todo", + "todomvc", + "token", + "tool", + "toolbox", + "toolkit", + "top-", + "top.", + "top_", + "tornado", + "touch", + "tower", + "tracker", + "tracking", + "traffic", + "training", + "transfer", + "translate", + "transport", + "tree", + "trello", + "try-", + "try.", + "try_", + "tumblr", + "tut-", + "tut.", + "tut_", + "tutorial", + "tweet", + "twig", + "twitter", + "type", + "typo", + "ubuntu", + "uiview", + "ultimate", + "under", + "unit", + "unity", + "universal", + "unix", + "update", + "updated", + "upgrade", + "upload", + "uploader", + "uri-", + "uri.", + "uri_", + "url-", + "url.", + "url_", + "usage", + "usb-", + "usb.", + "usb_", + "use-", + "use.", + "use_", + "used", + "useful", + "user", + "using", + "util", + "utilitie", + "utility", + "vagrant", + "validator", + "value", + "variou", + "varnish", + "version", + "via-", + "via.", + "via_", + "video", + "view", + "viewer", + "vim-", + "vim.", + "vim_", + "vimrc", + "virtual", + "vision", + "visual", + "vpn", + "want", + "warning", + "watch", + "watcher", + "wave", + "way-", + "way.", + "way_", + "weather", + "web-", + "web_", + "webapp", + "webgl", + "webhook", + "webkit", + "webrtc", + "website", + "websocket", + "welcome", + "what", + "what'", + "when", + "where", + "which", + "why-", + "why.", + "why_", + "widget", + "wifi", + "wiki", + "win-", + "win.", + "win_", + "window", + "wip-", + "wip.", + "wip_", + "within", + "without", + "wizard", + "word", + "wordpres", + "work", + "worker", + "workflow", + "working", + "workshop", + "world", + "wrapper", + "write", + "writer", + "writing", + "written", + "www-", + "www.", + "www_", + "xamarin", + "xcode", + "xml-", + "xml.", + "xml_", + "xmpp", + "xxxxxx", + "yahoo", + "yaml", + "yandex", + "yeoman", + "yet-", + "yet.", + "yet_", + "yii-", + "yii.", + "yii_", + "youtube", + "yui-", + "yui.", + "yui_", + "zend", + "zero", + "zip-", + "zip.", + "zip_", + "zsh-", + "zsh.", + "zsh_", +] +[[rules.allowlists]] +regexTarget = "line" +regexes = [ + '''--mount=type=secret,''', + '''import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]''', +] +[[rules.allowlists]] +condition = "AND" +paths = [ + '''\.bb$''','''\.bbappend$''','''\.bbclass$''','''\.inc$''', +] +regexTarget = "line" +regexes = [ + '''LICENSE[^=]*=\s*"[^"]+''', + '''LIC_FILES_CHKSUM[^=]*=\s*"[^"]+''', + '''SRC[^=]*=\s*"[a-zA-Z0-9]+''', +] + +[[rules]] +id = "github-app-token" +description = "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security." +regex = '''(?:ghu|ghs)_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = [ + "ghu_", + "ghs_", +] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-fine-grained-pat" +description = "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation." +regex = '''github_pat_\w{82}''' +entropy = 3 +keywords = ["github_pat_"] + +[[rules]] +id = "github-oauth" +description = "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks." +regex = '''gho_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["gho_"] + +[[rules]] +id = "github-pat" +description = "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure." +regex = '''ghp_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghp_"] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-refresh-token" +description = "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services." +regex = '''ghr_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghr_"] + +[[rules]] +id = "gitlab-cicd-job-token" +description = "Identified a GitLab CI/CD Job Token, potential access to projects and some APIs on behalf of a user while the CI job is running." +regex = '''glcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}''' +entropy = 3 +keywords = ["glcbt-"] + +[[rules]] +id = "gitlab-deploy-token" +description = "Identified a GitLab Deploy Token, risking access to repositories, packages and containers with write access." +regex = '''gldt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["gldt-"] + +[[rules]] +id = "gitlab-feature-flag-client-token" +description = "Identified a GitLab feature flag client token, risks exposing user lists and features flags used by an application." +regex = '''glffct-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glffct-"] + +[[rules]] +id = "gitlab-feed-token" +description = "Identified a GitLab feed token, risking exposure of user data." +regex = '''glft-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glft-"] + +[[rules]] +id = "gitlab-incoming-mail-token" +description = "Identified a GitLab incoming mail token, risking manipulation of data sent by mail." +regex = '''glimt-[0-9a-zA-Z_\-]{25}''' +entropy = 3 +keywords = ["glimt-"] + +[[rules]] +id = "gitlab-kubernetes-agent-token" +description = "Identified a GitLab Kubernetes Agent token, risking access to repos and registry of projects connected via agent." +regex = '''glagent-[0-9a-zA-Z_\-]{50}''' +entropy = 3 +keywords = ["glagent-"] + +[[rules]] +id = "gitlab-oauth-app-secret" +description = "Identified a GitLab OIDC Application Secret, risking access to apps using GitLab as authentication provider." +regex = '''gloas-[0-9a-zA-Z_\-]{64}''' +entropy = 3 +keywords = ["gloas-"] + +[[rules]] +id = "gitlab-pat" +description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''glpat-[\w-]{20}''' +entropy = 3 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-pat-routable" +description = "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-ptt" +description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security." +regex = '''glptt-[0-9a-f]{40}''' +entropy = 3 +keywords = ["glptt-"] + +[[rules]] +id = "gitlab-rrt" +description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''GR1348941[\w-]{20}''' +entropy = 3 +keywords = ["gr1348941"] + +[[rules]] +id = "gitlab-runner-authentication-token" +description = "Discovered a GitLab Runner Authentication Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''glrt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-runner-authentication-token-routable" +description = "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-scim-token" +description = "Discovered a GitLab SCIM Token, posing a risk to unauthorized access for a organization or instance." +regex = '''glsoat-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glsoat-"] + +[[rules]] +id = "gitlab-session-cookie" +description = "Discovered a GitLab Session Cookie, posing a risk to unauthorized access to a user account." +regex = '''_gitlab_session=[0-9a-z]{32}''' +entropy = 3 +keywords = ["_gitlab_session="] + +[[rules]] +id = "gitter-access-token" +description = "Uncovered a Gitter Access Token, which may lead to unauthorized access to chat and communication services." +regex = '''(?i)[\w.-]{0,50}?(?:gitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["gitter"] + +[[rules]] +id = "gocardless-api-token" +description = "Detected a GoCardless API token, potentially risking unauthorized direct debit payment operations and financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:gocardless)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(live_(?i)[a-z0-9\-_=]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "live_", + "gocardless", +] + +[[rules]] +id = "grafana-api-key" +description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics." +regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["eyjrijoi"] + +[[rules]] +id = "grafana-cloud-api-token" +description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure." +regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glc_"] + +[[rules]] +id = "grafana-service-account-token" +description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity." +regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glsa_"] + +[[rules]] +id = "harness-api-key" +description = "Identified a Harness Access Token (PAT or SAT), risking unauthorized access to a Harness account." +regex = '''(?:pat|sat)\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9]{24}\.[a-zA-Z0-9]{20}''' +keywords = [ + "pat.", + "sat.", +] + +[[rules]] +id = "hashicorp-tf-api-token" +description = "Uncovered a HashiCorp Terraform user/org API token, which may lead to unauthorized infrastructure management and security breaches." +regex = '''(?i)[a-z0-9]{14}\.(?-i:atlasv1)\.[a-z0-9\-_=]{60,70}''' +entropy = 3.5 +keywords = ["atlasv1"] + +[[rules]] +id = "hashicorp-tf-password" +description = "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches." +regex = '''(?i)[\w.-]{0,50}?(?:administrator_login_password|password)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}("[a-z0-9=_\-]{8,20}")(?:[\x60'"\s;]|\\[nr]|$)''' +path = '''(?i)\.(?:tf|hcl)$''' +entropy = 2 +keywords = [ + "administrator_login_password", + "password", +] + +[[rules]] +id = "heroku-api-key" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:heroku)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["heroku"] + +[[rules]] +id = "hubspot-api-key" +description = "Found a HubSpot API Token, posing a risk to CRM data integrity and unauthorized marketing operations." +regex = '''(?i)[\w.-]{0,50}?(?:hubspot)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["hubspot"] + +[[rules]] +id = "huggingface-access-token" +description = "Discovered a Hugging Face Access token, which could lead to unauthorized access to AI models and sensitive data." +regex = '''\b(hf_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["hf_"] + +[[rules]] +id = "huggingface-organization-api-token" +description = "Uncovered a Hugging Face Organization API token, potentially compromising AI organization accounts and associated data." +regex = '''\b(api_org_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["api_org_"] + +[[rules]] +id = "infracost-api-token" +description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data." +regex = '''\b(ico-[a-zA-Z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ico-"] + +[[rules]] +id = "intercom-api-key" +description = "Identified an Intercom API Token, which could compromise customer communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:intercom)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{60})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["intercom"] + +[[rules]] +id = "intra42-client-secret" +description = "Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data." +regex = '''\b(s-s4t2(?:ud|af)-(?i)[abcdef0123456789]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "intra", + "s-s4t2ud-", + "s-s4t2af-", +] + +[[rules]] +id = "jfrog-api-key" +description = "Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{73})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jfrog-identity-token" +description = "Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jwt" +description = "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data." +regex = '''\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ey"] + +[[rules]] +id = "jwt-base64" +description = "Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information." +regex = '''\bZXlK(?:(?PaGJHY2lPaU)|(?PaGNIVWlPaU)|(?PaGNIWWlPaU)|(?PaGRXUWlPaU)|(?PaU5qUWlP)|(?PamNtbDBJanBi)|(?PamRIa2lPaU)|(?PbGNHc2lPbn)|(?PbGJtTWlPaU)|(?PcWEzVWlPaU)|(?PcWQyc2lPb)|(?PcGMzTWlPaU)|(?PcGRpSTZJ)|(?PcmFXUWlP)|(?PclpYbGZiM0J6SWpwY)|(?PcmRIa2lPaUp)|(?PdWIyNWpaU0k2)|(?Pd01tTWlP)|(?Pd01uTWlPaU)|(?Pd2NIUWlPaU)|(?PemRXSWlPaU)|(?PemRuUWlP)|(?PMFlXY2lPaU)|(?PMGVYQWlPaUp)|(?PMWNtd2l)|(?PMWMyVWlPaUp)|(?PMlpYSWlPaU)|(?PMlpYSnphVzl1SWpv)|(?PNElqb2)|(?PNE5XTWlP)|(?PNE5YUWlPaU)|(?PNE5YUWpVekkxTmlJNkl)|(?PNE5YVWlPaU)|(?PNmFYQWlPaU))[a-zA-Z0-9\/\\_+\-\r\n]{40,}={0,2}''' +entropy = 2 +keywords = ["zxlk"] + +[[rules]] +id = "kraken-access-token" +description = "Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:kraken)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9\/=_\+\-]{80,90})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kraken"] + +[[rules]] +id = "kubernetes-secret-yaml" +description = "Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments" +regex = '''(?i)(?:\bkind:[ \t]*["']?\bsecret\b["']?(?s:.){0,200}?\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))|\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))(?s:.){0,200}?\bkind:[ \t]*["']?\bsecret\b["']?)''' +path = '''(?i)\.ya?ml$''' +keywords = ["secret"] +[[rules.allowlists]] +regexes = [ + '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''', +] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''(kind:(?s:.)+\n---\n(?s:.)+\bdata:|data:(?s:.)+\n---\n(?s:.)+\bkind:)''', +] + +[[rules]] +id = "kucoin-access-token" +description = "Found a Kucoin Access Token, risking unauthorized access to cryptocurrency exchange services and transactions." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "kucoin-secret-key" +description = "Discovered a Kucoin Secret Key, which could lead to compromised cryptocurrency operations and financial data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "launchdarkly-access-token" +description = "Uncovered a Launchdarkly Access Token, potentially compromising feature flag management and application functionality." +regex = '''(?i)[\w.-]{0,50}?(?:launchdarkly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["launchdarkly"] + +[[rules]] +id = "linear-api-key" +description = "Detected a Linear API Token, posing a risk to project management tools and sensitive task data." +regex = '''lin_api_(?i)[a-z0-9]{40}''' +entropy = 2 +keywords = ["lin_api_"] + +[[rules]] +id = "linear-client-secret" +description = "Identified a Linear Client Secret, which may compromise secure integrations and sensitive project management data." +regex = '''(?i)[\w.-]{0,50}?(?:linear)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["linear"] + +[[rules]] +id = "linkedin-client-id" +description = "Found a LinkedIn Client ID, risking unauthorized access to LinkedIn integrations and professional data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{14})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "linkedin-client-secret" +description = "Discovered a LinkedIn Client secret, potentially compromising LinkedIn application integrations and user data." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "lob-api-key" +description = "Uncovered a Lob API Key, which could lead to unauthorized access to mailing and address verification services." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((live|test)_[a-f0-9]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_", + "live_", +] + +[[rules]] +id = "lob-pub-api-key" +description = "Detected a Lob Publishable API Key, posing a risk of exposing mail and print service integrations." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((test|live)_pub_[a-f0-9]{31})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_pub", + "live_pub", + "_pub", +] + +[[rules]] +id = "mailchimp-api-key" +description = "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data." +regex = '''(?i)[\w.-]{0,50}?(?:MailchimpSDK.initialize|mailchimp)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32}-us\d\d)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailchimp"] + +[[rules]] +id = "mailgun-private-api-token" +description = "Found a Mailgun private API token, risking unauthorized email service operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(key-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-pub-key" +description = "Discovered a Mailgun public validation key, which could expose email verification processes and associated data." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pubkey-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-signing-key" +description = "Uncovered a Mailgun webhook signing key, potentially compromising email automation and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mapbox-api-token" +description = "Detected a MapBox API token, posing a risk to geospatial services and sensitive location data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:mapbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mapbox"] + +[[rules]] +id = "mattermost-access-token" +description = "Identified a Mattermost Access Token, which may compromise team communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:mattermost)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mattermost"] + +[[rules]] +id = "maxmind-license-key" +description = "Discovered a potential MaxMind license key." +regex = '''\b([A-Za-z0-9]{6}_[A-Za-z0-9]{29}_mmk)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["_mmk"] + +[[rules]] +id = "messagebird-api-token" +description = "Found a MessageBird API token, risking unauthorized access to communication platforms and message data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "messagebird-client-id" +description = "Discovered a MessageBird client ID, potentially compromising API integrations and sensitive communication data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "microsoft-teams-webhook" +description = "Uncovered a Microsoft Teams Webhook, which could lead to unauthorized access to team collaboration tools and data leaks." +regex = '''https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}/IncomingWebhook/[a-z0-9]{32}/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' +keywords = [ + "webhook.office.com", + "webhookb2", + "incomingwebhook", +] + +[[rules]] +id = "netlify-access-token" +description = "Detected a Netlify Access Token, potentially compromising web hosting services and site management." +regex = '''(?i)[\w.-]{0,50}?(?:netlify)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40,46})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["netlify"] + +[[rules]] +id = "new-relic-browser-api-token" +description = "Identified a New Relic ingest browser API token, risking unauthorized access to application performance data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRJS-[a-f0-9]{19})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrjs-"] + +[[rules]] +id = "new-relic-insert-key" +description = "Discovered a New Relic insight insert key, compromising data injection into the platform." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRII-[a-z0-9-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrii-"] + +[[rules]] +id = "new-relic-user-api-id" +description = "Found a New Relic user API ID, posing a risk to application monitoring services and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "new-relic", + "newrelic", + "new_relic", +] + +[[rules]] +id = "new-relic-user-api-key" +description = "Discovered a New Relic user API Key, which could lead to compromised application insights and performance monitoring." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRAK-[a-z0-9]{27})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrak"] + +[[rules]] +id = "npm-access-token" +description = "Uncovered an npm access token, potentially compromising package management and code repository access." +regex = '''(?i)\b(npm_[a-z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["npm_"] + +[[rules]] +id = "nuget-config-password" +description = "Identified a password within a Nuget config file, potentially compromising package management access." +regex = '''(?i)''' +path = '''(?i)nuget\.config$''' +entropy = 1 +keywords = ["|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "nytimes", + "new-york-times", + "newyorktimes", +] + +[[rules]] +id = "octopus-deploy-api-key" +description = "Discovered a potential Octopus Deploy API key, risking application deployments and operational security." +regex = '''\b(API-[A-Z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["api-"] + +[[rules]] +id = "okta-access-token" +description = "Identified an Okta Access Token, which may compromise identity management services and user authentication data." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Oo]kta|OKTA))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(00[\w=\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["okta"] + +[[rules]] +id = "openai-api-key" +description = "Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''\b(sk-(?:proj|svcacct|admin)-(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})T3BlbkFJ(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})\b|sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["t3blbkfj"] + +[[rules]] +id = "openshift-user-token" +description = "Found an OpenShift user token, potentially compromising an OpenShift/Kubernetes cluster." +regex = '''\b(sha256~[\w-]{43})(?:[^\w-]|\z)''' +entropy = 3.5 +keywords = ["sha256~"] + +[[rules]] +id = "perplexity-api-key" +description = "Detected a Perplexity API key, which could lead to unauthorized access to Perplexity AI services and data exposure." +regex = '''\b(pplx-[a-zA-Z0-9]{48})(?:[\x60'"\s;]|\\[nr]|$|\b)''' +entropy = 4 +keywords = ["pplx-"] + +[[rules]] +id = "pkcs12-file" +description = "Found a PKCS #12 file, which commonly contain bundled private keys." +path = '''(?i)(?:^|\/)[^\/]+\.p(?:12|fx)$''' + +[[rules]] +id = "plaid-api-token" +description = "Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["plaid"] + +[[rules]] +id = "plaid-client-id" +description = "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "plaid-secret-key" +description = "Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "planetscale-api-token" +description = "Identified a PlanetScale API token, potentially compromising database management and operations." +regex = '''\b(pscale_tkn_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_tkn_"] + +[[rules]] +id = "planetscale-oauth-token" +description = "Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity." +regex = '''\b(pscale_oauth_[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_oauth_"] + +[[rules]] +id = "planetscale-password" +description = "Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches." +regex = '''(?i)\b(pscale_pw_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_pw_"] + +[[rules]] +id = "postman-api-token" +description = "Uncovered a Postman API token, potentially compromising API testing and development workflows." +regex = '''\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pmak-"] + +[[rules]] +id = "prefect-api-token" +description = "Detected a Prefect API token, risking unauthorized access to workflow management and automation services." +regex = '''\b(pnu_[a-zA-Z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pnu_"] + +[[rules]] +id = "private-key" +description = "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption." +regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----''' +keywords = ["-----begin"] + +[[rules]] +id = "privateai-api-token" +description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:private[_-]?ai)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "privateai", + "private_ai", + "private-ai", +] + +[[rules]] +id = "pulumi-api-token" +description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management." +regex = '''\b(pul-[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pul-"] + +[[rules]] +id = "pypi-upload-token" +description = "Discovered a PyPI upload token, potentially compromising Python package distribution and repository integrity." +regex = '''pypi-AgEIcHlwaS5vcmc[\w-]{50,1000}''' +entropy = 3 +keywords = ["pypi-ageichlwas5vcmc"] + +[[rules]] +id = "rapidapi-access-token" +description = "Uncovered a RapidAPI Access Token, which could lead to unauthorized access to various APIs and data services." +regex = '''(?i)[\w.-]{0,50}?(?:rapidapi)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["rapidapi"] + +[[rules]] +id = "readme-api-token" +description = "Detected a Readme API token, risking unauthorized documentation management and content exposure." +regex = '''\b(rdme_[a-z0-9]{70})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rdme_"] + +[[rules]] +id = "rubygems-api-token" +description = "Identified a Rubygem API token, potentially compromising Ruby library distribution and package management." +regex = '''\b(rubygems_[a-f0-9]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rubygems_"] + +[[rules]] +id = "scalingo-api-token" +description = "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security." +regex = '''\b(tk-us-[\w-]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["tk-us-"] + +[[rules]] +id = "sendbird-access-id" +description = "Discovered a Sendbird Access ID, which could compromise chat and messaging platform integrations." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendbird-access-token" +description = "Uncovered a Sendbird Access Token, potentially risking unauthorized access to communication services and user data." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendgrid-api-token" +description = "Detected a SendGrid API token, posing a risk of unauthorized email service operations and data exposure." +regex = '''\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["sg."] + +[[rules]] +id = "sendinblue-api-token" +description = "Identified a Sendinblue API token, which may compromise email marketing services and subscriber data privacy." +regex = '''\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["xkeysib-"] + +[[rules]] +id = "sentry-access-token" +description = "Found a Sentry.io Access Token (old format), risking unauthorized access to error tracking services and sensitive application data." +regex = '''(?i)[\w.-]{0,50}?(?:sentry)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sentry"] + +[[rules]] +id = "sentry-org-token" +description = "Found a Sentry.io Organization Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\bsntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}(?:[^a-zA-Z0-9+/]|\z)''' +entropy = 4.5 +keywords = ["sntrys_eyjpyxqio"] + +[[rules]] +id = "sentry-user-token" +description = "Found a Sentry.io User Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\b(sntryu_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["sntryu_"] + +[[rules]] +id = "settlemint-application-access-token" +description = "Found a Settlemint Application Access Token." +regex = '''\b(sm_aat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_aat"] + +[[rules]] +id = "settlemint-personal-access-token" +description = "Found a Settlemint Personal Access Token." +regex = '''\b(sm_pat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_pat"] + +[[rules]] +id = "settlemint-service-access-token" +description = "Found a Settlemint Service Access Token." +regex = '''\b(sm_sat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_sat"] + +[[rules]] +id = "shippo-api-token" +description = "Discovered a Shippo API token, potentially compromising shipping services and customer order data." +regex = '''\b(shippo_(?:live|test)_[a-fA-F0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["shippo_"] + +[[rules]] +id = "shopify-access-token" +description = "Uncovered a Shopify access token, which could lead to unauthorized e-commerce platform access and data breaches." +regex = '''shpat_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpat_"] + +[[rules]] +id = "shopify-custom-access-token" +description = "Detected a Shopify custom access token, potentially compromising custom app integrations and e-commerce data security." +regex = '''shpca_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpca_"] + +[[rules]] +id = "shopify-private-app-access-token" +description = "Identified a Shopify private app access token, risking unauthorized access to private app data and store operations." +regex = '''shppa_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shppa_"] + +[[rules]] +id = "shopify-shared-secret" +description = "Found a Shopify shared secret, posing a risk to application authentication and e-commerce platform security." +regex = '''shpss_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpss_"] + +[[rules]] +id = "sidekiq-secret" +description = "Discovered a Sidekiq Secret, which could lead to compromised background job processing and application data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "bundle_enterprise__contribsys__com", + "bundle_gems__contribsys__com", +] + +[[rules]] +id = "sidekiq-sensitive-url" +description = "Uncovered a Sidekiq Sensitive URL, potentially exposing internal job queues and sensitive operation details." +regex = '''(?i)\bhttps?://([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' +keywords = [ + "gems.contribsys.com", + "enterprise.contribsys.com", +] + +[[rules]] +id = "slack-app-token" +description = "Detected a Slack App-level token, risking unauthorized access to Slack applications and workspace data." +regex = '''(?i)xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+''' +entropy = 2 +keywords = ["xapp"] + +[[rules]] +id = "slack-bot-token" +description = "Identified a Slack Bot token, which may compromise bot integrations and communication channel security." +regex = '''xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*''' +entropy = 3 +keywords = ["xoxb"] + +[[rules]] +id = "slack-config-access-token" +description = "Found a Slack Configuration access token, posing a risk to workspace configuration and sensitive data access." +regex = '''(?i)xoxe.xox[bp]-\d-[A-Z0-9]{163,166}''' +entropy = 2 +keywords = [ + "xoxe.xoxb-", + "xoxe.xoxp-", +] + +[[rules]] +id = "slack-config-refresh-token" +description = "Discovered a Slack Configuration refresh token, potentially allowing prolonged unauthorized access to configuration settings." +regex = '''(?i)xoxe-\d-[A-Z0-9]{146}''' +entropy = 2 +keywords = ["xoxe-"] + +[[rules]] +id = "slack-legacy-bot-token" +description = "Uncovered a Slack Legacy bot token, which could lead to compromised legacy bot operations and data exposure." +regex = '''xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26}''' +entropy = 2 +keywords = ["xoxb"] + +[[rules]] +id = "slack-legacy-token" +description = "Detected a Slack Legacy token, risking unauthorized access to older Slack integrations and user data." +regex = '''xox[os]-\d+-\d+-\d+-[a-fA-F\d]+''' +entropy = 2 +keywords = [ + "xoxo", + "xoxs", +] + +[[rules]] +id = "slack-legacy-workspace-token" +description = "Identified a Slack Legacy Workspace token, potentially compromising access to workspace data and legacy features." +regex = '''xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48}''' +entropy = 2 +keywords = [ + "xoxa", + "xoxr", +] + +[[rules]] +id = "slack-user-token" +description = "Found a Slack User token, posing a risk of unauthorized user impersonation and data access within Slack workspaces." +regex = '''xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34}''' +entropy = 2 +keywords = [ + "xoxp-", + "xoxe-", +] + +[[rules]] +id = "slack-webhook-url" +description = "Discovered a Slack Webhook, which could lead to unauthorized message posting and data leakage in Slack channels." +regex = '''(?:https?://)?hooks.slack.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}''' +keywords = ["hooks.slack.com"] + +[[rules]] +id = "snyk-api-token" +description = "Uncovered a Snyk API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["snyk"] + +[[rules]] +id = "sonar-api-token" +description = "Uncovered a Sonar API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:sonar[_.-]?(login|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sonar"] + +[[rules]] +id = "sourcegraph-access-token" +description = "Sourcegraph is a code search and navigation engine." +regex = '''(?i)\b(\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "sgp_", + "sourcegraph", +] + +[[rules]] +id = "square-access-token" +description = "Detected a Square Access Token, risking unauthorized payment processing and financial transaction exposure." +regex = '''\b((?:EAAA|sq0atp-)[\w-]{22,60})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sq0atp-", + "eaaa", +] + +[[rules]] +id = "squarespace-access-token" +description = "Identified a Squarespace Access Token, which may compromise website management and content control on Squarespace." +regex = '''(?i)[\w.-]{0,50}?(?:squarespace)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["squarespace"] + +[[rules]] +id = "stripe-access-token" +description = "Found a Stripe Access Token, posing a risk to payment processing services and sensitive financial data." +regex = '''\b((?:sk|rk)_(?:test|live|prod)_[a-zA-Z0-9]{10,99})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sk_test", + "sk_live", + "sk_prod", + "rk_test", + "rk_live", + "rk_prod", +] + +[[rules]] +id = "sumologic-access-id" +description = "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(su[a-zA-Z0-9]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "sumologic-access-token" +description = "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "telegram-bot-api-token" +description = "Detected a Telegram Bot API Token, risking unauthorized bot operations and message interception on Telegram." +regex = '''(?i)[\w.-]{0,50}?(?:telegr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{5,16}:(?-i:A)[a-z0-9_\-]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["telegr"] + +[[rules]] +id = "travisci-access-token" +description = "Identified a Travis CI Access Token, potentially compromising continuous integration services and codebase security." +regex = '''(?i)[\w.-]{0,50}?(?:travis)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["travis"] + +[[rules]] +id = "twilio-api-key" +description = "Found a Twilio API Key, posing a risk to communication services and sensitive customer interaction data." +regex = '''SK[0-9a-fA-F]{32}''' +entropy = 3 +keywords = ["sk"] + +[[rules]] +id = "twitch-api-token" +description = "Discovered a Twitch API token, which could compromise streaming services and account integrations." +regex = '''(?i)[\w.-]{0,50}?(?:twitch)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitch"] + +[[rules]] +id = "twitter-access-secret" +description = "Uncovered a Twitter Access Secret, potentially risking unauthorized Twitter integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{45})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-access-token" +description = "Detected a Twitter Access Token, posing a risk of unauthorized account operations and social media data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-key" +description = "Identified a Twitter API Key, which may compromise Twitter application integrations and user data security." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-secret" +description = "Found a Twitter API Secret, risking the security of Twitter app integrations and sensitive data access." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-bearer-token" +description = "Discovered a Twitter Bearer Token, potentially compromising API access and data retrieval from Twitter." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "typeform-api-token" +description = "Uncovered a Typeform API token, which could lead to unauthorized survey management and data collection." +regex = '''(?i)[\w.-]{0,50}?(?:typeform)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(tfp_[a-z0-9\-_\.=]{59})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["tfp_"] + +[[rules]] +id = "vault-batch-token" +description = "Detected a Vault Batch Token, risking unauthorized access to secret management services and sensitive data." +regex = '''\b(hvb\.[\w-]{138,300})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hvb."] + +[[rules]] +id = "vault-service-token" +description = "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials." +regex = '''\b((?:hvs\.[\w-]{90,120}|s\.(?i:[a-z0-9]{24})))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "hvs.", + "s.", +] +[[rules.allowlists]] +regexes = [ + '''s\.[A-Za-z]{24}''', +] + +[[rules]] +id = "yandex-access-token" +description = "Found a Yandex Access Token, posing a risk to Yandex service integrations and user data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-api-key" +description = "Discovered a Yandex API Key, which could lead to unauthorized access to Yandex services and data manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-aws-access-token" +description = "Uncovered a Yandex AWS Access Token, potentially compromising cloud resource access and data security on Yandex Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(YC[a-zA-Z0-9_\-]{38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "zendesk-secret-key" +description = "Detected a Zendesk Secret Key, risking unauthorized access to customer support services and sensitive ticketing data." +regex = '''(?i)[\w.-]{0,50}?(?:zendesk)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["zendesk"] + diff --git a/cli/detect/config/rule.go b/cli/detect/config/rule.go new file mode 100644 index 000000000..6d2b61326 --- /dev/null +++ b/cli/detect/config/rule.go @@ -0,0 +1,114 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package config + +import ( + "fmt" + "strings" + + "github.com/Infisical/infisical-merge/detect/regexp" +) + +// Rules contain information that define details on how to detect secrets +type Rule struct { + // RuleID is a unique identifier for this rule + RuleID string + + // Description is the description of the rule. + Description string + + // Entropy is a float representing the minimum shannon + // entropy a regex group must have to be considered a secret. + Entropy float64 + + // SecretGroup is an int used to extract secret from regex + // match and used as the group that will have its entropy + // checked if `entropy` is set. + SecretGroup int + + // Regex is a golang regular expression used to detect secrets. + Regex *regexp.Regexp + + // Path is a golang regular expression used to + // filter secrets by path + Path *regexp.Regexp + + // Tags is an array of strings used for metadata + // and reporting purposes. + Tags []string + + // Keywords are used for pre-regex check filtering. Rules that contain + // keywords will perform a quick string compare check to make sure the + // keyword(s) are in the content being scanned. + Keywords []string + + // Allowlists allows a rule to be ignored for specific commits, paths, regexes, and/or stopwords. + Allowlists []*Allowlist + + // validated is an internal flag to track whether `Validate()` has been called. + validated bool +} + +// Validate guards against common misconfigurations. +func (r *Rule) Validate() error { + if r.validated { + return nil + } + + // Ensure |id| is present. + if strings.TrimSpace(r.RuleID) == "" { + // Try to provide helpful context, since |id| is empty. + var context string + if r.Regex != nil { + context = ", regex: " + r.Regex.String() + } else if r.Path != nil { + context = ", path: " + r.Path.String() + } else if r.Description != "" { + context = ", description: " + r.Description + } + return fmt.Errorf("rule |id| is missing or empty" + context) + } + + // Ensure the rule actually matches something. + if r.Regex == nil && r.Path == nil { + return fmt.Errorf("%s: both |regex| and |path| are empty, this rule will have no effect", r.RuleID) + } + + // Ensure |secretGroup| works. + if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() { + return fmt.Errorf("%s: invalid regex secret group %d, max regex secret group %d", r.RuleID, r.SecretGroup, r.Regex.NumSubexp()) + } + + for _, allowlist := range r.Allowlists { + // This will probably never happen. + if allowlist == nil { + continue + } + if err := allowlist.Validate(); err != nil { + return fmt.Errorf("%s: %w", r.RuleID, err) + } + } + + r.validated = true + return nil +} diff --git a/cli/report/report.go b/cli/detect/config/utils.go similarity index 65% rename from cli/report/report.go rename to cli/detect/config/utils.go index 1191a4f33..e28a5cb37 100644 --- a/cli/report/report.go +++ b/cli/detect/config/utils.go @@ -20,35 +20,27 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -package report +package config import ( - "os" - "strings" - - "github.com/Infisical/infisical-merge/config" + "github.com/Infisical/infisical-merge/detect/regexp" ) -const ( - // https://cwe.mitre.org/data/definitions/798.html - CWE = "CWE-798" - CWE_DESCRIPTION = "Use of Hard-coded Credentials" -) - -func Write(findings []Finding, cfg config.Config, ext string, reportPath string) error { - file, err := os.Create(reportPath) - if err != nil { - return err +func anyRegexMatch(f string, res []*regexp.Regexp) bool { + for _, re := range res { + if regexMatched(f, re) { + return true + } } - ext = strings.ToLower(ext) - switch ext { - case ".json", "json": - err = writeJson(findings, file) - case ".csv", "csv": - err = writeCsv(findings, file) - case ".sarif", "sarif": - err = writeSarif(cfg, findings, file) - } - - return err + return false +} + +func regexMatched(f string, re *regexp.Regexp) bool { + if re == nil { + return false + } + if re.FindString(f) != "" { + return true + } + return false } diff --git a/cli/detect/decoder.go b/cli/detect/decoder.go new file mode 100644 index 000000000..6ec509757 --- /dev/null +++ b/cli/detect/decoder.go @@ -0,0 +1,328 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bytes" + "encoding/base64" + "fmt" + "regexp" + "unicode" + + "github.com/Infisical/infisical-merge/detect/logging" +) + +var b64LikelyChars [128]byte +var b64Regexp = regexp.MustCompile(`[\w/+-]{16,}={0,3}`) +var decoders = []func(string) ([]byte, error){ + base64.StdEncoding.DecodeString, + base64.RawURLEncoding.DecodeString, +} + +func init() { + // Basically look for anything that isn't just letters + for _, c := range `0123456789+/-_` { + b64LikelyChars[c] = 1 + } +} + +// EncodedSegment represents a portion of text that is encoded in some way. +// `decode` supports recusive decoding and can result in "segment trees". +// There can be multiple segments in the original text, so each can be thought +// of as its own tree with the root being the original segment. +type EncodedSegment struct { + // The parent segment in a segment tree. If nil, it is a root segment + parent *EncodedSegment + + // Relative start/end are the bounds of the encoded value in the current pass. + relativeStart int + relativeEnd int + + // Absolute start/end refer to the bounds of the root segment in this segment + // tree + absoluteStart int + absoluteEnd int + + // Decoded start/end refer to the bounds of the decoded value in the current + // pass. These can differ from relative values because decoding can shrink + // or grow the size of the segment. + decodedStart int + decodedEnd int + + // This is the actual decoded content in the segment + decodedValue string + + // This is the type of encoding + encoding string +} + +// isChildOf inspects the bounds of two segments to determine +// if one should be the child of another +func (s EncodedSegment) isChildOf(parent EncodedSegment) bool { + return parent.decodedStart <= s.relativeStart && parent.decodedEnd >= s.relativeEnd +} + +// decodedOverlaps checks if the decoded bounds of the segment overlaps a range +func (s EncodedSegment) decodedOverlaps(start, end int) bool { + return start <= s.decodedEnd && end >= s.decodedStart +} + +// adjustMatchIndex takes the matchIndex from the current decoding pass and +// updates it to match the absolute matchIndex in the original text. +func (s EncodedSegment) adjustMatchIndex(matchIndex []int) []int { + // The match is within the bounds of the segment so we just return + // the absolute start and end of the root segment. + if s.decodedStart <= matchIndex[0] && matchIndex[1] <= s.decodedEnd { + return []int{ + s.absoluteStart, + s.absoluteEnd, + } + } + + // Since it overlaps one side and/or the other, we're going to have to adjust + // and climb parents until we're either at the root or we've determined + // we're fully inside one of the parent segments. + adjustedMatchIndex := make([]int, 2) + + if matchIndex[0] < s.decodedStart { + // It starts before the encoded segment so adjust the start to match + // the location before it was decoded + matchStartDelta := s.decodedStart - matchIndex[0] + adjustedMatchIndex[0] = s.relativeStart - matchStartDelta + } else { + // It starts within the encoded segment so set the bound to the + // relative start + adjustedMatchIndex[0] = s.relativeStart + } + + if matchIndex[1] > s.decodedEnd { + // It ends after the encoded segment so adjust the end to match + // the location before it was decoded + matchEndDelta := matchIndex[1] - s.decodedEnd + adjustedMatchIndex[1] = s.relativeEnd + matchEndDelta + } else { + // It ends within the encoded segment so set the bound to the relative end + adjustedMatchIndex[1] = s.relativeEnd + } + + // We're still not at a root segment so we'll need to keep on adjusting + if s.parent != nil { + return s.parent.adjustMatchIndex(adjustedMatchIndex) + } + + return adjustedMatchIndex +} + +// depth reports how many levels of decoding needed to be done (default is 1) +func (s EncodedSegment) depth() int { + depth := 1 + + // Climb the tree and increment the depth + for current := &s; current.parent != nil; current = current.parent { + depth++ + } + + return depth +} + +// tags returns additional meta data tags related to the types of segments +func (s EncodedSegment) tags() []string { + return []string{ + fmt.Sprintf("decoded:%s", s.encoding), + fmt.Sprintf("decode-depth:%d", s.depth()), + } +} + +// Decoder decodes various types of data in place +type Decoder struct { + decodedMap map[string]string +} + +// NewDecoder creates a default decoder struct +func NewDecoder() *Decoder { + return &Decoder{ + decodedMap: make(map[string]string), + } +} + +// decode returns the data with the values decoded in-place +func (d *Decoder) decode(data string, parentSegments []EncodedSegment) (string, []EncodedSegment) { + segments := d.findEncodedSegments(data, parentSegments) + + if len(segments) > 0 { + result := bytes.NewBuffer(make([]byte, 0, len(data))) + + relativeStart := 0 + for _, segment := range segments { + result.WriteString(data[relativeStart:segment.relativeStart]) + result.WriteString(segment.decodedValue) + relativeStart = segment.relativeEnd + } + result.WriteString(data[relativeStart:]) + + return result.String(), segments + } + + return data, segments +} + +// findEncodedSegments finds the encoded segments in the data and updates the +// segment tree for this pass +func (d *Decoder) findEncodedSegments(data string, parentSegments []EncodedSegment) []EncodedSegment { + if len(data) == 0 { + return []EncodedSegment{} + } + + matchIndices := b64Regexp.FindAllStringIndex(data, -1) + if matchIndices == nil { + return []EncodedSegment{} + } + + segments := make([]EncodedSegment, 0, len(matchIndices)) + + // Keeps up with offsets from the text changing size as things are decoded + decodedShift := 0 + + for _, matchIndex := range matchIndices { + encodedValue := data[matchIndex[0]:matchIndex[1]] + + if !isLikelyB64(encodedValue) { + d.decodedMap[encodedValue] = "" + continue + } + + decodedValue, alreadyDecoded := d.decodedMap[encodedValue] + + // We haven't decoded this yet, so go ahead and decode it + if !alreadyDecoded { + decodedValue = decodeValue(encodedValue) + d.decodedMap[encodedValue] = decodedValue + } + + // Skip this segment because there was nothing to check + if len(decodedValue) == 0 { + continue + } + + // Create a segment for the encoded data + segment := EncodedSegment{ + relativeStart: matchIndex[0], + relativeEnd: matchIndex[1], + absoluteStart: matchIndex[0], + absoluteEnd: matchIndex[1], + decodedStart: matchIndex[0] + decodedShift, + decodedEnd: matchIndex[0] + decodedShift + len(decodedValue), + decodedValue: decodedValue, + encoding: "base64", + } + + // Shift decoded start and ends based on size changes + decodedShift += len(decodedValue) - len(encodedValue) + + // Adjust the absolute position of segments contained in parent segments + for _, parentSegment := range parentSegments { + if segment.isChildOf(parentSegment) { + segment.absoluteStart = parentSegment.absoluteStart + segment.absoluteEnd = parentSegment.absoluteEnd + segment.parent = &parentSegment + break + } + } + + logging.Debug().Msgf("segment found: %#v", segment) + segments = append(segments, segment) + } + + return segments +} + +// decoders tries a list of decoders and returns the first successful one +func decodeValue(encodedValue string) string { + for _, decoder := range decoders { + decodedValue, err := decoder(encodedValue) + + if err == nil && len(decodedValue) > 0 && isASCII(decodedValue) { + return string(decodedValue) + } + } + + return "" +} + +func isASCII(b []byte) bool { + for i := 0; i < len(b); i++ { + if b[i] > unicode.MaxASCII || b[i] < '\t' { + return false + } + } + + return true +} + +// Skip a lot of method signatures and things at the risk of missing about +// 1% of base64 +func isLikelyB64(s string) bool { + for _, c := range s { + if b64LikelyChars[c] != 0 { + return true + } + } + + return false +} + +// Find a segment where the decoded bounds overlaps a range +func segmentWithDecodedOverlap(encodedSegments []EncodedSegment, start, end int) *EncodedSegment { + for _, segment := range encodedSegments { + if segment.decodedOverlaps(start, end) { + return &segment + } + } + + return nil +} + +func (s EncodedSegment) currentLine(currentRaw string) string { + start := 0 + end := len(currentRaw) + + // Find the start of the range + for i := s.decodedStart; i > -1; i-- { + c := currentRaw[i] + if c == '\n' { + start = i + break + } + } + + // Find the end of the range + for i := s.decodedEnd; i < end; i++ { + c := currentRaw[i] + if c == '\n' { + end = i + break + } + } + + return currentRaw[start:end] +} diff --git a/cli/detect/detect.go b/cli/detect/detect.go index 84f058d4b..f2e42cccc 100644 --- a/cli/detect/detect.go +++ b/cli/detect/detect.go @@ -26,39 +26,37 @@ import ( "bufio" "context" "fmt" - "io" - "io/fs" "os" - "path/filepath" - "regexp" + "runtime" "strings" "sync" + "sync/atomic" + "time" - "github.com/h2non/filetype" - - "github.com/Infisical/infisical-merge/config" - "github.com/Infisical/infisical-merge/detect/git" - "github.com/Infisical/infisical-merge/report" + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/regexp" + "github.com/Infisical/infisical-merge/detect/report" + ahocorasick "github.com/BobuSumisu/aho-corasick" "github.com/fatih/semgroup" - "github.com/gitleaks/go-gitdiff/gitdiff" - ahocorasick "github.com/petar-dambovaliev/aho-corasick" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" "github.com/spf13/viper" + "golang.org/x/exp/maps" ) -// Type used to differentiate between git scan types: -// $ gitleaks detect -// $ gitleaks protect -// $ gitleaks protect staged -type GitScanType int - const ( - DetectType GitScanType = iota - ProtectType - ProtectStagedType + gitleaksAllowSignature = "gitleaks:allow" + chunkSize = 100 * 1_000 // 100kb - gitleaksAllowSignature = "infisical-scan:ignore" + // SlowWarningThreshold is the amount of time to wait before logging that a file is slow. + // This is useful for identifying problematic files and tuning the allowlist. + SlowWarningThreshold = 5 * time.Second +) + +var ( + newLineRegexp = regexp.MustCompile("\n") + isWindows = runtime.GOOS == "windows" ) // Detector is the main detector struct @@ -69,11 +67,14 @@ type Detector struct { // Redact is a flag to redact findings. This is exported // so users using gitleaks as a library can set this flag // without calling `detector.Start(cmd *cobra.Command)` - Redact bool + Redact uint // verbose is a flag to print findings Verbose bool + // MaxDecodeDepths limits how many recursive decoding passes are allowed + MaxDecodeDepth int + // files larger than this will be skipped MaxTargetMegaBytes int @@ -83,6 +84,9 @@ type Detector struct { // NoColor is a flag to disable color output NoColor bool + // IgnoreGitleaksAllow is a flag to ignore gitleaks:allow comments. + IgnoreGitleaksAllow bool + // commitMap is used to keep track of commits that have been scanned. // This is only used for logging purposes and git scans. commitMap map[string]bool @@ -98,7 +102,7 @@ type Detector struct { // prefilter is a ahocorasick struct used for doing efficient string // matching given a set of words (keywords from the rules in the config) - prefilter ahocorasick.AhoCorasick + prefilter ahocorasick.Trie // a list of known findings that should be ignored baseline []report.Finding @@ -107,7 +111,16 @@ type Detector struct { baselinePath string // gitleaksIgnore - gitleaksIgnore map[string]bool + gitleaksIgnore map[string]struct{} + + // Sema (https://github.com/fatih/semgroup) controls the concurrency + Sema *semgroup.Group + + // report-related settings. + ReportPath string + Reporter report.Reporter + + TotalBytes atomic.Uint64 } // Fragment contains the data to be scanned @@ -115,9 +128,15 @@ type Fragment struct { // Raw is the raw content of the fragment Raw string - // FilePath is the path to the file if applicable + Bytes []byte + + // FilePath is the path to the file, if applicable. + // The path separator MUST be normalized to `/`. FilePath string SymlinkFile string + // WindowsFilePath is the path with the original separator. + // This provides a backwards-compatible solution to https://github.com/gitleaks/gitleaks/issues/1565. + WindowsFilePath string `json:"-"` // TODO: remove this in v9. // CommitSHA is the SHA of the commit if applicable CommitSHA string @@ -125,28 +144,18 @@ type Fragment struct { // newlineIndices is a list of indices of newlines in the raw content. // This is used to calculate the line location of a finding newlineIndices [][]int - - // keywords is a map of all the keywords contain within the contents - // of this fragment - keywords map[string]bool } // NewDetector creates a new detector with the given config func NewDetector(cfg config.Config) *Detector { - builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{ - AsciiCaseInsensitive: true, - MatchOnlyWholeWords: false, - MatchKind: ahocorasick.LeftMostLongestMatch, - DFA: true, - }) - return &Detector{ commitMap: make(map[string]bool), - gitleaksIgnore: make(map[string]bool), + gitleaksIgnore: make(map[string]struct{}), findingMutex: &sync.Mutex{}, findings: make([]report.Finding, 0), Config: cfg, - prefilter: builder.Build(cfg.Keywords), + prefilter: *ahocorasick.NewTrieBuilder().AddStrings(maps.Keys(cfg.Keywords)).Build(), + Sema: semgroup.NewGroup(context.Background(), 40), } } @@ -170,58 +179,47 @@ func NewDetectorDefaultConfig() (*Detector, error) { } func (d *Detector) AddGitleaksIgnore(gitleaksIgnorePath string) error { - log.Debug().Msg("found .gitleaksignore file") + logging.Debug().Msgf("found .gitleaksignore file: %s", gitleaksIgnorePath) file, err := os.Open(gitleaksIgnorePath) - if err != nil { return err } - - // https://github.com/securego/gosec/issues/512 defer func() { + // https://github.com/securego/gosec/issues/512 if err := file.Close(); err != nil { - log.Warn().Msgf("Error closing .gitleaksignore file: %s\n", err) + logging.Warn().Msgf("Error closing .gitleaksignore file: %s\n", err) } }() + scanner := bufio.NewScanner(file) - + replacer := strings.NewReplacer("\\", "/") for scanner.Scan() { - d.gitleaksIgnore[scanner.Text()] = true + line := strings.TrimSpace(scanner.Text()) + // Skip lines that start with a comment + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Normalize the path. + // TODO: Make this a breaking change in v9. + s := strings.Split(line, ":") + switch len(s) { + case 3: + // Global fingerprint. + // `file:rule-id:start-line` + s[0] = replacer.Replace(s[0]) + case 4: + // Commit fingerprint. + // `commit:file:rule-id:start-line` + s[1] = replacer.Replace(s[1]) + default: + logging.Warn().Str("fingerprint", line).Msg("Invalid .gitleaksignore entry") + } + d.gitleaksIgnore[strings.Join(s, ":")] = struct{}{} } return nil } -func (d *Detector) AddBaseline(baselinePath string, source string) error { - if baselinePath != "" { - absoluteSource, err := filepath.Abs(source) - if err != nil { - return err - } - - absoluteBaseline, err := filepath.Abs(baselinePath) - if err != nil { - return err - } - - relativeBaseline, err := filepath.Rel(absoluteSource, absoluteBaseline) - if err != nil { - return err - } - - baseline, err := LoadBaseline(baselinePath) - if err != nil { - return err - } - - d.baseline = baseline - baselinePath = relativeBaseline - - } - - d.baselinePath = baselinePath - return nil -} - // DetectBytes scans the given bytes and returns a list of findings func (d *Detector) DetectBytes(content []byte) []report.Finding { return d.DetectString(string(content)) @@ -234,56 +232,179 @@ func (d *Detector) DetectString(content string) []report.Finding { }) } -// detectRule scans the given fragment for the given rule and returns a list of findings -func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Finding { - var findings []report.Finding +// Detect scans the given fragment and returns a list of findings +func (d *Detector) Detect(fragment Fragment) []report.Finding { + if fragment.Bytes == nil { + d.TotalBytes.Add(uint64(len(fragment.Raw))) + } + d.TotalBytes.Add(uint64(len(fragment.Bytes))) - // check if filepath or commit is allowed for this rule - if rule.Allowlist.CommitAllowed(fragment.CommitSHA) || - rule.Allowlist.PathAllowed(fragment.FilePath) { + var ( + findings []report.Finding + logger = func() zerolog.Logger { + l := logging.With().Str("path", fragment.FilePath) + if fragment.CommitSHA != "" { + l = l.Str("commit", fragment.CommitSHA) + } + return l.Logger() + }() + ) + + // check if filepath is allowed + if fragment.FilePath != "" { + // is the path our config or baseline file? + if fragment.FilePath == d.Config.Path || (d.baselinePath != "" && fragment.FilePath == d.baselinePath) { + logging.Trace().Msg("skipping file: matches config or baseline path") + return findings + } + } + // check if commit or filepath is allowed. + if isAllowed, event := checkCommitOrPathAllowed(logger, fragment, d.Config.Allowlists); isAllowed { + event.Msg("skipping file: global allowlist") return findings } - if rule.Path != nil && rule.Regex == nil { - // Path _only_ rule - if rule.Path.Match([]byte(fragment.FilePath)) { - finding := report.Finding{ - Description: rule.Description, - File: fragment.FilePath, - SymlinkFile: fragment.SymlinkFile, - RuleID: rule.RuleID, - Match: fmt.Sprintf("file detected: %s", fragment.FilePath), - Tags: rule.Tags, - } - return append(findings, finding) + // add newline indices for location calculation in detectRule + fragment.newlineIndices = newLineRegexp.FindAllStringIndex(fragment.Raw, -1) + + // setup variables to handle different decoding passes + currentRaw := fragment.Raw + encodedSegments := []EncodedSegment{} + currentDecodeDepth := 0 + decoder := NewDecoder() + + for { + // build keyword map for prefiltering rules + keywords := make(map[string]bool) + normalizedRaw := strings.ToLower(currentRaw) + matches := d.prefilter.MatchString(normalizedRaw) + for _, m := range matches { + keywords[normalizedRaw[m.Pos():int(m.Pos())+len(m.Match())]] = true } - } else if rule.Path != nil { - // if path is set _and_ a regex is set, then we need to check both - // so if the path does not match, then we should return early and not - // consider the regex - if !rule.Path.Match([]byte(fragment.FilePath)) { - return findings + + for _, rule := range d.Config.Rules { + if len(rule.Keywords) == 0 { + // if no keywords are associated with the rule always scan the + // fragment using the rule + findings = append(findings, d.detectRule(fragment, currentRaw, rule, encodedSegments)...) + continue + } + + // check if keywords are in the fragment + for _, k := range rule.Keywords { + if _, ok := keywords[strings.ToLower(k)]; ok { + findings = append(findings, d.detectRule(fragment, currentRaw, rule, encodedSegments)...) + break + } + } + } + + // increment the depth by 1 as we start our decoding pass + currentDecodeDepth++ + + // stop the loop if we've hit our max decoding depth + if currentDecodeDepth > d.MaxDecodeDepth { + break + } + + // decode the currentRaw for the next pass + currentRaw, encodedSegments = decoder.decode(currentRaw, encodedSegments) + + // stop the loop when there's nothing else to decode + if len(encodedSegments) == 0 { + break + } + } + + return filter(findings, d.Redact) +} + +// detectRule scans the given fragment for the given rule and returns a list of findings +func (d *Detector) detectRule(fragment Fragment, currentRaw string, r config.Rule, encodedSegments []EncodedSegment) []report.Finding { + var ( + findings []report.Finding + logger = func() zerolog.Logger { + l := logging.With().Str("rule-id", r.RuleID).Str("path", fragment.FilePath) + if fragment.CommitSHA != "" { + l = l.Str("commit", fragment.CommitSHA) + } + return l.Logger() + }() + ) + + // check if commit or file is allowed for this rule. + if isAllowed, event := checkCommitOrPathAllowed(logger, fragment, r.Allowlists); isAllowed { + event.Msg("skipping file: rule allowlist") + return findings + } + + if r.Path != nil { + if r.Regex == nil && len(encodedSegments) == 0 { + // Path _only_ rule + if r.Path.MatchString(fragment.FilePath) || (fragment.WindowsFilePath != "" && r.Path.MatchString(fragment.WindowsFilePath)) { + finding := report.Finding{ + RuleID: r.RuleID, + Description: r.Description, + File: fragment.FilePath, + SymlinkFile: fragment.SymlinkFile, + Match: fmt.Sprintf("file detected: %s", fragment.FilePath), + Tags: r.Tags, + } + return append(findings, finding) + } + } else { + // if path is set _and_ a regex is set, then we need to check both + // so if the path does not match, then we should return early and not + // consider the regex + if !(r.Path.MatchString(fragment.FilePath) || (fragment.WindowsFilePath != "" && r.Path.MatchString(fragment.WindowsFilePath))) { + return findings + } } } // if path only rule, skip content checks - if rule.Regex == nil { + if r.Regex == nil { return findings } - // If flag configure and raw data size bigger then the flag + // if flag configure and raw data size bigger then the flag if d.MaxTargetMegaBytes > 0 { - rawLength := len(fragment.Raw) / 1000000 + rawLength := len(currentRaw) / 1000000 if rawLength > d.MaxTargetMegaBytes { - log.Debug().Msgf("skipping file: %s scan due to size: %d", fragment.FilePath, rawLength) + logger.Debug(). + Int("size", rawLength). + Int("max-size", d.MaxTargetMegaBytes). + Msg("skipping fragment: size") return findings } } - matchIndices := rule.Regex.FindAllStringIndex(fragment.Raw, -1) - for _, matchIndex := range matchIndices { - // extract secret from match - secret := strings.Trim(fragment.Raw[matchIndex[0]:matchIndex[1]], "\n") + // use currentRaw instead of fragment.Raw since this represents the current + // decoding pass on the text + for _, matchIndex := range r.Regex.FindAllStringIndex(currentRaw, -1) { + // Extract secret from match + secret := strings.Trim(currentRaw[matchIndex[0]:matchIndex[1]], "\n") + + // For any meta data from decoding + var metaTags []string + currentLine := "" + + // Check if the decoded portions of the segment overlap with the match + // to see if its potentially a new match + if len(encodedSegments) > 0 { + if segment := segmentWithDecodedOverlap(encodedSegments, matchIndex[0], matchIndex[1]); segment != nil { + matchIndex = segment.adjustMatchIndex(matchIndex) + metaTags = append(metaTags, segment.tags()...) + currentLine = segment.currentLine(currentRaw) + } else { + // This item has already been added to a finding + continue + } + } else { + // Fixes: https://github.com/gitleaks/gitleaks/issues/1352 + // removes the incorrectly following line that was detected by regex expression '\n' + matchIndex[1] = matchIndex[0] + len(secret) + } // determine location of match. Note that the location // in the finding will be the line/column numbers of the _match_ @@ -296,345 +417,112 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find } finding := report.Finding{ - Description: rule.Description, - File: fragment.FilePath, - SymlinkFile: fragment.SymlinkFile, - RuleID: rule.RuleID, + RuleID: r.RuleID, + Description: r.Description, StartLine: loc.startLine, EndLine: loc.endLine, StartColumn: loc.startColumn, EndColumn: loc.endColumn, - Secret: secret, - Match: secret, - Tags: rule.Tags, Line: fragment.Raw[loc.startLineIndex:loc.endLineIndex], + Match: secret, + Secret: secret, + File: fragment.FilePath, + SymlinkFile: fragment.SymlinkFile, + Tags: append(r.Tags, metaTags...), } - if strings.Contains(fragment.Raw[loc.startLineIndex:loc.endLineIndex], - gitleaksAllowSignature) { + if !d.IgnoreGitleaksAllow && strings.Contains(finding.Line, gitleaksAllowSignature) { + logger.Trace(). + Str("finding", finding.Secret). + Msg("skipping finding: 'gitleaks:allow' signature") continue } - // extract secret from secret group if set - if rule.SecretGroup != 0 { - groups := rule.Regex.FindStringSubmatch(secret) - if len(groups) <= rule.SecretGroup || len(groups) == 0 { - // Config validation should prevent this - continue + if currentLine == "" { + currentLine = finding.Line + } + + // Set the value of |secret|, if the pattern contains at least one capture group. + // (The first element is the full match, hence we check >= 2.) + groups := r.Regex.FindStringSubmatch(finding.Secret) + if len(groups) >= 2 { + if r.SecretGroup > 0 { + if len(groups) <= r.SecretGroup { + // Config validation should prevent this + continue + } + finding.Secret = groups[r.SecretGroup] + } else { + // If |secretGroup| is not set, we will use the first suitable capture group. + for _, s := range groups[1:] { + if len(s) > 0 { + finding.Secret = s + break + } + } } - secret = groups[rule.SecretGroup] - finding.Secret = secret - } - - // check if the regexTarget is defined in the allowlist "regexes" entry - allowlistTarget := finding.Secret - switch rule.Allowlist.RegexTarget { - case "match": - allowlistTarget = finding.Match - case "line": - allowlistTarget = finding.Line - } - - globalAllowlistTarget := finding.Secret - switch d.Config.Allowlist.RegexTarget { - case "match": - globalAllowlistTarget = finding.Match - case "line": - globalAllowlistTarget = finding.Line - } - if rule.Allowlist.RegexAllowed(allowlistTarget) || - d.Config.Allowlist.RegexAllowed(globalAllowlistTarget) { - continue - } - - // check if the secret is in the list of stopwords - if rule.Allowlist.ContainsStopWord(finding.Secret) || - d.Config.Allowlist.ContainsStopWord(finding.Secret) { - continue } // check entropy entropy := shannonEntropy(finding.Secret) finding.Entropy = float32(entropy) - if rule.Entropy != 0.0 { - if entropy <= rule.Entropy { - // entropy is too low, skip this finding + if r.Entropy != 0.0 { + // entropy is too low, skip this finding + if entropy <= r.Entropy { + logger.Trace(). + Str("finding", finding.Secret). + Float32("entropy", finding.Entropy). + Msg("skipping finding: low entropy") continue } - // NOTE: this is a goofy hack to get around the fact there golang's regex engine - // does not support positive lookaheads. Ideally we would want to add a - // restriction on generic rules regex that requires the secret match group - // contains both numbers and alphabetical characters, not just alphabetical characters. - // What this bit of code does is check if the ruleid is prepended with "generic" and enforces the - // secret contains both digits and alphabetical characters. - // TODO: this should be replaced with stop words - if strings.HasPrefix(rule.RuleID, "generic") { - if !containsDigit(secret) { - continue - } - } } + // check if the result matches any of the global allowlists. + if isAllowed, event := checkFindingAllowed(logger, finding, fragment, currentLine, d.Config.Allowlists); isAllowed { + event.Msg("skipping finding: global allowlist") + continue + } + + // check if the result matches any of the rule allowlists. + if isAllowed, event := checkFindingAllowed(logger, finding, fragment, currentLine, r.Allowlists); isAllowed { + event.Msg("skipping finding: rule allowlist") + continue + } findings = append(findings, finding) } return findings } -// GitScan accepts a *gitdiff.File channel which contents a git history generated from -// the output of `git log -p ...`. startGitScan will look at each file (patch) in the history -// and determine if the patch contains any findings. -func (d *Detector) DetectGit(source string, logOpts string, gitScanType GitScanType) ([]report.Finding, error) { - var ( - gitdiffFiles <-chan *gitdiff.File - err error - ) - switch gitScanType { - case DetectType: - gitdiffFiles, err = git.GitLog(source, logOpts) - if err != nil { - return d.findings, err - } - case ProtectType: - gitdiffFiles, err = git.GitDiff(source, false) - if err != nil { - return d.findings, err - } - case ProtectStagedType: - gitdiffFiles, err = git.GitDiff(source, true) - if err != nil { - return d.findings, err - } - } - - s := semgroup.NewGroup(context.Background(), 4) - - for gitdiffFile := range gitdiffFiles { - gitdiffFile := gitdiffFile - - // skip binary files - if gitdiffFile.IsBinary || gitdiffFile.IsDelete { - continue - } - - // Check if commit is allowed - commitSHA := "" - if gitdiffFile.PatchHeader != nil { - commitSHA = gitdiffFile.PatchHeader.SHA - if d.Config.Allowlist.CommitAllowed(gitdiffFile.PatchHeader.SHA) { - continue - } - } - d.addCommit(commitSHA) - - s.Go(func() error { - for _, textFragment := range gitdiffFile.TextFragments { - if textFragment == nil { - return nil - } - - fragment := Fragment{ - Raw: textFragment.Raw(gitdiff.OpAdd), - CommitSHA: commitSHA, - FilePath: gitdiffFile.NewName, - } - - for _, finding := range d.Detect(fragment) { - d.addFinding(augmentGitFinding(finding, textFragment, gitdiffFile)) - } - } - return nil - }) - } - - if err := s.Wait(); err != nil { - return d.findings, err - } - log.Info().Msgf("%d commits scanned.", len(d.commitMap)) - log.Debug().Msg("Note: this number might be smaller than expected due to commits with no additions") - if git.ErrEncountered { - return d.findings, fmt.Errorf("%s", "git error encountered, see logs") - } - return d.findings, nil -} - -type scanTarget struct { - Path string - Symlink string -} - -// DetectFiles accepts a path to a source directory or file and begins a scan of the -// file or directory. -func (d *Detector) DetectFiles(source string) ([]report.Finding, error) { - s := semgroup.NewGroup(context.Background(), 4) - paths := make(chan scanTarget) - s.Go(func() error { - defer close(paths) - return filepath.Walk(source, - func(path string, fInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fInfo.Name() == ".git" && fInfo.IsDir() { - return filepath.SkipDir - } - if fInfo.Size() == 0 { - return nil - } - if fInfo.Mode().IsRegular() { - paths <- scanTarget{ - Path: path, - Symlink: "", - } - } - if fInfo.Mode().Type() == fs.ModeSymlink && d.FollowSymlinks { - realPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - realPathFileInfo, _ := os.Stat(realPath) - if realPathFileInfo.IsDir() { - log.Debug().Msgf("found symlinked directory: %s -> %s [skipping]", path, realPath) - return nil - } - paths <- scanTarget{ - Path: realPath, - Symlink: path, - } - } - return nil - }) - }) - for pa := range paths { - p := pa - s.Go(func() error { - b, err := os.ReadFile(p.Path) - if err != nil { - return err - } - - mimetype, err := filetype.Match(b) - if err != nil { - return err - } - if mimetype.MIME.Type == "application" { - return nil // skip binary files - } - - fragment := Fragment{ - Raw: string(b), - FilePath: p.Path, - } - if p.Symlink != "" { - fragment.SymlinkFile = p.Symlink - } - for _, finding := range d.Detect(fragment) { - // need to add 1 since line counting starts at 1 - finding.EndLine++ - finding.StartLine++ - d.addFinding(finding) - } - - return nil - }) - } - - if err := s.Wait(); err != nil { - return d.findings, err - } - - return d.findings, nil -} - -// DetectReader accepts an io.Reader and a buffer size for the reader in KB -func (d *Detector) DetectReader(r io.Reader, bufSize int) ([]report.Finding, error) { - reader := bufio.NewReader(r) - buf := make([]byte, 0, 1000*bufSize) - findings := []report.Finding{} - - for { - n, err := reader.Read(buf[:cap(buf)]) - buf = buf[:n] - if err != nil { - if err != io.EOF { - return findings, err - } - break - } - - fragment := Fragment{ - Raw: string(buf), - } - for _, finding := range d.Detect(fragment) { - findings = append(findings, finding) - if d.Verbose { - printFinding(finding, d.NoColor) - } - } - } - - return findings, nil -} - -// Detect scans the given fragment and returns a list of findings -func (d *Detector) Detect(fragment Fragment) []report.Finding { - var findings []report.Finding - - // initiate fragment keywords - fragment.keywords = make(map[string]bool) - - // check if filepath is allowed - if fragment.FilePath != "" && (d.Config.Allowlist.PathAllowed(fragment.FilePath) || - fragment.FilePath == d.Config.Path || (d.baselinePath != "" && fragment.FilePath == d.baselinePath)) { - return findings - } - - // add newline indices for location calculation in detectRule - fragment.newlineIndices = regexp.MustCompile("\n").FindAllStringIndex(fragment.Raw, -1) - - // build keyword map for prefiltering rules - normalizedRaw := strings.ToLower(fragment.Raw) - matches := d.prefilter.FindAll(normalizedRaw) - for _, m := range matches { - fragment.keywords[normalizedRaw[m.Start():m.End()]] = true - } - - for _, rule := range d.Config.Rules { - if len(rule.Keywords) == 0 { - // if not keywords are associated with the rule always scan the - // fragment using the rule - findings = append(findings, d.detectRule(fragment, rule)...) - continue - } - fragmentContainsKeyword := false - // check if keywords are in the fragment - for _, k := range rule.Keywords { - if _, ok := fragment.keywords[strings.ToLower(k)]; ok { - fragmentContainsKeyword = true - } - } - if fragmentContainsKeyword { - findings = append(findings, d.detectRule(fragment, rule)...) - } - } - return filter(findings, d.Redact) -} - -// addFinding synchronously adds a finding to the findings slice -func (d *Detector) addFinding(finding report.Finding) { - if finding.Commit == "" { - finding.Fingerprint = fmt.Sprintf("%s:%s:%d", finding.File, finding.RuleID, finding.StartLine) - } else { +// AddFinding synchronously adds a finding to the findings slice +func (d *Detector) AddFinding(finding report.Finding) { + globalFingerprint := fmt.Sprintf("%s:%s:%d", finding.File, finding.RuleID, finding.StartLine) + if finding.Commit != "" { finding.Fingerprint = fmt.Sprintf("%s:%s:%s:%d", finding.Commit, finding.File, finding.RuleID, finding.StartLine) - } - // check if we should ignore this finding - if _, ok := d.gitleaksIgnore[finding.Fingerprint]; ok { - log.Debug().Msgf("ignoring finding with Fingerprint %s", - finding.Fingerprint) - return + } else { + finding.Fingerprint = globalFingerprint } - if d.baseline != nil && !IsNew(finding, d.baseline) { - log.Debug().Msgf("baseline duplicate -- ignoring finding with Fingerprint %s", finding.Fingerprint) + // check if we should ignore this finding + logger := logging.With().Str("finding", finding.Secret).Logger() + if _, ok := d.gitleaksIgnore[globalFingerprint]; ok { + logger.Debug(). + Str("fingerprint", globalFingerprint). + Msg("skipping finding: global fingerprint") + return + } else if finding.Commit != "" { + // Awkward nested if because I'm not sure how to chain these two conditions. + if _, ok := d.gitleaksIgnore[finding.Fingerprint]; ok { + logger.Debug(). + Str("fingerprint", finding.Fingerprint). + Msgf("skipping finding: fingerprint") + return + } + } + + if d.baseline != nil && !IsNew(finding, d.Redact, d.baseline) { + logger.Debug(). + Str("fingerprint", finding.Fingerprint). + Msgf("skipping finding: baseline") return } @@ -646,7 +534,166 @@ func (d *Detector) addFinding(finding report.Finding) { d.findingMutex.Unlock() } -// addCommit synchronously adds a commit to the commit slice +// Findings returns the findings added to the detector +func (d *Detector) Findings() []report.Finding { + return d.findings +} + +// AddCommit synchronously adds a commit to the commit slice func (d *Detector) addCommit(commit string) { d.commitMap[commit] = true } + +// checkCommitOrPathAllowed evaluates |fragment| against all provided |allowlists|. +// +// If the match condition is "OR", only commit and path are checked. +// Otherwise, if regexes or stopwords are defined this will fail. +func checkCommitOrPathAllowed( + logger zerolog.Logger, + fragment Fragment, + allowlists []*config.Allowlist, +) (bool, *zerolog.Event) { + if fragment.FilePath == "" && fragment.CommitSHA == "" { + return false, nil + } + + for _, a := range allowlists { + var ( + isAllowed bool + allowlistChecks []bool + commitAllowed, _ = a.CommitAllowed(fragment.CommitSHA) + pathAllowed = a.PathAllowed(fragment.FilePath) || (fragment.WindowsFilePath != "" && a.PathAllowed(fragment.WindowsFilePath)) + ) + // If the condition is "AND" we need to check all conditions. + if a.MatchCondition == config.AllowlistMatchAnd { + if len(a.Commits) > 0 { + allowlistChecks = append(allowlistChecks, commitAllowed) + } + if len(a.Paths) > 0 { + allowlistChecks = append(allowlistChecks, pathAllowed) + } + // These will be checked later. + if len(a.Regexes) > 0 { + continue + } + if len(a.StopWords) > 0 { + continue + } + + isAllowed = allTrue(allowlistChecks) + } else { + isAllowed = commitAllowed || pathAllowed + } + if isAllowed { + event := logger.Trace().Str("condition", a.MatchCondition.String()) + if commitAllowed { + event.Bool("allowed-commit", commitAllowed) + } + if pathAllowed { + event.Bool("allowed-path", pathAllowed) + } + return true, event + } + } + return false, nil +} + +// checkFindingAllowed evaluates |finding| against all provided |allowlists|. +// +// If the match condition is "OR", only regex and stopwords are run. (Commit and path should be handled separately). +// Otherwise, all conditions are checked. +// +// TODO: The method signature is awkward. I can't think of a better way to log helpful info. +func checkFindingAllowed( + logger zerolog.Logger, + finding report.Finding, + fragment Fragment, + currentLine string, + allowlists []*config.Allowlist, +) (bool, *zerolog.Event) { + for _, a := range allowlists { + allowlistTarget := finding.Secret + switch a.RegexTarget { + case "match": + allowlistTarget = finding.Match + case "line": + allowlistTarget = currentLine + } + + var ( + checks []bool + isAllowed bool + commitAllowed bool + commit string + pathAllowed bool + regexAllowed = a.RegexAllowed(allowlistTarget) + containsStopword, word = a.ContainsStopWord(finding.Secret) + ) + // If the condition is "AND" we need to check all conditions. + if a.MatchCondition == config.AllowlistMatchAnd { + // Determine applicable checks. + if len(a.Commits) > 0 { + commitAllowed, commit = a.CommitAllowed(fragment.CommitSHA) + checks = append(checks, commitAllowed) + } + if len(a.Paths) > 0 { + pathAllowed = a.PathAllowed(fragment.FilePath) || (fragment.WindowsFilePath != "" && a.PathAllowed(fragment.WindowsFilePath)) + checks = append(checks, pathAllowed) + } + if len(a.Regexes) > 0 { + checks = append(checks, regexAllowed) + } + if len(a.StopWords) > 0 { + checks = append(checks, containsStopword) + } + + isAllowed = allTrue(checks) + } else { + isAllowed = regexAllowed || containsStopword + } + + if isAllowed { + event := logger.Trace(). + Str("finding", finding.Secret). + Str("condition", a.MatchCondition.String()) + if commitAllowed { + event.Str("allowed-commit", commit) + } + if pathAllowed { + event.Bool("allowed-path", pathAllowed) + } + if regexAllowed { + event.Bool("allowed-regex", regexAllowed) + } + if containsStopword { + event.Str("allowed-stopword", word) + } + return true, event + } + } + return false, nil +} + +func allTrue(bools []bool) bool { + for _, check := range bools { + if !check { + return false + } + } + return true +} + +func fileExists(fileName string) bool { + // check for a .infisicalignore file + info, err := os.Stat(fileName) + if err != nil && !os.IsNotExist(err) { + return false + } + + if info != nil && err == nil { + if !info.IsDir() { + return true + } + } + return false +} diff --git a/cli/detect/detect_test.go b/cli/detect/detect_test.go deleted file mode 100644 index 5a0f50828..000000000 --- a/cli/detect/detect_test.go +++ /dev/null @@ -1,754 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - - "github.com/Infisical/infisical-merge/config" - "github.com/Infisical/infisical-merge/report" -) - -const configPath = "../testdata/config/" -const repoBasePath = "../testdata/repos/" - -func TestDetect(t *testing.T) { - tests := []struct { - cfgName string - baselinePath string - fragment Fragment - // NOTE: for expected findings, all line numbers will be 0 - // because line deltas are added _after_ the finding is created. - // I.e, if the finding is from a --no-git file, the line number will be - // increase by 1 in DetectFromFiles(). If the finding is from git, - // the line number will be increased by the patch delta. - expectedFindings []report.Finding - wantError error - }{ - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OKIA\ // infisical-scan:ignore"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \ - - \"AKIALALEMEL33243OKIA\ // infisical-scan:ignore" - - `, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OKIA\" - - // infisical-scan:ignore" - - `, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - Secret: "AKIALALEMEL33243OKIA", - Match: "AKIALALEMEL33243OKIA", - File: "tmp.go", - Line: `awsToken := \"AKIALALEMEL33243OKIA\"`, - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - StartLine: 0, - EndLine: 0, - StartColumn: 15, - EndColumn: 34, - Entropy: 3.1464393, - }, - }, - }, - { - cfgName: "escaped_character_group", - fragment: Fragment{ - Raw: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "PyPI upload token", - Secret: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB", - Match: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB", - Line: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`, - File: "tmp.go", - RuleID: "pypi-upload-token", - Tags: []string{"key", "pypi"}, - StartLine: 0, - EndLine: 0, - StartColumn: 1, - EndColumn: 86, - Entropy: 1.9606875, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - Line: `awsToken := \"AKIALALEMEL33243OLIA\"`, - File: "tmp.go", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - StartLine: 0, - EndLine: 0, - StartColumn: 15, - EndColumn: 34, - Entropy: 3.0841837, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Secret", - Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;", - Secret: "cafebabe:deadbeef", - Line: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`, - File: "tmp.sh", - RuleID: "sidekiq-secret", - Tags: []string{}, - Entropy: 2.6098502, - StartLine: 0, - EndLine: 0, - StartColumn: 8, - EndColumn: 60, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Secret", - Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=\"cafebabe:deadbeef\"", - Secret: "cafebabe:deadbeef", - File: "tmp.sh", - Line: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`, - RuleID: "sidekiq-secret", - Tags: []string{}, - Entropy: 2.6098502, - StartLine: 0, - EndLine: 0, - StartColumn: 21, - EndColumn: 74, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `url = "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true¶m2=false#heading1"`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Sensitive URL", - Match: "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:", - Secret: "cafeb4b3:d3adb33f", - File: "tmp.sh", - Line: `url = "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true¶m2=false#heading1"`, - RuleID: "sidekiq-sensitive-url", - Tags: []string{}, - Entropy: 2.984234, - StartLine: 0, - EndLine: 0, - StartColumn: 8, - EndColumn: 58, - }, - }, - }, - { - cfgName: "allow_aws_re", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_path", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_commit", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - CommitSHA: "allowthiscommit", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "entropy_group", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "Discord API key", - Match: "Discord_Public_Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"", - Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5", - Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - File: "tmp.go", - RuleID: "discord-api-key", - Tags: []string{}, - Entropy: 3.7906237, - StartLine: 0, - EndLine: 0, - StartColumn: 7, - EndColumn: 93, - }, - }, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{ - { - Description: "Generic API Key", - Match: "Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"", - Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5", - Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - File: "tmp.py", - RuleID: "generic-api-key", - Tags: []string{}, - Entropy: 3.7906237, - StartLine: 0, - EndLine: 0, - StartColumn: 22, - EndColumn: 93, - }, - }, - }, - { - cfgName: "path_only", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{ - { - Description: "Python Files", - Match: "file detected: tmp.py", - File: "tmp.py", - RuleID: "python-files-only", - Tags: []string{}, - }, - }, - }, - { - cfgName: "bad_entropy_group", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - wantError: fmt.Errorf("Discord API key invalid regex secret group 5, max regex secret group 3"), - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: filepath.Join(configPath, "simple.toml"), - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_global_aws_re", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "load2523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "path_only", - baselinePath: ".baseline.json", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: ".baseline.json", - }, - expectedFindings: []report.Finding{}, - }, - } - - for _, tt := range tests { - viper.Reset() - viper.AddConfigPath(configPath) - viper.SetConfigName(tt.cfgName) - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - cfg.Path = filepath.Join(configPath, tt.cfgName+".toml") - if tt.wantError != nil { - if err == nil { - t.Errorf("expected error") - } - assert.Equal(t, tt.wantError, err) - } - d := NewDetector(cfg) - d.baselinePath = tt.baselinePath - - findings := d.Detect(tt.fragment) - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -// TestFromGit tests the FromGit function -func TestFromGit(t *testing.T) { - tests := []struct { - cfgName string - source string - logOpts string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "small"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 19, - EndColumn: 38, - Line: "\n awsToken := \"AKIALALEMEL33243OLIA\"", - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - File: "main.go", - Date: "2021-11-02T23:37:53Z", - Commit: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587", - Author: "Zachary Rice", - Email: "zricer@protonmail.com", - Message: "Accidentally add a secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587:main.go:aws-access-key:20", - }, - { - Description: "AWS Access Key", - StartLine: 9, - EndLine: 9, - StartColumn: 17, - EndColumn: 36, - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"", - File: "foo/foo.go", - Date: "2021-11-02T23:48:06Z", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "491504d5a31946ce75e22554cc34203d8e5ff3ca:foo/foo.go:aws-access-key:9", - }, - }, - }, - { - source: filepath.Join(repoBasePath, "small"), - logOpts: "--all foo...", - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 9, - EndLine: 9, - StartColumn: 17, - EndColumn: 36, - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"", - Match: "AKIALALEMEL33243OLIA", - Date: "2021-11-02T23:48:06Z", - File: "foo/foo.go", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "491504d5a31946ce75e22554cc34203d8e5ff3ca:foo/foo.go:aws-access-key:9", - }, - }, - }, - } - - err := moveDotGit("dotGit", ".git") - if err != nil { - t.Fatal(err) - } - defer func() { - if err := moveDotGit(".git", "dotGit"); err != nil { - t.Error(err) - } - }() - - for _, tt := range tests { - - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err = viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if err != nil { - t.Error(err) - } - detector := NewDetector(cfg) - findings, err := detector.DetectGit(tt.source, tt.logOpts, DetectType) - if err != nil { - t.Error(err) - } - - for _, f := range findings { - f.Match = "" // remove lines cause copying and pasting them has some wack formatting - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} -func TestFromGitStaged(t *testing.T) { - tests := []struct { - cfgName string - source string - logOpts string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "staged"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 7, - EndLine: 7, - StartColumn: 18, - EndColumn: 37, - Line: "\n\taws_token2 := \"AKIALALEMEL33243OLIA\" // this one is not", - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - File: "api/api.go", - SymlinkFile: "", - Commit: "", - Entropy: 3.0841837, - Author: "", - Email: "", - Date: "0001-01-01T00:00:00Z", - Message: "", - Tags: []string{ - "key", - "AWS", - }, - RuleID: "aws-access-key", - Fingerprint: "api/api.go:aws-access-key:7", - }, - }, - }, - } - - err := moveDotGit("dotGit", ".git") - if err != nil { - t.Fatal(err) - } - defer func() { - if err := moveDotGit(".git", "dotGit"); err != nil { - t.Error(err) - } - }() - - for _, tt := range tests { - - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err = viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if err != nil { - t.Error(err) - } - detector := NewDetector(cfg) - detector.AddGitleaksIgnore(filepath.Join(tt.source, ".gitleaksignore")) - findings, err := detector.DetectGit(tt.source, tt.logOpts, ProtectStagedType) - if err != nil { - t.Error(err) - } - - for _, f := range findings { - f.Match = "" // remove lines cause copying and pasting them has some wack formatting - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -// TestFromFiles tests the FromFiles function -func TestFromFiles(t *testing.T) { - tests := []struct { - cfgName string - source string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "nogit"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 16, - EndColumn: 35, - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"", - File: "../testdata/repos/nogit/main.go", - SymlinkFile: "", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "../testdata/repos/nogit/main.go:aws-access-key:20", - }, - }, - }, - { - source: filepath.Join(repoBasePath, "nogit", "main.go"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 16, - EndColumn: 35, - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"", - File: "../testdata/repos/nogit/main.go", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "../testdata/repos/nogit/main.go:aws-access-key:20", - }, - }, - }, - } - - for _, tt := range tests { - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, _ := vc.Translate() - detector := NewDetector(cfg) - detector.FollowSymlinks = true - findings, err := detector.DetectFiles(tt.source) - if err != nil { - t.Error(err) - } - - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -func TestDetectWithSymlinks(t *testing.T) { - tests := []struct { - cfgName string - source string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "symlinks/file_symlink"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "Asymmetric Private Key", - StartLine: 1, - EndLine: 1, - StartColumn: 1, - EndColumn: 35, - Match: "-----BEGIN OPENSSH PRIVATE KEY-----", - Secret: "-----BEGIN OPENSSH PRIVATE KEY-----", - Line: "-----BEGIN OPENSSH PRIVATE KEY-----", - File: "../testdata/repos/symlinks/source_file/id_ed25519", - SymlinkFile: "../testdata/repos/symlinks/file_symlink/symlinked_id_ed25519", - RuleID: "apkey", - Tags: []string{"key", "AsymmetricPrivateKey"}, - Entropy: 3.587164, - Fingerprint: "../testdata/repos/symlinks/source_file/id_ed25519:apkey:1", - }, - }, - }, - } - - for _, tt := range tests { - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, _ := vc.Translate() - detector := NewDetector(cfg) - detector.FollowSymlinks = true - findings, err := detector.DetectFiles(tt.source) - if err != nil { - t.Error(err) - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -func moveDotGit(from, to string) error { - repoDirs, err := os.ReadDir("../testdata/repos") - if err != nil { - return err - } - for _, dir := range repoDirs { - if to == ".git" { - _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), "dotGit")) - if os.IsNotExist(err) { - // dont want to delete the only copy of .git accidentally - continue - } - os.RemoveAll(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), ".git")) - } - if !dir.IsDir() { - continue - } - _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from)) - if os.IsNotExist(err) { - continue - } - - err = os.Rename(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from), - fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), to)) - if err != nil { - return err - } - } - return nil -} diff --git a/cli/detect/directory.go b/cli/detect/directory.go new file mode 100644 index 000000000..56f4999f2 --- /dev/null +++ b/cli/detect/directory.go @@ -0,0 +1,225 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bufio" + "bytes" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/h2non/filetype" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" +) + +const maxPeekSize = 25 * 1_000 // 10kb + +func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Finding, error) { + for pa := range paths { + d.Sema.Go(func() error { + logger := logging.With().Str("path", pa.Path).Logger() + logger.Trace().Msg("Scanning path") + + f, err := os.Open(pa.Path) + if err != nil { + if os.IsPermission(err) { + logger.Warn().Msg("Skipping file: permission denied") + return nil + } + return err + } + defer func() { + _ = f.Close() + }() + + // Get file size + fileInfo, err := f.Stat() + if err != nil { + return err + } + fileSize := fileInfo.Size() + if d.MaxTargetMegaBytes > 0 { + rawLength := fileSize / 1000000 + if rawLength > int64(d.MaxTargetMegaBytes) { + logger.Debug(). + Int64("size", rawLength). + Msg("Skipping file: exceeds --max-target-megabytes") + return nil + } + } + + var ( + // Buffer to hold file chunks + reader = bufio.NewReaderSize(f, chunkSize) + buf = make([]byte, chunkSize) + totalLines = 0 + ) + for { + n, err := reader.Read(buf) + + // "Callers should always process the n > 0 bytes returned before considering the error err." + // https://pkg.go.dev/io#Reader + if n > 0 { + // Only check the filetype at the start of file. + if totalLines == 0 { + // TODO: could other optimizations be introduced here? + if mimetype, err := filetype.Match(buf[:n]); err != nil { + return nil + } else if mimetype.MIME.Type == "application" { + return nil // skip binary files + } + } + + // Try to split chunks across large areas of whitespace, if possible. + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + return readErr + } + + // Count the number of newlines in this chunk + chunk := peekBuf.String() + linesInChunk := strings.Count(chunk, "\n") + totalLines += linesInChunk + fragment := Fragment{ + Raw: chunk, + Bytes: peekBuf.Bytes(), + } + if pa.Symlink != "" { + fragment.SymlinkFile = pa.Symlink + } + + if isWindows { + fragment.FilePath = filepath.ToSlash(pa.Path) + fragment.SymlinkFile = filepath.ToSlash(fragment.SymlinkFile) + fragment.WindowsFilePath = pa.Path + } else { + fragment.FilePath = pa.Path + } + + timer := time.AfterFunc(SlowWarningThreshold, func() { + logger.Debug().Msgf("Taking longer than %s to inspect fragment", SlowWarningThreshold.String()) + }) + for _, finding := range d.Detect(fragment) { + // need to add 1 since line counting starts at 1 + finding.StartLine += (totalLines - linesInChunk) + 1 + finding.EndLine += (totalLines - linesInChunk) + 1 + d.AddFinding(finding) + } + if timer != nil { + timer.Stop() + timer = nil + } + } + + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } + }) + } + + if err := d.Sema.Wait(); err != nil { + return d.findings, err + } + + return d.findings, nil +} + +// readUntilSafeBoundary consumes |f| until it finds two consecutive `\n` characters, up to |maxPeekSize|. +// This hopefully avoids splitting. (https://github.com/gitleaks/gitleaks/issues/1651) +func readUntilSafeBoundary(r *bufio.Reader, n int, maxPeekSize int, peekBuf *bytes.Buffer) error { + if peekBuf.Len() == 0 { + return nil + } + + // Does the buffer end in consecutive newlines? + var ( + data = peekBuf.Bytes() + lastChar = data[len(data)-1] + newlineCount = 0 // Tracks consecutive newlines + ) + if isWhitespace(lastChar) { + for i := len(data) - 1; i >= 0; i-- { + lastChar = data[i] + if lastChar == '\n' { + newlineCount++ + + // Stop if two consecutive newlines are found + if newlineCount >= 2 { + return nil + } + } else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' { + // The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count. + // (Intentionally do nothing.) + } else { + break + } + } + } + + // If not, read ahead until we (hopefully) find some. + newlineCount = 0 + for { + data = peekBuf.Bytes() + // Check if the last character is a newline. + lastChar = data[len(data)-1] + if lastChar == '\n' { + newlineCount++ + + // Stop if two consecutive newlines are found + if newlineCount >= 2 { + break + } + } else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' { + // The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count. + // (Intentionally do nothing.) + } else { + newlineCount = 0 // Reset if a non-newline character is found + } + + // Stop growing the buffer if it reaches maxSize + if (peekBuf.Len() - n) >= maxPeekSize { + break + } + + // Read additional data into a temporary buffer + b, err := r.ReadByte() + if err != nil { + if err == io.EOF { + break + } + return err + } + peekBuf.WriteByte(b) + } + return nil +} diff --git a/cli/detect/git.go b/cli/detect/git.go new file mode 100644 index 000000000..ddde0757d --- /dev/null +++ b/cli/detect/git.go @@ -0,0 +1,214 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bytes" + "errors" + "fmt" + "net/url" + "os/exec" + "regexp" + "strings" + "time" + + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/gitleaks/go-gitdiff/gitdiff" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" +) + +func (d *Detector) DetectGit(cmd *sources.GitCmd, remote *RemoteInfo) ([]report.Finding, error) { + defer cmd.Wait() + var ( + diffFilesCh = cmd.DiffFilesCh() + errCh = cmd.ErrCh() + ) + + // loop to range over both DiffFiles (stdout) and ErrCh (stderr) + for diffFilesCh != nil || errCh != nil { + select { + case gitdiffFile, open := <-diffFilesCh: + if !open { + diffFilesCh = nil + break + } + + // skip binary files + if gitdiffFile.IsBinary || gitdiffFile.IsDelete { + continue + } + + // Check if commit is allowed + commitSHA := "" + if gitdiffFile.PatchHeader != nil { + commitSHA = gitdiffFile.PatchHeader.SHA + for _, a := range d.Config.Allowlists { + if ok, c := a.CommitAllowed(gitdiffFile.PatchHeader.SHA); ok { + logging.Trace().Str("allowed-commit", c).Msg("skipping commit: global allowlist") + continue + } + } + } + d.addCommit(commitSHA) + + d.Sema.Go(func() error { + for _, textFragment := range gitdiffFile.TextFragments { + if textFragment == nil { + return nil + } + + fragment := Fragment{ + Raw: textFragment.Raw(gitdiff.OpAdd), + CommitSHA: commitSHA, + FilePath: gitdiffFile.NewName, + } + + timer := time.AfterFunc(SlowWarningThreshold, func() { + logging.Debug(). + Str("commit", commitSHA[:7]). + Str("path", fragment.FilePath). + Msgf("Taking longer than %s to inspect fragment", SlowWarningThreshold.String()) + }) + for _, finding := range d.Detect(fragment) { + d.AddFinding(augmentGitFinding(remote, finding, textFragment, gitdiffFile)) + } + if timer != nil { + timer.Stop() + timer = nil + } + } + return nil + }) + case err, open := <-errCh: + if !open { + errCh = nil + break + } + + return d.findings, err + } + } + + if err := d.Sema.Wait(); err != nil { + return d.findings, err + } + logging.Info().Msgf("%d commits scanned.", len(d.commitMap)) + logging.Debug().Msg("Note: this number might be smaller than expected due to commits with no additions") + return d.findings, nil +} + +type RemoteInfo struct { + Platform scm.Platform + Url string +} + +func NewRemoteInfo(platform scm.Platform, source string) *RemoteInfo { + if platform == scm.NoPlatform { + return &RemoteInfo{Platform: platform} + } + + remoteUrl, err := getRemoteUrl(source) + if err != nil { + if strings.Contains(err.Error(), "No remote configured") { + logging.Debug().Msg("skipping finding links: repository has no configured remote.") + platform = scm.NoPlatform + } else { + logging.Error().Err(err).Msg("skipping finding links: unable to parse remote URL") + } + goto End + } + + if platform == scm.UnknownPlatform { + platform = platformFromHost(remoteUrl) + if platform == scm.UnknownPlatform { + logging.Info(). + Str("host", remoteUrl.Hostname()). + Msg("Unknown SCM platform. Use --platform to include links in findings.") + } else { + logging.Debug(). + Str("host", remoteUrl.Hostname()). + Str("platform", platform.String()). + Msg("SCM platform parsed from host") + } + } + +End: + var rUrl string + if remoteUrl != nil { + rUrl = remoteUrl.String() + } + return &RemoteInfo{ + Platform: platform, + Url: rUrl, + } +} + +var sshUrlpat = regexp.MustCompile(`^git@([a-zA-Z0-9.-]+):([\w/.-]+?)(?:\.git)?$`) + +func getRemoteUrl(source string) (*url.URL, error) { + // This will return the first remote — typically, "origin". + cmd := exec.Command("git", "ls-remote", "--quiet", "--get-url") + if source != "." { + cmd.Dir = source + } + + stdout, err := cmd.Output() + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return nil, fmt.Errorf("command failed (%d): %w, stderr: %s", exitError.ExitCode(), err, string(bytes.TrimSpace(exitError.Stderr))) + } + return nil, err + } + + remoteUrl := string(bytes.TrimSpace(stdout)) + if matches := sshUrlpat.FindStringSubmatch(remoteUrl); matches != nil { + remoteUrl = fmt.Sprintf("https://%s/%s", matches[1], matches[2]) + } + remoteUrl = strings.TrimSuffix(remoteUrl, ".git") + + parsedUrl, err := url.Parse(remoteUrl) + if err != nil { + return nil, fmt.Errorf("unable to parse remote URL: %w", err) + } + + // Remove any user info. + parsedUrl.User = nil + return parsedUrl, nil +} + +func platformFromHost(u *url.URL) scm.Platform { + switch strings.ToLower(u.Hostname()) { + case "github.com": + return scm.GitHubPlatform + case "gitlab.com": + return scm.GitLabPlatform + case "dev.azure.com", "visualstudio.com": + return scm.AzureDevOpsPlatform + default: + return scm.UnknownPlatform + } +} diff --git a/cli/detect/git/git.go b/cli/detect/git/git.go deleted file mode 100644 index 910384497..000000000 --- a/cli/detect/git/git.go +++ /dev/null @@ -1,147 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package git - -import ( - "bufio" - "io" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/gitleaks/go-gitdiff/gitdiff" - "github.com/rs/zerolog/log" -) - -var ErrEncountered bool - -// GitLog returns a channel of gitdiff.File objects from the -// git log -p command for the given source. -func GitLog(source string, logOpts string) (<-chan *gitdiff.File, error) { - sourceClean := filepath.Clean(source) - var cmd *exec.Cmd - if logOpts != "" { - args := []string{"-C", sourceClean, "log", "-p", "-U0"} - args = append(args, strings.Split(logOpts, " ")...) - cmd = exec.Command("git", args...) - } else { - cmd = exec.Command("git", "-C", sourceClean, "log", "-p", "-U0", - "--full-history", "--all") - } - - log.Debug().Msgf("executing: %s", cmd.String()) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - go listenForStdErr(stderr) - - if err := cmd.Start(); err != nil { - return nil, err - } - // HACK: to avoid https://github.com/zricethezav/gitleaks/issues/722 - time.Sleep(50 * time.Millisecond) - - reader := bufio.NewReader(stdout) - - return gitdiff.Parse(reader) -} - -// GitDiff returns a channel of gitdiff.File objects from -// the git diff command for the given source. -func GitDiff(source string, staged bool) (<-chan *gitdiff.File, error) { - sourceClean := filepath.Clean(source) - var cmd *exec.Cmd - cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", ".") - if staged { - cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", - "--staged", ".") - } - log.Debug().Msgf("executing: %s", cmd.String()) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - go listenForStdErr(stderr) - - if err := cmd.Start(); err != nil { - return nil, err - } - // HACK: to avoid https://github.com/zricethezav/gitleaks/issues/722 - time.Sleep(50 * time.Millisecond) - - reader := bufio.NewReader(stdout) - - return gitdiff.Parse(reader) -} - -// listenForStdErr listens for stderr output from git and prints it to stdout -// then exits with exit code 1 -func listenForStdErr(stderr io.ReadCloser) { - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - // if git throws one of the following errors: - // - // exhaustive rename detection was skipped due to too many files. - // you may want to set your diff.renameLimit variable to at least - // (some large number) and retry the command. - // - // inexact rename detection was skipped due to too many files. - // you may want to set your diff.renameLimit variable to at least - // (some large number) and retry the command. - // - // we skip exiting the program as git log -p/git diff will continue - // to send data to stdout and finish executing. This next bit of - // code prevents gitleaks from stopping mid scan if this error is - // encountered - if strings.Contains(scanner.Text(), - "exhaustive rename detection was skipped") || - strings.Contains(scanner.Text(), - "inexact rename detection was skipped") || - strings.Contains(scanner.Text(), - "you may want to set your diff.renameLimit") { - log.Warn().Msg(scanner.Text()) - } else { - log.Error().Msgf("[git] %s", scanner.Text()) - - // asynchronously set this error flag to true so that we can - // capture a log message and exit with a non-zero exit code - // This value should get set before the `git` command exits so it's - // safe-ish, although I know I know, bad practice. - ErrEncountered = true - } - } -} diff --git a/cli/detect/git/git_test.go b/cli/detect/git/git_test.go deleted file mode 100644 index 3a2ea9c35..000000000 --- a/cli/detect/git/git_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package git_test - -// TODO: commenting out this test for now because it's flaky. Alternatives to consider to get this working: -// -- use `git stash` instead of `restore()` - -// const repoBasePath = "../../testdata/repos/" - -// const expectPath = "../../testdata/expected/" - -// func TestGitLog(t *testing.T) { -// tests := []struct { -// source string -// logOpts string -// expected string -// }{ -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: filepath.Join(expectPath, "git", "small.txt"), -// }, -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: filepath.Join(expectPath, "git", "small-branch-foo.txt"), -// logOpts: "--all foo...", -// }, -// } - -// err := moveDotGit("dotGit", ".git") -// if err != nil { -// t.Fatal(err) -// } -// defer func() { -// if err = moveDotGit(".git", "dotGit"); err != nil { -// t.Fatal(err) -// } -// }() - -// for _, tt := range tests { -// files, err := git.GitLog(tt.source, tt.logOpts) -// if err != nil { -// t.Error(err) -// } - -// var diffSb strings.Builder -// for f := range files { -// for _, tf := range f.TextFragments { -// diffSb.WriteString(tf.Raw(gitdiff.OpAdd)) -// } -// } - -// expectedBytes, err := os.ReadFile(tt.expected) -// if err != nil { -// t.Error(err) -// } -// expected := string(expectedBytes) -// if expected != diffSb.String() { -// // write string builder to .got file using os.Create -// err = os.WriteFile(strings.Replace(tt.expected, ".txt", ".got.txt", 1), []byte(diffSb.String()), 0644) -// if err != nil { -// t.Error(err) -// } -// t.Error("expected: ", expected, "got: ", diffSb.String()) -// } -// } -// } - -// func TestGitDiff(t *testing.T) { -// tests := []struct { -// source string -// expected string -// additions string -// target string -// }{ -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: "this line is added\nand another one", -// additions: "this line is added\nand another one", -// target: filepath.Join(repoBasePath, "small", "main.go"), -// }, -// } - -// err := moveDotGit("dotGit", ".git") -// if err != nil { -// t.Fatal(err) -// } -// defer func() { -// if err = moveDotGit(".git", "dotGit"); err != nil { -// t.Fatal(err) -// } -// }() - -// for _, tt := range tests { -// noChanges, err := os.ReadFile(tt.target) -// if err != nil { -// t.Error(err) -// } -// err = os.WriteFile(tt.target, []byte(tt.additions), 0644) -// if err != nil { -// restore(tt.target, noChanges, t) -// t.Error(err) -// } - -// files, err := git.GitDiff(tt.source, false) -// if err != nil { -// restore(tt.target, noChanges, t) -// t.Error(err) -// } - -// for f := range files { -// sb := strings.Builder{} -// for _, tf := range f.TextFragments { -// sb.WriteString(tf.Raw(gitdiff.OpAdd)) -// } -// if sb.String() != tt.expected { -// restore(tt.target, noChanges, t) -// t.Error("expected: ", tt.expected, "got: ", sb.String()) -// } -// } -// restore(tt.target, noChanges, t) -// } -// } - -// func restore(path string, data []byte, t *testing.T) { -// err := os.WriteFile(path, data, 0644) -// if err != nil { -// t.Fatal(err) -// } -// } - -// func moveDotGit(from, to string) error { -// repoDirs, err := os.ReadDir("../../testdata/repos") -// if err != nil { -// return err -// } -// for _, dir := range repoDirs { -// if to == ".git" { -// _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), "dotGit")) -// if os.IsNotExist(err) { -// // dont want to delete the only copy of .git accidentally -// continue -// } -// os.RemoveAll(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), ".git")) -// } -// if !dir.IsDir() { -// continue -// } -// _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from)) -// if os.IsNotExist(err) { -// continue -// } - -// err = os.Rename(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from), -// fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), to)) -// if err != nil { -// return err -// } -// } -// return nil -// } diff --git a/cli/detect/location.go b/cli/detect/location.go index 418af83f6..81419511c 100644 --- a/cli/detect/location.go +++ b/cli/detect/location.go @@ -72,6 +72,7 @@ func location(fragment Fragment, matchIndex []int) Location { location.endColumn = (end - prevNewLine) location.endLineIndex = newLineByteIndex } + prevNewLine = pair[0] } diff --git a/cli/detect/location_test.go b/cli/detect/location_test.go deleted file mode 100644 index f76a6f814..000000000 --- a/cli/detect/location_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "testing" -) - -// TestGetLocation tests the getLocation function. -func TestGetLocation(t *testing.T) { - tests := []struct { - linePairs [][]int - start int - end int - wantLocation Location - }{ - { - linePairs: [][]int{ - {0, 39}, - {40, 55}, - {56, 57}, - }, - start: 35, - end: 38, - wantLocation: Location{ - startLine: 1, - startColumn: 36, - endLine: 1, - endColumn: 38, - startLineIndex: 0, - endLineIndex: 40, - }, - }, - { - linePairs: [][]int{ - {0, 39}, - {40, 55}, - {56, 57}, - }, - start: 40, - end: 44, - wantLocation: Location{ - startLine: 2, - startColumn: 1, - endLine: 2, - endColumn: 4, - startLineIndex: 40, - endLineIndex: 56, - }, - }, - } - - for _, test := range tests { - loc := location(Fragment{newlineIndices: test.linePairs}, []int{test.start, test.end}) - if loc != test.wantLocation { - t.Errorf("\nstartLine %d\nstartColumn: %d\nendLine: %d\nendColumn: %d\nstartLineIndex: %d\nendlineIndex %d", - loc.startLine, loc.startColumn, loc.endLine, loc.endColumn, loc.startLineIndex, loc.endLineIndex) - - t.Error("got", loc, "want", test.wantLocation) - } - } -} diff --git a/cli/detect/logging/log.go b/cli/detect/logging/log.go new file mode 100644 index 000000000..efac01725 --- /dev/null +++ b/cli/detect/logging/log.go @@ -0,0 +1,72 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package logging + +import ( + "os" + + "github.com/rs/zerolog" +) + +var Logger zerolog.Logger + +func init() { + // send all logs to stdout + Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). + Level(zerolog.InfoLevel). + With().Timestamp().Logger() +} + +func With() zerolog.Context { + return Logger.With() +} + +func Trace() *zerolog.Event { + return Logger.Trace() +} + +func Debug() *zerolog.Event { + return Logger.Debug() +} +func Info() *zerolog.Event { + return Logger.Info() +} +func Warn() *zerolog.Event { + return Logger.Warn() +} + +func Error() *zerolog.Event { + return Logger.Error() +} + +func Err(err error) *zerolog.Event { + return Logger.Err(err) +} + +func Fatal() *zerolog.Event { + return Logger.Fatal() +} + +func Panic() *zerolog.Event { + return Logger.Panic() +} diff --git a/cli/detect/reader.go b/cli/detect/reader.go new file mode 100644 index 000000000..d3559b68a --- /dev/null +++ b/cli/detect/reader.go @@ -0,0 +1,149 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bufio" + "bytes" + "errors" + "io" + + "github.com/Infisical/infisical-merge/detect/report" +) + +// DetectReader accepts an io.Reader and a buffer size for the reader in KB +func (d *Detector) DetectReader(r io.Reader, bufSize int) ([]report.Finding, error) { + reader := bufio.NewReader(r) + buf := make([]byte, 1000*bufSize) + findings := []report.Finding{} + + for { + n, err := reader.Read(buf) + + // "Callers should always process the n > 0 bytes returned before considering the error err." + // https://pkg.go.dev/io#Reader + if n > 0 { + // Try to split chunks across large areas of whitespace, if possible. + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + return findings, readErr + } + + fragment := Fragment{ + Raw: peekBuf.String(), + } + for _, finding := range d.Detect(fragment) { + findings = append(findings, finding) + if d.Verbose { + printFinding(finding, d.NoColor) + } + } + } + + if err != nil { + if err == io.EOF { + break + } + return findings, err + } + } + + return findings, nil +} + +// StreamDetectReader streams the detection results from the provided io.Reader. +// It reads data using the specified buffer size (in KB) and processes each chunk through +// the existing detection logic. Findings are sent down the returned findings channel as soon as +// they are detected, while a separate error channel signals a terminal error (or nil upon successful completion). +// The function returns two channels: +// - findingsCh: a receive-only channel that emits report.Finding objects as they are found. +// - errCh: a receive-only channel that emits a single final error (or nil if no error occurred) +// once the stream ends. +// +// Recommended Usage: +// +// Since there will only ever be a single value on the errCh, it is recommended to consume the findingsCh +// first. Once findingsCh is closed, the consumer should then read from errCh to determine +// if the stream completed successfully or if an error occurred. +// +// This design avoids the need for a select loop, keeping client code simple. +// +// Example: +// +// // Assume detector is an instance of *Detector and myReader implements io.Reader. +// findingsCh, errCh := detector.StreamDetectReader(myReader, 64) // using 64 KB buffer size +// +// // Process findings as they arrive. +// for finding := range findingsCh { +// fmt.Printf("Found secret: %+v\n", finding) +// } +// +// // After the findings channel is closed, check the final error. +// if err := <-errCh; err != nil { +// log.Fatalf("StreamDetectReader encountered an error: %v", err) +// } else { +// fmt.Println("Scanning completed successfully.") +// } +func (d *Detector) StreamDetectReader(r io.Reader, bufSize int) (<-chan report.Finding, <-chan error) { + findingsCh := make(chan report.Finding, 1) + errCh := make(chan error, 1) + + go func() { + defer close(findingsCh) + defer close(errCh) + + reader := bufio.NewReader(r) + buf := make([]byte, 1000*bufSize) + + for { + n, err := reader.Read(buf) + + if n > 0 { + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + errCh <- readErr + return + } + + fragment := Fragment{Raw: peekBuf.String()} + for _, finding := range d.Detect(fragment) { + findingsCh <- finding + if d.Verbose { + printFinding(finding, d.NoColor) + } + } + } + + if err != nil { + if errors.Is(err, io.EOF) { + errCh <- nil + return + } + errCh <- err + return + } + } + }() + + return findingsCh, errCh +} diff --git a/cli/detect/regexp/stdlib_regex.go b/cli/detect/regexp/stdlib_regex.go new file mode 100644 index 000000000..81e2089b7 --- /dev/null +++ b/cli/detect/regexp/stdlib_regex.go @@ -0,0 +1,37 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +//go:build !gore2regex + +package regexp + +import ( + re "regexp" +) + +const Version = "stdlib" + +type Regexp = re.Regexp + +func MustCompile(str string) *re.Regexp { + return re.MustCompile(str) +} diff --git a/cli/detect/regexp/wasilibs_regex.go b/cli/detect/regexp/wasilibs_regex.go new file mode 100644 index 000000000..bc64fb14b --- /dev/null +++ b/cli/detect/regexp/wasilibs_regex.go @@ -0,0 +1,37 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +//go:build gore2regex + +package regexp + +import ( + re "github.com/wasilibs/go-re2" +) + +const Version = "github.com/wasilibs/go-re2" + +type Regexp = re.Regexp + +func MustCompile(str string) *re.Regexp { + return re.MustCompile(str) +} diff --git a/cli/report/constants.go b/cli/detect/report/constants.go similarity index 99% rename from cli/report/constants.go rename to cli/detect/report/constants.go index 8bad495cb..c4f06a9a3 100644 --- a/cli/report/constants.go +++ b/cli/detect/report/constants.go @@ -19,6 +19,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. + package report const version = "v8.0.0" diff --git a/cli/report/csv.go b/cli/detect/report/csv.go similarity index 72% rename from cli/report/csv.go rename to cli/detect/report/csv.go index 0a30c9fd5..1f8812f97 100644 --- a/cli/report/csv.go +++ b/cli/detect/report/csv.go @@ -26,16 +26,24 @@ import ( "encoding/csv" "io" "strconv" + "strings" ) -// writeCsv writes the list of findings to a writeCloser. -func writeCsv(f []Finding, w io.WriteCloser) error { - if len(f) == 0 { +type CsvReporter struct { +} + +var _ Reporter = (*CsvReporter)(nil) + +func (r *CsvReporter) Write(w io.WriteCloser, findings []Finding) error { + if len(findings) == 0 { return nil } - defer w.Close() - cw := csv.NewWriter(w) - err := cw.Write([]string{"RuleID", + + var ( + cw = csv.NewWriter(w) + err error + ) + columns := []string{"RuleID", "Commit", "File", "SymlinkFile", @@ -50,12 +58,18 @@ func writeCsv(f []Finding, w io.WriteCloser) error { "Date", "Email", "Fingerprint", - }) - if err != nil { + "Tags", + } + // A miserable attempt at "omitempty" so tests don't yell at me. + if findings[0].Link != "" { + columns = append(columns, "Link") + } + + if err = cw.Write(columns); err != nil { return err } - for _, f := range f { - err = cw.Write([]string{f.RuleID, + for _, f := range findings { + row := []string{f.RuleID, f.Commit, f.File, f.SymlinkFile, @@ -70,8 +84,13 @@ func writeCsv(f []Finding, w io.WriteCloser) error { f.Date, f.Email, f.Fingerprint, - }) - if err != nil { + strings.Join(f.Tags, " "), + } + if findings[0].Link != "" { + row = append(row, f.Link) + } + + if err = cw.Write(row); err != nil { return err } } diff --git a/cli/report/finding.go b/cli/detect/report/finding.go similarity index 75% rename from cli/report/finding.go rename to cli/detect/report/finding.go index be461072b..c53f16ee7 100644 --- a/cli/report/finding.go +++ b/cli/detect/report/finding.go @@ -23,13 +23,17 @@ package report import ( + "math" "strings" ) // Finding contains information about strings that // have been captured by a tree-sitter query. type Finding struct { + // Rule is the name of the rule that was matched + RuleID string Description string + StartLine int EndLine int StartColumn int @@ -47,6 +51,7 @@ type Finding struct { File string SymlinkFile string Commit string + Link string `json:",omitempty"` // Entropy is the shannon entropy of Value Entropy float32 @@ -57,16 +62,31 @@ type Finding struct { Message string Tags []string - // Rule is the name of the rule that was matched - RuleID string - - // unique identifer + // unique identifier Fingerprint string } // Redact removes sensitive information from a finding. -func (f *Finding) Redact() { - f.Line = strings.Replace(f.Line, f.Secret, "REDACTED", -1) - f.Match = strings.Replace(f.Match, f.Secret, "REDACTED", -1) - f.Secret = "REDACTED" +func (f *Finding) Redact(percent uint) { + secret := maskSecret(f.Secret, percent) + if percent >= 100 { + secret = "REDACTED" + } + f.Line = strings.Replace(f.Line, f.Secret, secret, -1) + f.Match = strings.Replace(f.Match, f.Secret, secret, -1) + f.Secret = secret +} + +func maskSecret(secret string, percent uint) string { + if percent > 100 { + percent = 100 + } + len := float64(len(secret)) + if len <= 0 { + return secret + } + prc := float64(100 - percent) + lth := int64(math.RoundToEven(len * prc / float64(100))) + + return secret[:lth] + "..." } diff --git a/cli/report/json.go b/cli/detect/report/json.go similarity index 89% rename from cli/report/json.go rename to cli/detect/report/json.go index d091ac3c5..f47b7eee0 100644 --- a/cli/report/json.go +++ b/cli/detect/report/json.go @@ -27,10 +27,12 @@ import ( "io" ) -func writeJson(findings []Finding, w io.WriteCloser) error { - if len(findings) == 0 { - findings = []Finding{} - } +type JsonReporter struct { +} + +var _ Reporter = (*JsonReporter)(nil) + +func (t *JsonReporter) Write(w io.WriteCloser, findings []Finding) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") return encoder.Encode(findings) diff --git a/cli/detect/report/junit.go b/cli/detect/report/junit.go new file mode 100644 index 000000000..0862a45f1 --- /dev/null +++ b/cli/detect/report/junit.go @@ -0,0 +1,129 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package report + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "strconv" +) + +type JunitReporter struct { +} + +var _ Reporter = (*JunitReporter)(nil) + +func (r *JunitReporter) Write(w io.WriteCloser, findings []Finding) error { + testSuites := TestSuites{ + TestSuites: getTestSuites(findings), + } + + io.WriteString(w, xml.Header) + encoder := xml.NewEncoder(w) + encoder.Indent("", "\t") + return encoder.Encode(testSuites) +} + +func getTestSuites(findings []Finding) []TestSuite { + return []TestSuite{ + { + Failures: strconv.Itoa(len(findings)), + Name: "gitleaks", + Tests: strconv.Itoa(len(findings)), + TestCases: getTestCases(findings), + Time: "", + }, + } +} + +func getTestCases(findings []Finding) []TestCase { + testCases := []TestCase{} + for _, f := range findings { + testCase := TestCase{ + Classname: f.Description, + Failure: getFailure(f), + File: f.File, + Name: getMessage(f), + Time: "", + } + testCases = append(testCases, testCase) + } + return testCases +} + +func getFailure(f Finding) Failure { + return Failure{ + Data: getData(f), + Message: getMessage(f), + Type: f.Description, + } +} + +func getData(f Finding) string { + data, err := json.MarshalIndent(f, "", "\t") + if err != nil { + fmt.Println(err) + return "" + } + return string(data) +} + +func getMessage(f Finding) string { + if f.Commit == "" { + return fmt.Sprintf("%s has detected a secret in file %s, line %s.", f.RuleID, f.File, strconv.Itoa(f.StartLine)) + } + + return fmt.Sprintf("%s has detected a secret in file %s, line %s, at commit %s.", f.RuleID, f.File, strconv.Itoa(f.StartLine), f.Commit) +} + +type TestSuites struct { + XMLName xml.Name `xml:"testsuites"` + TestSuites []TestSuite +} + +type TestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Failures string `xml:"failures,attr"` + Name string `xml:"name,attr"` + Tests string `xml:"tests,attr"` + TestCases []TestCase `xml:"testcase"` + Time string `xml:"time,attr"` +} + +type TestCase struct { + XMLName xml.Name `xml:"testcase"` + Classname string `xml:"classname,attr"` + Failure Failure `xml:"failure"` + File string `xml:"file,attr"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` +} + +type Failure struct { + XMLName xml.Name `xml:"failure"` + Data string `xml:",chardata"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` +} diff --git a/cli/report/finding_test.go b/cli/detect/report/report.go similarity index 73% rename from cli/report/finding_test.go rename to cli/detect/report/report.go index cdb74a329..120841bb8 100644 --- a/cli/report/finding_test.go +++ b/cli/detect/report/report.go @@ -19,30 +19,20 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. + package report -import "testing" +import ( + "io" +) -func TestRedact(t *testing.T) { - tests := []struct { - findings []Finding - redact bool - }{ - { - redact: true, - findings: []Finding{ - { - Secret: "line containing secret", - Match: "secret", - }, - }}, - } - for _, test := range tests { - for _, f := range test.findings { - f.Redact() - if f.Secret != "REDACTED" { - t.Error("redact not redacting: ", f.Secret) - } - } - } +const ( + // https://cwe.mitre.org/data/definitions/798.html + CWE = "CWE-798" + CWE_DESCRIPTION = "Use of Hard-coded Credentials" + StdoutReportPath = "-" +) + +type Reporter interface { + Write(w io.WriteCloser, findings []Finding) error } diff --git a/cli/report/sarif.go b/cli/detect/report/sarif.go similarity index 84% rename from cli/report/sarif.go rename to cli/detect/report/sarif.go index e5120887a..f7457eb57 100644 --- a/cli/report/sarif.go +++ b/cli/detect/report/sarif.go @@ -27,14 +27,20 @@ import ( "fmt" "io" - "github.com/Infisical/infisical-merge/config" + "github.com/Infisical/infisical-merge/detect/config" ) -func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error { +type SarifReporter struct { + OrderedRules []config.Rule +} + +var _ Reporter = (*SarifReporter)(nil) + +func (r *SarifReporter) Write(w io.WriteCloser, findings []Finding) error { sarif := Sarif{ Schema: "https://json.schemastore.org/sarif-2.1.0.json", Version: "2.1.0", - Runs: getRuns(cfg, findings), + Runs: r.getRuns(findings), } encoder := json.NewEncoder(w) @@ -42,22 +48,22 @@ func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error { return encoder.Encode(sarif) } -func getRuns(cfg config.Config, findings []Finding) []Runs { +func (r *SarifReporter) getRuns(findings []Finding) []Runs { return []Runs{ { - Tool: getTool(cfg), + Tool: r.getTool(), Results: getResults(findings), }, } } -func getTool(cfg config.Config) Tool { +func (r *SarifReporter) getTool() Tool { tool := Tool{ Driver: Driver{ Name: driver, SemanticVersion: version, - InformationUri: "https://github.com/Infisical/infisical", - Rules: getRules(cfg), + InformationUri: "https://github.com/gitleaks/gitleaks", + Rules: r.getRules(), }, } @@ -73,26 +79,15 @@ func hasEmptyRules(tool Tool) bool { return len(tool.Driver.Rules) == 0 } -func getRules(cfg config.Config) []Rules { +func (r *SarifReporter) getRules() []Rules { // TODO for _, rule := range cfg.Rules { var rules []Rules - for _, rule := range cfg.OrderedRules() { - shortDescription := ShortDescription{ - Text: rule.Description, - } - if rule.Regex != nil { - shortDescription = ShortDescription{ - Text: rule.Regex.String(), - } - } else if rule.Path != nil { - shortDescription = ShortDescription{ - Text: rule.Path.String(), - } - } + for _, rule := range r.OrderedRules { rules = append(rules, Rules{ - ID: rule.RuleID, - Name: rule.Description, - Description: shortDescription, + ID: rule.RuleID, + Description: ShortDescription{ + Text: rule.Description, + }, }) } return rules @@ -125,6 +120,9 @@ func getResults(findings []Finding) []Results { Date: f.Date, Author: f.Author, }, + Properties: Properties{ + Tags: f.Tags, + }, } results = append(results, r) } @@ -180,7 +178,6 @@ type FullDescription struct { type Rules struct { ID string `json:"id"` - Name string `json:"name"` Description ShortDescription `json:"shortDescription"` } @@ -224,11 +221,16 @@ type Locations struct { PhysicalLocation PhysicalLocation `json:"physicalLocation"` } +type Properties struct { + Tags []string `json:"tags"` +} + type Results struct { Message Message `json:"message"` RuleId string `json:"ruleId"` Locations []Locations `json:"locations"` PartialFingerPrints `json:"partialFingerprints"` + Properties Properties `json:"properties"` } type Runs struct { diff --git a/cli/detect/report/template.go b/cli/detect/report/template.go new file mode 100644 index 000000000..094aaaea9 --- /dev/null +++ b/cli/detect/report/template.go @@ -0,0 +1,68 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package report + +import ( + "fmt" + "io" + "os" + "text/template" + + "github.com/Masterminds/sprig/v3" +) + +type TemplateReporter struct { + template *template.Template +} + +var _ Reporter = (*TemplateReporter)(nil) + +func NewTemplateReporter(templatePath string) (*TemplateReporter, error) { + if templatePath == "" { + return nil, fmt.Errorf("template path cannot be empty") + } + + file, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + templateText := string(file) + + // TODO: Add helper functions like escaping for JSON, XML, etc. + t := template.New("custom") + t = t.Funcs(sprig.TxtFuncMap()) + t, err = t.Parse(templateText) + if err != nil { + return nil, fmt.Errorf("error parsing file: %w", err) + } + return &TemplateReporter{template: t}, nil +} + +// writeTemplate renders the findings using the user-provided template. +// https://www.digitalocean.com/community/tutorials/how-to-use-templates-in-go +func (t *TemplateReporter) Write(w io.WriteCloser, findings []Finding) error { + if err := t.template.Execute(w, findings); err != nil { + return err + } + return nil +} diff --git a/cli/detect/sources/directory.go b/cli/detect/sources/directory.go new file mode 100644 index 000000000..0ad46c3d8 --- /dev/null +++ b/cli/detect/sources/directory.go @@ -0,0 +1,127 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package sources + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + + "github.com/fatih/semgroup" + + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" +) + +type ScanTarget struct { + Path string + Symlink string +} + +var isWindows = runtime.GOOS == "windows" + +func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool, allowlists []*config.Allowlist) (<-chan ScanTarget, error) { + paths := make(chan ScanTarget) + s.Go(func() error { + defer close(paths) + return filepath.Walk(source, + func(path string, fInfo os.FileInfo, err error) error { + logger := logging.With().Str("path", path).Logger() + + if err != nil { + if os.IsPermission(err) { + // This seems to only fail on directories at this stage. + logger.Warn().Msg("Skipping directory: permission denied") + return filepath.SkipDir + } + return err + } + + // Empty; nothing to do here. + if fInfo.Size() == 0 { + return nil + } + + // Unwrap symlinks, if |followSymlinks| is set. + scanTarget := ScanTarget{ + Path: path, + } + if fInfo.Mode().Type() == fs.ModeSymlink { + if !followSymlinks { + logger.Debug().Msg("Skipping symlink") + return nil + } + + realPath, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + + realPathFileInfo, _ := os.Stat(realPath) + if realPathFileInfo.IsDir() { + logger.Warn().Str("target", realPath).Msg("Skipping symlinked directory") + return nil + } + + scanTarget.Path = realPath + scanTarget.Symlink = path + } + + // TODO: Also run this check against the resolved symlink? + var skip bool + for _, a := range allowlists { + skip = a.PathAllowed(path) || + // TODO: Remove this in v9. + // This is an awkward hack to mitigate https://github.com/gitleaks/gitleaks/issues/1641. + (isWindows && a.PathAllowed(filepath.ToSlash(path))) + if skip { + break + } + } + if fInfo.IsDir() { + // Directory + if skip { + logger.Debug().Msg("Skipping directory due to global allowlist") + return filepath.SkipDir + } + + if fInfo.Name() == ".git" { + // Don't scan .git directories. + // TODO: Add this to the config allowlist, instead of hard-coding it. + return filepath.SkipDir + } + } else { + // File + if skip { + logger.Debug().Msg("Skipping file due to global allowlist") + return nil + } + + paths <- scanTarget + } + return nil + }) + }) + return paths, nil +} diff --git a/cli/detect/sources/git.go b/cli/detect/sources/git.go new file mode 100644 index 000000000..95b829a9a --- /dev/null +++ b/cli/detect/sources/git.go @@ -0,0 +1,211 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package sources + +import ( + "bufio" + "errors" + "io" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/gitleaks/go-gitdiff/gitdiff" + + "github.com/Infisical/infisical-merge/detect/logging" +) + +var quotedOptPattern = regexp.MustCompile(`^(?:"[^"]+"|'[^']+')$`) + +// GitCmd helps to work with Git's output. +type GitCmd struct { + cmd *exec.Cmd + diffFilesCh <-chan *gitdiff.File + errCh <-chan error +} + +// NewGitLogCmd returns `*DiffFilesCmd` with two channels: `<-chan *gitdiff.File` and `<-chan error`. +// Caller should read everything from channels until receiving a signal about their closure and call +// the `func (*DiffFilesCmd) Wait()` error in order to release resources. +func NewGitLogCmd(source string, logOpts string) (*GitCmd, error) { + sourceClean := filepath.Clean(source) + var cmd *exec.Cmd + if logOpts != "" { + args := []string{"-C", sourceClean, "log", "-p", "-U0"} + + // Ensure that the user-provided |logOpts| aren't wrapped in quotes. + // https://github.com/gitleaks/gitleaks/issues/1153 + userArgs := strings.Split(logOpts, " ") + var quotedOpts []string + for _, element := range userArgs { + if quotedOptPattern.MatchString(element) { + quotedOpts = append(quotedOpts, element) + } + } + if len(quotedOpts) > 0 { + logging.Warn().Msgf("the following `--log-opts` values may not work as expected: %v\n\tsee https://github.com/gitleaks/gitleaks/issues/1153 for more information", quotedOpts) + } + + args = append(args, userArgs...) + cmd = exec.Command("git", args...) + } else { + cmd = exec.Command("git", "-C", sourceClean, "log", "-p", "-U0", + "--full-history", "--all") + } + + logging.Debug().Msgf("executing: %s", cmd.String()) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + errCh := make(chan error) + go listenForStdErr(stderr, errCh) + + gitdiffFiles, err := gitdiff.Parse(stdout) + if err != nil { + return nil, err + } + + return &GitCmd{ + cmd: cmd, + diffFilesCh: gitdiffFiles, + errCh: errCh, + }, nil +} + +// NewGitDiffCmd returns `*DiffFilesCmd` with two channels: `<-chan *gitdiff.File` and `<-chan error`. +// Caller should read everything from channels until receiving a signal about their closure and call +// the `func (*DiffFilesCmd) Wait()` error in order to release resources. +func NewGitDiffCmd(source string, staged bool) (*GitCmd, error) { + sourceClean := filepath.Clean(source) + var cmd *exec.Cmd + cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", "--no-ext-diff", ".") + if staged { + cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", "--no-ext-diff", + "--staged", ".") + } + logging.Debug().Msgf("executing: %s", cmd.String()) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + errCh := make(chan error) + go listenForStdErr(stderr, errCh) + + gitdiffFiles, err := gitdiff.Parse(stdout) + if err != nil { + return nil, err + } + + return &GitCmd{ + cmd: cmd, + diffFilesCh: gitdiffFiles, + errCh: errCh, + }, nil +} + +// DiffFilesCh returns a channel with *gitdiff.File. +func (c *GitCmd) DiffFilesCh() <-chan *gitdiff.File { + return c.diffFilesCh +} + +// ErrCh returns a channel that could produce an error if there is something in stderr. +func (c *GitCmd) ErrCh() <-chan error { + return c.errCh +} + +// Wait waits for the command to exit and waits for any copying to +// stdin or copying from stdout or stderr to complete. +// +// Wait also closes underlying stdout and stderr. +func (c *GitCmd) Wait() (err error) { + return c.cmd.Wait() +} + +// listenForStdErr listens for stderr output from git, prints it to stdout, +// sends to errCh and closes it. +func listenForStdErr(stderr io.ReadCloser, errCh chan<- error) { + defer close(errCh) + + var errEncountered bool + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + // if git throws one of the following errors: + // + // exhaustive rename detection was skipped due to too many files. + // you may want to set your diff.renameLimit variable to at least + // (some large number) and retry the command. + // + // inexact rename detection was skipped due to too many files. + // you may want to set your diff.renameLimit variable to at least + // (some large number) and retry the command. + // + // Auto packing the repository in background for optimum performance. + // See "git help gc" for manual housekeeping. + // + // we skip exiting the program as git log -p/git diff will continue + // to send data to stdout and finish executing. This next bit of + // code prevents gitleaks from stopping mid scan if this error is + // encountered + if strings.Contains(scanner.Text(), + "exhaustive rename detection was skipped") || + strings.Contains(scanner.Text(), + "inexact rename detection was skipped") || + strings.Contains(scanner.Text(), + "you may want to set your diff.renameLimit") || + strings.Contains(scanner.Text(), + "See \"git help gc\" for manual housekeeping") || + strings.Contains(scanner.Text(), + "Auto packing the repository in background for optimum performance") { + logging.Warn().Msg(scanner.Text()) + } else { + logging.Error().Msgf("[git] %s", scanner.Text()) + errEncountered = true + } + } + + if errEncountered { + errCh <- errors.New("stderr is not empty") + return + } +} diff --git a/cli/detect/utils.go b/cli/detect/utils.go index 462716239..255d01fbe 100644 --- a/cli/detect/utils.go +++ b/cli/detect/utils.go @@ -26,20 +26,21 @@ import ( // "encoding/json" "fmt" "math" + "path/filepath" "strings" "time" + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/charmbracelet/lipgloss" - - "github.com/Infisical/infisical-merge/report" - "github.com/gitleaks/go-gitdiff/gitdiff" - "github.com/rs/zerolog/log" ) // augmentGitFinding updates the start and end line numbers of a finding to include the // delta from the git diff -func augmentGitFinding(finding report.Finding, textFragment *gitdiff.TextFragment, f *gitdiff.File) report.Finding { +func augmentGitFinding(remote *RemoteInfo, finding report.Finding, textFragment *gitdiff.TextFragment, f *gitdiff.File) report.Finding { if !strings.HasPrefix(finding.Match, "file detected") { finding.StartLine += int(textFragment.NewPosition) finding.EndLine += int(textFragment.NewPosition) @@ -47,16 +48,76 @@ func augmentGitFinding(finding report.Finding, textFragment *gitdiff.TextFragmen if f.PatchHeader != nil { finding.Commit = f.PatchHeader.SHA - finding.Message = f.PatchHeader.Message() if f.PatchHeader.Author != nil { finding.Author = f.PatchHeader.Author.Name finding.Email = f.PatchHeader.Author.Email } finding.Date = f.PatchHeader.AuthorDate.UTC().Format(time.RFC3339) + finding.Message = f.PatchHeader.Message() + // Results from `git diff` shouldn't have a link. + if finding.Commit != "" { + finding.Link = createScmLink(remote.Platform, remote.Url, finding) + } } return finding } +var linkCleaner = strings.NewReplacer( + " ", "%20", + "%", "%25", +) + +func createScmLink(scmPlatform scm.Platform, remoteUrl string, finding report.Finding) string { + if scmPlatform == scm.UnknownPlatform || scmPlatform == scm.NoPlatform { + return "" + } + + // Clean the path. + var ( + filePath = linkCleaner.Replace(finding.File) + ext = strings.ToLower(filepath.Ext(filePath)) + ) + + switch scmPlatform { + case scm.GitHubPlatform: + link := fmt.Sprintf("%s/blob/%s/%s", remoteUrl, finding.Commit, filePath) + if ext == ".ipynb" || ext == ".md" { + link += "?plain=1" + } + if finding.StartLine != 0 { + link += fmt.Sprintf("#L%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("-L%d", finding.EndLine) + } + return link + case scm.GitLabPlatform: + link := fmt.Sprintf("%s/blob/%s/%s", remoteUrl, finding.Commit, filePath) + if finding.StartLine != 0 { + link += fmt.Sprintf("#L%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("-%d", finding.EndLine) + } + return link + case scm.AzureDevOpsPlatform: + link := fmt.Sprintf("%s/commit/%s?path=/%s", remoteUrl, finding.Commit, filePath) + // Add line information if applicable + if finding.StartLine != 0 { + link += fmt.Sprintf("&line=%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("&lineEnd=%d", finding.EndLine) + } + // This is a bit dirty, but Azure DevOps does not highlight the line when the lineStartColumn and lineEndColumn are not provided + link += "&lineStartColumn=1&lineEndColumn=10000000&type=2&lineStyle=plain&_a=files" + return link + default: + // This should never happen. + return "" + } +} + // shannonEntropy calculates the entropy of data using the formula defined here: // https://en.wiktionary.org/wiki/Shannon_entropy // Another way to think about what this is doing is calculating the number of bits @@ -82,7 +143,7 @@ func shannonEntropy(data string) (entropy float64) { } // filter will dedupe and redact findings -func filter(findings []report.Finding, redact bool) []report.Finding { +func filter(findings []report.Finding, redact uint) []report.Finding { var retFindings []report.Finding for _, f := range findings { include := true @@ -96,15 +157,15 @@ func filter(findings []report.Finding, redact bool) []report.Finding { genericMatch := strings.Replace(f.Match, f.Secret, "REDACTED", -1) betterMatch := strings.Replace(fPrime.Match, fPrime.Secret, "REDACTED", -1) - log.Trace().Msgf("skipping %s finding (%s), %s rule takes precendence (%s)", f.RuleID, genericMatch, fPrime.RuleID, betterMatch) + logging.Trace().Msgf("skipping %s finding (%s), %s rule takes precedence (%s)", f.RuleID, genericMatch, fPrime.RuleID, betterMatch) include = false break } } } - if redact { - f.Redact() + if redact > 0 { + f.Redact(redact) } if include { retFindings = append(retFindings, f) @@ -152,7 +213,7 @@ func printFinding(f report.Finding, noColor bool) { lineEndIdx := matchInLineIDX + len(f.Match) if len(f.Line)-1 <= lineEndIdx { - lineEndIdx = len(f.Line) - 1 + lineEndIdx = len(f.Line) } lineEnd := f.Line[lineEndIdx:] @@ -184,6 +245,9 @@ func printFinding(f report.Finding, noColor bool) { fmt.Println("") return } + if len(f.Tags) > 0 { + fmt.Printf("%-12s %s\n", "Tags:", f.Tags) + } fmt.Printf("%-12s %s\n", "File:", f.File) fmt.Printf("%-12s %d\n", "Line:", f.StartLine) if f.Commit == "" { @@ -196,16 +260,12 @@ func printFinding(f report.Finding, noColor bool) { fmt.Printf("%-12s %s\n", "Email:", f.Email) fmt.Printf("%-12s %s\n", "Date:", f.Date) fmt.Printf("%-12s %s\n", "Fingerprint:", f.Fingerprint) + if f.Link != "" { + fmt.Printf("%-12s %s\n", "Link:", f.Link) + } fmt.Println("") } -func containsDigit(s string) bool { - for _, c := range s { - switch c { - case '1', '2', '3', '4', '5', '6', '7', '8', '9': - return true - } - - } - return false +func isWhitespace(ch byte) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } diff --git a/cli/go.mod b/cli/go.mod index cc38c6859..a2b256f8a 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -42,6 +42,11 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.4.0 // indirect cloud.google.com/go/iam v1.1.11 // indirect + dario.cat/mergo v1.0.1 // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/aws/aws-sdk-go-v2 v1.27.2 // indirect @@ -74,17 +79,21 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/pprof v0.0.0-20250302191652-9094ed2288e7 // indirect github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/muesli/mango v0.1.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect @@ -98,8 +107,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cast v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/wlynxg/anet v0.0.5 // indirect diff --git a/cli/go.sum b/cli/go.sum index f5ddebade..cb1b1c1cf 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -44,13 +44,23 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= +github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Infisical/go-keyring v1.0.2 h1:dWOkI/pB/7RocfSJgGXbXxLDcVYsdslgjEPmVhb+nl8= github.com/Infisical/go-keyring v1.0.2/go.mod h1:LWOnn/sw9FxDW/0VY+jHFAfOFEe03xmwBVSfJnBowto= github.com/Infisical/turn/v4 v4.0.1 h1:omdelNsnFfzS5cu86W5OBR68by68a8sva4ogR0lQQnw= github.com/Infisical/turn/v4 v4.0.1/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -275,6 +285,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= @@ -317,6 +329,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -326,6 +340,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -395,6 +411,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -404,6 +422,8 @@ github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= diff --git a/cli/packages/cmd/scan.go b/cli/packages/cmd/scan.go index 1226e3319..42ff0f1e1 100644 --- a/cli/packages/cmd/scan.go +++ b/cli/packages/cmd/scan.go @@ -32,10 +32,13 @@ import ( "strings" "time" - "github.com/Infisical/infisical-merge/config" "github.com/Infisical/infisical-merge/detect" + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" "github.com/Infisical/infisical-merge/packages/util" - "github.com/Infisical/infisical-merge/report" "github.com/manifoldco/promptui" "github.com/posthog/posthog-go" "github.com/rs/zerolog/log" @@ -240,9 +243,17 @@ var scanCmd = &cobra.Command{ log.Fatal().Err(err).Msg("") } // set redact flag - if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil { + + redactFlag, err := cmd.Flags().GetBool("redact") + if err != nil { log.Fatal().Err(err).Msg("") } + if redactFlag { + detector.Redact = 100 + } else { + detector.Redact = 0 + } + if detector.MaxTargetMegaBytes, err = cmd.Flags().GetInt("max-target-megabytes"); err != nil { log.Fatal().Err(err).Msg("") } @@ -293,31 +304,49 @@ var scanCmd = &cobra.Command{ // start the detector scan if noGit { - findings, err = detector.DetectFiles(source) + paths, err := sources.DirectoryTargets( + source, + detector.Sema, + detector.FollowSymlinks, + detector.Config.Allowlists, + ) if err != nil { + logging.Fatal().Err(err).Send() + } + + if findings, err = detector.DetectFiles(paths); err != nil { // don't exit on error, just log it - log.Error().Err(err).Msg("") + logging.Error().Err(err).Msg("failed scan directory") } } else if fromPipe { - findings, err = detector.DetectReader(os.Stdin, 10) - if err != nil { + if findings, err = detector.DetectReader(os.Stdin, 10); err != nil { // log fatal to exit, no need to continue since a report // will not be generated when scanning from a pipe...for now - log.Fatal().Err(err).Msg("") + logging.Fatal().Err(err).Msg("failed scan input from stdin") } } else { + var ( + gitCmd *sources.GitCmd + scmPlatform scm.Platform + remote *detect.RemoteInfo + ) + var logOpts string logOpts, err = cmd.Flags().GetString("log-opts") - if err != nil { - log.Fatal().Err(err).Msg("") + + if gitCmd, err = sources.NewGitLogCmd(source, logOpts); err != nil { + logging.Fatal().Err(err).Msg("could not create Git cmd") } - findings, err = detector.DetectGit(source, logOpts, detect.DetectType) - if err != nil { + if scmPlatform, err = scm.PlatformFromString("github"); err != nil { + logging.Fatal().Err(err).Send() + } + remote = detect.NewRemoteInfo(scmPlatform, source) + + if findings, err = detector.DetectGit(gitCmd, remote); err != nil { // don't exit on error, just log it - log.Error().Err(err).Msg("") + logging.Error().Err(err).Msg("failed to scan Git repository") } } - // log info about the scan if err == nil { log.Info().Msgf("scan completed in %s", FormatDuration(time.Since(start))) @@ -341,9 +370,7 @@ var scanCmd = &cobra.Command{ reportPath, _ := cmd.Flags().GetString("report-path") ext, _ := cmd.Flags().GetString("report-format") if reportPath != "" { - if err := report.Write(findings, cfg, ext, reportPath); err != nil { - log.Fatal().Err(err).Msg("could not write") - } + reportFindings(findings, reportPath, ext, &cfg) } if err != nil { @@ -375,7 +402,6 @@ var scanGitChangesCmd = &cobra.Command{ cfg.Path, _ = cmd.Flags().GetString("config") exitCode, _ := cmd.Flags().GetInt("exit-code") staged, _ := cmd.Flags().GetBool("staged") - start := time.Now() // Setup detector detector := detect.NewDetector(cfg) @@ -397,9 +423,17 @@ var scanGitChangesCmd = &cobra.Command{ log.Fatal().Err(err).Msg("") } // set redact flag - if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil { + + redactFlag, err := cmd.Flags().GetBool("redact") + if err != nil { log.Fatal().Err(err).Msg("") } + if redactFlag { + detector.Redact = 100 + } else { + detector.Redact = 0 + } + if detector.MaxTargetMegaBytes, err = cmd.Flags().GetInt("max-target-megabytes"); err != nil { log.Fatal().Err(err).Msg("") } @@ -414,32 +448,22 @@ var scanGitChangesCmd = &cobra.Command{ } } - // get log options for git scan - logOpts, err := cmd.Flags().GetString("log-opts") - if err != nil { - log.Fatal().Err(err).Msg("") - } - - log.Info().Msgf("scanning for exposed secrets...") - // start git scan - var findings []report.Finding - if staged { - findings, err = detector.DetectGit(source, logOpts, detect.ProtectStagedType) - } else { - findings, err = detector.DetectGit(source, logOpts, detect.ProtectType) - } - if err != nil { - // don't exit on error, just log it - log.Error().Err(err).Msg("") - } + var ( + findings []report.Finding - // log info about the scan - log.Info().Msgf("scan completed in %s", FormatDuration(time.Since(start))) - if len(findings) != 0 { - log.Warn().Msgf("leaks found: %d", len(findings)) - } else { - log.Info().Msg("no leaks found") + gitCmd *sources.GitCmd + remote *detect.RemoteInfo + ) + + if gitCmd, err = sources.NewGitDiffCmd(source, staged); err != nil { + logging.Fatal().Err(err).Msg("could not create Git diff cmd") + } + remote = &detect.RemoteInfo{Platform: scm.NoPlatform} + + if findings, err = detector.DetectGit(gitCmd, remote); err != nil { + // don't exit on error, just log it + logging.Error().Err(err).Msg("failed to scan Git repository") } Telemetry.CaptureEvent("cli-command:scan git-changes", posthog.NewProperties().Set("risks", len(findings)).Set("version", util.CLI_VERSION)) @@ -447,9 +471,7 @@ var scanGitChangesCmd = &cobra.Command{ reportPath, _ := cmd.Flags().GetString("report-path") ext, _ := cmd.Flags().GetString("report-format") if reportPath != "" { - if err = report.Write(findings, cfg, ext, reportPath); err != nil { - log.Fatal().Err(err).Msg("") - } + reportFindings(findings, reportPath, ext, &cfg) } if len(findings) != 0 { os.Exit(exitCode) @@ -457,6 +479,36 @@ var scanGitChangesCmd = &cobra.Command{ }, } +func reportFindings(findings []report.Finding, reportPath string, ext string, cfg *config.Config) { + + var reporter report.Reporter + + switch ext { + case "csv": + reporter = &report.CsvReporter{} + case "json": + reporter = &report.JsonReporter{} + case "junit": + reporter = &report.JunitReporter{} + case "sarif": + reporter = &report.SarifReporter{ + OrderedRules: cfg.GetOrderedRules(), + } + default: + logging.Fatal().Msgf("unknown report format %s", ext) + } + + file, err := os.Create(reportPath) + if err != nil { + log.Fatal().Err(err).Msg("could not create file") + } + + if err := reporter.Write(file, findings); err != nil { + log.Fatal().Err(err).Msg("could not write") + } + +} + func fileExists(fileName string) bool { // check for a .infisicalignore file info, err := os.Stat(fileName) diff --git a/cli/report/csv_test.go b/cli/report/csv_test.go deleted file mode 100644 index 967026519..000000000 --- a/cli/report/csv_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestWriteCSV(t *testing.T) { - tests := []struct { - findings []Finding - testReportName string - expected string - wantEmpty bool - }{ - { - testReportName: "simple", - expected: filepath.Join(expectPath, "report", "csv_simple.csv"), - findings: []Finding{ - { - RuleID: "test-rule", - Match: "line containing secret", - Secret: "a secret", - StartLine: 1, - EndLine: 2, - StartColumn: 1, - EndColumn: 2, - Message: "opps", - File: "auth.py", - SymlinkFile: "", - Commit: "0000000000000000", - Author: "John Doe", - Email: "johndoe@gmail.com", - Date: "10-19-2003", - Fingerprint: "fingerprint", - }, - }}, - { - - wantEmpty: true, - testReportName: "empty", - expected: filepath.Join(expectPath, "report", "this_should_not_exist.csv"), - findings: []Finding{}}, - } - - for _, test := range tests { - tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".csv")) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = writeCsv(test.findings, tmpfile) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - if test.wantEmpty { - if len(got) > 0 { - t.Errorf("Expected empty file, got %s", got) - } - os.Remove(tmpfile.Name()) - continue - } - want, err := os.ReadFile(test.expected) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - - if string(got) != string(want) { - err = os.WriteFile(strings.Replace(test.expected, ".csv", ".got.csv", 1), got, 0644) - if err != nil { - t.Error(err) - } - t.Errorf("got %s, want %s", string(got), string(want)) - } - - os.Remove(tmpfile.Name()) - } -} diff --git a/cli/report/json_test.go b/cli/report/json_test.go deleted file mode 100644 index e81aaa827..000000000 --- a/cli/report/json_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestWriteJSON(t *testing.T) { - tests := []struct { - findings []Finding - testReportName string - expected string - wantEmpty bool - }{ - { - testReportName: "simple", - expected: filepath.Join(expectPath, "report", "json_simple.json"), - findings: []Finding{ - { - - Description: "", - RuleID: "test-rule", - Match: "line containing secret", - Secret: "a secret", - StartLine: 1, - EndLine: 2, - StartColumn: 1, - EndColumn: 2, - Message: "opps", - File: "auth.py", - SymlinkFile: "", - Commit: "0000000000000000", - Author: "John Doe", - Email: "johndoe@gmail.com", - Date: "10-19-2003", - Tags: []string{}, - }, - }}, - { - - testReportName: "empty", - expected: filepath.Join(expectPath, "report", "empty.json"), - findings: []Finding{}}, - } - - for _, test := range tests { - // create tmp file using os.TempDir() - tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".json")) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = writeJson(test.findings, tmpfile) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - if test.wantEmpty { - if len(got) > 0 { - os.Remove(tmpfile.Name()) - t.Errorf("Expected empty file, got %s", got) - } - os.Remove(tmpfile.Name()) - continue - } - want, err := os.ReadFile(test.expected) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - - if string(got) != string(want) { - err = os.WriteFile(strings.Replace(test.expected, ".json", ".got.json", 1), got, 0644) - if err != nil { - t.Error(err) - } - t.Errorf("got %s, want %s", string(got), string(want)) - } - - os.Remove(tmpfile.Name()) - } -} diff --git a/cli/report/report_test.go b/cli/report/report_test.go deleted file mode 100644 index ef38f19c2..000000000 --- a/cli/report/report_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/Infisical/infisical-merge/config" -) - -const ( - expectPath = "../testdata/expected/" - tmpPath = "../testdata/tmp" -) - -func TestReport(t *testing.T) { - tests := []struct { - findings []Finding - ext string - wantEmpty bool - }{ - { - ext: "json", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: ".json", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: ".jsonj", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - wantEmpty: true, - }, - { - ext: ".csv", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: "csv", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: "CSV", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - // { - // ext: "SARIF", - // findings: []Finding{ - // { - // RuleID: "test-rule", - // }, - // }, - // }, - } - - for i, test := range tests { - tmpfile, err := os.Create(filepath.Join(tmpPath, strconv.Itoa(i)+test.ext)) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = Write(test.findings, config.Config{}, test.ext, tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - os.Remove(tmpfile.Name()) - - if len(got) == 0 && !test.wantEmpty { - t.Errorf("got empty file with extension " + test.ext) - } - - if test.wantEmpty { - if len(got) > 0 { - t.Errorf("Expected empty file, got %s", got) - } - continue - } - } -} diff --git a/cli/report/sarif_test.go b/cli/report/sarif_test.go deleted file mode 100644 index 9331060f1..000000000 --- a/cli/report/sarif_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -const configPath = "../testdata/config/" - -// func TestWriteSarif(t *testing.T) { -// tests := []struct { -// findings []Finding -// testReportName string -// expected string -// wantEmpty bool -// cfgName string -// }{ -// { -// cfgName: "simple", -// testReportName: "simple", -// expected: filepath.Join(expectPath, "report", "sarif_simple.sarif"), -// findings: []Finding{ -// { - -// Description: "A test rule", -// RuleID: "test-rule", -// Match: "line containing secret", -// Secret: "a secret", -// StartLine: 1, -// EndLine: 2, -// StartColumn: 1, -// EndColumn: 2, -// Message: "opps", -// File: "auth.py", -// Commit: "0000000000000000", -// Author: "John Doe", -// Email: "johndoe@gmail.com", -// Date: "10-19-2003", -// Tags: []string{}, -// }, -// }}, -// } - -// for _, test := range tests { -// // create tmp file using os.TempDir() -// tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".json")) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// viper.Reset() -// viper.AddConfigPath(configPath) -// viper.SetConfigName(test.cfgName) -// viper.SetConfigType("toml") -// err = viper.ReadInConfig() -// if err != nil { -// t.Error(err) -// } - -// var vc config.ViperConfig -// err = viper.Unmarshal(&vc) -// if err != nil { -// t.Error(err) -// } - -// cfg, err := vc.Translate() -// if err != nil { -// t.Error(err) -// } -// err = writeSarif(cfg, test.findings, tmpfile) -// fmt.Println(cfg) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// got, err := os.ReadFile(tmpfile.Name()) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// if test.wantEmpty { -// if len(got) > 0 { -// os.Remove(tmpfile.Name()) -// t.Errorf("Expected empty file, got %s", got) -// } -// os.Remove(tmpfile.Name()) -// continue -// } -// want, err := os.ReadFile(test.expected) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } - -// if string(got) != string(want) { -// err = os.WriteFile(strings.Replace(test.expected, ".sarif", ".got.sarif", 1), got, 0644) -// if err != nil { -// t.Error(err) -// } -// t.Errorf("got %s, want %s", string(got), string(want)) -// } - -// os.Remove(tmpfile.Name()) -// } -// }