add gitleak to cli

cli/config/allowlist.go

@@ -0,0 +1,85 @@
+// MIT License
+
+// Copyright (c) 2019 Zachary Rice
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package config
+
+import (
+	"regexp"
+	"strings"
+)
+
+// Allowlist allows a rule to be ignored for specific
+// regexes, paths, and/or commits
+type Allowlist struct {
+	// Short human readable description of the allowlist.
+	Description string
+
+	// Regexes is slice of content regular expressions that are allowed to be ignored.
+	Regexes []*regexp.Regexp
+
+	// RegexTarget
+	RegexTarget string
+
+	// Paths is a slice of path regular expressions that are allowed to be ignored.
+	Paths []*regexp.Regexp
+
+	// Commits is a slice of commit SHAs that are allowed to be ignored.
+	Commits []string
+
+	// StopWords is a slice of stop words that are allowed to be ignored.
+	// This targets the _secret_, not the content of the regex match like the
+	// Regexes slice.
+	StopWords []string
+}
+
+// CommitAllowed returns true if the commit is allowed to be ignored.
+func (a *Allowlist) CommitAllowed(c string) bool {
+	if c == "" {
+		return false
+	}
+	for _, commit := range a.Commits {
+		if commit == c {
+			return true
+		}
+	}
+	return false
+}
+
+// PathAllowed returns true if the path is allowed to be ignored.
+func (a *Allowlist) PathAllowed(path string) bool {
+	return anyRegexMatch(path, a.Paths)
+}
+
+// RegexAllowed returns true if the regex is allowed to be ignored.
+func (a *Allowlist) RegexAllowed(s string) bool {
+	return anyRegexMatch(s, a.Regexes)
+}
+
+func (a *Allowlist) ContainsStopWord(s string) bool {
+	s = strings.ToLower(s)
+	for _, stopWord := range a.StopWords {
+		if strings.Contains(s, strings.ToLower(stopWord)) {
+			return true
+		}
+	}
+	return false
+}

cli/config/allowlist_test.go

@@ -0,0 +1,115 @@
+// MIT License
+
+// Copyright (c) 2019 Zachary Rice
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package config
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCommitAllowed(t *testing.T) {
+	tests := []struct {
+		allowlist     Allowlist
+		commit        string
+		commitAllowed bool
+	}{
+		{
+			allowlist: Allowlist{
+				Commits: []string{"commitA"},
+			},
+			commit:        "commitA",
+			commitAllowed: true,
+		},
+		{
+			allowlist: Allowlist{
+				Commits: []string{"commitB"},
+			},
+			commit:        "commitA",
+			commitAllowed: false,
+		},
+		{
+			allowlist: Allowlist{
+				Commits: []string{"commitB"},
+			},
+			commit:        "",
+			commitAllowed: false,
+		},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.commitAllowed, tt.allowlist.CommitAllowed(tt.commit))
+	}
+}
+
+func TestRegexAllowed(t *testing.T) {
+	tests := []struct {
+		allowlist    Allowlist
+		secret       string
+		regexAllowed bool
+	}{
+		{
+			allowlist: Allowlist{
+				Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")},
+			},
+			secret:       "a secret: matchthis, done",
+			regexAllowed: true,
+		},
+		{
+			allowlist: Allowlist{
+				Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")},
+			},
+			secret:       "a secret",
+			regexAllowed: false,
+		},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.regexAllowed, tt.allowlist.RegexAllowed(tt.secret))
+	}
+}
+
+func TestPathAllowed(t *testing.T) {
+	tests := []struct {
+		allowlist   Allowlist
+		path        string
+		pathAllowed bool
+	}{
+		{
+			allowlist: Allowlist{
+				Paths: []*regexp.Regexp{regexp.MustCompile("path")},
+			},
+			path:        "a path",
+			pathAllowed: true,
+		},
+		{
+			allowlist: Allowlist{
+				Paths: []*regexp.Regexp{regexp.MustCompile("path")},
+			},
+			path:        "a ???",
+			pathAllowed: false,
+		},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.pathAllowed, tt.allowlist.PathAllowed(tt.path))
+	}
+}

cli/config/config.go

@@ -0,0 +1,275 @@
+// MIT License
+
+// Copyright (c) 2019 Zachary Rice
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package config
+
+import (
+	_ "embed"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+)
+
+//go:embed gitleaks.toml
+var DefaultConfig string
+
+// use to keep track of how many configs we can extend
+// yea I know, globals bad
+var extendDepth int
+
+const maxExtendDepth = 2
+
+// ViperConfig is the config struct used by the Viper config package
+// to parse the config file. This struct does not include regular expressions.
+// It is used as an intermediary to convert the Viper config to the Config struct.
+type ViperConfig struct {
+	Description string
+	Extend      Extend
+	Rules       []struct {
+		ID          string
+		Description string
+		Entropy     float64
+		SecretGroup int
+		Regex       string
+		Keywords    []string
+		Path        string
+		Tags        []string
+
+		Allowlist struct {
+			RegexTarget string
+			Regexes     []string
+			Paths       []string
+			Commits     []string
+			StopWords   []string
+		}
+	}
+	Allowlist struct {
+		RegexTarget string
+		Regexes     []string
+		Paths       []string
+		Commits     []string
+		StopWords   []string
+	}
+}
+
+// Config is a configuration struct that contains rules and an allowlist if present.
+type Config struct {
+	Extend      Extend
+	Path        string
+	Description string
+	Rules       map[string]Rule
+	Allowlist   Allowlist
+	Keywords    []string
+
+	// used to keep sarif results consistent
+	orderedRules []string
+}
+
+// Extend is a struct that allows users to define how they want their
+// configuration extended by other configuration files.
+type Extend struct {
+	Path       string
+	URL        string
+	UseDefault bool
+}
+
+func (vc *ViperConfig) Translate() (Config, error) {
+	var (
+		keywords     []string
+		orderedRules []string
+	)
+	rulesMap := make(map[string]Rule)
+
+	for _, r := range vc.Rules {
+		var allowlistRegexes []*regexp.Regexp
+		for _, a := range r.Allowlist.Regexes {
+			allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a))
+		}
+		var allowlistPaths []*regexp.Regexp
+		for _, a := range r.Allowlist.Paths {
+			allowlistPaths = append(allowlistPaths, regexp.MustCompile(a))
+		}
+
+		if r.Keywords == nil {
+			r.Keywords = []string{}
+		} else {
+			for _, k := range r.Keywords {
+				keywords = append(keywords, strings.ToLower(k))
+			}
+		}
+
+		if r.Tags == nil {
+			r.Tags = []string{}
+		}
+
+		var configRegex *regexp.Regexp
+		var configPathRegex *regexp.Regexp
+		if r.Regex == "" {
+			configRegex = nil
+		} else {
+			configRegex = regexp.MustCompile(r.Regex)
+		}
+		if r.Path == "" {
+			configPathRegex = nil
+		} else {
+			configPathRegex = regexp.MustCompile(r.Path)
+		}
+		r := Rule{
+			Description: r.Description,
+			RuleID:      r.ID,
+			Regex:       configRegex,
+			Path:        configPathRegex,
+			SecretGroup: r.SecretGroup,
+			Entropy:     r.Entropy,
+			Tags:        r.Tags,
+			Keywords:    r.Keywords,
+			Allowlist: Allowlist{
+				RegexTarget: r.Allowlist.RegexTarget,
+				Regexes:     allowlistRegexes,
+				Paths:       allowlistPaths,
+				Commits:     r.Allowlist.Commits,
+				StopWords:   r.Allowlist.StopWords,
+			},
+		}
+		orderedRules = append(orderedRules, r.RuleID)
+
+		if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() {
+			return Config{}, fmt.Errorf("%s invalid regex secret group %d, max regex secret group %d", r.Description, r.SecretGroup, r.Regex.NumSubexp())
+		}
+		rulesMap[r.RuleID] = r
+	}
+	var allowlistRegexes []*regexp.Regexp
+	for _, a := range vc.Allowlist.Regexes {
+		allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a))
+	}
+	var allowlistPaths []*regexp.Regexp
+	for _, a := range vc.Allowlist.Paths {
+		allowlistPaths = append(allowlistPaths, regexp.MustCompile(a))
+	}
+	c := Config{
+		Description: vc.Description,
+		Extend:      vc.Extend,
+		Rules:       rulesMap,
+		Allowlist: Allowlist{
+			RegexTarget: vc.Allowlist.RegexTarget,
+			Regexes:     allowlistRegexes,
+			Paths:       allowlistPaths,
+			Commits:     vc.Allowlist.Commits,
+			StopWords:   vc.Allowlist.StopWords,
+		},
+		Keywords:     keywords,
+		orderedRules: orderedRules,
+	}
+
+	if maxExtendDepth != extendDepth {
+		// disallow both usedefault and path from being set
+		if c.Extend.Path != "" && c.Extend.UseDefault {
+			log.Fatal().Msg("unable to load config due to extend.path and extend.useDefault being set")
+		}
+		if c.Extend.UseDefault {
+			c.extendDefault()
+		} else if c.Extend.Path != "" {
+			c.extendPath()
+		}
+
+	}
+
+	return c, nil
+}
+
+func (c *Config) OrderedRules() []Rule {
+	var orderedRules []Rule
+	for _, id := range c.orderedRules {
+		if _, ok := c.Rules[id]; ok {
+			orderedRules = append(orderedRules, c.Rules[id])
+		}
+	}
+	return orderedRules
+}
+
+func (c *Config) extendDefault() {
+	extendDepth++
+	viper.SetConfigType("toml")
+	if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	defaultViperConfig := ViperConfig{}
+	if err := viper.Unmarshal(&defaultViperConfig); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	cfg, err := defaultViperConfig.Translate()
+	if err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	log.Debug().Msg("extending config with default config")
+	c.extend(cfg)
+
+}
+
+func (c *Config) extendPath() {
+	extendDepth++
+	viper.SetConfigFile(c.Extend.Path)
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	extensionViperConfig := ViperConfig{}
+	if err := viper.Unmarshal(&extensionViperConfig); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	cfg, err := extensionViperConfig.Translate()
+	if err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	log.Debug().Msgf("extending config with %s", c.Extend.Path)
+	c.extend(cfg)
+}
+
+func (c *Config) extendURL() {
+	// TODO
+}
+
+func (c *Config) extend(extensionConfig Config) {
+	for ruleID, rule := range extensionConfig.Rules {
+		if _, ok := c.Rules[ruleID]; !ok {
+			log.Trace().Msgf("adding %s to base config", ruleID)
+			c.Rules[ruleID] = rule
+			c.Keywords = append(c.Keywords, rule.Keywords...)
+		}
+	}
+
+	// append allowlists, not attempting to merge
+	c.Allowlist.Commits = append(c.Allowlist.Commits,
+		extensionConfig.Allowlist.Commits...)
+	c.Allowlist.Paths = append(c.Allowlist.Paths,
+		extensionConfig.Allowlist.Paths...)
+	c.Allowlist.Regexes = append(c.Allowlist.Regexes,
+		extensionConfig.Allowlist.Regexes...)
+}

cli/config/config_test.go

@@ -0,0 +1,170 @@
+// MIT License
+
+// Copyright (c) 2019 Zachary Rice
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+const configPath = "../testdata/config/"
+
+func TestTranslate(t *testing.T) {
+	tests := []struct {
+		cfgName   string
+		cfg       Config
+		wantError error
+	}{
+		{
+			cfgName: "allow_aws_re",
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"),
+					Tags:        []string{"key", "AWS"},
+					Keywords:    []string{},
+					RuleID:      "aws-access-key",
+					Allowlist: Allowlist{
+						Regexes: []*regexp.Regexp{
+							regexp.MustCompile("AKIALALEMEL33243OLIA"),
+						},
+					},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "allow_commit",
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"),
+					Tags:        []string{"key", "AWS"},
+					Keywords:    []string{},
+					RuleID:      "aws-access-key",
+					Allowlist: Allowlist{
+						Commits: []string{"allowthiscommit"},
+					},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "allow_path",
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"),
+					Tags:        []string{"key", "AWS"},
+					Keywords:    []string{},
+					RuleID:      "aws-access-key",
+					Allowlist: Allowlist{
+						Paths: []*regexp.Regexp{
+							regexp.MustCompile(".go"),
+						},
+					},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "entropy_group",
+			cfg: Config{
+				Rules: map[string]Rule{"discord-api-key": {
+					Description: "Discord API key",
+					Regex:       regexp.MustCompile(`(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]`),
+					RuleID:      "discord-api-key",
+					Allowlist:   Allowlist{},
+					Entropy:     3.5,
+					SecretGroup: 3,
+					Tags:        []string{},
+					Keywords:    []string{},
+				},
+				},
+			},
+		},
+		{
+			cfgName:   "bad_entropy_group",
+			cfg:       Config{},
+			wantError: fmt.Errorf("Discord API key invalid regex secret group 5, max regex secret group 3"),
+		},
+		{
+			cfgName: "base",
+			cfg: Config{
+				Rules: map[string]Rule{
+					"aws-access-key": {
+						Description: "AWS Access Key",
+						Regex:       regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"),
+						Tags:        []string{"key", "AWS"},
+						Keywords:    []string{},
+						RuleID:      "aws-access-key",
+					},
+					"aws-secret-key": {
+						Description: "AWS Secret Key",
+						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
+						Tags:        []string{"key", "AWS"},
+						Keywords:    []string{},
+						RuleID:      "aws-secret-key",
+					},
+					"aws-secret-key-again": {
+						Description: "AWS Secret Key",
+						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
+						Tags:        []string{"key", "AWS"},
+						Keywords:    []string{},
+						RuleID:      "aws-secret-key-again",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		viper.Reset()
+		viper.AddConfigPath(configPath)
+		viper.SetConfigName(tt.cfgName)
+		viper.SetConfigType("toml")
+		err := viper.ReadInConfig()
+		if err != nil {
+			t.Error(err)
+		}
+
+		var vc ViperConfig
+		err = viper.Unmarshal(&vc)
+		if err != nil {
+			t.Error(err)
+		}
+		cfg, err := vc.Translate()
+		if tt.wantError != nil {
+			if err == nil {
+				t.Errorf("expected error")
+			}
+			assert.Equal(t, tt.wantError, err)
+		}
+
+		assert.Equal(t, cfg.Rules, tt.cfg.Rules)
+	}
+}

cli/config/rule.go

@@ -0,0 +1,43 @@
+package config
+
+import (
+	"regexp"
+)
+
+// Rules contain information that define details on how to detect secrets
+type Rule struct {
+	// Description is the description of the rule.
+	Description string
+
+	// RuleID is a unique identifier for this rule
+	RuleID string
+
+	// Entropy is a float representing the minimum shannon
+	// entropy a regex group must have to be considered a secret.
+	Entropy float64
+
+	// SecretGroup is an int used to extract secret from regex
+	// match and used as the group that will have its entropy
+	// checked if `entropy` is set.
+	SecretGroup int
+
+	// Regex is a golang regular expression used to detect secrets.
+	Regex *regexp.Regexp
+
+	// Path is a golang regular expression used to
+	// filter secrets by path
+	Path *regexp.Regexp
+
+	// Tags is an array of strings used for metadata
+	// and reporting purposes.
+	Tags []string
+
+	// Keywords are used for pre-regex check filtering. Rules that contain
+	// keywords will perform a quick string compare check to make sure the
+	// keyword(s) are in the content being scanned.
+	Keywords []string
+
+	// Allowlist allows a rule to be ignored for specific
+	// regexes, paths, and/or commits
+	Allowlist Allowlist
+}

cli/config/utils.go

@@ -0,0 +1,24 @@
+package config
+
+import (
+	"regexp"
+)
+
+func anyRegexMatch(f string, res []*regexp.Regexp) bool {
+	for _, re := range res {
+		if regexMatched(f, re) {
+			return true
+		}
+	}
+	return false
+}
+
+func regexMatched(f string, re *regexp.Regexp) bool {
+	if re == nil {
+		return false
+	}
+	if re.FindString(f) != "" {
+		return true
+	}
+	return false
+}