mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
feat(cli): upgrade secret scanner
This commit is contained in:
Daniel Hougaard
225
cli/detect/directory.go
Normal file
225
cli/detect/directory.go
Normal file
@@ -0,0 +1,225 @@
|
||||
// MIT License
|
||||
|
||||
// Copyright (c) 2019 Zachary Rice
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
package detect
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/h2non/filetype"
|
||||
|
||||
"github.com/Infisical/infisical-merge/detect/logging"
|
||||
"github.com/Infisical/infisical-merge/detect/report"
|
||||
"github.com/Infisical/infisical-merge/detect/sources"
|
||||
)
|
||||
|
||||
const maxPeekSize = 25 * 1_000 // 10kb
|
||||
|
||||
func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Finding, error) {
|
||||
for pa := range paths {
|
||||
d.Sema.Go(func() error {
|
||||
logger := logging.With().Str("path", pa.Path).Logger()
|
||||
logger.Trace().Msg("Scanning path")
|
||||
|
||||
f, err := os.Open(pa.Path)
|
||||
if err != nil {
|
||||
if os.IsPermission(err) {
|
||||
logger.Warn().Msg("Skipping file: permission denied")
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = f.Close()
|
||||
}()
|
||||
|
||||
// Get file size
|
||||
fileInfo, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileSize := fileInfo.Size()
|
||||
if d.MaxTargetMegaBytes > 0 {
|
||||
rawLength := fileSize / 1000000
|
||||
if rawLength > int64(d.MaxTargetMegaBytes) {
|
||||
logger.Debug().
|
||||
Int64("size", rawLength).
|
||||
Msg("Skipping file: exceeds --max-target-megabytes")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
// Buffer to hold file chunks
|
||||
reader = bufio.NewReaderSize(f, chunkSize)
|
||||
buf = make([]byte, chunkSize)
|
||||
totalLines = 0
|
||||
)
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
|
||||
// "Callers should always process the n > 0 bytes returned before considering the error err."
|
||||
// https://pkg.go.dev/io#Reader
|
||||
if n > 0 {
|
||||
// Only check the filetype at the start of file.
|
||||
if totalLines == 0 {
|
||||
// TODO: could other optimizations be introduced here?
|
||||
if mimetype, err := filetype.Match(buf[:n]); err != nil {
|
||||
return nil
|
||||
} else if mimetype.MIME.Type == "application" {
|
||||
return nil // skip binary files
|
||||
}
|
||||
}
|
||||
|
||||
// Try to split chunks across large areas of whitespace, if possible.
|
||||
peekBuf := bytes.NewBuffer(buf[:n])
|
||||
if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
|
||||
// Count the number of newlines in this chunk
|
||||
chunk := peekBuf.String()
|
||||
linesInChunk := strings.Count(chunk, "\n")
|
||||
totalLines += linesInChunk
|
||||
fragment := Fragment{
|
||||
Raw: chunk,
|
||||
Bytes: peekBuf.Bytes(),
|
||||
}
|
||||
if pa.Symlink != "" {
|
||||
fragment.SymlinkFile = pa.Symlink
|
||||
}
|
||||
|
||||
if isWindows {
|
||||
fragment.FilePath = filepath.ToSlash(pa.Path)
|
||||
fragment.SymlinkFile = filepath.ToSlash(fragment.SymlinkFile)
|
||||
fragment.WindowsFilePath = pa.Path
|
||||
} else {
|
||||
fragment.FilePath = pa.Path
|
||||
}
|
||||
|
||||
timer := time.AfterFunc(SlowWarningThreshold, func() {
|
||||
logger.Debug().Msgf("Taking longer than %s to inspect fragment", SlowWarningThreshold.String())
|
||||
})
|
||||
for _, finding := range d.Detect(fragment) {
|
||||
// need to add 1 since line counting starts at 1
|
||||
finding.StartLine += (totalLines - linesInChunk) + 1
|
||||
finding.EndLine += (totalLines - linesInChunk) + 1
|
||||
d.AddFinding(finding)
|
||||
}
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if err := d.Sema.Wait(); err != nil {
|
||||
return d.findings, err
|
||||
}
|
||||
|
||||
return d.findings, nil
|
||||
}
|
||||
|
||||
// readUntilSafeBoundary consumes |f| until it finds two consecutive `\n` characters, up to |maxPeekSize|.
|
||||
// This hopefully avoids splitting. (https://github.com/gitleaks/gitleaks/issues/1651)
|
||||
func readUntilSafeBoundary(r *bufio.Reader, n int, maxPeekSize int, peekBuf *bytes.Buffer) error {
|
||||
if peekBuf.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Does the buffer end in consecutive newlines?
|
||||
var (
|
||||
data = peekBuf.Bytes()
|
||||
lastChar = data[len(data)-1]
|
||||
newlineCount = 0 // Tracks consecutive newlines
|
||||
)
|
||||
if isWhitespace(lastChar) {
|
||||
for i := len(data) - 1; i >= 0; i-- {
|
||||
lastChar = data[i]
|
||||
if lastChar == '\n' {
|
||||
newlineCount++
|
||||
|
||||
// Stop if two consecutive newlines are found
|
||||
if newlineCount >= 2 {
|
||||
return nil
|
||||
}
|
||||
} else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' {
|
||||
// The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count.
|
||||
// (Intentionally do nothing.)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not, read ahead until we (hopefully) find some.
|
||||
newlineCount = 0
|
||||
for {
|
||||
data = peekBuf.Bytes()
|
||||
// Check if the last character is a newline.
|
||||
lastChar = data[len(data)-1]
|
||||
if lastChar == '\n' {
|
||||
newlineCount++
|
||||
|
||||
// Stop if two consecutive newlines are found
|
||||
if newlineCount >= 2 {
|
||||
break
|
||||
}
|
||||
} else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' {
|
||||
// The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count.
|
||||
// (Intentionally do nothing.)
|
||||
} else {
|
||||
newlineCount = 0 // Reset if a non-newline character is found
|
||||
}
|
||||
|
||||
// Stop growing the buffer if it reaches maxSize
|
||||
if (peekBuf.Len() - n) >= maxPeekSize {
|
||||
break
|
||||
}
|
||||
|
||||
// Read additional data into a temporary buffer
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
peekBuf.WriteByte(b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user