diff --git a/Dockerfile.fips.standalone-infisical b/Dockerfile.fips.standalone-infisical index 33360bf45..0c67b40c8 100644 --- a/Dockerfile.fips.standalone-infisical +++ b/Dockerfile.fips.standalone-infisical @@ -134,7 +134,7 @@ RUN printf "[FreeTDS]\nDescription = FreeTDS Driver\nDriver = /usr/lib/x86_64-li # Install Infisical CLI RUN curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash \ - && apt-get update && apt-get install -y infisical=0.31.1 \ + && apt-get update && apt-get install -y infisical=0.41.2 \ && rm -rf /var/lib/apt/lists/* RUN groupadd -r -g 1001 nodejs && useradd -r -u 1001 -g nodejs non-root-user @@ -171,6 +171,7 @@ ENV NODE_ENV production ENV STANDALONE_BUILD true ENV STANDALONE_MODE true ENV ChrystokiConfigurationPath=/usr/safenet/lunaclient/ +ENV NODE_OPTIONS="--max-old-space-size=1024" WORKDIR /backend diff --git a/Dockerfile.standalone-infisical b/Dockerfile.standalone-infisical index 6d582ce76..404326376 100644 --- a/Dockerfile.standalone-infisical +++ b/Dockerfile.standalone-infisical @@ -128,7 +128,7 @@ RUN apt-get update && apt-get install -y \ # Install Infisical CLI RUN curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash \ - && apt-get update && apt-get install -y infisical=0.31.1 \ + && apt-get update && apt-get install -y infisical=0.41.2 \ && rm -rf /var/lib/apt/lists/* WORKDIR / @@ -168,6 +168,7 @@ ENV HTTPS_ENABLED false ENV NODE_ENV production ENV STANDALONE_BUILD true ENV STANDALONE_MODE true +ENV NODE_OPTIONS="--max-old-space-size=1024" WORKDIR /backend diff --git a/backend/Dockerfile b/backend/Dockerfile index b9edf8b98..420f1a488 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -55,7 +55,7 @@ COPY --from=build /app . # Install Infisical CLI RUN apt-get install -y curl bash && \ curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash && \ - apt-get update && apt-get install -y infisical=0.8.1 git + apt-get update && apt-get install -y infisical=0.41.2 git HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \ CMD node healthcheck.js diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev index 3435672e7..bcd924f6e 100644 --- a/backend/Dockerfile.dev +++ b/backend/Dockerfile.dev @@ -57,7 +57,7 @@ RUN mkdir -p /etc/softhsm2/tokens && \ # Install Infisical CLI RUN curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash && \ apt-get update && \ - apt-get install -y infisical=0.8.1 + apt-get install -y infisical=0.41.2 WORKDIR /app diff --git a/backend/Dockerfile.dev.fips b/backend/Dockerfile.dev.fips index 8c40404dc..a9ba9c9f7 100644 --- a/backend/Dockerfile.dev.fips +++ b/backend/Dockerfile.dev.fips @@ -66,7 +66,7 @@ RUN wget https://www.openssl.org/source/openssl-3.1.2.tar.gz \ # Install Infisical CLI RUN curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.deb.sh' | bash && \ apt-get update && \ - apt-get install -y infisical=0.8.1 + apt-get install -y infisical=0.41.2 WORKDIR /app diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index 48f52f9e7..f4f251616 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -1,4 +1,8 @@ +import RE2 from "re2"; + import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { applyJitter } from "@app/lib/dates"; +import { delay as delayMs } from "@app/lib/delay"; import { Lock } from "@app/lib/red-lock"; export const mockKeyStore = (): TKeyStoreFactory => { @@ -18,6 +22,27 @@ export const mockKeyStore = (): TKeyStoreFactory => { delete store[key]; return 1; }, + deleteItems: async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }) => { + const regex = new RE2(`^${pattern.replace(/[-[\]/{}()+?.\\^$|]/g, "\\$&").replace(/\*/g, ".*")}$`); + let totalDeleted = 0; + const keys = Object.keys(store); + + for (let i = 0; i < keys.length; i += batchSize) { + const batch = keys.slice(i, i + batchSize); + + for (const key of batch) { + if (regex.test(key)) { + delete store[key]; + totalDeleted += 1; + } + } + + // eslint-disable-next-line no-await-in-loop + await delayMs(Math.max(0, applyJitter(delay, jitter))); + } + + return totalDeleted; + }, getItem: async (key) => { const value = store[key]; if (typeof value === "string") { diff --git a/backend/package-lock.json b/backend/package-lock.json index 93c28c9e2..0bc3aadd1 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -33,7 +33,8 @@ "@infisical/quic": "^1.0.8", "@node-saml/passport-saml": "^5.0.1", "@octokit/auth-app": "^7.1.1", - "@octokit/plugin-paginate-graphql": "^5.2.4", + "@octokit/core": "^5.2.1", + "@octokit/plugin-paginate-graphql": "^4.0.1", "@octokit/plugin-retry": "^5.0.5", "@octokit/rest": "^20.0.2", "@octokit/webhooks-types": "^7.3.1", @@ -121,7 +122,7 @@ "tweetnacl-util": "^0.15.1", "uuid": "^9.0.1", "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.4" + "zod-to-json-schema": "^3.24.5" }, "bin": { "backend": "dist/main.js" @@ -7805,119 +7806,38 @@ } }, "node_modules/@octokit/core": { - "version": "6.1.5", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-6.1.5.tgz", - "integrity": "sha512-vvmsN0r7rguA+FySiCsbaTTobSftpIDIpPW81trAmsv9TGxg3YCujAxRYp/Uy8xmDgYCzzgulG62H7KYUFmeIg==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", + "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", "license": "MIT", - "peer": true, "dependencies": { - "@octokit/auth-token": "^5.0.0", - "@octokit/graphql": "^8.2.2", - "@octokit/request": "^9.2.3", - "@octokit/request-error": "^6.1.8", - "@octokit/types": "^14.0.0", - "before-after-hook": "^3.0.2", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/core/node_modules/@octokit/auth-token": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-5.1.2.tgz", - "integrity": "sha512-JcQDsBdg49Yky2w2ld20IHAlwr8d/d8N6NiOXbtuoPCqzbsiJgF633mVUw3x4mo0H5ypataQIX7SFu3yy44Mpw==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/core/node_modules/@octokit/endpoint": { - "version": "10.1.4", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.4.tgz", - "integrity": "sha512-OlYOlZIsfEVZm5HCSR8aSg02T2lbUWOsCQoPKfTXJwDzcHQBrVBGdGXb89dv2Kw2ToZaRtudp8O3ZIYoaOjKlA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/types": "^14.0.0", - "universal-user-agent": "^7.0.2" + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" }, "engines": { "node": ">= 18" } }, "node_modules/@octokit/core/node_modules/@octokit/openapi-types": { - "version": "25.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-25.0.0.tgz", - "integrity": "sha512-FZvktFu7HfOIJf2BScLKIEYjDsw6RKc7rBJCdvCTfKsVnx2GEB/Nbzjr29DUdb7vQhlzS/j8qDzdditP0OC6aw==", - "license": "MIT", - "peer": true - }, - "node_modules/@octokit/core/node_modules/@octokit/request": { - "version": "9.2.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.2.3.tgz", - "integrity": "sha512-Ma+pZU8PXLOEYzsWf0cn/gY+ME57Wq8f49WTXA8FMHp2Ps9djKw//xYJ1je8Hm0pR2lU9FUGeJRWOtxq6olt4w==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/endpoint": "^10.1.4", - "@octokit/request-error": "^6.1.8", - "@octokit/types": "^14.0.0", - "fast-content-type-parse": "^2.0.0", - "universal-user-agent": "^7.0.2" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/core/node_modules/@octokit/request-error": { - "version": "6.1.8", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.8.tgz", - "integrity": "sha512-WEi/R0Jmq+IJKydWlKDmryPcmdYSVjL3ekaiEL1L9eo1sUnqMJ+grqmC9cjk7CA7+b2/T397tO5d8YLOH3qYpQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/types": "^14.0.0" - }, - "engines": { - "node": ">= 18" - } + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" }, "node_modules/@octokit/core/node_modules/@octokit/types": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-14.0.0.tgz", - "integrity": "sha512-VVmZP0lEhbo2O1pdq63gZFiGCKkm8PPp8AUOijlwPO6hojEVjspA0MWKP7E4hbvGxzFKNqKr6p0IYtOH/Wf/zA==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", "license": "MIT", - "peer": true, "dependencies": { - "@octokit/openapi-types": "^25.0.0" + "@octokit/openapi-types": "^24.2.0" } }, - "node_modules/@octokit/core/node_modules/fast-content-type-parse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-2.0.1.tgz", - "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "peer": true - }, - "node_modules/@octokit/core/node_modules/universal-user-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", - "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==", - "license": "ISC", - "peer": true - }, "node_modules/@octokit/endpoint": { "version": "9.0.6", "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", @@ -7947,105 +7867,34 @@ } }, "node_modules/@octokit/graphql": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-8.2.2.tgz", - "integrity": "sha512-Yi8hcoqsrXGdt0yObxbebHXFOiUA+2v3n53epuOg1QUgOB6c4XzvisBNVXJSl8RYA5KrDuSL2yq9Qmqe5N0ryA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", "license": "MIT", - "peer": true, "dependencies": { - "@octokit/request": "^9.2.3", - "@octokit/types": "^14.0.0", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/graphql/node_modules/@octokit/endpoint": { - "version": "10.1.4", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.4.tgz", - "integrity": "sha512-OlYOlZIsfEVZm5HCSR8aSg02T2lbUWOsCQoPKfTXJwDzcHQBrVBGdGXb89dv2Kw2ToZaRtudp8O3ZIYoaOjKlA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/types": "^14.0.0", - "universal-user-agent": "^7.0.2" + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" }, "engines": { "node": ">= 18" } }, "node_modules/@octokit/graphql/node_modules/@octokit/openapi-types": { - "version": "25.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-25.0.0.tgz", - "integrity": "sha512-FZvktFu7HfOIJf2BScLKIEYjDsw6RKc7rBJCdvCTfKsVnx2GEB/Nbzjr29DUdb7vQhlzS/j8qDzdditP0OC6aw==", - "license": "MIT", - "peer": true - }, - "node_modules/@octokit/graphql/node_modules/@octokit/request": { - "version": "9.2.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.2.3.tgz", - "integrity": "sha512-Ma+pZU8PXLOEYzsWf0cn/gY+ME57Wq8f49WTXA8FMHp2Ps9djKw//xYJ1je8Hm0pR2lU9FUGeJRWOtxq6olt4w==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/endpoint": "^10.1.4", - "@octokit/request-error": "^6.1.8", - "@octokit/types": "^14.0.0", - "fast-content-type-parse": "^2.0.0", - "universal-user-agent": "^7.0.2" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/graphql/node_modules/@octokit/request-error": { - "version": "6.1.8", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.8.tgz", - "integrity": "sha512-WEi/R0Jmq+IJKydWlKDmryPcmdYSVjL3ekaiEL1L9eo1sUnqMJ+grqmC9cjk7CA7+b2/T397tO5d8YLOH3qYpQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@octokit/types": "^14.0.0" - }, - "engines": { - "node": ">= 18" - } + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" }, "node_modules/@octokit/graphql/node_modules/@octokit/types": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-14.0.0.tgz", - "integrity": "sha512-VVmZP0lEhbo2O1pdq63gZFiGCKkm8PPp8AUOijlwPO6hojEVjspA0MWKP7E4hbvGxzFKNqKr6p0IYtOH/Wf/zA==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", "license": "MIT", - "peer": true, "dependencies": { - "@octokit/openapi-types": "^25.0.0" + "@octokit/openapi-types": "^24.2.0" } }, - "node_modules/@octokit/graphql/node_modules/fast-content-type-parse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-2.0.1.tgz", - "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "peer": true - }, - "node_modules/@octokit/graphql/node_modules/universal-user-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", - "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==", - "license": "ISC", - "peer": true - }, "node_modules/@octokit/oauth-authorization-url": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-7.1.1.tgz", @@ -8141,15 +7990,15 @@ } }, "node_modules/@octokit/plugin-paginate-graphql": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-5.2.4.tgz", - "integrity": "sha512-pLZES1jWaOynXKHOqdnwZ5ULeVR6tVVCMm+AUbp0htdcyXDU95WbkYdU4R2ej1wKj5Tu94Mee2Ne0PjPO9cCyA==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-4.0.1.tgz", + "integrity": "sha512-R8ZQNmrIKKpHWC6V2gum4x9LG2qF1RxRjo27gjQcG3j+vf2tLsEfE7I/wRWEPzYMaenr1M+qDAtNcwZve1ce1A==", "license": "MIT", "engines": { "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=6" + "@octokit/core": ">=5" } }, "node_modules/@octokit/plugin-paginate-rest": { @@ -8302,59 +8151,6 @@ "node": ">= 18" } }, - "node_modules/@octokit/rest/node_modules/@octokit/core": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", - "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", - "license": "MIT", - "dependencies": { - "@octokit/auth-token": "^4.0.0", - "@octokit/graphql": "^7.1.0", - "@octokit/request": "^8.4.1", - "@octokit/request-error": "^5.1.1", - "@octokit/types": "^13.0.0", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/graphql": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", - "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", - "license": "MIT", - "dependencies": { - "@octokit/request": "^8.4.1", - "@octokit/types": "^13.0.0", - "universal-user-agent": "^6.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/openapi-types": { - "version": "24.2.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", - "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", - "license": "MIT" - }, - "node_modules/@octokit/rest/node_modules/@octokit/types": { - "version": "13.10.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", - "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^24.2.0" - } - }, - "node_modules/@octokit/rest/node_modules/before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", - "license": "Apache-2.0" - }, "node_modules/@octokit/types": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.4.0.tgz", @@ -12799,11 +12595,10 @@ "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==" }, "node_modules/before-after-hook": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-3.0.2.tgz", - "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==", - "license": "Apache-2.0", - "peer": true + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "license": "Apache-2.0" }, "node_modules/big-integer": { "version": "1.6.52", @@ -21602,62 +21397,6 @@ "node": ">=18" } }, - "node_modules/probot/node_modules/@octokit/core": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", - "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", - "license": "MIT", - "dependencies": { - "@octokit/auth-token": "^4.0.0", - "@octokit/graphql": "^7.1.0", - "@octokit/request": "^8.4.1", - "@octokit/request-error": "^5.1.1", - "@octokit/types": "^13.0.0", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/probot/node_modules/@octokit/core/node_modules/@octokit/types": { - "version": "13.10.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", - "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^24.2.0" - } - }, - "node_modules/probot/node_modules/@octokit/graphql": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", - "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", - "license": "MIT", - "dependencies": { - "@octokit/request": "^8.4.1", - "@octokit/types": "^13.0.0", - "universal-user-agent": "^6.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/probot/node_modules/@octokit/graphql/node_modules/@octokit/types": { - "version": "13.10.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", - "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^24.2.0" - } - }, - "node_modules/probot/node_modules/@octokit/openapi-types": { - "version": "24.2.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", - "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", - "license": "MIT" - }, "node_modules/probot/node_modules/@octokit/plugin-retry": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz", @@ -21690,12 +21429,6 @@ "@octokit/core": "^5.0.0" } }, - "node_modules/probot/node_modules/before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", - "license": "Apache-2.0" - }, "node_modules/probot/node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -27709,11 +27442,12 @@ } }, "node_modules/zod-to-json-schema": { - "version": "3.22.4", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.22.4.tgz", - "integrity": "sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ==", + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", + "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", + "license": "ISC", "peerDependencies": { - "zod": "^3.22.4" + "zod": "^3.24.1" } } } diff --git a/backend/package.json b/backend/package.json index c19d30441..a8ab76cf8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -152,7 +152,8 @@ "@infisical/quic": "^1.0.8", "@node-saml/passport-saml": "^5.0.1", "@octokit/auth-app": "^7.1.1", - "@octokit/plugin-paginate-graphql": "^5.2.4", + "@octokit/core": "^5.2.1", + "@octokit/plugin-paginate-graphql": "^4.0.1", "@octokit/plugin-retry": "^5.0.5", "@octokit/rest": "^20.0.2", "@octokit/webhooks-types": "^7.3.1", @@ -240,6 +241,6 @@ "tweetnacl-util": "^0.15.1", "uuid": "^9.0.1", "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.4" + "zod-to-json-schema": "^3.24.5" } } diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 6ec542c6b..748a7d431 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -66,6 +66,8 @@ import { TIdentityAzureAuthServiceFactory } from "@app/services/identity-azure-a import { TIdentityGcpAuthServiceFactory } from "@app/services/identity-gcp-auth/identity-gcp-auth-service"; import { TIdentityJwtAuthServiceFactory } from "@app/services/identity-jwt-auth/identity-jwt-auth-service"; import { TIdentityKubernetesAuthServiceFactory } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-service"; +import { TIdentityLdapAuthServiceFactory } from "@app/services/identity-ldap-auth/identity-ldap-auth-service"; +import { TAllowedFields } from "@app/services/identity-ldap-auth/identity-ldap-auth-types"; import { TIdentityOidcAuthServiceFactory } from "@app/services/identity-oidc-auth/identity-oidc-auth-service"; import { TIdentityProjectServiceFactory } from "@app/services/identity-project/identity-project-service"; import { TIdentityTokenAuthServiceFactory } from "@app/services/identity-token-auth/identity-token-auth-service"; @@ -146,6 +148,13 @@ declare module "fastify" { providerAuthToken: string; externalProviderAccessToken?: string; }; + passportMachineIdentity: { + identityId: string; + user: { + uid: string; + mail?: string; + }; + }; kmipUser: { projectId: string; clientId: string; @@ -153,7 +162,9 @@ declare module "fastify" { }; auditLogInfo: Pick; ssoConfig: Awaited>; - ldapConfig: Awaited>; + ldapConfig: Awaited> & { + allowedFields?: TAllowedFields[]; + }; } interface FastifyInstance { @@ -199,6 +210,7 @@ declare module "fastify" { identityAzureAuth: TIdentityAzureAuthServiceFactory; identityOidcAuth: TIdentityOidcAuthServiceFactory; identityJwtAuth: TIdentityJwtAuthServiceFactory; + identityLdapAuth: TIdentityLdapAuthServiceFactory; accessApprovalPolicy: TAccessApprovalPolicyServiceFactory; accessApprovalRequest: TAccessApprovalRequestServiceFactory; secretApprovalPolicy: TSecretApprovalPolicyServiceFactory; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 13f3bc306..c26f1128e 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -432,6 +432,11 @@ import { TWorkflowIntegrationsInsert, TWorkflowIntegrationsUpdate } from "@app/db/schemas"; +import { + TIdentityLdapAuths, + TIdentityLdapAuthsInsert, + TIdentityLdapAuthsUpdate +} from "@app/db/schemas/identity-ldap-auths"; import { TMicrosoftTeamsIntegrations, TMicrosoftTeamsIntegrationsInsert, @@ -735,6 +740,11 @@ declare module "knex/types/tables" { TIdentityJwtAuthsInsert, TIdentityJwtAuthsUpdate >; + [TableName.IdentityLdapAuth]: KnexOriginal.CompositeTableType< + TIdentityLdapAuths, + TIdentityLdapAuthsInsert, + TIdentityLdapAuthsUpdate + >; [TableName.IdentityUaClientSecret]: KnexOriginal.CompositeTableType< TIdentityUaClientSecrets, TIdentityUaClientSecretsInsert, diff --git a/backend/src/db/migrations/20250429232917_store-cert-secret-key-and-chain.ts b/backend/src/db/migrations/20250429232917_store-cert-secret-key-and-chain.ts index a627bf83c..f90b2d593 100644 --- a/backend/src/db/migrations/20250429232917_store-cert-secret-key-and-chain.ts +++ b/backend/src/db/migrations/20250429232917_store-cert-secret-key-and-chain.ts @@ -3,13 +3,10 @@ import { Knex } from "knex"; import { TableName } from "../schemas"; export async function up(knex: Knex): Promise { - if (await knex.schema.hasTable(TableName.CertificateBody)) { - const hasColumn = await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain"); - if (!hasColumn) { - await knex.schema.alterTable(TableName.CertificateBody, (t) => { - t.binary("encryptedCertificateChain").nullable(); - }); - } + if (!(await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain"))) { + await knex.schema.alterTable(TableName.CertificateBody, (t) => { + t.binary("encryptedCertificateChain").nullable(); + }); } if (!(await knex.schema.hasTable(TableName.CertificateSecret))) { @@ -28,12 +25,9 @@ export async function down(knex: Knex): Promise { await knex.schema.dropTable(TableName.CertificateSecret); } - if (await knex.schema.hasTable(TableName.CertificateBody)) { - const hasColumn = await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain"); - if (hasColumn) { - await knex.schema.alterTable(TableName.CertificateBody, (t) => { - t.dropColumn("encryptedCertificateChain"); - }); - } + if (await knex.schema.hasColumn(TableName.CertificateBody, "encryptedCertificateChain")) { + await knex.schema.alterTable(TableName.CertificateBody, (t) => { + t.dropColumn("encryptedCertificateChain"); + }); } } diff --git a/backend/src/db/migrations/20250501164905_add-groups-to-ssh-host-login-user-mappings.ts b/backend/src/db/migrations/20250501164905_add-groups-to-ssh-host-login-user-mappings.ts new file mode 100644 index 000000000..4f08146f9 --- /dev/null +++ b/backend/src/db/migrations/20250501164905_add-groups-to-ssh-host-login-user-mappings.ts @@ -0,0 +1,22 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.SshHostLoginUserMapping, "groupId"))) { + await knex.schema.alterTable(TableName.SshHostLoginUserMapping, (t) => { + t.uuid("groupId").nullable(); + t.foreign("groupId").references("id").inTable(TableName.Groups).onDelete("CASCADE"); + t.unique(["sshHostLoginUserId", "groupId"]); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.SshHostLoginUserMapping, "groupId")) { + await knex.schema.alterTable(TableName.SshHostLoginUserMapping, (t) => { + t.dropUnique(["sshHostLoginUserId", "groupId"]); + t.dropColumn("groupId"); + }); + } +} diff --git a/backend/src/db/migrations/20250505203703_project-templates-type-col.ts b/backend/src/db/migrations/20250505203703_project-templates-type-col.ts new file mode 100644 index 000000000..d1ef14d72 --- /dev/null +++ b/backend/src/db/migrations/20250505203703_project-templates-type-col.ts @@ -0,0 +1,22 @@ +import { Knex } from "knex"; + +import { ProjectType, TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.ProjectTemplates, "type"))) { + await knex.schema.alterTable(TableName.ProjectTemplates, (t) => { + // defaulting to sm for migration to set existing, new ones will always be specified on creation + t.string("type").defaultTo(ProjectType.SecretManager).notNullable(); + t.jsonb("environments").nullable().alter(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.ProjectTemplates, "type")) { + await knex.schema.alterTable(TableName.ProjectTemplates, (t) => { + t.dropColumn("type"); + // not reverting nullable environments + }); + } +} diff --git a/backend/src/db/migrations/20250507003056_identity-ldap-auth.ts b/backend/src/db/migrations/20250507003056_identity-ldap-auth.ts new file mode 100644 index 000000000..da9912022 --- /dev/null +++ b/backend/src/db/migrations/20250507003056_identity-ldap-auth.ts @@ -0,0 +1,39 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.IdentityLdapAuth))) { + await knex.schema.createTable(TableName.IdentityLdapAuth, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + + t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable(); + t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable(); + t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable(); + t.jsonb("accessTokenTrustedIps").notNullable(); + + t.uuid("identityId").notNullable().unique(); + t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE"); + + t.binary("encryptedBindDN").notNullable(); + t.binary("encryptedBindPass").notNullable(); + t.binary("encryptedLdapCaCertificate").nullable(); + + t.string("url").notNullable(); + t.string("searchBase").notNullable(); + t.string("searchFilter").notNullable(); + + t.jsonb("allowedFields").nullable(); + + t.timestamps(true, true, true); + }); + } + + await createOnUpdateTrigger(knex, TableName.IdentityLdapAuth); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.IdentityLdapAuth); + await dropOnUpdateTrigger(knex, TableName.IdentityLdapAuth); +} diff --git a/backend/src/db/schemas/identity-ldap-auths.ts b/backend/src/db/schemas/identity-ldap-auths.ts new file mode 100644 index 000000000..d5b15fc6a --- /dev/null +++ b/backend/src/db/schemas/identity-ldap-auths.ts @@ -0,0 +1,32 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const IdentityLdapAuthsSchema = z.object({ + id: z.string().uuid(), + accessTokenTTL: z.coerce.number().default(7200), + accessTokenMaxTTL: z.coerce.number().default(7200), + accessTokenNumUsesLimit: z.coerce.number().default(0), + accessTokenTrustedIps: z.unknown(), + identityId: z.string().uuid(), + encryptedBindDN: zodBuffer, + encryptedBindPass: zodBuffer, + encryptedLdapCaCertificate: zodBuffer.nullable().optional(), + url: z.string(), + searchBase: z.string(), + searchFilter: z.string(), + allowedFields: z.unknown().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TIdentityLdapAuths = z.infer; +export type TIdentityLdapAuthsInsert = Omit, TImmutableDBKeys>; +export type TIdentityLdapAuthsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 7fd77da6c..81d5319e1 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -80,6 +80,7 @@ export enum TableName { IdentityAwsAuth = "identity_aws_auths", IdentityOidcAuth = "identity_oidc_auths", IdentityJwtAuth = "identity_jwt_auths", + IdentityLdapAuth = "identity_ldap_auths", IdentityOrgMembership = "identity_org_memberships", IdentityProjectMembership = "identity_project_memberships", IdentityProjectMembershipRole = "identity_project_membership_role", @@ -185,11 +186,16 @@ export enum OrgMembershipStatus { } export enum ProjectMembershipRole { + // general Admin = "admin", Member = "member", Custom = "custom", Viewer = "viewer", - NoAccess = "no-access" + NoAccess = "no-access", + // ssh + SshHostBootstrapper = "ssh-host-bootstrapper", + // kms + KmsCryptographicOperator = "cryptographic-operator" } export enum SecretEncryptionAlgo { @@ -227,7 +233,8 @@ export enum IdentityAuthMethod { AWS_AUTH = "aws-auth", AZURE_AUTH = "azure-auth", OIDC_AUTH = "oidc-auth", - JWT_AUTH = "jwt-auth" + JWT_AUTH = "jwt-auth", + LDAP_AUTH = "ldap-auth" } export enum ProjectType { diff --git a/backend/src/db/schemas/project-templates.ts b/backend/src/db/schemas/project-templates.ts index 68f37d256..f12386165 100644 --- a/backend/src/db/schemas/project-templates.ts +++ b/backend/src/db/schemas/project-templates.ts @@ -12,10 +12,11 @@ export const ProjectTemplatesSchema = z.object({ name: z.string(), description: z.string().nullable().optional(), roles: z.unknown(), - environments: z.unknown(), + environments: z.unknown().nullable().optional(), orgId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + type: z.string().default("secret-manager") }); export type TProjectTemplates = z.infer; diff --git a/backend/src/db/schemas/ssh-host-login-user-mappings.ts b/backend/src/db/schemas/ssh-host-login-user-mappings.ts index 6edb0d5a3..fd5fa460c 100644 --- a/backend/src/db/schemas/ssh-host-login-user-mappings.ts +++ b/backend/src/db/schemas/ssh-host-login-user-mappings.ts @@ -12,7 +12,8 @@ export const SshHostLoginUserMappingsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), sshHostLoginUserId: z.string().uuid(), - userId: z.string().uuid().nullable().optional() + userId: z.string().uuid().nullable().optional(), + groupId: z.string().uuid().nullable().optional() }); export type TSshHostLoginUserMappings = z.infer; diff --git a/backend/src/ee/routes/v1/access-approval-request-router.ts b/backend/src/ee/routes/v1/access-approval-request-router.ts index 8a7ccfdef..b0914d5c4 100644 --- a/backend/src/ee/routes/v1/access-approval-request-router.ts +++ b/backend/src/ee/routes/v1/access-approval-request-router.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { AccessApprovalRequestsReviewersSchema, AccessApprovalRequestsSchema, UsersSchema } from "@app/db/schemas"; import { ApprovalStatus } from "@app/ee/services/access-approval-request/access-approval-request-types"; +import { writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -18,6 +19,9 @@ export const registerAccessApprovalRequestRouter = async (server: FastifyZodProv server.route({ url: "/", method: "POST", + config: { + rateLimit: writeLimit + }, schema: { body: z.object({ permissions: z.any().array(), diff --git a/backend/src/ee/routes/v1/ldap-router.ts b/backend/src/ee/routes/v1/ldap-router.ts index 5f80ad02b..57c5736df 100644 --- a/backend/src/ee/routes/v1/ldap-router.ts +++ b/backend/src/ee/routes/v1/ldap-router.ts @@ -98,6 +98,9 @@ export const registerLdapRouter = async (server: FastifyZodProvider) => { server.route({ url: "/login", method: "POST", + config: { + rateLimit: writeLimit + }, schema: { body: z.object({ organizationSlug: z.string().trim() diff --git a/backend/src/ee/routes/v1/project-template-router.ts b/backend/src/ee/routes/v1/project-template-router.ts index 08d16414b..5d33b4d58 100644 --- a/backend/src/ee/routes/v1/project-template-router.ts +++ b/backend/src/ee/routes/v1/project-template-router.ts @@ -1,9 +1,8 @@ import { z } from "zod"; -import { ProjectMembershipRole, ProjectTemplatesSchema } from "@app/db/schemas"; +import { ProjectMembershipRole, ProjectTemplatesSchema, ProjectType } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { ProjectPermissionV2Schema } from "@app/ee/services/permission/project-permission"; -import { ProjectTemplateDefaultEnvironments } from "@app/ee/services/project-template/project-template-constants"; import { isInfisicalProjectTemplate } from "@app/ee/services/project-template/project-template-fns"; import { ApiDocsTags, ProjectTemplates } from "@app/lib/api-docs"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; @@ -35,6 +34,7 @@ const SanitizedProjectTemplateSchema = ProjectTemplatesSchema.extend({ position: z.number().min(1) }) .array() + .nullable() }); const ProjectTemplateRolesSchema = z @@ -104,6 +104,9 @@ export const registerProjectTemplateRouter = async (server: FastifyZodProvider) hide: false, tags: [ApiDocsTags.ProjectTemplates], description: "List project templates for the current organization.", + querystring: z.object({ + type: z.nativeEnum(ProjectType).optional().describe(ProjectTemplates.LIST.type) + }), response: { 200: z.object({ projectTemplates: SanitizedProjectTemplateSchema.array() @@ -112,7 +115,8 @@ export const registerProjectTemplateRouter = async (server: FastifyZodProvider) }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const projectTemplates = await server.services.projectTemplate.listProjectTemplatesByOrg(req.permission); + const { type } = req.query; + const projectTemplates = await server.services.projectTemplate.listProjectTemplatesByOrg(req.permission, type); const auditTemplates = projectTemplates.filter((template) => !isInfisicalProjectTemplate(template.name)); @@ -184,6 +188,7 @@ export const registerProjectTemplateRouter = async (server: FastifyZodProvider) tags: [ApiDocsTags.ProjectTemplates], description: "Create a project template.", body: z.object({ + type: z.nativeEnum(ProjectType).describe(ProjectTemplates.CREATE.type), name: slugSchema({ field: "name" }) .refine((val) => !isInfisicalProjectTemplate(val), { message: `The requested project template name is reserved.` @@ -191,9 +196,7 @@ export const registerProjectTemplateRouter = async (server: FastifyZodProvider) .describe(ProjectTemplates.CREATE.name), description: z.string().max(256).trim().optional().describe(ProjectTemplates.CREATE.description), roles: ProjectTemplateRolesSchema.default([]).describe(ProjectTemplates.CREATE.roles), - environments: ProjectTemplateEnvironmentsSchema.default(ProjectTemplateDefaultEnvironments).describe( - ProjectTemplates.CREATE.environments - ) + environments: ProjectTemplateEnvironmentsSchema.describe(ProjectTemplates.CREATE.environments).optional() }), response: { 200: z.object({ diff --git a/backend/src/ee/routes/v1/saml-router.ts b/backend/src/ee/routes/v1/saml-router.ts index f2df2fb89..8648ade7c 100644 --- a/backend/src/ee/routes/v1/saml-router.ts +++ b/backend/src/ee/routes/v1/saml-router.ts @@ -166,6 +166,9 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => { server.route({ url: "/redirect/saml2/organizations/:orgSlug", method: "GET", + config: { + rateLimit: readLimit + }, schema: { params: z.object({ orgSlug: z.string().trim() @@ -192,6 +195,9 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => { server.route({ url: "/redirect/saml2/:samlConfigId", method: "GET", + config: { + rateLimit: readLimit + }, schema: { params: z.object({ samlConfigId: z.string().trim() @@ -218,6 +224,9 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => { server.route({ url: "/saml2/:samlConfigId", method: "POST", + config: { + rateLimit: writeLimit + }, schema: { params: z.object({ samlConfigId: z.string().trim() diff --git a/backend/src/ee/routes/v1/scim-router.ts b/backend/src/ee/routes/v1/scim-router.ts index cd5f2f9f3..5fa0d19e8 100644 --- a/backend/src/ee/routes/v1/scim-router.ts +++ b/backend/src/ee/routes/v1/scim-router.ts @@ -196,6 +196,9 @@ export const registerScimRouter = async (server: FastifyZodProvider) => { server.route({ url: "/Users", method: "POST", + config: { + rateLimit: writeLimit + }, schema: { body: z.object({ schemas: z.array(z.string()), diff --git a/backend/src/ee/routes/v1/secret-scanning-router.ts b/backend/src/ee/routes/v1/secret-scanning-router.ts index f144a6c00..1bc8e3998 100644 --- a/backend/src/ee/routes/v1/secret-scanning-router.ts +++ b/backend/src/ee/routes/v1/secret-scanning-router.ts @@ -1,11 +1,11 @@ import { z } from "zod"; import { GitAppOrgSchema, SecretScanningGitRisksSchema } from "@app/db/schemas"; +import { canUseSecretScanning } from "@app/ee/services/secret-scanning/secret-scanning-fns"; import { SecretScanningResolvedStatus, SecretScanningRiskStatus } from "@app/ee/services/secret-scanning/secret-scanning-types"; -import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; import { OrderByDirection } from "@app/lib/types"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; @@ -23,14 +23,14 @@ export const registerSecretScanningRouter = async (server: FastifyZodProvider) = body: z.object({ organizationId: z.string().trim() }), response: { 200: z.object({ - sessionId: z.string() + sessionId: z.string(), + gitAppSlug: z.string() }) } }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const appCfg = getConfig(); - if (!appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(req.auth.orgId)) { + if (!canUseSecretScanning(req.auth.orgId)) { throw new BadRequestError({ message: "Secret scanning is temporarily unavailable." }); diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 5d27b37f3..df727f491 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -22,6 +22,7 @@ import { ActorType } from "@app/services/auth/auth-type"; import { CertKeyAlgorithm } from "@app/services/certificate/certificate-types"; import { CaStatus } from "@app/services/certificate-authority/certificate-authority-types"; import { TIdentityTrustedIp } from "@app/services/identity/identity-types"; +import { TAllowedFields } from "@app/services/identity-ldap-auth/identity-ldap-auth-types"; import { PkiItemType } from "@app/services/pki-collection/pki-collection-types"; import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums"; import { @@ -119,44 +120,60 @@ export enum EventType { CREATE_TOKEN_IDENTITY_TOKEN_AUTH = "create-token-identity-token-auth", UPDATE_TOKEN_IDENTITY_TOKEN_AUTH = "update-token-identity-token-auth", GET_TOKENS_IDENTITY_TOKEN_AUTH = "get-tokens-identity-token-auth", + ADD_IDENTITY_TOKEN_AUTH = "add-identity-token-auth", UPDATE_IDENTITY_TOKEN_AUTH = "update-identity-token-auth", GET_IDENTITY_TOKEN_AUTH = "get-identity-token-auth", REVOKE_IDENTITY_TOKEN_AUTH = "revoke-identity-token-auth", + LOGIN_IDENTITY_KUBERNETES_AUTH = "login-identity-kubernetes-auth", ADD_IDENTITY_KUBERNETES_AUTH = "add-identity-kubernetes-auth", UPDATE_IDENTITY_KUBENETES_AUTH = "update-identity-kubernetes-auth", GET_IDENTITY_KUBERNETES_AUTH = "get-identity-kubernetes-auth", REVOKE_IDENTITY_KUBERNETES_AUTH = "revoke-identity-kubernetes-auth", + LOGIN_IDENTITY_OIDC_AUTH = "login-identity-oidc-auth", ADD_IDENTITY_OIDC_AUTH = "add-identity-oidc-auth", UPDATE_IDENTITY_OIDC_AUTH = "update-identity-oidc-auth", GET_IDENTITY_OIDC_AUTH = "get-identity-oidc-auth", REVOKE_IDENTITY_OIDC_AUTH = "revoke-identity-oidc-auth", + LOGIN_IDENTITY_JWT_AUTH = "login-identity-jwt-auth", ADD_IDENTITY_JWT_AUTH = "add-identity-jwt-auth", UPDATE_IDENTITY_JWT_AUTH = "update-identity-jwt-auth", GET_IDENTITY_JWT_AUTH = "get-identity-jwt-auth", REVOKE_IDENTITY_JWT_AUTH = "revoke-identity-jwt-auth", + CREATE_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRET = "create-identity-universal-auth-client-secret", REVOKE_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRET = "revoke-identity-universal-auth-client-secret", + GET_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRETS = "get-identity-universal-auth-client-secret", GET_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRET_BY_ID = "get-identity-universal-auth-client-secret-by-id", + LOGIN_IDENTITY_GCP_AUTH = "login-identity-gcp-auth", ADD_IDENTITY_GCP_AUTH = "add-identity-gcp-auth", UPDATE_IDENTITY_GCP_AUTH = "update-identity-gcp-auth", REVOKE_IDENTITY_GCP_AUTH = "revoke-identity-gcp-auth", GET_IDENTITY_GCP_AUTH = "get-identity-gcp-auth", + LOGIN_IDENTITY_AWS_AUTH = "login-identity-aws-auth", ADD_IDENTITY_AWS_AUTH = "add-identity-aws-auth", UPDATE_IDENTITY_AWS_AUTH = "update-identity-aws-auth", REVOKE_IDENTITY_AWS_AUTH = "revoke-identity-aws-auth", GET_IDENTITY_AWS_AUTH = "get-identity-aws-auth", + LOGIN_IDENTITY_AZURE_AUTH = "login-identity-azure-auth", ADD_IDENTITY_AZURE_AUTH = "add-identity-azure-auth", UPDATE_IDENTITY_AZURE_AUTH = "update-identity-azure-auth", GET_IDENTITY_AZURE_AUTH = "get-identity-azure-auth", REVOKE_IDENTITY_AZURE_AUTH = "revoke-identity-azure-auth", + + LOGIN_IDENTITY_LDAP_AUTH = "login-identity-ldap-auth", + ADD_IDENTITY_LDAP_AUTH = "add-identity-ldap-auth", + UPDATE_IDENTITY_LDAP_AUTH = "update-identity-ldap-auth", + GET_IDENTITY_LDAP_AUTH = "get-identity-ldap-auth", + REVOKE_IDENTITY_LDAP_AUTH = "revoke-identity-ldap-auth", + CREATE_ENVIRONMENT = "create-environment", UPDATE_ENVIRONMENT = "update-environment", DELETE_ENVIRONMENT = "delete-environment", @@ -1035,6 +1052,55 @@ interface GetIdentityAzureAuthEvent { }; } +interface LoginIdentityLdapAuthEvent { + type: EventType.LOGIN_IDENTITY_LDAP_AUTH; + metadata: { + identityId: string; + ldapUsername: string; + ldapEmail?: string; + }; +} + +interface AddIdentityLdapAuthEvent { + type: EventType.ADD_IDENTITY_LDAP_AUTH; + metadata: { + identityId: string; + accessTokenTTL?: number; + accessTokenMaxTTL?: number; + accessTokenNumUsesLimit?: number; + accessTokenTrustedIps?: Array; + allowedFields?: TAllowedFields[]; + url: string; + }; +} + +interface UpdateIdentityLdapAuthEvent { + type: EventType.UPDATE_IDENTITY_LDAP_AUTH; + metadata: { + identityId: string; + accessTokenTTL?: number; + accessTokenMaxTTL?: number; + accessTokenNumUsesLimit?: number; + accessTokenTrustedIps?: Array; + allowedFields?: TAllowedFields[]; + url?: string; + }; +} + +interface GetIdentityLdapAuthEvent { + type: EventType.GET_IDENTITY_LDAP_AUTH; + metadata: { + identityId: string; + }; +} + +interface RevokeIdentityLdapAuthEvent { + type: EventType.REVOKE_IDENTITY_LDAP_AUTH; + metadata: { + identityId: string; + }; +} + interface LoginIdentityOidcAuthEvent { type: EventType.LOGIN_IDENTITY_OIDC_AUTH; metadata: { @@ -2795,6 +2861,11 @@ export type Event = | UpdateIdentityJwtAuthEvent | GetIdentityJwtAuthEvent | DeleteIdentityJwtAuthEvent + | LoginIdentityLdapAuthEvent + | AddIdentityLdapAuthEvent + | UpdateIdentityLdapAuthEvent + | GetIdentityLdapAuthEvent + | RevokeIdentityLdapAuthEvent | CreateEnvironmentEvent | GetEnvironmentEvent | UpdateEnvironmentEvent diff --git a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts index 22a078399..867feb4a4 100644 --- a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts +++ b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts @@ -1,6 +1,6 @@ import { ForbiddenError } from "@casl/ability"; import { Octokit } from "@octokit/core"; -import { paginateGraphQL } from "@octokit/plugin-paginate-graphql"; +import { paginateGraphql } from "@octokit/plugin-paginate-graphql"; import { Octokit as OctokitRest } from "@octokit/rest"; import { OrgMembershipRole } from "@app/db/schemas"; @@ -18,7 +18,7 @@ import { TPermissionServiceFactory } from "../permission/permission-service"; import { TGithubOrgSyncDALFactory } from "./github-org-sync-dal"; import { TCreateGithubOrgSyncDTO, TDeleteGithubOrgSyncDTO, TUpdateGithubOrgSyncDTO } from "./github-org-sync-types"; -const OctokitWithPlugin = Octokit.plugin(paginateGraphQL); +const OctokitWithPlugin = Octokit.plugin(paginateGraphql); type TGithubOrgSyncServiceFactoryDep = { githubOrgSyncDAL: TGithubOrgSyncDALFactory; diff --git a/backend/src/ee/services/group/group-dal.ts b/backend/src/ee/services/group/group-dal.ts index 2458454da..1d33cafd6 100644 --- a/backend/src/ee/services/group/group-dal.ts +++ b/backend/src/ee/services/group/group-dal.ts @@ -157,10 +157,23 @@ export const groupDALFactory = (db: TDbClient) => { } }; + const findGroupsByProjectId = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.Groups) + .join(TableName.GroupProjectMembership, `${TableName.Groups}.id`, `${TableName.GroupProjectMembership}.groupId`) + .where(`${TableName.GroupProjectMembership}.projectId`, projectId) + .select(selectAllTableCols(TableName.Groups)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "Find groups by project id" }); + } + }; + return { findGroups, findByOrgId, findAllGroupPossibleMembers, + findGroupsByProjectId, ...groupOrm }; }; diff --git a/backend/src/ee/services/group/user-group-membership-dal.ts b/backend/src/ee/services/group/user-group-membership-dal.ts index be654b338..5ee97e457 100644 --- a/backend/src/ee/services/group/user-group-membership-dal.ts +++ b/backend/src/ee/services/group/user-group-membership-dal.ts @@ -176,7 +176,8 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { db.ref("name").withSchema(TableName.Groups).as("groupName"), db.ref("id").withSchema(TableName.OrgMembership).as("orgMembershipId"), db.ref("firstName").withSchema(TableName.Users).as("firstName"), - db.ref("lastName").withSchema(TableName.Users).as("lastName") + db.ref("lastName").withSchema(TableName.Users).as("lastName"), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ); return docs; diff --git a/backend/src/ee/services/ldap-config/ldap-config-types.ts b/backend/src/ee/services/ldap-config/ldap-config-types.ts index 86f4bf0d5..941335fa4 100644 --- a/backend/src/ee/services/ldap-config/ldap-config-types.ts +++ b/backend/src/ee/services/ldap-config/ldap-config-types.ts @@ -14,6 +14,11 @@ export type TLDAPConfig = { caCert: string; }; +export type TTestLDAPConfigDTO = Omit< + TLDAPConfig, + "organization" | "id" | "groupSearchBase" | "groupSearchFilter" | "isActive" | "uniqueUserAttribute" | "searchBase" +>; + export type TCreateLdapCfgDTO = { orgId: string; isActive: boolean; diff --git a/backend/src/ee/services/ldap-config/ldap-fns.ts b/backend/src/ee/services/ldap-config/ldap-fns.ts index 44af718ed..01b70b4db 100644 --- a/backend/src/ee/services/ldap-config/ldap-fns.ts +++ b/backend/src/ee/services/ldap-config/ldap-fns.ts @@ -2,15 +2,14 @@ import ldapjs from "ldapjs"; import { logger } from "@app/lib/logger"; -import { TLDAPConfig } from "./ldap-config-types"; +import { TLDAPConfig, TTestLDAPConfigDTO } from "./ldap-config-types"; export const isValidLdapFilter = (filter: string) => { try { ldapjs.parseFilter(filter); return true; } catch (error) { - logger.error("Invalid LDAP filter"); - logger.error(error); + logger.error(error, "Invalid LDAP filter"); return false; } }; @@ -20,7 +19,7 @@ export const isValidLdapFilter = (filter: string) => { * @param ldapConfig - The LDAP configuration to test * @returns {Boolean} isConnected - Whether or not the connection was successful */ -export const testLDAPConfig = async (ldapConfig: TLDAPConfig): Promise => { +export const testLDAPConfig = async (ldapConfig: TTestLDAPConfigDTO): Promise => { return new Promise((resolve) => { const ldapClient = ldapjs.createClient({ url: ldapConfig.url, diff --git a/backend/src/ee/services/permission/default-roles.ts b/backend/src/ee/services/permission/default-roles.ts new file mode 100644 index 000000000..44fcb7825 --- /dev/null +++ b/backend/src/ee/services/permission/default-roles.ts @@ -0,0 +1,448 @@ +import { AbilityBuilder, createMongoAbility, MongoAbility } from "@casl/ability"; + +import { + ProjectPermissionActions, + ProjectPermissionCertificateActions, + ProjectPermissionCmekActions, + ProjectPermissionDynamicSecretActions, + ProjectPermissionGroupActions, + ProjectPermissionIdentityActions, + ProjectPermissionKmipActions, + ProjectPermissionMemberActions, + ProjectPermissionSecretActions, + ProjectPermissionSecretRotationActions, + ProjectPermissionSecretSyncActions, + ProjectPermissionSet, + ProjectPermissionSshHostActions, + ProjectPermissionSub +} from "@app/ee/services/permission/project-permission"; + +const buildAdminPermissionRules = () => { + const { can, rules } = new AbilityBuilder>(createMongoAbility); + + // Admins get full access to everything + [ + ProjectPermissionSub.SecretFolders, + ProjectPermissionSub.SecretImports, + ProjectPermissionSub.SecretApproval, + ProjectPermissionSub.Role, + ProjectPermissionSub.Integrations, + ProjectPermissionSub.Webhooks, + ProjectPermissionSub.ServiceTokens, + ProjectPermissionSub.Settings, + ProjectPermissionSub.Environments, + ProjectPermissionSub.Tags, + ProjectPermissionSub.AuditLogs, + ProjectPermissionSub.IpAllowList, + ProjectPermissionSub.CertificateAuthorities, + ProjectPermissionSub.CertificateTemplates, + ProjectPermissionSub.PkiAlerts, + ProjectPermissionSub.PkiCollections, + ProjectPermissionSub.SshCertificateAuthorities, + ProjectPermissionSub.SshCertificates, + ProjectPermissionSub.SshCertificateTemplates, + ProjectPermissionSub.SshHostGroups + ].forEach((el) => { + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + el + ); + }); + + can( + [ + ProjectPermissionCertificateActions.Read, + ProjectPermissionCertificateActions.Edit, + ProjectPermissionCertificateActions.Create, + ProjectPermissionCertificateActions.Delete, + ProjectPermissionCertificateActions.ReadPrivateKey + ], + ProjectPermissionSub.Certificates + ); + + can( + [ + ProjectPermissionSshHostActions.Edit, + ProjectPermissionSshHostActions.Read, + ProjectPermissionSshHostActions.Create, + ProjectPermissionSshHostActions.Delete, + ProjectPermissionSshHostActions.IssueHostCert + ], + ProjectPermissionSub.SshHosts + ); + + can( + [ + ProjectPermissionMemberActions.Create, + ProjectPermissionMemberActions.Edit, + ProjectPermissionMemberActions.Delete, + ProjectPermissionMemberActions.Read, + ProjectPermissionMemberActions.GrantPrivileges, + ProjectPermissionMemberActions.AssumePrivileges + ], + ProjectPermissionSub.Member + ); + + can( + [ + ProjectPermissionGroupActions.Create, + ProjectPermissionGroupActions.Edit, + ProjectPermissionGroupActions.Delete, + ProjectPermissionGroupActions.Read, + ProjectPermissionGroupActions.GrantPrivileges + ], + ProjectPermissionSub.Groups + ); + + can( + [ + ProjectPermissionIdentityActions.Create, + ProjectPermissionIdentityActions.Edit, + ProjectPermissionIdentityActions.Delete, + ProjectPermissionIdentityActions.Read, + ProjectPermissionIdentityActions.GrantPrivileges, + ProjectPermissionIdentityActions.AssumePrivileges + ], + ProjectPermissionSub.Identity + ); + + can( + [ + ProjectPermissionSecretActions.DescribeAndReadValue, + ProjectPermissionSecretActions.DescribeSecret, + ProjectPermissionSecretActions.ReadValue, + ProjectPermissionSecretActions.Create, + ProjectPermissionSecretActions.Edit, + ProjectPermissionSecretActions.Delete + ], + ProjectPermissionSub.Secrets + ); + + can( + [ + ProjectPermissionDynamicSecretActions.ReadRootCredential, + ProjectPermissionDynamicSecretActions.EditRootCredential, + ProjectPermissionDynamicSecretActions.CreateRootCredential, + ProjectPermissionDynamicSecretActions.DeleteRootCredential, + ProjectPermissionDynamicSecretActions.Lease + ], + ProjectPermissionSub.DynamicSecrets + ); + + can([ProjectPermissionActions.Edit, ProjectPermissionActions.Delete], ProjectPermissionSub.Project); + can([ProjectPermissionActions.Read, ProjectPermissionActions.Create], ProjectPermissionSub.SecretRollback); + can([ProjectPermissionActions.Edit], ProjectPermissionSub.Kms); + can( + [ + ProjectPermissionCmekActions.Create, + ProjectPermissionCmekActions.Edit, + ProjectPermissionCmekActions.Delete, + ProjectPermissionCmekActions.Read, + ProjectPermissionCmekActions.Encrypt, + ProjectPermissionCmekActions.Decrypt, + ProjectPermissionCmekActions.Sign, + ProjectPermissionCmekActions.Verify + ], + ProjectPermissionSub.Cmek + ); + can( + [ + ProjectPermissionSecretSyncActions.Create, + ProjectPermissionSecretSyncActions.Edit, + ProjectPermissionSecretSyncActions.Delete, + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSecretSyncActions.SyncSecrets, + ProjectPermissionSecretSyncActions.ImportSecrets, + ProjectPermissionSecretSyncActions.RemoveSecrets + ], + ProjectPermissionSub.SecretSyncs + ); + + can( + [ + ProjectPermissionKmipActions.CreateClients, + ProjectPermissionKmipActions.UpdateClients, + ProjectPermissionKmipActions.DeleteClients, + ProjectPermissionKmipActions.ReadClients, + ProjectPermissionKmipActions.GenerateClientCertificates + ], + ProjectPermissionSub.Kmip + ); + + can( + [ + ProjectPermissionSecretRotationActions.Create, + ProjectPermissionSecretRotationActions.Edit, + ProjectPermissionSecretRotationActions.Delete, + ProjectPermissionSecretRotationActions.Read, + ProjectPermissionSecretRotationActions.ReadGeneratedCredentials, + ProjectPermissionSecretRotationActions.RotateSecrets + ], + ProjectPermissionSub.SecretRotation + ); + + return rules; +}; + +const buildMemberPermissionRules = () => { + const { can, rules } = new AbilityBuilder>(createMongoAbility); + + can( + [ + ProjectPermissionSecretActions.DescribeAndReadValue, + ProjectPermissionSecretActions.DescribeSecret, + ProjectPermissionSecretActions.ReadValue, + ProjectPermissionSecretActions.Edit, + ProjectPermissionSecretActions.Create, + ProjectPermissionSecretActions.Delete + ], + ProjectPermissionSub.Secrets + ); + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.SecretFolders + ); + can( + [ + ProjectPermissionDynamicSecretActions.ReadRootCredential, + ProjectPermissionDynamicSecretActions.EditRootCredential, + ProjectPermissionDynamicSecretActions.CreateRootCredential, + ProjectPermissionDynamicSecretActions.DeleteRootCredential, + ProjectPermissionDynamicSecretActions.Lease + ], + ProjectPermissionSub.DynamicSecrets + ); + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.SecretImports + ); + + can([ProjectPermissionActions.Read], ProjectPermissionSub.SecretApproval); + can([ProjectPermissionSecretRotationActions.Read], ProjectPermissionSub.SecretRotation); + + can([ProjectPermissionActions.Read, ProjectPermissionActions.Create], ProjectPermissionSub.SecretRollback); + + can([ProjectPermissionMemberActions.Read, ProjectPermissionMemberActions.Create], ProjectPermissionSub.Member); + + can([ProjectPermissionGroupActions.Read], ProjectPermissionSub.Groups); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.Integrations + ); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.Webhooks + ); + + can( + [ + ProjectPermissionIdentityActions.Read, + ProjectPermissionIdentityActions.Edit, + ProjectPermissionIdentityActions.Create, + ProjectPermissionIdentityActions.Delete + ], + ProjectPermissionSub.Identity + ); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.ServiceTokens + ); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.Settings + ); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.Environments + ); + + can( + [ + ProjectPermissionActions.Read, + ProjectPermissionActions.Edit, + ProjectPermissionActions.Create, + ProjectPermissionActions.Delete + ], + ProjectPermissionSub.Tags + ); + + can([ProjectPermissionActions.Read], ProjectPermissionSub.Role); + can([ProjectPermissionActions.Read], ProjectPermissionSub.AuditLogs); + can([ProjectPermissionActions.Read], ProjectPermissionSub.IpAllowList); + + // double check if all CRUD are needed for CA and Certificates + can([ProjectPermissionActions.Read], ProjectPermissionSub.CertificateAuthorities); + + can( + [ + ProjectPermissionCertificateActions.Read, + ProjectPermissionCertificateActions.Edit, + ProjectPermissionCertificateActions.Create, + ProjectPermissionCertificateActions.Delete + ], + ProjectPermissionSub.Certificates + ); + + can([ProjectPermissionActions.Read], ProjectPermissionSub.CertificateTemplates); + + can([ProjectPermissionActions.Read], ProjectPermissionSub.PkiAlerts); + can([ProjectPermissionActions.Read], ProjectPermissionSub.PkiCollections); + + can([ProjectPermissionActions.Read], ProjectPermissionSub.SshCertificates); + can([ProjectPermissionActions.Create], ProjectPermissionSub.SshCertificates); + can([ProjectPermissionActions.Read], ProjectPermissionSub.SshCertificateTemplates); + + can([ProjectPermissionSshHostActions.Read], ProjectPermissionSub.SshHosts); + + can( + [ + ProjectPermissionCmekActions.Create, + ProjectPermissionCmekActions.Edit, + ProjectPermissionCmekActions.Delete, + ProjectPermissionCmekActions.Read, + ProjectPermissionCmekActions.Encrypt, + ProjectPermissionCmekActions.Decrypt, + ProjectPermissionCmekActions.Sign, + ProjectPermissionCmekActions.Verify + ], + ProjectPermissionSub.Cmek + ); + + can( + [ + ProjectPermissionSecretSyncActions.Create, + ProjectPermissionSecretSyncActions.Edit, + ProjectPermissionSecretSyncActions.Delete, + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSecretSyncActions.SyncSecrets, + ProjectPermissionSecretSyncActions.ImportSecrets, + ProjectPermissionSecretSyncActions.RemoveSecrets + ], + ProjectPermissionSub.SecretSyncs + ); + + return rules; +}; + +const buildViewerPermissionRules = () => { + const { can, rules } = new AbilityBuilder>(createMongoAbility); + + can(ProjectPermissionSecretActions.DescribeAndReadValue, ProjectPermissionSub.Secrets); + can(ProjectPermissionSecretActions.DescribeSecret, ProjectPermissionSub.Secrets); + can(ProjectPermissionSecretActions.ReadValue, ProjectPermissionSub.Secrets); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretFolders); + can(ProjectPermissionDynamicSecretActions.ReadRootCredential, ProjectPermissionSub.DynamicSecrets); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretImports); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretApproval); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + can(ProjectPermissionSecretRotationActions.Read, ProjectPermissionSub.SecretRotation); + can(ProjectPermissionMemberActions.Read, ProjectPermissionSub.Member); + can(ProjectPermissionGroupActions.Read, ProjectPermissionSub.Groups); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Role); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Webhooks); + can(ProjectPermissionIdentityActions.Read, ProjectPermissionSub.Identity); + can(ProjectPermissionActions.Read, ProjectPermissionSub.ServiceTokens); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Settings); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Environments); + can(ProjectPermissionActions.Read, ProjectPermissionSub.Tags); + can(ProjectPermissionActions.Read, ProjectPermissionSub.AuditLogs); + can(ProjectPermissionActions.Read, ProjectPermissionSub.IpAllowList); + can(ProjectPermissionActions.Read, ProjectPermissionSub.CertificateAuthorities); + can(ProjectPermissionCertificateActions.Read, ProjectPermissionSub.Certificates); + can(ProjectPermissionCmekActions.Read, ProjectPermissionSub.Cmek); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); + can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); + can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + + return rules; +}; + +const buildNoAccessProjectPermission = () => { + const { rules } = new AbilityBuilder>(createMongoAbility); + return rules; +}; + +const buildSshHostBootstrapPermissionRules = () => { + const { can, rules } = new AbilityBuilder>(createMongoAbility); + + can( + [ProjectPermissionSshHostActions.Create, ProjectPermissionSshHostActions.IssueHostCert], + ProjectPermissionSub.SshHosts + ); + + return rules; +}; + +const buildCryptographicOperatorPermissionRules = () => { + const { can, rules } = new AbilityBuilder>(createMongoAbility); + + can( + [ + ProjectPermissionCmekActions.Encrypt, + ProjectPermissionCmekActions.Decrypt, + ProjectPermissionCmekActions.Sign, + ProjectPermissionCmekActions.Verify + ], + ProjectPermissionSub.Cmek + ); + + return rules; +}; + +// General +export const projectAdminPermissions = buildAdminPermissionRules(); +export const projectMemberPermissions = buildMemberPermissionRules(); +export const projectViewerPermission = buildViewerPermissionRules(); +export const projectNoAccessPermissions = buildNoAccessProjectPermission(); + +// SSH +export const sshHostBootstrapPermissions = buildSshHostBootstrapPermissionRules(); + +// KMS +export const cryptographicOperatorPermissions = buildCryptographicOperatorPermissionRules(); diff --git a/backend/src/ee/services/permission/permission-dal.ts b/backend/src/ee/services/permission/permission-dal.ts index 891d7193e..7a17108a2 100644 --- a/backend/src/ee/services/permission/permission-dal.ts +++ b/backend/src/ee/services/permission/permission-dal.ts @@ -132,7 +132,7 @@ export const permissionDALFactory = (db: TDbClient) => { } }; - const getProjectGroupPermissions = async (projectId: string) => { + const getProjectGroupPermissions = async (projectId: string, filterGroupId?: string) => { try { const docs = await db .replicaNode()(TableName.GroupProjectMembership) @@ -148,6 +148,11 @@ export const permissionDALFactory = (db: TDbClient) => { `groupCustomRoles.id` ) .where(`${TableName.GroupProjectMembership}.projectId`, "=", projectId) + .where((bd) => { + if (filterGroupId) { + void bd.where(`${TableName.GroupProjectMembership}.groupId`, "=", filterGroupId); + } + }) .select( db.ref("id").withSchema(TableName.GroupProjectMembership).as("membershipId"), db.ref("id").withSchema(TableName.Groups).as("groupId"), diff --git a/backend/src/ee/services/permission/permission-service.ts b/backend/src/ee/services/permission/permission-service.ts index 3d2f96f82..a1acaeb21 100644 --- a/backend/src/ee/services/permission/permission-service.ts +++ b/backend/src/ee/services/permission/permission-service.ts @@ -12,6 +12,14 @@ import { TIdentityProjectMemberships, TProjectMemberships } from "@app/db/schemas"; +import { + cryptographicOperatorPermissions, + projectAdminPermissions, + projectMemberPermissions, + projectNoAccessPermissions, + projectViewerPermission, + sshHostBootstrapPermissions +} from "@app/ee/services/permission/default-roles"; import { conditionsMatcher } from "@app/lib/casl"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; import { objectify } from "@app/lib/fn"; @@ -32,14 +40,7 @@ import { TGetServiceTokenProjectPermissionArg, TGetUserProjectPermissionArg } from "./permission-service-types"; -import { - buildServiceTokenProjectPermission, - projectAdminPermissions, - projectMemberPermissions, - projectNoAccessPermissions, - ProjectPermissionSet, - projectViewerPermission -} from "./project-permission"; +import { buildServiceTokenProjectPermission, ProjectPermissionSet } from "./project-permission"; type TPermissionServiceFactoryDep = { orgRoleDAL: Pick; @@ -95,6 +96,10 @@ export const permissionServiceFactory = ({ return projectViewerPermission; case ProjectMembershipRole.NoAccess: return projectNoAccessPermissions; + case ProjectMembershipRole.SshHostBootstrapper: + return sshHostBootstrapPermissions; + case ProjectMembershipRole.KmsCryptographicOperator: + return cryptographicOperatorPermissions; case ProjectMembershipRole.Custom: { return unpackRules>>( permissions as PackRule>>[] @@ -625,6 +630,34 @@ export const permissionServiceFactory = ({ return { permission }; }; + const checkGroupProjectPermission = async ({ + groupId, + projectId, + checkPermissions + }: { + groupId: string; + projectId: string; + checkPermissions: ProjectPermissionSet; + }) => { + const rawGroupProjectPermissions = await permissionDAL.getProjectGroupPermissions(projectId, groupId); + const groupPermissions = rawGroupProjectPermissions.map((groupProjectPermission) => { + const rolePermissions = + groupProjectPermission.roles?.map(({ role, permissions }) => ({ role, permissions })) || []; + const rules = buildProjectPermissionRules(rolePermissions); + const permission = createMongoAbility(rules, { + conditionsMatcher + }); + + return { + permission, + id: groupProjectPermission.groupId, + name: groupProjectPermission.username, + membershipId: groupProjectPermission.id + }; + }); + return groupPermissions.some((groupPermission) => groupPermission.permission.can(...checkPermissions)); + }; + return { getUserOrgPermission, getOrgPermission, @@ -634,6 +667,7 @@ export const permissionServiceFactory = ({ getOrgPermissionByRole, getProjectPermissionByRole, buildOrgPermission, - buildProjectPermissionRules + buildProjectPermissionRules, + checkGroupProjectPermission }; }; diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 993653045..7463c35f6 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -678,403 +678,6 @@ export const ProjectPermissionV2Schema = z.discriminatedUnion("subject", [ export type TProjectPermissionV2Schema = z.infer; -const buildAdminPermissionRules = () => { - const { can, rules } = new AbilityBuilder>(createMongoAbility); - - // Admins get full access to everything - [ - ProjectPermissionSub.SecretFolders, - ProjectPermissionSub.SecretImports, - ProjectPermissionSub.SecretApproval, - ProjectPermissionSub.Role, - ProjectPermissionSub.Integrations, - ProjectPermissionSub.Webhooks, - ProjectPermissionSub.ServiceTokens, - ProjectPermissionSub.Settings, - ProjectPermissionSub.Environments, - ProjectPermissionSub.Tags, - ProjectPermissionSub.AuditLogs, - ProjectPermissionSub.IpAllowList, - ProjectPermissionSub.CertificateAuthorities, - ProjectPermissionSub.CertificateTemplates, - ProjectPermissionSub.PkiAlerts, - ProjectPermissionSub.PkiCollections, - ProjectPermissionSub.SshCertificateAuthorities, - ProjectPermissionSub.SshCertificates, - ProjectPermissionSub.SshCertificateTemplates, - ProjectPermissionSub.SshHostGroups - ].forEach((el) => { - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - el - ); - }); - - can( - [ - ProjectPermissionCertificateActions.Read, - ProjectPermissionCertificateActions.Edit, - ProjectPermissionCertificateActions.Create, - ProjectPermissionCertificateActions.Delete, - ProjectPermissionCertificateActions.ReadPrivateKey - ], - ProjectPermissionSub.Certificates - ); - - can( - [ - ProjectPermissionSshHostActions.Edit, - ProjectPermissionSshHostActions.Read, - ProjectPermissionSshHostActions.Create, - ProjectPermissionSshHostActions.Delete, - ProjectPermissionSshHostActions.IssueHostCert - ], - ProjectPermissionSub.SshHosts - ); - - can( - [ - ProjectPermissionMemberActions.Create, - ProjectPermissionMemberActions.Edit, - ProjectPermissionMemberActions.Delete, - ProjectPermissionMemberActions.Read, - ProjectPermissionMemberActions.GrantPrivileges, - ProjectPermissionMemberActions.AssumePrivileges - ], - ProjectPermissionSub.Member - ); - - can( - [ - ProjectPermissionGroupActions.Create, - ProjectPermissionGroupActions.Edit, - ProjectPermissionGroupActions.Delete, - ProjectPermissionGroupActions.Read, - ProjectPermissionGroupActions.GrantPrivileges - ], - ProjectPermissionSub.Groups - ); - - can( - [ - ProjectPermissionIdentityActions.Create, - ProjectPermissionIdentityActions.Edit, - ProjectPermissionIdentityActions.Delete, - ProjectPermissionIdentityActions.Read, - ProjectPermissionIdentityActions.GrantPrivileges, - ProjectPermissionIdentityActions.AssumePrivileges - ], - ProjectPermissionSub.Identity - ); - - can( - [ - ProjectPermissionSecretActions.DescribeAndReadValue, - ProjectPermissionSecretActions.DescribeSecret, - ProjectPermissionSecretActions.ReadValue, - ProjectPermissionSecretActions.Create, - ProjectPermissionSecretActions.Edit, - ProjectPermissionSecretActions.Delete - ], - ProjectPermissionSub.Secrets - ); - - can( - [ - ProjectPermissionDynamicSecretActions.ReadRootCredential, - ProjectPermissionDynamicSecretActions.EditRootCredential, - ProjectPermissionDynamicSecretActions.CreateRootCredential, - ProjectPermissionDynamicSecretActions.DeleteRootCredential, - ProjectPermissionDynamicSecretActions.Lease - ], - ProjectPermissionSub.DynamicSecrets - ); - - can([ProjectPermissionActions.Edit, ProjectPermissionActions.Delete], ProjectPermissionSub.Project); - can([ProjectPermissionActions.Read, ProjectPermissionActions.Create], ProjectPermissionSub.SecretRollback); - can([ProjectPermissionActions.Edit], ProjectPermissionSub.Kms); - can( - [ - ProjectPermissionCmekActions.Create, - ProjectPermissionCmekActions.Edit, - ProjectPermissionCmekActions.Delete, - ProjectPermissionCmekActions.Read, - ProjectPermissionCmekActions.Encrypt, - ProjectPermissionCmekActions.Decrypt, - ProjectPermissionCmekActions.Sign, - ProjectPermissionCmekActions.Verify - ], - ProjectPermissionSub.Cmek - ); - can( - [ - ProjectPermissionSecretSyncActions.Create, - ProjectPermissionSecretSyncActions.Edit, - ProjectPermissionSecretSyncActions.Delete, - ProjectPermissionSecretSyncActions.Read, - ProjectPermissionSecretSyncActions.SyncSecrets, - ProjectPermissionSecretSyncActions.ImportSecrets, - ProjectPermissionSecretSyncActions.RemoveSecrets - ], - ProjectPermissionSub.SecretSyncs - ); - - can( - [ - ProjectPermissionKmipActions.CreateClients, - ProjectPermissionKmipActions.UpdateClients, - ProjectPermissionKmipActions.DeleteClients, - ProjectPermissionKmipActions.ReadClients, - ProjectPermissionKmipActions.GenerateClientCertificates - ], - ProjectPermissionSub.Kmip - ); - - can( - [ - ProjectPermissionSecretRotationActions.Create, - ProjectPermissionSecretRotationActions.Edit, - ProjectPermissionSecretRotationActions.Delete, - ProjectPermissionSecretRotationActions.Read, - ProjectPermissionSecretRotationActions.ReadGeneratedCredentials, - ProjectPermissionSecretRotationActions.RotateSecrets - ], - ProjectPermissionSub.SecretRotation - ); - - return rules; -}; - -export const projectAdminPermissions = buildAdminPermissionRules(); - -const buildMemberPermissionRules = () => { - const { can, rules } = new AbilityBuilder>(createMongoAbility); - - can( - [ - ProjectPermissionSecretActions.DescribeAndReadValue, - ProjectPermissionSecretActions.DescribeSecret, - ProjectPermissionSecretActions.ReadValue, - ProjectPermissionSecretActions.Edit, - ProjectPermissionSecretActions.Create, - ProjectPermissionSecretActions.Delete - ], - ProjectPermissionSub.Secrets - ); - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.SecretFolders - ); - can( - [ - ProjectPermissionDynamicSecretActions.ReadRootCredential, - ProjectPermissionDynamicSecretActions.EditRootCredential, - ProjectPermissionDynamicSecretActions.CreateRootCredential, - ProjectPermissionDynamicSecretActions.DeleteRootCredential, - ProjectPermissionDynamicSecretActions.Lease - ], - ProjectPermissionSub.DynamicSecrets - ); - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.SecretImports - ); - - can([ProjectPermissionActions.Read], ProjectPermissionSub.SecretApproval); - can([ProjectPermissionSecretRotationActions.Read], ProjectPermissionSub.SecretRotation); - - can([ProjectPermissionActions.Read, ProjectPermissionActions.Create], ProjectPermissionSub.SecretRollback); - - can([ProjectPermissionMemberActions.Read, ProjectPermissionMemberActions.Create], ProjectPermissionSub.Member); - - can([ProjectPermissionGroupActions.Read], ProjectPermissionSub.Groups); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.Integrations - ); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.Webhooks - ); - - can( - [ - ProjectPermissionIdentityActions.Read, - ProjectPermissionIdentityActions.Edit, - ProjectPermissionIdentityActions.Create, - ProjectPermissionIdentityActions.Delete - ], - ProjectPermissionSub.Identity - ); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.ServiceTokens - ); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.Settings - ); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.Environments - ); - - can( - [ - ProjectPermissionActions.Read, - ProjectPermissionActions.Edit, - ProjectPermissionActions.Create, - ProjectPermissionActions.Delete - ], - ProjectPermissionSub.Tags - ); - - can([ProjectPermissionActions.Read], ProjectPermissionSub.Role); - can([ProjectPermissionActions.Read], ProjectPermissionSub.AuditLogs); - can([ProjectPermissionActions.Read], ProjectPermissionSub.IpAllowList); - - // double check if all CRUD are needed for CA and Certificates - can([ProjectPermissionActions.Read], ProjectPermissionSub.CertificateAuthorities); - - can( - [ - ProjectPermissionCertificateActions.Read, - ProjectPermissionCertificateActions.Edit, - ProjectPermissionCertificateActions.Create, - ProjectPermissionCertificateActions.Delete - ], - ProjectPermissionSub.Certificates - ); - - can([ProjectPermissionActions.Read], ProjectPermissionSub.CertificateTemplates); - - can([ProjectPermissionActions.Read], ProjectPermissionSub.PkiAlerts); - can([ProjectPermissionActions.Read], ProjectPermissionSub.PkiCollections); - - can([ProjectPermissionActions.Read], ProjectPermissionSub.SshCertificates); - can([ProjectPermissionActions.Create], ProjectPermissionSub.SshCertificates); - can([ProjectPermissionActions.Read], ProjectPermissionSub.SshCertificateTemplates); - - can([ProjectPermissionSshHostActions.Read], ProjectPermissionSub.SshHosts); - - can( - [ - ProjectPermissionCmekActions.Create, - ProjectPermissionCmekActions.Edit, - ProjectPermissionCmekActions.Delete, - ProjectPermissionCmekActions.Read, - ProjectPermissionCmekActions.Encrypt, - ProjectPermissionCmekActions.Decrypt, - ProjectPermissionCmekActions.Sign, - ProjectPermissionCmekActions.Verify - ], - ProjectPermissionSub.Cmek - ); - - can( - [ - ProjectPermissionSecretSyncActions.Create, - ProjectPermissionSecretSyncActions.Edit, - ProjectPermissionSecretSyncActions.Delete, - ProjectPermissionSecretSyncActions.Read, - ProjectPermissionSecretSyncActions.SyncSecrets, - ProjectPermissionSecretSyncActions.ImportSecrets, - ProjectPermissionSecretSyncActions.RemoveSecrets - ], - ProjectPermissionSub.SecretSyncs - ); - - return rules; -}; - -export const projectMemberPermissions = buildMemberPermissionRules(); - -const buildViewerPermissionRules = () => { - const { can, rules } = new AbilityBuilder>(createMongoAbility); - - can(ProjectPermissionSecretActions.DescribeAndReadValue, ProjectPermissionSub.Secrets); - can(ProjectPermissionSecretActions.DescribeSecret, ProjectPermissionSub.Secrets); - can(ProjectPermissionSecretActions.ReadValue, ProjectPermissionSub.Secrets); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretFolders); - can(ProjectPermissionDynamicSecretActions.ReadRootCredential, ProjectPermissionSub.DynamicSecrets); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretImports); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretApproval); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - can(ProjectPermissionSecretRotationActions.Read, ProjectPermissionSub.SecretRotation); - can(ProjectPermissionMemberActions.Read, ProjectPermissionSub.Member); - can(ProjectPermissionGroupActions.Read, ProjectPermissionSub.Groups); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Role); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Webhooks); - can(ProjectPermissionIdentityActions.Read, ProjectPermissionSub.Identity); - can(ProjectPermissionActions.Read, ProjectPermissionSub.ServiceTokens); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Settings); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Environments); - can(ProjectPermissionActions.Read, ProjectPermissionSub.Tags); - can(ProjectPermissionActions.Read, ProjectPermissionSub.AuditLogs); - can(ProjectPermissionActions.Read, ProjectPermissionSub.IpAllowList); - can(ProjectPermissionActions.Read, ProjectPermissionSub.CertificateAuthorities); - can(ProjectPermissionCertificateActions.Read, ProjectPermissionSub.Certificates); - can(ProjectPermissionCmekActions.Read, ProjectPermissionSub.Cmek); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); - can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); - can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); - - return rules; -}; - -export const projectViewerPermission = buildViewerPermissionRules(); - -const buildNoAccessProjectPermission = () => { - const { rules } = new AbilityBuilder>(createMongoAbility); - return rules; -}; - export const buildServiceTokenProjectPermission = ( scopes: Array<{ secretPath: string; environment: string }>, permission: string[] @@ -1116,8 +719,6 @@ export const buildServiceTokenProjectPermission = ( return build({ conditionsMatcher }); }; -export const projectNoAccessPermissions = buildNoAccessProjectPermission(); - /* eslint-disable */ /** diff --git a/backend/src/ee/services/project-template/project-template-fns.ts b/backend/src/ee/services/project-template/project-template-fns.ts index 2ca78e876..8e8ebfa13 100644 --- a/backend/src/ee/services/project-template/project-template-fns.ts +++ b/backend/src/ee/services/project-template/project-template-fns.ts @@ -1,22 +1,27 @@ -import { ProjectTemplateDefaultEnvironments } from "@app/ee/services/project-template/project-template-constants"; +import { ProjectType } from "@app/db/schemas"; import { InfisicalProjectTemplate, TUnpackedPermission } from "@app/ee/services/project-template/project-template-types"; import { getPredefinedRoles } from "@app/services/project-role/project-role-fns"; -export const getDefaultProjectTemplate = (orgId: string) => ({ +import { ProjectTemplateDefaultEnvironments } from "./project-template-constants"; + +export const getDefaultProjectTemplate = (orgId: string, type: ProjectType) => ({ id: "b11b49a9-09a9-4443-916a-4246f9ff2c69", // random ID to appease zod + type, name: InfisicalProjectTemplate.Default, createdAt: new Date(), updatedAt: new Date(), - description: "Infisical's default project template", - environments: ProjectTemplateDefaultEnvironments, - roles: [...getPredefinedRoles("project-template")].map(({ name, slug, permissions }) => ({ - name, - slug, - permissions: permissions as TUnpackedPermission[] - })), + description: `Infisical's ${type} default project template`, + environments: type === ProjectType.SecretManager ? ProjectTemplateDefaultEnvironments : null, + roles: [...getPredefinedRoles({ projectId: "project-template", projectType: type })].map( + ({ name, slug, permissions }) => ({ + name, + slug, + permissions: permissions as TUnpackedPermission[] + }) + ), orgId }); diff --git a/backend/src/ee/services/project-template/project-template-service.ts b/backend/src/ee/services/project-template/project-template-service.ts index b2430ac14..5b6163977 100644 --- a/backend/src/ee/services/project-template/project-template-service.ts +++ b/backend/src/ee/services/project-template/project-template-service.ts @@ -1,10 +1,11 @@ import { ForbiddenError } from "@casl/ability"; import { packRules } from "@casl/ability/extra"; -import { TProjectTemplates } from "@app/db/schemas"; +import { ProjectType, TProjectTemplates } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectTemplateDefaultEnvironments } from "@app/ee/services/project-template/project-template-constants"; import { getDefaultProjectTemplate } from "@app/ee/services/project-template/project-template-fns"; import { TCreateProjectTemplateDTO, @@ -32,11 +33,13 @@ const $unpackProjectTemplate = ({ roles, environments, ...rest }: TProjectTempla ...rest, environments: environments as TProjectTemplateEnvironment[], roles: [ - ...getPredefinedRoles("project-template").map(({ name, slug, permissions }) => ({ - name, - slug, - permissions: permissions as TUnpackedPermission[] - })), + ...getPredefinedRoles({ projectId: "project-template", projectType: rest.type as ProjectType }).map( + ({ name, slug, permissions }) => ({ + name, + slug, + permissions: permissions as TUnpackedPermission[] + }) + ), ...(roles as TProjectTemplateRole[]).map((role) => ({ ...role, permissions: unpackPermissions(role.permissions) @@ -49,7 +52,7 @@ export const projectTemplateServiceFactory = ({ permissionService, projectTemplateDAL }: TProjectTemplatesServiceFactoryDep) => { - const listProjectTemplatesByOrg = async (actor: OrgServiceActor) => { + const listProjectTemplatesByOrg = async (actor: OrgServiceActor, type?: ProjectType) => { const plan = await licenseService.getPlan(actor.orgId); if (!plan.projectTemplates) @@ -68,11 +71,14 @@ export const projectTemplateServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.ProjectTemplates); const projectTemplates = await projectTemplateDAL.find({ - orgId: actor.orgId + orgId: actor.orgId, + ...(type ? { type } : {}) }); return [ - getDefaultProjectTemplate(actor.orgId), + ...(type + ? [getDefaultProjectTemplate(actor.orgId, type)] + : Object.values(ProjectType).map((projectType) => getDefaultProjectTemplate(actor.orgId, projectType))), ...projectTemplates.map((template) => $unpackProjectTemplate(template)) ]; }; @@ -134,7 +140,7 @@ export const projectTemplateServiceFactory = ({ }; const createProjectTemplate = async ( - { roles, environments, ...params }: TCreateProjectTemplateDTO, + { roles, environments, type, ...params }: TCreateProjectTemplateDTO, actor: OrgServiceActor ) => { const plan = await licenseService.getPlan(actor.orgId); @@ -154,6 +160,17 @@ export const projectTemplateServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.ProjectTemplates); + if (environments && type !== ProjectType.SecretManager) { + throw new BadRequestError({ message: "Cannot configure environments for non-SecretManager project templates" }); + } + + if (environments && plan.environmentLimit !== null && environments.length > plan.environmentLimit) { + throw new BadRequestError({ + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + message: `Failed to create project template due to environment count exceeding your current limit of ${plan.environmentLimit}. Contact Infisical to increase limit.` + }); + } + const isConflictingName = Boolean( await projectTemplateDAL.findOne({ name: params.name, @@ -169,8 +186,10 @@ export const projectTemplateServiceFactory = ({ const projectTemplate = await projectTemplateDAL.create({ ...params, roles: JSON.stringify(roles.map((role) => ({ ...role, permissions: packRules(role.permissions) }))), - environments: JSON.stringify(environments), - orgId: actor.orgId + environments: + type === ProjectType.SecretManager ? JSON.stringify(environments ?? ProjectTemplateDefaultEnvironments) : null, + orgId: actor.orgId, + type }); return $unpackProjectTemplate(projectTemplate); @@ -202,6 +221,19 @@ export const projectTemplateServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.ProjectTemplates); + if (projectTemplate.type !== ProjectType.SecretManager && environments) + throw new BadRequestError({ message: "Cannot configure environments for non-SecretManager project templates" }); + + if (projectTemplate.type === ProjectType.SecretManager && environments === null) + throw new BadRequestError({ message: "Environments cannot be removed for SecretManager project templates" }); + + if (environments && plan.environmentLimit !== null && environments.length > plan.environmentLimit) { + throw new BadRequestError({ + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + message: `Failed to update project template due to environment count exceeding your current limit of ${plan.environmentLimit}. Contact Infisical to increase limit.` + }); + } + if (params.name && projectTemplate.name !== params.name) { const isConflictingName = Boolean( await projectTemplateDAL.findOne({ diff --git a/backend/src/ee/services/project-template/project-template-types.ts b/backend/src/ee/services/project-template/project-template-types.ts index c2764dc53..d53b2375e 100644 --- a/backend/src/ee/services/project-template/project-template-types.ts +++ b/backend/src/ee/services/project-template/project-template-types.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { TProjectEnvironments } from "@app/db/schemas"; +import { ProjectType, TProjectEnvironments } from "@app/db/schemas"; import { TProjectPermissionV2Schema } from "@app/ee/services/permission/project-permission"; import { UnpackedPermissionSchema } from "@app/server/routes/sanitizedSchema/permission"; @@ -15,8 +15,9 @@ export type TProjectTemplateRole = { export type TCreateProjectTemplateDTO = { name: string; description?: string; + type: ProjectType; roles: TProjectTemplateRole[]; - environments: TProjectTemplateEnvironment[]; + environments?: TProjectTemplateEnvironment[] | null; }; export type TUpdateProjectTemplateDTO = Partial; diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-fns.ts b/backend/src/ee/services/secret-scanning/secret-scanning-fns.ts new file mode 100644 index 000000000..b1e2e0bbb --- /dev/null +++ b/backend/src/ee/services/secret-scanning/secret-scanning-fns.ts @@ -0,0 +1,11 @@ +import { getConfig } from "@app/lib/config/env"; + +export const canUseSecretScanning = (orgId: string) => { + const appCfg = getConfig(); + + if (!appCfg.isCloud) { + return true; + } + + return appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(orgId); +}; diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts index c5e7be9d8..7d41091fc 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts @@ -12,6 +12,7 @@ import { NotFoundError } from "@app/lib/errors"; import { TGitAppDALFactory } from "./git-app-dal"; import { TGitAppInstallSessionDALFactory } from "./git-app-install-session-dal"; import { TSecretScanningDALFactory } from "./secret-scanning-dal"; +import { canUseSecretScanning } from "./secret-scanning-fns"; import { TSecretScanningQueueFactory } from "./secret-scanning-queue"; import { SecretScanningRiskStatus, @@ -47,12 +48,14 @@ export const secretScanningServiceFactory = ({ actorAuthMethod, actorOrgId }: TInstallAppSessionDTO) => { + const appCfg = getConfig(); + const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.SecretScanning); const sessionId = crypto.randomBytes(16).toString("hex"); await gitAppInstallSessionDAL.upsert({ orgId, sessionId, userId: actorId }); - return { sessionId }; + return { sessionId, gitAppSlug: appCfg.SECRET_SCANNING_GIT_APP_SLUG }; }; const linkInstallationToOrg = async ({ @@ -91,7 +94,8 @@ export const secretScanningServiceFactory = ({ const { data: { repositories } } = await octokit.apps.listReposAccessibleToInstallation(); - if (appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(actorOrgId)) { + + if (canUseSecretScanning(actorOrgId)) { await Promise.all( repositories.map(({ id, full_name }) => secretScanningQueue.startFullRepoScan({ @@ -102,6 +106,7 @@ export const secretScanningServiceFactory = ({ ) ); } + return { installatedApp }; }; @@ -164,7 +169,6 @@ export const secretScanningServiceFactory = ({ }; const handleRepoPushEvent = async (payload: WebhookEventMap["push"]) => { - const appCfg = getConfig(); const { commits, repository, installation, pusher } = payload; if (!commits || !repository || !installation || !pusher) { return; @@ -175,7 +179,7 @@ export const secretScanningServiceFactory = ({ }); if (!installationLink) return; - if (appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(installationLink.orgId)) { + if (canUseSecretScanning(installationLink.orgId)) { await secretScanningQueue.startPushEventScan({ commits, pusher: { name: pusher.name, email: pusher.email }, diff --git a/backend/src/ee/services/ssh-host-group/ssh-host-group-dal.ts b/backend/src/ee/services/ssh-host-group/ssh-host-group-dal.ts index 08242d4cb..2f57cce3a 100644 --- a/backend/src/ee/services/ssh-host-group/ssh-host-group-dal.ts +++ b/backend/src/ee/services/ssh-host-group/ssh-host-group-dal.ts @@ -28,6 +28,7 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .where(`${TableName.SshHostGroup}.projectId`, projectId) .select( db.ref("id").withSchema(TableName.SshHostGroup).as("sshHostGroupId"), @@ -35,7 +36,8 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { db.ref("name").withSchema(TableName.SshHostGroup), db.ref("loginUser").withSchema(TableName.SshHostLoginUser), db.ref("username").withSchema(TableName.Users), - db.ref("userId").withSchema(TableName.SshHostLoginUserMapping) + db.ref("userId").withSchema(TableName.SshHostLoginUserMapping), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ) .orderBy(`${TableName.SshHostGroup}.updatedAt`, "desc"); @@ -69,7 +71,8 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { const loginMappings = Object.entries(loginMappingGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) } })); return { @@ -99,6 +102,7 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .where(`${TableName.SshHostGroup}.id`, sshHostGroupId) .select( db.ref("id").withSchema(TableName.SshHostGroup).as("sshHostGroupId"), @@ -106,7 +110,8 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { db.ref("name").withSchema(TableName.SshHostGroup), db.ref("loginUser").withSchema(TableName.SshHostLoginUser), db.ref("username").withSchema(TableName.Users), - db.ref("userId").withSchema(TableName.SshHostLoginUserMapping) + db.ref("userId").withSchema(TableName.SshHostLoginUserMapping), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ); if (rows.length === 0) return null; @@ -121,7 +126,8 @@ export const sshHostGroupDALFactory = (db: TDbClient) => { const loginMappings = Object.entries(loginMappingGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) } })); diff --git a/backend/src/ee/services/ssh-host-group/ssh-host-group-service.ts b/backend/src/ee/services/ssh-host-group/ssh-host-group-service.ts index 751116895..1eacc7602 100644 --- a/backend/src/ee/services/ssh-host-group/ssh-host-group-service.ts +++ b/backend/src/ee/services/ssh-host-group/ssh-host-group-service.ts @@ -12,6 +12,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TUserDALFactory } from "@app/services/user/user-dal"; +import { TGroupDALFactory } from "../group/group-dal"; import { TLicenseServiceFactory } from "../license/license-service"; import { createSshLoginMappings } from "../ssh-host/ssh-host-fns"; import { @@ -43,8 +44,12 @@ type TSshHostGroupServiceFactoryDep = { sshHostLoginUserDAL: Pick; sshHostLoginUserMappingDAL: Pick; userDAL: Pick; - permissionService: Pick; + permissionService: Pick< + TPermissionServiceFactory, + "getProjectPermission" | "getUserProjectPermission" | "checkGroupProjectPermission" + >; licenseService: Pick; + groupDAL: Pick; }; export type TSshHostGroupServiceFactory = ReturnType; @@ -58,7 +63,8 @@ export const sshHostGroupServiceFactory = ({ sshHostLoginUserMappingDAL, userDAL, permissionService, - licenseService + licenseService, + groupDAL }: TSshHostGroupServiceFactoryDep) => { const createSshHostGroup = async ({ projectId, @@ -127,6 +133,7 @@ export const sshHostGroupServiceFactory = ({ loginMappings, sshHostLoginUserDAL, sshHostLoginUserMappingDAL, + groupDAL, userDAL, permissionService, projectId, @@ -194,6 +201,7 @@ export const sshHostGroupServiceFactory = ({ loginMappings, sshHostLoginUserDAL, sshHostLoginUserMappingDAL, + groupDAL, userDAL, permissionService, projectId: sshHostGroup.projectId, diff --git a/backend/src/ee/services/ssh-host-group/ssh-host-group-types.ts b/backend/src/ee/services/ssh-host-group/ssh-host-group-types.ts index 3485b5d26..52f805f02 100644 --- a/backend/src/ee/services/ssh-host-group/ssh-host-group-types.ts +++ b/backend/src/ee/services/ssh-host-group/ssh-host-group-types.ts @@ -9,12 +9,7 @@ export type TCreateSshHostGroupDTO = { export type TUpdateSshHostGroupDTO = { sshHostGroupId: string; name?: string; - loginMappings?: { - loginUser: string; - allowedPrincipals: { - usernames: string[]; - }; - }[]; + loginMappings?: TLoginMapping[]; } & Omit; export type TGetSshHostGroupDTO = { diff --git a/backend/src/ee/services/ssh-host/ssh-host-dal.ts b/backend/src/ee/services/ssh-host/ssh-host-dal.ts index e66f7da7a..3b8564ce2 100644 --- a/backend/src/ee/services/ssh-host/ssh-host-dal.ts +++ b/backend/src/ee/services/ssh-host/ssh-host-dal.ts @@ -31,8 +31,18 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUser}.id`, `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) + .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SshHostLoginUserMapping}.userId`) + .leftJoin( + TableName.UserGroupMembership, + `${TableName.UserGroupMembership}.groupId`, + `${TableName.SshHostLoginUserMapping}.groupId` + ) .whereIn(`${TableName.SshHost}.projectId`, projectIds) - .andWhere(`${TableName.SshHostLoginUserMapping}.userId`, userId) + .andWhere((bd) => { + void bd + .where(`${TableName.SshHostLoginUserMapping}.userId`, userId) + .orWhere(`${TableName.UserGroupMembership}.userId`, userId); + }) .select( db.ref("id").withSchema(TableName.SshHost).as("sshHostId"), db.ref("projectId").withSchema(TableName.SshHost), @@ -58,8 +68,17 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .join(TableName.SshHost, `${TableName.SshHostGroupMembership}.sshHostId`, `${TableName.SshHost}.id`) + .leftJoin( + TableName.UserGroupMembership, + `${TableName.UserGroupMembership}.groupId`, + `${TableName.SshHostLoginUserMapping}.groupId` + ) .whereIn(`${TableName.SshHost}.projectId`, projectIds) - .andWhere(`${TableName.SshHostLoginUserMapping}.userId`, userId) + .andWhere((bd) => { + void bd + .where(`${TableName.SshHostLoginUserMapping}.userId`, userId) + .orWhere(`${TableName.UserGroupMembership}.userId`, userId); + }) .select( db.ref("id").withSchema(TableName.SshHost).as("sshHostId"), db.ref("projectId").withSchema(TableName.SshHost), @@ -133,6 +152,7 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .where(`${TableName.SshHost}.projectId`, projectId) .select( db.ref("id").withSchema(TableName.SshHost).as("sshHostId"), @@ -144,6 +164,7 @@ export const sshHostDALFactory = (db: TDbClient) => { db.ref("loginUser").withSchema(TableName.SshHostLoginUser), db.ref("username").withSchema(TableName.Users), db.ref("userId").withSchema(TableName.SshHostLoginUserMapping), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug"), db.ref("userSshCaId").withSchema(TableName.SshHost), db.ref("hostSshCaId").withSchema(TableName.SshHost) ) @@ -163,10 +184,12 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .select( db.ref("sshHostId").withSchema(TableName.SshHostGroupMembership), db.ref("loginUser").withSchema(TableName.SshHostLoginUser), - db.ref("username").withSchema(TableName.Users) + db.ref("username").withSchema(TableName.Users), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ) .whereIn(`${TableName.SshHostGroupMembership}.sshHostId`, hostIds); @@ -185,7 +208,8 @@ export const sshHostDALFactory = (db: TDbClient) => { const directMappings = Object.entries(loginMappingGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) }, source: LoginMappingSource.HOST })); @@ -197,7 +221,8 @@ export const sshHostDALFactory = (db: TDbClient) => { const groupMappings = Object.entries(inheritedGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) }, source: LoginMappingSource.HOST_GROUP })); @@ -229,6 +254,7 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .where(`${TableName.SshHost}.id`, sshHostId) .select( db.ref("id").withSchema(TableName.SshHost).as("sshHostId"), @@ -241,7 +267,8 @@ export const sshHostDALFactory = (db: TDbClient) => { db.ref("username").withSchema(TableName.Users), db.ref("userId").withSchema(TableName.SshHostLoginUserMapping), db.ref("userSshCaId").withSchema(TableName.SshHost), - db.ref("hostSshCaId").withSchema(TableName.SshHost) + db.ref("hostSshCaId").withSchema(TableName.SshHost), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ); if (rows.length === 0) return null; @@ -257,7 +284,8 @@ export const sshHostDALFactory = (db: TDbClient) => { const directMappings = Object.entries(directGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) }, source: LoginMappingSource.HOST })); @@ -275,10 +303,12 @@ export const sshHostDALFactory = (db: TDbClient) => { `${TableName.SshHostLoginUserMapping}.sshHostLoginUserId` ) .leftJoin(TableName.Users, `${TableName.SshHostLoginUserMapping}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.Groups, `${TableName.SshHostLoginUserMapping}.groupId`, `${TableName.Groups}.id`) .where(`${TableName.SshHostGroupMembership}.sshHostId`, sshHostId) .select( db.ref("loginUser").withSchema(TableName.SshHostLoginUser), - db.ref("username").withSchema(TableName.Users) + db.ref("username").withSchema(TableName.Users), + db.ref("slug").withSchema(TableName.Groups).as("groupSlug") ); const groupGrouped = groupBy( @@ -289,7 +319,8 @@ export const sshHostDALFactory = (db: TDbClient) => { const groupMappings = Object.entries(groupGrouped).map(([loginUser, entries]) => ({ loginUser, allowedPrincipals: { - usernames: unique(entries.map((e) => e.username)).filter(Boolean) + usernames: unique(entries.map((e) => e.username)).filter(Boolean), + groups: unique(entries.map((e) => e.groupSlug)).filter(Boolean) }, source: LoginMappingSource.HOST_GROUP })); diff --git a/backend/src/ee/services/ssh-host/ssh-host-fns.ts b/backend/src/ee/services/ssh-host/ssh-host-fns.ts index 9b9ce2642..dec15e093 100644 --- a/backend/src/ee/services/ssh-host/ssh-host-fns.ts +++ b/backend/src/ee/services/ssh-host/ssh-host-fns.ts @@ -3,6 +3,7 @@ import { Knex } from "knex"; import { ActionProjectType } from "@app/db/schemas"; import { BadRequestError } from "@app/lib/errors"; +import { ProjectPermissionSshHostActions, ProjectPermissionSub } from "../permission/project-permission"; import { TCreateSshLoginMappingsDTO } from "./ssh-host-types"; /** @@ -15,6 +16,7 @@ export const createSshLoginMappings = async ({ loginMappings, sshHostLoginUserDAL, sshHostLoginUserMappingDAL, + groupDAL, userDAL, permissionService, projectId, @@ -35,7 +37,7 @@ export const createSshLoginMappings = async ({ tx ); - if (allowedPrincipals.usernames.length > 0) { + if (allowedPrincipals.usernames && allowedPrincipals.usernames.length > 0) { const users = await userDAL.find( { $in: { @@ -74,6 +76,41 @@ export const createSshLoginMappings = async ({ tx ); } + + if (allowedPrincipals.groups && allowedPrincipals.groups.length > 0) { + const projectGroups = await groupDAL.findGroupsByProjectId(projectId); + const groups = projectGroups.filter((g) => allowedPrincipals.groups?.includes(g.slug)); + + if (groups.length !== allowedPrincipals.groups?.length) { + throw new BadRequestError({ + message: `Invalid group slugs: ${allowedPrincipals.groups + .filter((g) => !projectGroups.some((pg) => pg.slug === g)) + .join(", ")}` + }); + } + + for await (const group of groups) { + // check that each group has access to the SSH project and have read access to hosts + const hasPermission = await permissionService.checkGroupProjectPermission({ + groupId: group.id, + projectId, + checkPermissions: [ProjectPermissionSshHostActions.Read, ProjectPermissionSub.SshHosts] + }); + if (!hasPermission) { + throw new BadRequestError({ + message: `Group ${group.slug} does not have access to the SSH project` + }); + } + } + + await sshHostLoginUserMappingDAL.insertMany( + groups.map((group) => ({ + sshHostLoginUserId: sshHostLoginUser.id, + groupId: group.id + })), + tx + ); + } } }; diff --git a/backend/src/ee/services/ssh-host/ssh-host-schema.ts b/backend/src/ee/services/ssh-host/ssh-host-schema.ts index a9b674991..c8acb37bf 100644 --- a/backend/src/ee/services/ssh-host/ssh-host-schema.ts +++ b/backend/src/ee/services/ssh-host/ssh-host-schema.ts @@ -15,7 +15,24 @@ export const sanitizedSshHost = SshHostsSchema.pick({ export const loginMappingSchema = z.object({ loginUser: z.string().trim(), - allowedPrincipals: z.object({ - usernames: z.array(z.string().trim()).transform((usernames) => Array.from(new Set(usernames))) - }) + allowedPrincipals: z + .object({ + usernames: z + .array(z.string().trim()) + .transform((usernames) => Array.from(new Set(usernames))) + .optional(), + groups: z + .array(z.string().trim()) + .transform((groups) => Array.from(new Set(groups))) + .optional() + }) + .refine( + (data) => { + return (data.usernames && data.usernames.length > 0) || (data.groups && data.groups.length > 0); + }, + { + message: "At least one username or group must be provided", + path: ["allowedPrincipals"] + } + ) }); diff --git a/backend/src/ee/services/ssh-host/ssh-host-service.ts b/backend/src/ee/services/ssh-host/ssh-host-service.ts index 87f4862bb..79c74cd57 100644 --- a/backend/src/ee/services/ssh-host/ssh-host-service.ts +++ b/backend/src/ee/services/ssh-host/ssh-host-service.ts @@ -1,6 +1,7 @@ import { ForbiddenError, subject } from "@casl/ability"; import { ActionProjectType, ProjectType } from "@app/db/schemas"; +import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionSshHostActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TSshCertificateAuthorityDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-dal"; @@ -19,6 +20,7 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectSshConfigDALFactory } from "@app/services/project/project-ssh-config-dal"; import { TUserDALFactory } from "@app/services/user/user-dal"; +import { TUserGroupMembershipDALFactory } from "../group/user-group-membership-dal"; import { convertActorToPrincipals, createSshCert, @@ -39,12 +41,14 @@ import { type TSshHostServiceFactoryDep = { userDAL: Pick; + groupDAL: Pick; projectDAL: Pick; projectSshConfigDAL: Pick; sshCertificateAuthorityDAL: Pick; sshCertificateAuthoritySecretDAL: Pick; sshCertificateDAL: Pick; sshCertificateBodyDAL: Pick; + userGroupMembershipDAL: Pick; sshHostDAL: Pick< TSshHostDALFactory, | "transaction" @@ -58,7 +62,10 @@ type TSshHostServiceFactoryDep = { >; sshHostLoginUserDAL: TSshHostLoginUserDALFactory; sshHostLoginUserMappingDAL: TSshHostLoginUserMappingDALFactory; - permissionService: Pick; + permissionService: Pick< + TPermissionServiceFactory, + "getProjectPermission" | "getUserProjectPermission" | "checkGroupProjectPermission" + >; kmsService: Pick; }; @@ -66,6 +73,8 @@ export type TSshHostServiceFactory = ReturnType; export const sshHostServiceFactory = ({ userDAL, + userGroupMembershipDAL, + groupDAL, projectDAL, projectSshConfigDAL, sshCertificateAuthorityDAL, @@ -208,6 +217,7 @@ export const sshHostServiceFactory = ({ loginMappings, sshHostLoginUserDAL, sshHostLoginUserMappingDAL, + groupDAL, userDAL, permissionService, projectId, @@ -278,6 +288,7 @@ export const sshHostServiceFactory = ({ loginMappings, sshHostLoginUserDAL, sshHostLoginUserMappingDAL, + groupDAL, userDAL, permissionService, projectId: host.projectId, @@ -387,10 +398,14 @@ export const sshHostServiceFactory = ({ userDAL }); + const userGroups = await userGroupMembershipDAL.findGroupMembershipsByUserIdInOrg(actorId, actorOrgId); + const userGroupSlugs = userGroups.map((g) => g.groupSlug); + const mapping = host.loginMappings.find( (m) => m.loginUser === loginUser && - m.allowedPrincipals.usernames.some((allowed) => internalPrincipals.includes(allowed)) + (m.allowedPrincipals.usernames?.some((allowed) => internalPrincipals.includes(allowed)) || + m.allowedPrincipals.groups?.some((allowed) => userGroupSlugs.includes(allowed))) ); if (!mapping) { diff --git a/backend/src/ee/services/ssh-host/ssh-host-types.ts b/backend/src/ee/services/ssh-host/ssh-host-types.ts index 9846920b7..c0a780fbb 100644 --- a/backend/src/ee/services/ssh-host/ssh-host-types.ts +++ b/backend/src/ee/services/ssh-host/ssh-host-types.ts @@ -7,12 +7,15 @@ import { TProjectPermission } from "@app/lib/types"; import { ActorAuthMethod } from "@app/services/auth/auth-type"; import { TUserDALFactory } from "@app/services/user/user-dal"; +import { TGroupDALFactory } from "../group/group-dal"; + export type TListSshHostsDTO = Omit; export type TLoginMapping = { loginUser: string; allowedPrincipals: { - usernames: string[]; + usernames?: string[]; + groups?: string[]; }; }; @@ -63,7 +66,8 @@ type BaseCreateSshLoginMappingsDTO = { sshHostLoginUserDAL: Pick; sshHostLoginUserMappingDAL: Pick; userDAL: Pick; - permissionService: Pick; + permissionService: Pick; + groupDAL: Pick; projectId: string; actorAuthMethod: ActorAuthMethod; actorOrgId: string; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index ac28e9ade..6da6c4fa4 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,6 +1,8 @@ import { Redis } from "ioredis"; import { pgAdvisoryLockHashText } from "@app/lib/crypto/hashtext"; +import { applyJitter } from "@app/lib/dates"; +import { delay as delayMs } from "@app/lib/delay"; import { Redlock, Settings } from "@app/lib/red-lock"; export const PgSqlLock = { @@ -48,6 +50,13 @@ export const KeyStoreTtls = { AccessTokenStatusUpdateInSeconds: 120 }; +type TDeleteItems = { + pattern: string; + batchSize?: number; + delay?: number; + jitter?: number; +}; + type TWaitTillReady = { key: string; waitingCb?: () => void; @@ -75,6 +84,35 @@ export const keyStoreFactory = (redisUrl: string) => { const deleteItem = async (key: string) => redis.del(key); + const deleteItems = async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }: TDeleteItems) => { + let cursor = "0"; + let totalDeleted = 0; + + do { + // Await in loop is needed so that Redis is not overwhelmed + // eslint-disable-next-line no-await-in-loop + const [nextCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); // Count should be 1000 - 5000 for prod loads + cursor = nextCursor; + + for (let i = 0; i < keys.length; i += batchSize) { + const batch = keys.slice(i, i + batchSize); + const pipeline = redis.pipeline(); + for (const key of batch) { + pipeline.unlink(key); + } + // eslint-disable-next-line no-await-in-loop + await pipeline.exec(); + totalDeleted += batch.length; + console.log("BATCH DONE"); + + // eslint-disable-next-line no-await-in-loop + await delayMs(Math.max(0, applyJitter(delay, jitter))); + } + } while (cursor !== "0"); + + return totalDeleted; + }; + const incrementBy = async (key: string, value: number) => redis.incrby(key, value); const setExpiry = async (key: string, expiryInSeconds: number) => redis.expire(key, expiryInSeconds); @@ -94,7 +132,7 @@ export const keyStoreFactory = (redisUrl: string) => { // eslint-disable-next-line await new Promise((resolve) => { waitingCb?.(); - setTimeout(resolve, Math.max(0, delay + Math.floor((Math.random() * 2 - 1) * jitter))); + setTimeout(resolve, Math.max(0, applyJitter(delay, jitter))); }); attempts += 1; // eslint-disable-next-line @@ -108,6 +146,7 @@ export const keyStoreFactory = (redisUrl: string) => { setExpiry, setItemWithExpiry, deleteItem, + deleteItems, incrementBy, acquireLock(resources: string[], duration: number, settings?: Partial) { return redisLock.acquire(resources, duration, settings); diff --git a/backend/src/keystore/memory.ts b/backend/src/keystore/memory.ts index 10b28ffec..84cd06c03 100644 --- a/backend/src/keystore/memory.ts +++ b/backend/src/keystore/memory.ts @@ -1,3 +1,7 @@ +import RE2 from "re2"; + +import { applyJitter } from "@app/lib/dates"; +import { delay as delayMs } from "@app/lib/delay"; import { Lock } from "@app/lib/red-lock"; import { TKeyStoreFactory } from "./keystore"; @@ -19,6 +23,27 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => { delete store[key]; return 1; }, + deleteItems: async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }) => { + const regex = new RE2(`^${pattern.replace(/[-[\]/{}()+?.\\^$|]/g, "\\$&").replace(/\*/g, ".*")}$`); + let totalDeleted = 0; + const keys = Object.keys(store); + + for (let i = 0; i < keys.length; i += batchSize) { + const batch = keys.slice(i, i + batchSize); + + for (const key of batch) { + if (regex.test(key)) { + delete store[key]; + totalDeleted += 1; + } + } + + // eslint-disable-next-line no-await-in-loop + await delayMs(Math.max(0, applyJitter(delay, jitter))); + } + + return totalDeleted; + }, getItem: async (key) => { const value = store[key]; if (typeof value === "string") { diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 6ecb748c8..1a5db1315 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -18,6 +18,7 @@ export enum ApiDocsTags { KubernetesAuth = "Kubernetes Auth", JwtAuth = "JWT Auth", OidcAuth = "OIDC Auth", + LdapAuth = "LDAP Auth", Groups = "Groups", Organizations = "Organizations", Projects = "Projects", @@ -184,6 +185,49 @@ export const UNIVERSAL_AUTH = { } } as const; +export const LDAP_AUTH = { + LOGIN: { + identityId: "The ID of the identity to login.", + username: "The username of the LDAP user to login.", + password: "The password of the LDAP user to login." + }, + ATTACH: { + identityId: "The ID of the identity to attach the configuration onto.", + url: "The URL of the LDAP server.", + allowedFields: + "The comma-separated array of key/value pairs of required fields that the LDAP entry must have in order to authenticate.", + searchBase: "The base DN to search for the LDAP user.", + searchFilter: "The filter to use to search for the LDAP user.", + bindDN: "The DN of the user to bind to the LDAP server.", + bindPass: "The password of the user to bind to the LDAP server.", + ldapCaCertificate: "The PEM-encoded CA certificate for the LDAP server.", + accessTokenTTL: "The lifetime for an access token in seconds.", + accessTokenMaxTTL: "The maximum lifetime for an access token in seconds.", + accessTokenNumUsesLimit: "The maximum number of times that an access token can be used.", + accessTokenTrustedIps: "The IPs or CIDR ranges that access tokens can be used from." + }, + UPDATE: { + identityId: "The ID of the identity to update the configuration for.", + url: "The new URL of the LDAP server.", + allowedFields: "The comma-separated list of allowed fields to return from the LDAP user.", + searchBase: "The new base DN to search for the LDAP user.", + searchFilter: "The new filter to use to search for the LDAP user.", + bindDN: "The new DN of the user to bind to the LDAP server.", + bindPass: "The new password of the user to bind to the LDAP server.", + ldapCaCertificate: "The new PEM-encoded CA certificate for the LDAP server.", + accessTokenTTL: "The new lifetime for an access token in seconds.", + accessTokenMaxTTL: "The new maximum lifetime for an access token in seconds.", + accessTokenNumUsesLimit: "The new maximum number of times that an access token can be used.", + accessTokenTrustedIps: "The new IPs or CIDR ranges that access tokens can be used from." + }, + RETRIEVE: { + identityId: "The ID of the identity to retrieve the configuration for." + }, + REVOKE: { + identityId: "The ID of the identity to revoke the configuration for." + } +} as const; + export const AWS_AUTH = { LOGIN: { identityId: "The ID of the identity to login.", @@ -1434,7 +1478,7 @@ export const SSH_HOSTS = { loginUser: "A login user on the remote machine (e.g. 'ec2-user', 'deploy', 'admin')", allowedPrincipals: "A list of allowed principals that can log in as the login user.", loginMappings: - "A list of login mappings for the SSH host. Each login mapping contains a login user and a list of corresponding allowed principals being usernames of users in the Infisical SSH project.", + "A list of login mappings for the SSH host. Each login mapping contains a login user and a list of corresponding allowed principals being usernames of users or groups slugs in the Infisical SSH project.", userSshCaId: "The ID of the SSH CA to use for user certificates. If not specified, the default user SSH CA will be used if it exists.", hostSshCaId: @@ -1449,7 +1493,7 @@ export const SSH_HOSTS = { loginUser: "A login user on the remote machine (e.g. 'ec2-user', 'deploy', 'admin')", allowedPrincipals: "A list of allowed principals that can log in as the login user.", loginMappings: - "A list of login mappings for the SSH host. Each login mapping contains a login user and a list of corresponding allowed principals being usernames of users in the Infisical SSH project." + "A list of login mappings for the SSH host. Each login mapping contains a login user and a list of corresponding allowed principals being usernames of users or groups slugs in the Infisical SSH project." }, DELETE: { sshHostId: "The ID of the SSH host to delete." @@ -1835,8 +1879,12 @@ export const KMS = { }; export const ProjectTemplates = { + LIST: { + type: "The type of project template to list." + }, CREATE: { name: "The name of the project template to be created. Must be slug-friendly.", + type: "The type of project template to be created.", description: "An optional description of the project template.", roles: "The roles to be created when the template is applied to a project.", environments: "The environments to be created when the template is applied to a project." diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 907884433..e38dbcfb5 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -146,6 +146,7 @@ const envSchema = z SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()), SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional()), SECRET_SCANNING_ORG_WHITELIST: zpStr(z.string().optional()), + SECRET_SCANNING_GIT_APP_SLUG: zpStr(z.string().default("infisical-radar")), // LICENSE LICENSE_SERVER_URL: zpStr(z.string().optional().default("https://portal.infisical.com")), LICENSE_SERVER_KEY: zpStr(z.string().optional()), diff --git a/backend/src/lib/delay/index.ts b/backend/src/lib/delay/index.ts new file mode 100644 index 000000000..32cb8ebfc --- /dev/null +++ b/backend/src/lib/delay/index.ts @@ -0,0 +1,4 @@ +export const delay = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); diff --git a/backend/src/lib/logger/logger.ts b/backend/src/lib/logger/logger.ts index 170a0285f..afde8ef97 100644 --- a/backend/src/lib/logger/logger.ts +++ b/backend/src/lib/logger/logger.ts @@ -84,7 +84,9 @@ const redactedKeys = [ "secrets", "key", "password", - "config" + "config", + "bindPass", + "bindDN" ]; const UNKNOWN_REQUEST_ID = "UNKNOWN_REQUEST_ID"; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index ae1a3e821..807d6c286 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -25,6 +25,7 @@ import { TQueueSecretSyncSyncSecretsByIdDTO, TQueueSendSecretSyncActionFailedNotificationsDTO } from "@app/services/secret-sync/secret-sync-types"; +import { CacheType } from "@app/services/super-admin/super-admin-types"; import { TWebhookPayloads } from "@app/services/webhook/webhook-types"; export enum QueueName { @@ -49,7 +50,8 @@ export enum QueueName { AccessTokenStatusUpdate = "access-token-status-update", ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", - SecretRotationV2 = "secret-rotation-v2" + SecretRotationV2 = "secret-rotation-v2", + InvalidateCache = "invalidate-cache" } export enum QueueJobs { @@ -81,7 +83,8 @@ export enum QueueJobs { SecretSyncSendActionFailedNotifications = "secret-sync-send-action-failed-notifications", SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", - SecretRotationV2SendNotification = "secret-rotation-v2-send-notification" + SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", + InvalidateCache = "invalidate-cache" } export type TQueueJobTypes = { @@ -234,6 +237,14 @@ export type TQueueJobTypes = { name: QueueJobs.SecretRotationV2SendNotification; payload: TSecretRotationSendNotificationJobPayload; }; + [QueueName.InvalidateCache]: { + name: QueueJobs.InvalidateCache; + payload: { + data: { + type: CacheType; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 681442d1b..b12c9b0d3 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -100,3 +100,18 @@ export const publicSshCaLimit: RateLimitOptions = { max: 30, // conservative default keyGenerator: (req) => req.realIp }; + +export const invalidateCacheLimit: RateLimitOptions = { + timeWindow: 60 * 1000, + hook: "preValidation", + max: 2, + keyGenerator: (req) => req.realIp +}; + +// Makes spamming "request access" harder, preventing email DDoS +export const requestAccessLimit: RateLimitOptions = { + timeWindow: 60 * 1000, + hook: "preValidation", + max: 10, + keyGenerator: (req) => req.realIp +}; diff --git a/backend/src/server/plugins/fastify-zod.ts b/backend/src/server/plugins/fastify-zod.ts index 4e898a9b5..f9b9f4f59 100644 --- a/backend/src/server/plugins/fastify-zod.ts +++ b/backend/src/server/plugins/fastify-zod.ts @@ -5,7 +5,7 @@ import type { FastifySchema, FastifySchemaCompiler, FastifyTypeProvider } from "fastify"; import type { FastifySerializerCompiler } from "fastify/types/schema"; import type { z, ZodAny, ZodTypeAny } from "zod"; -import { zodToJsonSchema } from "zod-to-json-schema"; +import { PostProcessCallback, zodToJsonSchema } from "zod-to-json-schema"; // eslint-disable-next-line @typescript-eslint/no-explicit-any type FreeformRecord = Record; @@ -28,9 +28,25 @@ interface Schema extends FastifySchema { hide?: boolean; } +// Credit: https://github.com/StefanTerdell/zod-to-json-schema +const jsonDescription: PostProcessCallback = (jsonSchema, def) => { + if (def.description) { + try { + return { + ...jsonSchema, + description: undefined, + ...JSON.parse(def.description) + }; + } catch {} + } + + return jsonSchema; +}; + const zodToJsonSchemaOptions = { target: "openApi3", - $refStrategy: "none" + $refStrategy: "none", + postProcess: jsonDescription } as const; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index a0b6bb5bf..5f415564e 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -160,6 +160,8 @@ import { identityJwtAuthDALFactory } from "@app/services/identity-jwt-auth/ident import { identityJwtAuthServiceFactory } from "@app/services/identity-jwt-auth/identity-jwt-auth-service"; import { identityKubernetesAuthDALFactory } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-dal"; import { identityKubernetesAuthServiceFactory } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-service"; +import { identityLdapAuthDALFactory } from "@app/services/identity-ldap-auth/identity-ldap-auth-dal"; +import { identityLdapAuthServiceFactory } from "@app/services/identity-ldap-auth/identity-ldap-auth-service"; import { identityOidcAuthDALFactory } from "@app/services/identity-oidc-auth/identity-oidc-auth-dal"; import { identityOidcAuthServiceFactory } from "@app/services/identity-oidc-auth/identity-oidc-auth-service"; import { identityProjectDALFactory } from "@app/services/identity-project/identity-project-dal"; @@ -242,6 +244,7 @@ import { projectSlackConfigDALFactory } from "@app/services/slack/project-slack- import { slackIntegrationDALFactory } from "@app/services/slack/slack-integration-dal"; import { slackServiceFactory } from "@app/services/slack/slack-service"; import { TSmtpService } from "@app/services/smtp/smtp-service"; +import { invalidateCacheQueueFactory } from "@app/services/super-admin/invalidate-cache-queue"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service"; import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal"; @@ -353,6 +356,7 @@ export const registerRoutes = async ( const identityOidcAuthDAL = identityOidcAuthDALFactory(db); const identityJwtAuthDAL = identityJwtAuthDALFactory(db); const identityAzureAuthDAL = identityAzureAuthDALFactory(db); + const identityLdapAuthDAL = identityLdapAuthDALFactory(db); const auditLogDAL = auditLogDALFactory(auditLogDb ?? db); const auditLogStreamDAL = auditLogStreamDALFactory(db); @@ -611,6 +615,11 @@ export const registerRoutes = async ( queueService }); + const invalidateCacheQueue = invalidateCacheQueueFactory({ + keyStore, + queueService + }); + const userService = userServiceFactory({ userDAL, userAliasDAL, @@ -722,7 +731,8 @@ export const registerRoutes = async ( keyStore, licenseService, kmsService, - microsoftTeamsService + microsoftTeamsService, + invalidateCacheQueue }); const orgAdminService = orgAdminServiceFactory({ @@ -862,6 +872,8 @@ export const registerRoutes = async ( const sshHostService = sshHostServiceFactory({ userDAL, + groupDAL, + userGroupMembershipDAL, projectDAL, projectSshConfigDAL, sshCertificateAuthorityDAL, @@ -884,7 +896,8 @@ export const registerRoutes = async ( sshHostLoginUserMappingDAL, userDAL, permissionService, - licenseService + licenseService, + groupDAL }); const certificateAuthorityService = certificateAuthorityServiceFactory({ @@ -1440,6 +1453,16 @@ export const registerRoutes = async ( kmsService }); + const identityLdapAuthService = identityLdapAuthServiceFactory({ + identityLdapAuthDAL, + permissionService, + kmsService, + identityAccessTokenDAL, + identityOrgMembershipDAL, + licenseService, + identityDAL + }); + const gatewayService = gatewayServiceFactory({ permissionService, gatewayDAL, @@ -1700,6 +1723,7 @@ export const registerRoutes = async ( identityAzureAuth: identityAzureAuthService, identityOidcAuth: identityOidcAuthService, identityJwtAuth: identityJwtAuthService, + identityLdapAuth: identityLdapAuthService, accessApprovalPolicy: accessApprovalPolicyService, accessApprovalRequest: accessApprovalRequestService, secretApprovalPolicy: secretApprovalPolicyService, @@ -1765,6 +1789,10 @@ export const registerRoutes = async ( if (licenseSyncJob) { cronJobs.push(licenseSyncJob); } + const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync(); + if (microsoftTeamsSyncJob) { + cronJobs.push(microsoftTeamsSyncJob); + } } server.decorate("store", { diff --git a/backend/src/server/routes/v1/admin-router.ts b/backend/src/server/routes/v1/admin-router.ts index a55aa2ba4..8610a611b 100644 --- a/backend/src/server/routes/v1/admin-router.ts +++ b/backend/src/server/routes/v1/admin-router.ts @@ -4,13 +4,14 @@ import { z } from "zod"; import { IdentitiesSchema, OrganizationsSchema, SuperAdminSchema, UsersSchema } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; -import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { invalidateCacheLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { getTelemetryDistinctId } from "@app/server/lib/telemetry"; import { verifySuperAdmin } from "@app/server/plugins/auth/superAdmin"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { RootKeyEncryptionStrategy } from "@app/services/kms/kms-types"; import { getServerCfg } from "@app/services/super-admin/super-admin-service"; -import { LoginMethod } from "@app/services/super-admin/super-admin-types"; +import { CacheType, LoginMethod } from "@app/services/super-admin/super-admin-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; export const registerAdminRouter = async (server: FastifyZodProvider) => { @@ -548,4 +549,69 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { }; } }); + + server.route({ + method: "POST", + url: "/invalidate-cache", + config: { + rateLimit: invalidateCacheLimit + }, + schema: { + body: z.object({ + type: z.nativeEnum(CacheType) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: (req, res, done) => { + verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN])(req, res, () => { + verifySuperAdmin(req, res, done); + }); + }, + handler: async (req) => { + await server.services.superAdmin.invalidateCache(req.body.type); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.InvalidateCache, + distinctId: getTelemetryDistinctId(req), + properties: { + ...req.auditLogInfo + } + }); + + return { + message: "Cache invalidation job started" + }; + } + }); + + server.route({ + method: "GET", + url: "/invalidating-cache-status", + config: { + rateLimit: readLimit + }, + schema: { + response: { + 200: z.object({ + invalidating: z.boolean() + }) + } + }, + onRequest: (req, res, done) => { + verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN])(req, res, () => { + verifySuperAdmin(req, res, done); + }); + }, + handler: async () => { + const invalidating = await server.services.superAdmin.checkIfInvalidatingCache(); + + return { + invalidating + }; + } + }); }; diff --git a/backend/src/server/routes/v1/identity-ldap-auth-router.ts b/backend/src/server/routes/v1/identity-ldap-auth-router.ts new file mode 100644 index 000000000..3da8a425b --- /dev/null +++ b/backend/src/server/routes/v1/identity-ldap-auth-router.ts @@ -0,0 +1,497 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +// All the any rules are disabled because passport typesense with fastify is really poor + +import { Authenticator } from "@fastify/passport"; +import fastifySession from "@fastify/session"; +import { FastifyRequest } from "fastify"; +import { IncomingMessage } from "http"; +import LdapStrategy from "passport-ldapauth"; +import { z } from "zod"; + +import { IdentityLdapAuthsSchema } from "@app/db/schemas/identity-ldap-auths"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { isValidLdapFilter } from "@app/ee/services/ldap-config/ldap-fns"; +import { ApiDocsTags, LDAP_AUTH } from "@app/lib/api-docs"; +import { getConfig } from "@app/lib/config/env"; +import { UnauthorizedError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { TIdentityTrustedIp } from "@app/services/identity/identity-types"; +import { AllowedFieldsSchema } from "@app/services/identity-ldap-auth/identity-ldap-auth-types"; +import { isSuperAdmin } from "@app/services/super-admin/super-admin-fns"; + +export const registerIdentityLdapAuthRouter = async (server: FastifyZodProvider) => { + const appCfg = getConfig(); + const passport = new Authenticator({ key: "ldap-identity-auth", userProperty: "passportMachineIdentity" }); + await server.register(fastifySession, { secret: appCfg.COOKIE_SECRET_SIGN_KEY }); + await server.register(passport.initialize()); + await server.register(passport.secureSession()); + + const getLdapPassportOpts = (req: FastifyRequest, done: any) => { + const { identityId } = req.body as { + identityId: string; + }; + + process.nextTick(async () => { + try { + const { ldapConfig, opts } = await server.services.identityLdapAuth.getLdapConfig(identityId); + req.ldapConfig = { + ...ldapConfig, + isActive: true, + groupSearchBase: "", + uniqueUserAttribute: "", + groupSearchFilter: "" + }; + + done(null, opts); + } catch (err) { + logger.error(err, "Error in LDAP verification callback"); + done(err); + } + }); + }; + + passport.use( + new LdapStrategy( + getLdapPassportOpts as any, + // eslint-disable-next-line + async (req: IncomingMessage, user, cb) => { + try { + const requestBody = (req as unknown as FastifyRequest).body as { + username: string; + password: string; + identityId: string; + }; + + if (!requestBody.username || !requestBody.password) { + return cb(new UnauthorizedError({ message: "Invalid request. Missing username or password." }), false); + } + + if (!requestBody.identityId) { + return cb(new UnauthorizedError({ message: "Invalid request. Missing identity ID." }), false); + } + + const { ldapConfig } = req as unknown as FastifyRequest; + + if (ldapConfig.allowedFields) { + for (const field of ldapConfig.allowedFields) { + if (!user[field.key]) { + return cb( + new UnauthorizedError({ message: `Invalid request. Missing field ${field.key} on user.` }), + false + ); + } + + const value = field.value.split(","); + + if (!value.includes(user[field.key])) { + return cb( + new UnauthorizedError({ + message: `Invalid request. User field '${field.key}' does not match required fields.` + }), + false + ); + } + } + } + + return cb(null, { identityId: requestBody.identityId, user }); + } catch (error) { + logger.error(error, "Error in LDAP verification callback"); + return cb(error, false); + } + } + ) + ); + + server.route({ + method: "POST", + url: "/ldap-auth/login", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.LdapAuth], + description: "Login with LDAP Auth", + body: z.object({ + identityId: z.string().trim().describe(LDAP_AUTH.LOGIN.identityId), + username: z.string().describe(LDAP_AUTH.LOGIN.username), + password: z.string().describe(LDAP_AUTH.LOGIN.password) + }), + response: { + 200: z.object({ + accessToken: z.string(), + expiresIn: z.coerce.number(), + accessTokenMaxTTL: z.coerce.number(), + tokenType: z.literal("Bearer") + }) + } + }, + preValidation: passport.authenticate("ldapauth", { + failWithError: true, + session: false + }) as any, + + errorHandler: (error) => { + if (error.name === "AuthenticationError") { + throw new UnauthorizedError({ message: "Invalid credentials" }); + } + + throw error; + }, + + handler: async (req) => { + if (!req.passportMachineIdentity?.identityId) { + throw new UnauthorizedError({ message: "Invalid request. Missing identity ID or LDAP entry details." }); + } + + const { identityId, user } = req.passportMachineIdentity; + + const { accessToken, identityLdapAuth, identityMembershipOrg } = await server.services.identityLdapAuth.login({ + identityId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: identityMembershipOrg?.orgId, + event: { + type: EventType.LOGIN_IDENTITY_LDAP_AUTH, + metadata: { + identityId, + ldapEmail: user.mail, + ldapUsername: user.uid + } + } + }); + + return { + accessToken, + tokenType: "Bearer" as const, + expiresIn: identityLdapAuth.accessTokenTTL, + accessTokenMaxTTL: identityLdapAuth.accessTokenMaxTTL + }; + } + }); + + server.route({ + method: "POST", + url: "/ldap-auth/identities/:identityId", + config: { + rateLimit: writeLimit + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + schema: { + hide: false, + tags: [ApiDocsTags.LdapAuth], + description: "Attach LDAP Auth configuration onto identity", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + identityId: z.string().trim().describe(LDAP_AUTH.ATTACH.identityId) + }), + body: z + .object({ + url: z.string().trim().min(1).describe(LDAP_AUTH.ATTACH.url), + bindDN: z.string().trim().min(1).describe(LDAP_AUTH.ATTACH.bindDN), + bindPass: z.string().trim().min(1).describe(LDAP_AUTH.ATTACH.bindPass), + searchBase: z.string().trim().min(1).describe(LDAP_AUTH.ATTACH.searchBase), + searchFilter: z + .string() + .trim() + .min(1) + .default("(uid={{username}})") + .refine(isValidLdapFilter, "Invalid LDAP search filter") + .describe(LDAP_AUTH.ATTACH.searchFilter), + allowedFields: AllowedFieldsSchema.array().optional().describe(LDAP_AUTH.ATTACH.allowedFields), + ldapCaCertificate: z.string().trim().optional().describe(LDAP_AUTH.ATTACH.ldapCaCertificate), + accessTokenTrustedIps: z + .object({ + ipAddress: z.string().trim() + }) + .array() + .min(1) + .default([{ ipAddress: "0.0.0.0/0" }, { ipAddress: "::/0" }]) + .describe(LDAP_AUTH.ATTACH.accessTokenTrustedIps), + accessTokenTTL: z + .number() + .int() + .min(0) + .max(315360000) + .default(2592000) + .describe(LDAP_AUTH.ATTACH.accessTokenTTL), + accessTokenMaxTTL: z + .number() + .int() + .min(1) + .max(315360000) + .default(2592000) + .describe(LDAP_AUTH.ATTACH.accessTokenMaxTTL), + accessTokenNumUsesLimit: z.number().int().min(0).default(0).describe(LDAP_AUTH.ATTACH.accessTokenNumUsesLimit) + }) + .refine( + (val) => val.accessTokenTTL <= val.accessTokenMaxTTL, + "Access Token TTL cannot be greater than Access Token Max TTL." + ), + response: { + 200: z.object({ + identityLdapAuth: IdentityLdapAuthsSchema.omit({ + encryptedBindDN: true, + encryptedBindPass: true, + encryptedLdapCaCertificate: true + }) + }) + } + }, + handler: async (req) => { + const identityLdapAuth = await server.services.identityLdapAuth.attachLdapAuth({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + ...req.body, + identityId: req.params.identityId, + isActorSuperAdmin: isSuperAdmin(req.auth) + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.ADD_IDENTITY_LDAP_AUTH, + metadata: { + identityId: req.params.identityId, + url: identityLdapAuth.url, + accessTokenMaxTTL: identityLdapAuth.accessTokenMaxTTL, + accessTokenTTL: identityLdapAuth.accessTokenTTL, + accessTokenNumUsesLimit: identityLdapAuth.accessTokenNumUsesLimit, + allowedFields: req.body.allowedFields + } + } + }); + + return { identityLdapAuth }; + } + }); + + server.route({ + method: "PATCH", + url: "/ldap-auth/identities/:identityId", + config: { + rateLimit: writeLimit + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + schema: { + hide: false, + tags: [ApiDocsTags.LdapAuth], + description: "Update LDAP Auth configuration on identity", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + identityId: z.string().trim().describe(LDAP_AUTH.UPDATE.identityId) + }), + body: z + .object({ + url: z.string().trim().min(1).optional().describe(LDAP_AUTH.UPDATE.url), + bindDN: z.string().trim().min(1).optional().describe(LDAP_AUTH.UPDATE.bindDN), + bindPass: z.string().trim().min(1).optional().describe(LDAP_AUTH.UPDATE.bindPass), + searchBase: z.string().trim().min(1).optional().describe(LDAP_AUTH.UPDATE.searchBase), + searchFilter: z + .string() + .trim() + .min(1) + .optional() + .refine((v) => v === undefined || isValidLdapFilter(v), "Invalid LDAP search filter") + .describe(LDAP_AUTH.UPDATE.searchFilter), + allowedFields: AllowedFieldsSchema.array().optional().describe(LDAP_AUTH.UPDATE.allowedFields), + accessTokenTrustedIps: z + .object({ + ipAddress: z.string().trim() + }) + .array() + .min(1) + .optional() + .describe(LDAP_AUTH.UPDATE.accessTokenTrustedIps), + accessTokenTTL: z.number().int().min(0).max(315360000).optional().describe(LDAP_AUTH.UPDATE.accessTokenTTL), + accessTokenNumUsesLimit: z + .number() + .int() + .min(0) + .optional() + .describe(LDAP_AUTH.UPDATE.accessTokenNumUsesLimit), + accessTokenMaxTTL: z + .number() + .int() + .max(315360000) + .min(0) + .optional() + .describe(LDAP_AUTH.UPDATE.accessTokenMaxTTL) + }) + .refine( + (val) => (val.accessTokenMaxTTL && val.accessTokenTTL ? val.accessTokenTTL <= val.accessTokenMaxTTL : true), + "Access Token TTL cannot be greater than Access Token Max TTL." + ), + response: { + 200: z.object({ + identityLdapAuth: IdentityLdapAuthsSchema.omit({ + encryptedBindDN: true, + encryptedBindPass: true, + encryptedLdapCaCertificate: true + }) + }) + } + }, + handler: async (req) => { + const identityLdapAuth = await server.services.identityLdapAuth.updateLdapAuth({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + ...req.body, + identityId: req.params.identityId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.UPDATE_IDENTITY_LDAP_AUTH, + metadata: { + identityId: req.params.identityId, + url: identityLdapAuth.url, + accessTokenMaxTTL: identityLdapAuth.accessTokenMaxTTL, + accessTokenTTL: identityLdapAuth.accessTokenTTL, + accessTokenNumUsesLimit: identityLdapAuth.accessTokenNumUsesLimit, + accessTokenTrustedIps: identityLdapAuth.accessTokenTrustedIps as TIdentityTrustedIp[], + allowedFields: req.body.allowedFields + } + } + }); + + return { identityLdapAuth }; + } + }); + + server.route({ + method: "GET", + url: "/ldap-auth/identities/:identityId", + config: { + rateLimit: readLimit + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + schema: { + hide: false, + tags: [ApiDocsTags.LdapAuth], + description: "Retrieve LDAP Auth configuration on identity", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + identityId: z.string().trim().describe(LDAP_AUTH.RETRIEVE.identityId) + }), + response: { + 200: z.object({ + identityLdapAuth: IdentityLdapAuthsSchema.omit({ + encryptedBindDN: true, + encryptedBindPass: true, + encryptedLdapCaCertificate: true + }).extend({ + bindDN: z.string(), + bindPass: z.string(), + ldapCaCertificate: z.string().optional() + }) + }) + } + }, + handler: async (req) => { + const identityLdapAuth = await server.services.identityLdapAuth.getLdapAuth({ + identityId: req.params.identityId, + actor: req.permission.type, + actorId: req.permission.id, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.GET_IDENTITY_LDAP_AUTH, + metadata: { + identityId: identityLdapAuth.identityId + } + } + }); + + return { identityLdapAuth }; + } + }); + + server.route({ + method: "DELETE", + url: "/ldap-auth/identities/:identityId", + config: { + rateLimit: writeLimit + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + schema: { + hide: false, + tags: [ApiDocsTags.LdapAuth], + description: "Delete LDAP Auth configuration on identity", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + identityId: z.string().trim().describe(LDAP_AUTH.REVOKE.identityId) + }), + response: { + 200: z.object({ + identityLdapAuth: IdentityLdapAuthsSchema.omit({ + encryptedBindDN: true, + encryptedBindPass: true, + encryptedLdapCaCertificate: true + }) + }) + } + }, + handler: async (req) => { + const identityLdapAuth = await server.services.identityLdapAuth.revokeIdentityLdapAuth({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + identityId: req.params.identityId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.REVOKE_IDENTITY_LDAP_AUTH, + metadata: { + identityId: identityLdapAuth.identityId + } + } + }); + + return { identityLdapAuth }; + } + }); +}; diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index a50299555..b1a49f815 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -19,6 +19,7 @@ import { registerIdentityAzureAuthRouter } from "./identity-azure-auth-router"; import { registerIdentityGcpAuthRouter } from "./identity-gcp-auth-router"; import { registerIdentityJwtAuthRouter } from "./identity-jwt-auth-router"; import { registerIdentityKubernetesRouter } from "./identity-kubernetes-auth-router"; +import { registerIdentityLdapAuthRouter } from "./identity-ldap-auth-router"; import { registerIdentityOidcAuthRouter } from "./identity-oidc-auth-router"; import { registerIdentityRouter } from "./identity-router"; import { registerIdentityTokenAuthRouter } from "./identity-token-auth-router"; @@ -63,6 +64,7 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await authRouter.register(registerIdentityAzureAuthRouter); await authRouter.register(registerIdentityOidcAuthRouter); await authRouter.register(registerIdentityJwtAuthRouter); + await authRouter.register(registerIdentityLdapAuthRouter); }, { prefix: "/auth" } ); diff --git a/backend/src/server/routes/v1/org-admin-router.ts b/backend/src/server/routes/v1/org-admin-router.ts index 2d28b09bd..cc0543d4c 100644 --- a/backend/src/server/routes/v1/org-admin-router.ts +++ b/backend/src/server/routes/v1/org-admin-router.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { ProjectMembershipsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; -import { readLimit } from "@app/server/config/rateLimiter"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -47,7 +47,7 @@ export const registerOrgAdminRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/projects/:projectId/grant-admin-access", config: { - rateLimit: readLimit + rateLimit: writeLimit }, schema: { params: z.object({ diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index fdc729548..99e681764 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -19,7 +19,7 @@ import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { ApiDocsTags, PROJECTS } from "@app/lib/api-docs"; import { CharacterType, characterValidator } from "@app/lib/validator/validate-string"; import { re2Validator } from "@app/lib/zod"; -import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { readLimit, requestAccessLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { validateMicrosoftTeamsChannelsSchema } from "@app/services/microsoft-teams/microsoft-teams-fns"; @@ -1006,7 +1006,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/:workspaceId/project-access", config: { - rateLimit: writeLimit + rateLimit: requestAccessLimit }, schema: { params: z.object({ diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index a223004a9..498fb8e8b 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -170,7 +170,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { .optional() .default(InfisicalProjectTemplate.Default) .describe(PROJECTS.CREATE.template), - type: z.nativeEnum(ProjectType).default(ProjectType.SecretManager) + type: z.nativeEnum(ProjectType).default(ProjectType.SecretManager), + shouldCreateDefaultEnvs: z.boolean().optional().default(true) }), response: { 200: z.object({ @@ -190,7 +191,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { slug: req.body.slug, kmsKeyId: req.body.kmsKeyId, template: req.body.template, - type: req.body.type + type: req.body.type, + createDefaultEnvs: req.body.shouldCreateDefaultEnvs }); await server.services.telemetry.sendPostHogEvents({ @@ -272,7 +274,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }, schema: { params: z.object({ - slug: slugSchema({ min: 5, max: 36 }).describe("The slug of the project to get.") + slug: slugSchema({ max: 36 }).describe("The slug of the project to get.") }), response: { 200: projectWithEnv diff --git a/backend/src/services/identity-access-token/identity-access-token-dal.ts b/backend/src/services/identity-access-token/identity-access-token-dal.ts index 57517c706..a2a067cad 100644 --- a/backend/src/services/identity-access-token/identity-access-token-dal.ts +++ b/backend/src/services/identity-access-token/identity-access-token-dal.ts @@ -30,6 +30,7 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => { .leftJoin(TableName.IdentityGcpAuth, `${TableName.Identity}.id`, `${TableName.IdentityGcpAuth}.identityId`) .leftJoin(TableName.IdentityAwsAuth, `${TableName.Identity}.id`, `${TableName.IdentityAwsAuth}.identityId`) .leftJoin(TableName.IdentityAzureAuth, `${TableName.Identity}.id`, `${TableName.IdentityAzureAuth}.identityId`) + .leftJoin(TableName.IdentityLdapAuth, `${TableName.Identity}.id`, `${TableName.IdentityLdapAuth}.identityId`) .leftJoin( TableName.IdentityKubernetesAuth, `${TableName.Identity}.id`, @@ -48,6 +49,7 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => { db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityOidcAuth).as("accessTokenTrustedIpsOidc"), db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityTokenAuth).as("accessTokenTrustedIpsToken"), db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityJwtAuth).as("accessTokenTrustedIpsJwt"), + db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityLdapAuth).as("accessTokenTrustedIpsLdap"), db.ref("name").withSchema(TableName.Identity) ) .first(); @@ -63,7 +65,8 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => { trustedIpsKubernetesAuth: doc.accessTokenTrustedIpsK8s, trustedIpsOidcAuth: doc.accessTokenTrustedIpsOidc, trustedIpsAccessTokenAuth: doc.accessTokenTrustedIpsToken, - trustedIpsAccessJwtAuth: doc.accessTokenTrustedIpsJwt + trustedIpsAccessJwtAuth: doc.accessTokenTrustedIpsJwt, + trustedIpsAccessLdapAuth: doc.accessTokenTrustedIpsLdap }; } catch (error) { throw new DatabaseError({ error, name: "IdAccessTokenFindOne" }); diff --git a/backend/src/services/identity-access-token/identity-access-token-service.ts b/backend/src/services/identity-access-token/identity-access-token-service.ts index a51d80e41..cd79981fe 100644 --- a/backend/src/services/identity-access-token/identity-access-token-service.ts +++ b/backend/src/services/identity-access-token/identity-access-token-service.ts @@ -186,7 +186,8 @@ export const identityAccessTokenServiceFactory = ({ [IdentityAuthMethod.KUBERNETES_AUTH]: identityAccessToken.trustedIpsKubernetesAuth, [IdentityAuthMethod.OIDC_AUTH]: identityAccessToken.trustedIpsOidcAuth, [IdentityAuthMethod.TOKEN_AUTH]: identityAccessToken.trustedIpsAccessTokenAuth, - [IdentityAuthMethod.JWT_AUTH]: identityAccessToken.trustedIpsAccessJwtAuth + [IdentityAuthMethod.JWT_AUTH]: identityAccessToken.trustedIpsAccessJwtAuth, + [IdentityAuthMethod.LDAP_AUTH]: identityAccessToken.trustedIpsAccessLdapAuth }; const trustedIps = trustedIpsMap[identityAccessToken.authMethod as IdentityAuthMethod]; diff --git a/backend/src/services/identity-ldap-auth/identity-ldap-auth-dal.ts b/backend/src/services/identity-ldap-auth/identity-ldap-auth-dal.ts new file mode 100644 index 000000000..0d998dbe9 --- /dev/null +++ b/backend/src/services/identity-ldap-auth/identity-ldap-auth-dal.ts @@ -0,0 +1,11 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TIdentityLdapAuthDALFactory = ReturnType; + +export const identityLdapAuthDALFactory = (db: TDbClient) => { + const ldapAuthOrm = ormify(db, TableName.IdentityLdapAuth); + + return ldapAuthOrm; +}; diff --git a/backend/src/services/identity-ldap-auth/identity-ldap-auth-service.ts b/backend/src/services/identity-ldap-auth/identity-ldap-auth-service.ts new file mode 100644 index 000000000..7462c9228 --- /dev/null +++ b/backend/src/services/identity-ldap-auth/identity-ldap-auth-service.ts @@ -0,0 +1,543 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { ForbiddenError } from "@casl/ability"; +import jwt from "jsonwebtoken"; + +import { IdentityAuthMethod } from "@app/db/schemas"; +import { testLDAPConfig } from "@app/ee/services/ldap-config/ldap-fns"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { OrgPermissionIdentityActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; +import { + constructPermissionErrorMessage, + validatePrivilegeChangeOperation +} from "@app/ee/services/permission/permission-fns"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError, NotFoundError, PermissionBoundaryError } from "@app/lib/errors"; +import { extractIPDetails, isValidIpOrCidr } from "@app/lib/ip"; + +import { ActorType, AuthTokenType } from "../auth/auth-type"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; +import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal"; +import { TIdentityAccessTokenJwtPayload } from "../identity-access-token/identity-access-token-types"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { validateIdentityUpdateForSuperAdminPrivileges } from "../super-admin/super-admin-fns"; +import { TIdentityLdapAuthDALFactory } from "./identity-ldap-auth-dal"; +import { + AllowedFieldsSchema, + TAttachLdapAuthDTO, + TGetLdapAuthDTO, + TLoginLdapAuthDTO, + TRevokeLdapAuthDTO, + TUpdateLdapAuthDTO +} from "./identity-ldap-auth-types"; + +type TIdentityLdapAuthServiceFactoryDep = { + identityAccessTokenDAL: Pick; + identityLdapAuthDAL: Pick< + TIdentityLdapAuthDALFactory, + "findOne" | "transaction" | "create" | "updateById" | "delete" + >; + identityOrgMembershipDAL: Pick; + licenseService: Pick; + permissionService: Pick; + kmsService: TKmsServiceFactory; + identityDAL: TIdentityDALFactory; +}; + +export type TIdentityLdapAuthServiceFactory = ReturnType; + +export const identityLdapAuthServiceFactory = ({ + identityAccessTokenDAL, + identityDAL, + identityLdapAuthDAL, + identityOrgMembershipDAL, + licenseService, + permissionService, + kmsService +}: TIdentityLdapAuthServiceFactoryDep) => { + const getLdapConfig = async (identityId: string) => { + const identity = await identityDAL.findOne({ id: identityId }); + if (!identity) throw new NotFoundError({ message: `Identity with ID '${identityId}' not found` }); + + const identityOrgMembership = await identityOrgMembershipDAL.findOne({ identityId: identity.id }); + if (!identityOrgMembership) throw new NotFoundError({ message: `Identity with ID '${identityId}' not found` }); + + const ldapAuth = await identityLdapAuthDAL.findOne({ identityId: identity.id }); + if (!ldapAuth) throw new NotFoundError({ message: `LDAP auth with ID '${identityId}' not found` }); + + const parsedAllowedFields = ldapAuth.allowedFields + ? AllowedFieldsSchema.array().parse(ldapAuth.allowedFields) + : undefined; + + const { decryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: identityOrgMembership.orgId + }); + + const bindDN = decryptor({ cipherTextBlob: ldapAuth.encryptedBindDN }).toString(); + const bindPass = decryptor({ cipherTextBlob: ldapAuth.encryptedBindPass }).toString(); + const ldapCaCertificate = ldapAuth.encryptedLdapCaCertificate + ? decryptor({ cipherTextBlob: ldapAuth.encryptedLdapCaCertificate }).toString() + : undefined; + + const ldapConfig = { + id: ldapAuth.id, + organization: identityOrgMembership.orgId, + url: ldapAuth.url, + bindDN, + bindPass, + searchBase: ldapAuth.searchBase, + searchFilter: ldapAuth.searchFilter, + caCert: ldapCaCertificate || "", + allowedFields: parsedAllowedFields + }; + + const opts = { + server: { + url: ldapAuth.url, + bindDN, + bindCredentials: bindPass, + searchBase: ldapAuth.searchBase, + searchFilter: ldapAuth.searchFilter, + ...(ldapCaCertificate + ? { + tlsOptions: { + ca: [ldapCaCertificate] + } + } + : {}) + }, + passReqToCallback: true + }; + + return { opts, ldapConfig }; + }; + + const login = async ({ identityId }: TLoginLdapAuthDTO) => { + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId }); + + if (!identityMembershipOrg) { + throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` }); + } + + const identityLdapAuth = await identityLdapAuthDAL.findOne({ identityId }); + + if (!identityLdapAuth) { + throw new NotFoundError({ message: `Failed to find LDAP auth for identity with ID ${identityId}` }); + } + + const plan = await licenseService.getPlan(identityMembershipOrg.orgId); + if (!plan.ldap) { + throw new BadRequestError({ + message: + "Failed to login to identity due to plan restriction. Upgrade plan to login to use LDAP authentication." + }); + } + + const identityAccessToken = await identityLdapAuthDAL.transaction(async (tx) => { + const newToken = await identityAccessTokenDAL.create( + { + identityId: identityLdapAuth.identityId, + isAccessTokenRevoked: false, + accessTokenTTL: identityLdapAuth.accessTokenTTL, + accessTokenMaxTTL: identityLdapAuth.accessTokenMaxTTL, + accessTokenNumUses: 0, + accessTokenNumUsesLimit: identityLdapAuth.accessTokenNumUsesLimit, + authMethod: IdentityAuthMethod.LDAP_AUTH + }, + tx + ); + return newToken; + }); + + const appCfg = getConfig(); + const accessToken = jwt.sign( + { + identityId: identityLdapAuth.identityId, + identityAccessTokenId: identityAccessToken.id, + authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN + } as TIdentityAccessTokenJwtPayload, + appCfg.AUTH_SECRET, + // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error + Number(identityAccessToken.accessTokenTTL) === 0 + ? undefined + : { + expiresIn: Number(identityAccessToken.accessTokenTTL) + } + ); + + return { accessToken, identityLdapAuth, identityAccessToken, identityMembershipOrg }; + }; + + const attachLdapAuth = async ({ + identityId, + url, + searchBase, + searchFilter, + bindDN, + bindPass, + ldapCaCertificate, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps, + actorId, + actorAuthMethod, + actor, + actorOrgId, + isActorSuperAdmin, + allowedFields + }: TAttachLdapAuthDTO) => { + await validateIdentityUpdateForSuperAdminPrivileges(identityId, isActorSuperAdmin); + + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId }); + if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` }); + + if (identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.LDAP_AUTH)) { + throw new BadRequestError({ + message: "Failed to add LDAP Auth to already configured identity" + }); + } + + if (accessTokenMaxTTL > 0 && accessTokenTTL > accessTokenMaxTTL) { + throw new BadRequestError({ message: "Access token TTL cannot be greater than max TTL" }); + } + + const { permission } = await permissionService.getOrgPermission( + actor, + actorId, + identityMembershipOrg.orgId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Create, OrgPermissionSubjects.Identity); + const plan = await licenseService.getPlan(identityMembershipOrg.orgId); + + if (!plan.ldap) { + throw new BadRequestError({ + message: "Failed to add LDAP Auth to identity due to plan restriction. Upgrade plan to add LDAP Auth." + }); + } + + const reformattedAccessTokenTrustedIps = accessTokenTrustedIps.map((accessTokenTrustedIp) => { + if ( + !plan.ipAllowlisting && + accessTokenTrustedIp.ipAddress !== "0.0.0.0/0" && + accessTokenTrustedIp.ipAddress !== "::/0" + ) + throw new BadRequestError({ + message: + "Failed to add IP access range to access token due to plan restriction. Upgrade plan to add IP access range." + }); + if (!isValidIpOrCidr(accessTokenTrustedIp.ipAddress)) + throw new BadRequestError({ + message: "The IP is not a valid IPv4, IPv6, or CIDR block" + }); + return extractIPDetails(accessTokenTrustedIp.ipAddress); + }); + + if (allowedFields) AllowedFieldsSchema.array().parse(allowedFields); + + const identityLdapAuth = await identityLdapAuthDAL.transaction(async (tx) => { + const { encryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: identityMembershipOrg.orgId + }); + + const { cipherTextBlob: encryptedBindPass } = encryptor({ + plainText: Buffer.from(bindPass) + }); + + let encryptedLdapCaCertificate: Buffer | undefined; + if (ldapCaCertificate) { + const { cipherTextBlob: encryptedCertificate } = encryptor({ + plainText: Buffer.from(ldapCaCertificate) + }); + + encryptedLdapCaCertificate = encryptedCertificate; + } + + const { cipherTextBlob: encryptedBindDN } = encryptor({ + plainText: Buffer.from(bindDN) + }); + + const isConnected = await testLDAPConfig({ + bindDN, + bindPass, + caCert: ldapCaCertificate || "", + url + }); + + if (!isConnected) { + throw new BadRequestError({ + message: + "Failed to connect to LDAP server. Please ensure that the LDAP server is running and your credentials are correct." + }); + } + + const doc = await identityLdapAuthDAL.create( + { + identityId: identityMembershipOrg.identityId, + encryptedBindDN, + encryptedBindPass, + searchBase, + searchFilter, + url, + encryptedLdapCaCertificate, + accessTokenMaxTTL, + accessTokenTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps: JSON.stringify(reformattedAccessTokenTrustedIps), + allowedFields: allowedFields ? JSON.stringify(allowedFields) : undefined + }, + tx + ); + return doc; + }); + return { ...identityLdapAuth, orgId: identityMembershipOrg.orgId }; + }; + + const updateLdapAuth = async ({ + identityId, + url, + searchBase, + searchFilter, + bindDN, + bindPass, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps, + actorId, + actorAuthMethod, + actor, + actorOrgId + }: TUpdateLdapAuthDTO) => { + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId }); + if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` }); + + if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.LDAP_AUTH)) { + throw new NotFoundError({ + message: "The identity does not have LDAP Auth attached" + }); + } + + const identityLdapAuth = await identityLdapAuthDAL.findOne({ identityId }); + + if ( + (accessTokenMaxTTL || identityLdapAuth.accessTokenMaxTTL) > 0 && + (accessTokenTTL || identityLdapAuth.accessTokenTTL) > (accessTokenMaxTTL || identityLdapAuth.accessTokenMaxTTL) + ) { + throw new BadRequestError({ message: "Access token TTL cannot be greater than max TTL" }); + } + + const { permission } = await permissionService.getOrgPermission( + actor, + actorId, + identityMembershipOrg.orgId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Edit, OrgPermissionSubjects.Identity); + + const plan = await licenseService.getPlan(identityMembershipOrg.orgId); + + if (!plan.ldap) { + throw new BadRequestError({ + message: "Failed to update LDAP Auth due to plan restriction. Upgrade plan to update LDAP Auth." + }); + } + + const reformattedAccessTokenTrustedIps = accessTokenTrustedIps?.map((accessTokenTrustedIp) => { + if ( + !plan.ipAllowlisting && + accessTokenTrustedIp.ipAddress !== "0.0.0.0/0" && + accessTokenTrustedIp.ipAddress !== "::/0" + ) + throw new BadRequestError({ + message: + "Failed to add IP access range to access token due to plan restriction. Upgrade plan to add IP access range." + }); + if (!isValidIpOrCidr(accessTokenTrustedIp.ipAddress)) + throw new BadRequestError({ + message: "The IP is not a valid IPv4, IPv6, or CIDR block" + }); + return extractIPDetails(accessTokenTrustedIp.ipAddress); + }); + + if (allowedFields) AllowedFieldsSchema.array().parse(allowedFields); + + const { encryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: identityMembershipOrg.orgId + }); + + let encryptedBindPass: Buffer | undefined; + if (bindPass) { + const { cipherTextBlob: bindPassCiphertext } = encryptor({ + plainText: Buffer.from(bindPass) + }); + + encryptedBindPass = bindPassCiphertext; + } + + let encryptedLdapCaCertificate: Buffer | undefined; + if (ldapCaCertificate) { + const { cipherTextBlob: ldapCaCertificateCiphertext } = encryptor({ + plainText: Buffer.from(ldapCaCertificate) + }); + + encryptedLdapCaCertificate = ldapCaCertificateCiphertext; + } + + let encryptedBindDN: Buffer | undefined; + if (bindDN) { + const { cipherTextBlob: bindDNCiphertext } = encryptor({ + plainText: Buffer.from(bindDN) + }); + + encryptedBindDN = bindDNCiphertext; + } + + const { ldapConfig } = await getLdapConfig(identityId); + + const isConnected = await testLDAPConfig({ + bindDN: bindDN || ldapConfig.bindDN, + bindPass: bindPass || ldapConfig.bindPass, + caCert: ldapCaCertificate || ldapConfig.caCert, + url: url || ldapConfig.url + }); + + if (!isConnected) { + throw new BadRequestError({ + message: + "Failed to connect to LDAP server. Please ensure that the LDAP server is running and your credentials are correct." + }); + } + + const updatedLdapAuth = await identityLdapAuthDAL.updateById(identityLdapAuth.id, { + url, + searchBase, + searchFilter, + encryptedBindDN, + encryptedBindPass, + encryptedLdapCaCertificate, + allowedFields: allowedFields ? JSON.stringify(allowedFields) : undefined, + accessTokenMaxTTL, + accessTokenTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps: reformattedAccessTokenTrustedIps + ? JSON.stringify(reformattedAccessTokenTrustedIps) + : undefined + }); + + return { ...updatedLdapAuth, orgId: identityMembershipOrg.orgId }; + }; + + const getLdapAuth = async ({ identityId, actorId, actor, actorAuthMethod, actorOrgId }: TGetLdapAuthDTO) => { + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId }); + if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` }); + + if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.LDAP_AUTH)) { + throw new BadRequestError({ + message: "The identity does not have LDAP Auth attached" + }); + } + + const ldapIdentityAuth = await identityLdapAuthDAL.findOne({ identityId }); + + const { permission } = await permissionService.getOrgPermission( + actor, + actorId, + identityMembershipOrg.orgId, + actorAuthMethod, + actorOrgId + ); + + const { decryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: identityMembershipOrg.orgId + }); + + const bindDN = decryptor({ cipherTextBlob: ldapIdentityAuth.encryptedBindDN }).toString(); + const bindPass = decryptor({ cipherTextBlob: ldapIdentityAuth.encryptedBindPass }).toString(); + const ldapCaCertificate = ldapIdentityAuth.encryptedLdapCaCertificate + ? decryptor({ cipherTextBlob: ldapIdentityAuth.encryptedLdapCaCertificate }).toString() + : undefined; + + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Read, OrgPermissionSubjects.Identity); + return { ...ldapIdentityAuth, orgId: identityMembershipOrg.orgId, bindDN, bindPass, ldapCaCertificate }; + }; + + const revokeIdentityLdapAuth = async ({ + identityId, + actorId, + actor, + actorAuthMethod, + actorOrgId + }: TRevokeLdapAuthDTO) => { + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId }); + if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` }); + if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.LDAP_AUTH)) { + throw new BadRequestError({ + message: "The identity does not have LDAP Auth attached" + }); + } + const { permission, membership } = await permissionService.getOrgPermission( + actor, + actorId, + identityMembershipOrg.orgId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Edit, OrgPermissionSubjects.Identity); + + const { permission: rolePermission } = await permissionService.getOrgPermission( + ActorType.IDENTITY, + identityMembershipOrg.identityId, + identityMembershipOrg.orgId, + actorAuthMethod, + actorOrgId + ); + + const permissionBoundary = validatePrivilegeChangeOperation( + membership.shouldUseNewPrivilegeSystem, + OrgPermissionIdentityActions.RevokeAuth, + OrgPermissionSubjects.Identity, + permission, + rolePermission + ); + + if (!permissionBoundary.isValid) + throw new PermissionBoundaryError({ + message: constructPermissionErrorMessage( + "Failed to revoke LDAP auth of identity with more privileged role", + membership.shouldUseNewPrivilegeSystem, + OrgPermissionIdentityActions.RevokeAuth, + OrgPermissionSubjects.Identity + ), + details: { missingPermissions: permissionBoundary.missingPermissions } + }); + + const revokedIdentityLdapAuth = await identityLdapAuthDAL.transaction(async (tx) => { + const [deletedLdapAuth] = await identityLdapAuthDAL.delete({ identityId }, tx); + await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.LDAP_AUTH }, tx); + + return { ...deletedLdapAuth, orgId: identityMembershipOrg.orgId }; + }); + return revokedIdentityLdapAuth; + }; + + return { + attachLdapAuth, + getLdapConfig, + updateLdapAuth, + login, + revokeIdentityLdapAuth, + getLdapAuth + }; +}; diff --git a/backend/src/services/identity-ldap-auth/identity-ldap-auth-types.ts b/backend/src/services/identity-ldap-auth/identity-ldap-auth-types.ts new file mode 100644 index 000000000..0e6feb5fb --- /dev/null +++ b/backend/src/services/identity-ldap-auth/identity-ldap-auth-types.ts @@ -0,0 +1,56 @@ +import { z } from "zod"; + +import { TProjectPermission } from "@app/lib/types"; + +export const AllowedFieldsSchema = z.object({ + key: z.string().trim(), + value: z + .string() + .trim() + .transform((val) => val.replace(/\s/g, "")) +}); + +export type TAllowedFields = z.infer; + +export type TAttachLdapAuthDTO = { + identityId: string; + url: string; + searchBase: string; + searchFilter: string; + bindDN: string; + bindPass: string; + ldapCaCertificate?: string; + allowedFields?: TAllowedFields[]; + accessTokenTTL: number; + accessTokenMaxTTL: number; + accessTokenNumUsesLimit: number; + accessTokenTrustedIps: { ipAddress: string }[]; + isActorSuperAdmin?: boolean; +} & Omit; + +export type TUpdateLdapAuthDTO = { + identityId: string; + url?: string; + searchBase?: string; + searchFilter?: string; + bindDN?: string; + bindPass?: string; + allowedFields?: TAllowedFields[]; + ldapCaCertificate?: string; + accessTokenTTL?: number; + accessTokenMaxTTL?: number; + accessTokenNumUsesLimit?: number; + accessTokenTrustedIps?: { ipAddress: string }[]; +} & Omit; + +export type TGetLdapAuthDTO = { + identityId: string; +} & Omit; + +export type TLoginLdapAuthDTO = { + identityId: string; +}; + +export type TRevokeLdapAuthDTO = { + identityId: string; +} & Omit; diff --git a/backend/src/services/identity/identity-fns.ts b/backend/src/services/identity/identity-fns.ts index 2d77e6544..6c77618e4 100644 --- a/backend/src/services/identity/identity-fns.ts +++ b/backend/src/services/identity/identity-fns.ts @@ -8,7 +8,8 @@ export const buildAuthMethods = ({ oidcId, azureId, tokenId, - jwtId + jwtId, + ldapId }: { uaId?: string; gcpId?: string; @@ -18,6 +19,7 @@ export const buildAuthMethods = ({ azureId?: string; tokenId?: string; jwtId?: string; + ldapId?: string; }) => { return [ ...[uaId ? IdentityAuthMethod.UNIVERSAL_AUTH : null], @@ -27,6 +29,7 @@ export const buildAuthMethods = ({ ...[oidcId ? IdentityAuthMethod.OIDC_AUTH : null], ...[azureId ? IdentityAuthMethod.AZURE_AUTH : null], ...[tokenId ? IdentityAuthMethod.TOKEN_AUTH : null], - ...[jwtId ? IdentityAuthMethod.JWT_AUTH : null] + ...[jwtId ? IdentityAuthMethod.JWT_AUTH : null], + ...[ldapId ? IdentityAuthMethod.LDAP_AUTH : null] ].filter((authMethod) => authMethod) as IdentityAuthMethod[]; }; diff --git a/backend/src/services/identity/identity-org-dal.ts b/backend/src/services/identity/identity-org-dal.ts index dbae59bbe..8b5032945 100644 --- a/backend/src/services/identity/identity-org-dal.ts +++ b/backend/src/services/identity/identity-org-dal.ts @@ -14,6 +14,7 @@ import { TIdentityUniversalAuths, TOrgRoles } from "@app/db/schemas"; +import { TIdentityLdapAuths } from "@app/db/schemas/identity-ldap-auths"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { buildKnexFilterForSearchResource } from "@app/lib/search-resource/db"; @@ -81,6 +82,11 @@ export const identityOrgDALFactory = (db: TDbClient) => { `${TableName.IdentityOrgMembership}.identityId`, `${TableName.IdentityJwtAuth}.identityId` ) + .leftJoin( + TableName.IdentityLdapAuth, + `${TableName.IdentityOrgMembership}.identityId`, + `${TableName.IdentityLdapAuth}.identityId` + ) .select( selectAllTableCols(TableName.IdentityOrgMembership), @@ -93,7 +99,7 @@ export const identityOrgDALFactory = (db: TDbClient) => { db.ref("id").as("azureId").withSchema(TableName.IdentityAzureAuth), db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth), db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth), - + db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth), db.ref("name").withSchema(TableName.Identity) ); @@ -200,6 +206,12 @@ export const identityOrgDALFactory = (db: TDbClient) => { "paginatedIdentity.identityId", `${TableName.IdentityJwtAuth}.identityId` ) + .leftJoin( + TableName.IdentityLdapAuth, + "paginatedIdentity.identityId", + `${TableName.IdentityLdapAuth}.identityId` + ) + .select( db.ref("id").withSchema("paginatedIdentity"), db.ref("role").withSchema("paginatedIdentity"), @@ -217,7 +229,8 @@ export const identityOrgDALFactory = (db: TDbClient) => { db.ref("id").as("oidcId").withSchema(TableName.IdentityOidcAuth), db.ref("id").as("azureId").withSchema(TableName.IdentityAzureAuth), db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth), - db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth) + db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth), + db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth) ) // cr stands for custom role .select(db.ref("id").as("crId").withSchema(TableName.OrgRoles)) @@ -259,6 +272,7 @@ export const identityOrgDALFactory = (db: TDbClient) => { oidcId, azureId, tokenId, + ldapId, createdAt, updatedAt }) => ({ @@ -290,7 +304,8 @@ export const identityOrgDALFactory = (db: TDbClient) => { oidcId, azureId, tokenId, - jwtId + jwtId, + ldapId }) } }), @@ -406,6 +421,11 @@ export const identityOrgDALFactory = (db: TDbClient) => { `${TableName.IdentityOrgMembership}.identityId`, `${TableName.IdentityJwtAuth}.identityId` ) + .leftJoin( + TableName.IdentityLdapAuth, + `${TableName.IdentityOrgMembership}.identityId`, + `${TableName.IdentityLdapAuth}.identityId` + ) .select( db.ref("id").withSchema(TableName.IdentityOrgMembership), db.ref("total_count").withSchema("searchedIdentities"), @@ -424,7 +444,8 @@ export const identityOrgDALFactory = (db: TDbClient) => { db.ref("id").as("oidcId").withSchema(TableName.IdentityOidcAuth), db.ref("id").as("azureId").withSchema(TableName.IdentityAzureAuth), db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth), - db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth) + db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth), + db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth) ) // cr stands for custom role .select(db.ref("id").as("crId").withSchema(TableName.OrgRoles)) @@ -467,6 +488,7 @@ export const identityOrgDALFactory = (db: TDbClient) => { oidcId, azureId, tokenId, + ldapId, createdAt, updatedAt }) => ({ @@ -498,7 +520,8 @@ export const identityOrgDALFactory = (db: TDbClient) => { oidcId, azureId, tokenId, - jwtId + jwtId, + ldapId }) } }), diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index 46c5ed2bd..f77becb02 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -177,7 +177,6 @@ export const deleteGithubSecrets = async ({ selected_repositories_url?: string | undefined; } - // @ts-expect-error just octokit ts compatiability issue const OctokitWithRetry = Octokit.plugin(retry); let octokit: Octokit; const appCfg = getConfig(); diff --git a/backend/src/services/microsoft-teams/microsoft-teams-service.ts b/backend/src/services/microsoft-teams/microsoft-teams-service.ts index 3712a0793..1413a3199 100644 --- a/backend/src/services/microsoft-teams/microsoft-teams-service.ts +++ b/backend/src/services/microsoft-teams/microsoft-teams-service.ts @@ -6,6 +6,7 @@ import { Request, Response } from "botbuilder"; +import { CronJob } from "cron"; import { FastifyReply, FastifyRequest } from "fastify"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; @@ -86,8 +87,17 @@ export const microsoftTeamsServiceFactory = ({ }: TMicrosoftTeamsServiceFactoryDep) => { let teamsBot: TeamsBot | null = null; let adapter: CloudAdapter | null = null; + let lastKnownUpdatedAt = new Date(); - const initializeTeamsBot = async ({ botAppId, botAppPassword }: { botAppId: string; botAppPassword: string }) => { + const initializeTeamsBot = async ({ + botAppId, + botAppPassword, + lastUpdatedAt + }: { + botAppId: string; + botAppPassword: string; + lastUpdatedAt?: Date; + }) => { logger.info("Initializing Microsoft Teams bot"); teamsBot = new TeamsBot({ botAppId, @@ -106,6 +116,57 @@ export const microsoftTeamsServiceFactory = ({ }) ) ); + + if (lastUpdatedAt) { + lastKnownUpdatedAt = lastUpdatedAt; + } + }; + + const $syncMicrosoftTeamsIntegrationConfiguration = async () => { + try { + const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID); + if (!serverCfg) { + throw new BadRequestError({ + message: "Failed to get server configuration." + }); + } + + if (lastKnownUpdatedAt.getTime() === serverCfg.updatedAt.getTime()) { + logger.info("No changes to Microsoft Teams integration configuration, skipping sync"); + return; + } + + lastKnownUpdatedAt = serverCfg.updatedAt; + + if ( + serverCfg.encryptedMicrosoftTeamsAppId && + serverCfg.encryptedMicrosoftTeamsClientSecret && + serverCfg.encryptedMicrosoftTeamsBotId + ) { + const decryptWithRoot = kmsService.decryptWithRootKey(); + const decryptedAppId = decryptWithRoot(serverCfg.encryptedMicrosoftTeamsAppId); + const decryptedAppPassword = decryptWithRoot(serverCfg.encryptedMicrosoftTeamsClientSecret); + + await initializeTeamsBot({ + botAppId: decryptedAppId.toString(), + botAppPassword: decryptedAppPassword.toString() + }); + } + } catch (err) { + logger.error(err, "Error syncing Microsoft Teams integration configuration"); + } + }; + + const initializeBackgroundSync = async () => { + logger.info("Setting up background sync process for Microsoft Teams workflow integration configuration"); + // initial sync upon startup + await $syncMicrosoftTeamsIntegrationConfiguration(); + + // sync rate limits configuration every 5 minutes + const job = new CronJob("*/5 * * * *", $syncMicrosoftTeamsIntegrationConfiguration); + job.start(); + + return job; }; const start = async () => { @@ -703,6 +764,7 @@ export const microsoftTeamsServiceFactory = ({ getTeams, handleMessageEndpoint, start, + initializeBackgroundSync, sendNotification, checkInstallationStatus, getClientId diff --git a/backend/src/services/project-role/project-role-fns.ts b/backend/src/services/project-role/project-role-fns.ts index c465715a7..4dfcf960b 100644 --- a/backend/src/services/project-role/project-role-fns.ts +++ b/backend/src/services/project-role/project-role-fns.ts @@ -1,15 +1,20 @@ -import { ProjectMembershipRole } from "@app/db/schemas"; +import { v4 as uuidv4 } from "uuid"; + +import { ProjectMembershipRole, ProjectType } from "@app/db/schemas"; import { + cryptographicOperatorPermissions, projectAdminPermissions, projectMemberPermissions, projectNoAccessPermissions, - projectViewerPermission -} from "@app/ee/services/permission/project-permission"; + projectViewerPermission, + sshHostBootstrapPermissions +} from "@app/ee/services/permission/default-roles"; +import { TGetPredefinedRolesDTO } from "@app/services/project-role/project-role-types"; -export const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMembershipRole) => { +export const getPredefinedRoles = ({ projectId, projectType, roleFilter }: TGetPredefinedRolesDTO) => { return [ { - id: "b11b49a9-09a9-4443-916a-4246f9ff2c69", // dummy userid + id: uuidv4(), projectId, name: "Admin", slug: ProjectMembershipRole.Admin, @@ -19,7 +24,7 @@ export const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMember updatedAt: new Date() }, { - id: "b11b49a9-09a9-4443-916a-4246f9ff2c70", // dummy user for zod validation in response + id: uuidv4(), projectId, name: "Developer", slug: ProjectMembershipRole.Member, @@ -29,7 +34,29 @@ export const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMember updatedAt: new Date() }, { - id: "b11b49a9-09a9-4443-916a-4246f9ff2c71", // dummy user for zod validation in response + id: uuidv4(), + projectId, + name: "SSH Host Bootstrapper", + slug: ProjectMembershipRole.SshHostBootstrapper, + permissions: sshHostBootstrapPermissions, + description: "Create and issue SSH Hosts in a project", + createdAt: new Date(), + updatedAt: new Date(), + type: ProjectType.SSH + }, + { + id: uuidv4(), + projectId, + name: "Cryptographic Operator", + slug: ProjectMembershipRole.KmsCryptographicOperator, + permissions: cryptographicOperatorPermissions, + description: "Perform cryptographic operations, such as encryption and signing, in a project", + createdAt: new Date(), + updatedAt: new Date(), + type: ProjectType.KMS + }, + { + id: uuidv4(), projectId, name: "Viewer", slug: ProjectMembershipRole.Viewer, @@ -39,7 +66,7 @@ export const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMember updatedAt: new Date() }, { - id: "b11b49a9-09a9-4443-916a-4246f9ff2c72", // dummy user for zod validation in response + id: uuidv4(), projectId, name: "No Access", slug: ProjectMembershipRole.NoAccess, @@ -48,5 +75,5 @@ export const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMember createdAt: new Date(), updatedAt: new Date() } - ].filter(({ slug }) => !roleFilter || roleFilter.includes(slug)); + ].filter(({ slug, type }) => (type ? type === projectType : true) && (!roleFilter || roleFilter === slug)); }; diff --git a/backend/src/services/project-role/project-role-service.ts b/backend/src/services/project-role/project-role-service.ts index 211dcff4f..babcf7d9c 100644 --- a/backend/src/services/project-role/project-role-service.ts +++ b/backend/src/services/project-role/project-role-service.ts @@ -2,7 +2,7 @@ import { ForbiddenError, MongoAbility, RawRuleOf } from "@casl/ability"; import { PackRule, packRules, unpackRules } from "@casl/ability/extra"; import { requestContext } from "@fastify/request-context"; -import { ActionProjectType, ProjectMembershipRole, TableName } from "@app/db/schemas"; +import { ActionProjectType, ProjectMembershipRole, ProjectType, TableName, TProjects } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, @@ -34,7 +34,7 @@ type TProjectRoleServiceFactoryDep = { projectRoleDAL: TProjectRoleDALFactory; identityDAL: Pick; userDAL: Pick; - projectDAL: Pick; + projectDAL: Pick; permissionService: Pick; identityProjectMembershipRoleDAL: TIdentityProjectMembershipRoleDALFactory; projectUserMembershipRoleDAL: TProjectUserMembershipRoleDALFactory; @@ -98,30 +98,37 @@ export const projectRoleServiceFactory = ({ roleSlug, filter }: TGetRoleDetailsDTO) => { - let projectId = ""; + let project: TProjects; if (filter.type === ProjectRoleServiceIdentifierType.SLUG) { - const project = await projectDAL.findProjectBySlug(filter.projectSlug, actorOrgId); - if (!project) throw new NotFoundError({ message: "Project not found" }); - projectId = project.id; + project = await projectDAL.findProjectBySlug(filter.projectSlug, actorOrgId); } else { - projectId = filter.projectId; + project = await projectDAL.findProjectById(filter.projectId); } + if (!project) throw new NotFoundError({ message: "Project not found" }); + const { permission } = await permissionService.getProjectPermission({ actor, actorId, - projectId, + projectId: project.id, actorAuthMethod, actorOrgId, actionProjectType: ActionProjectType.Any }); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Role); if (roleSlug !== "custom" && Object.values(ProjectMembershipRole).includes(roleSlug as ProjectMembershipRole)) { - const predefinedRole = getPredefinedRoles(projectId, roleSlug as ProjectMembershipRole)[0]; + const [predefinedRole] = getPredefinedRoles({ + projectId: project.id, + projectType: project.type as ProjectType, + roleFilter: roleSlug as ProjectMembershipRole + }); + + if (!predefinedRole) throw new NotFoundError({ message: `Default role with slug '${roleSlug}' not found` }); + return { ...predefinedRole, permissions: UnpackedPermissionSchema.array().parse(predefinedRole.permissions) }; } - const customRole = await projectRoleDAL.findOne({ slug: roleSlug, projectId }); + const customRole = await projectRoleDAL.findOne({ slug: roleSlug, projectId: project.id }); if (!customRole) throw new NotFoundError({ message: `Project role with slug '${roleSlug}' not found` }); return { ...customRole, permissions: unpackPermissions(customRole.permissions) }; }; @@ -194,29 +201,32 @@ export const projectRoleServiceFactory = ({ }; const listRoles = async ({ actorOrgId, actorAuthMethod, actorId, actor, filter }: TListRolesDTO) => { - let projectId = ""; + let project: TProjects; if (filter.type === ProjectRoleServiceIdentifierType.SLUG) { - const project = await projectDAL.findProjectBySlug(filter.projectSlug, actorOrgId); - if (!project) throw new BadRequestError({ message: "Project not found" }); - projectId = project.id; + project = await projectDAL.findProjectBySlug(filter.projectSlug, actorOrgId); } else { - projectId = filter.projectId; + project = await projectDAL.findProjectById(filter.projectId); } + if (!project) throw new BadRequestError({ message: "Project not found" }); + const { permission } = await permissionService.getProjectPermission({ actor, actorId, - projectId, + projectId: project.id, actorAuthMethod, actorOrgId, actionProjectType: ActionProjectType.Any }); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Role); const customRoles = await projectRoleDAL.find( - { projectId }, + { projectId: project.id }, { sort: [[`${TableName.ProjectRoles}.slug` as "slug", "asc"]] } ); - const roles = [...getPredefinedRoles(projectId), ...(customRoles || [])]; + const roles = [ + ...getPredefinedRoles({ projectId: project.id, projectType: project.type as ProjectType }), + ...(customRoles || []) + ]; return roles; }; diff --git a/backend/src/services/project-role/project-role-types.ts b/backend/src/services/project-role/project-role-types.ts index a71c73113..508623a0c 100644 --- a/backend/src/services/project-role/project-role-types.ts +++ b/backend/src/services/project-role/project-role-types.ts @@ -1,4 +1,4 @@ -import { TOrgRolesUpdate, TProjectRolesInsert } from "@app/db/schemas"; +import { ProjectMembershipRole, ProjectType, TOrgRolesUpdate, TProjectRolesInsert } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; export enum ProjectRoleServiceIdentifierType { @@ -34,3 +34,9 @@ export type TListRolesDTO = { | { type: ProjectRoleServiceIdentifierType.SLUG; projectSlug: string } | { type: ProjectRoleServiceIdentifierType.ID; projectId: string }; } & Omit; + +export type TGetPredefinedRolesDTO = { + projectId: string; + projectType: ProjectType; + roleFilter?: ProjectMembershipRole; +}; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 6e17bf489..2ae2c849d 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -329,14 +329,16 @@ export const projectServiceFactory = ({ // set default environments and root folder for provided environments let envs: TProjectEnvironments[] = []; if (projectTemplate) { - envs = await projectEnvDAL.insertMany( - projectTemplate.environments.map((env) => ({ ...env, projectId: project.id })), - tx - ); - await folderDAL.insertMany( - envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })), - tx - ); + if (projectTemplate.environments) { + envs = await projectEnvDAL.insertMany( + projectTemplate.environments.map((env) => ({ ...env, projectId: project.id })), + tx + ); + await folderDAL.insertMany( + envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })), + tx + ); + } await projectRoleDAL.insertMany( projectTemplate.packedRoles.map((role) => ({ ...role, @@ -592,7 +594,10 @@ export const projectServiceFactory = ({ workspaces.map(async (workspace) => { return { ...workspace, - roles: [...(workspaceMappedToRoles[workspace.id] || []), ...getPredefinedRoles(workspace.id)] + roles: [ + ...(workspaceMappedToRoles[workspace.id] || []), + ...getPredefinedRoles({ projectId: workspace.id, projectType: workspace.type as ProjectType }) + ] }; }) ); diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts index 7e77bd256..abc4dcf82 100644 --- a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -169,7 +169,7 @@ const getParameterStoreTagsRecord = async ( throw new SecretSyncError({ message: - "IAM role has inadequate permissions to manage resource tags. Ensure the following polices are present: ssm:ListTagsForResource, ssm:AddTagsToResource, and ssm:RemoveTagsFromResource", + "IAM role has inadequate permissions to manage resource tags. Ensure the following policies are present: ssm:ListTagsForResource, ssm:AddTagsToResource, and ssm:RemoveTagsFromResource", shouldRetry: false }); } diff --git a/backend/src/services/super-admin/invalidate-cache-queue.ts b/backend/src/services/super-admin/invalidate-cache-queue.ts new file mode 100644 index 000000000..c2a12f5d5 --- /dev/null +++ b/backend/src/services/super-admin/invalidate-cache-queue.ts @@ -0,0 +1,49 @@ +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { CacheType } from "./super-admin-types"; + +export type TInvalidateCacheQueueFactoryDep = { + queueService: TQueueServiceFactory; + + keyStore: Pick; +}; + +export type TInvalidateCacheQueueFactory = ReturnType; + +export const invalidateCacheQueueFactory = ({ queueService, keyStore }: TInvalidateCacheQueueFactoryDep) => { + const startInvalidate = async (dto: { + data: { + type: CacheType; + }; + }) => { + await queueService.queue(QueueName.InvalidateCache, QueueJobs.InvalidateCache, dto, { + removeOnComplete: true, + removeOnFail: true, + jobId: `invalidate-cache-${dto.data.type}` + }); + }; + + queueService.start(QueueName.InvalidateCache, async (job) => { + try { + const { + data: { type } + } = job.data; + + await keyStore.setItemWithExpiry("invalidating-cache", 1800, "true"); // 30 minutes max (in case the job somehow silently fails) + + if (type === CacheType.ALL || type === CacheType.SECRETS) + await keyStore.deleteItems({ pattern: "secret-manager:*" }); + + await keyStore.deleteItem("invalidating-cache"); + } catch (err) { + logger.error(err, "Failed to invalidate cache"); + await keyStore.deleteItem("invalidating-cache"); + } + }); + + return { + startInvalidate + }; +}; diff --git a/backend/src/services/super-admin/super-admin-service.ts b/backend/src/services/super-admin/super-admin-service.ts index 8687826c8..22fc51d1a 100644 --- a/backend/src/services/super-admin/super-admin-service.ts +++ b/backend/src/services/super-admin/super-admin-service.ts @@ -25,8 +25,10 @@ import { TOrgServiceFactory } from "../org/org-service"; import { TUserDALFactory } from "../user/user-dal"; import { TUserAliasDALFactory } from "../user-alias/user-alias-dal"; import { UserAliasType } from "../user-alias/user-alias-types"; +import { TInvalidateCacheQueueFactory } from "./invalidate-cache-queue"; import { TSuperAdminDALFactory } from "./super-admin-dal"; import { + CacheType, LoginMethod, TAdminBootstrapInstanceDTO, TAdminGetIdentitiesDTO, @@ -46,9 +48,10 @@ type TSuperAdminServiceFactoryDep = { kmsService: Pick; kmsRootConfigDAL: TKmsRootConfigDALFactory; orgService: Pick; - keyStore: Pick; + keyStore: Pick; licenseService: Pick; microsoftTeamsService: Pick; + invalidateCacheQueue: TInvalidateCacheQueueFactory; }; export type TSuperAdminServiceFactory = ReturnType; @@ -64,7 +67,7 @@ export let getServerCfg: () => Promise< const ADMIN_CONFIG_KEY = "infisical-admin-cfg"; const ADMIN_CONFIG_KEY_EXP = 60; // 60s -const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000"; +export const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000"; export const superAdminServiceFactory = ({ serverCfgDAL, @@ -80,7 +83,8 @@ export const superAdminServiceFactory = ({ identityAccessTokenDAL, identityTokenAuthDAL, identityOrgMembershipDAL, - microsoftTeamsService + microsoftTeamsService, + invalidateCacheQueue }: TSuperAdminServiceFactoryDep) => { const initServerCfg = async () => { // TODO(akhilmhdh): bad pattern time less change this later to me itself @@ -242,7 +246,8 @@ export const superAdminServiceFactory = ({ await microsoftTeamsService.initializeTeamsBot({ botAppId: decryptedAppId.toString(), - botAppPassword: decryptedAppPassword.toString() + botAppPassword: decryptedAppPassword.toString(), + lastUpdatedAt: updatedServerCfg.updatedAt }); } @@ -631,6 +636,16 @@ export const superAdminServiceFactory = ({ await kmsService.updateEncryptionStrategy(strategy); }; + const invalidateCache = async (type: CacheType) => { + await invalidateCacheQueue.startInvalidate({ + data: { type } + }); + }; + + const checkIfInvalidatingCache = async () => { + return (await keyStore.getItem("invalidating-cache")) !== null; + }; + return { initServerCfg, updateServerCfg, @@ -644,6 +659,8 @@ export const superAdminServiceFactory = ({ getConfiguredEncryptionStrategies, grantServerAdminAccessToUser, deleteIdentitySuperAdminAccess, - deleteUserSuperAdminAccess + deleteUserSuperAdminAccess, + invalidateCache, + checkIfInvalidatingCache }; }; diff --git a/backend/src/services/super-admin/super-admin-types.ts b/backend/src/services/super-admin/super-admin-types.ts index 64ec92632..c804bed74 100644 --- a/backend/src/services/super-admin/super-admin-types.ts +++ b/backend/src/services/super-admin/super-admin-types.ts @@ -44,3 +44,8 @@ export enum LoginMethod { LDAP = "ldap", OIDC = "oidc" } + +export enum CacheType { + ALL = "all", + SECRETS = "secrets" +} diff --git a/backend/src/services/telemetry/telemetry-types.ts b/backend/src/services/telemetry/telemetry-types.ts index ab90a71d4..9e046cdbd 100644 --- a/backend/src/services/telemetry/telemetry-types.ts +++ b/backend/src/services/telemetry/telemetry-types.ts @@ -21,7 +21,8 @@ export enum PostHogEventTypes { IssueSshHostUserCert = "Issue SSH Host User Certificate", IssueSshHostHostCert = "Issue SSH Host Host Certificate", SignCert = "Sign PKI Certificate", - IssueCert = "Issue PKI Certificate" + IssueCert = "Issue PKI Certificate", + InvalidateCache = "Invalidate Cache" } export type TSecretModifiedEvent = { @@ -203,6 +204,13 @@ export type TIssueCertificateEvent = { }; }; +export type TInvalidateCacheEvent = { + event: PostHogEventTypes.InvalidateCache; + properties: { + userAgent?: string; + }; +}; + export type TPostHogEvent = { distinctId: string } & ( | TSecretModifiedEvent | TAdminInitEvent @@ -221,4 +229,5 @@ export type TPostHogEvent = { distinctId: string } & ( | TIssueSshHostHostCertEvent | TSignCertificateEvent | TIssueCertificateEvent + | TInvalidateCacheEvent ); diff --git a/cli/config/allowlist_test.go b/cli/config/allowlist_test.go deleted file mode 100644 index 52766e3cd..000000000 --- a/cli/config/allowlist_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - "regexp" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCommitAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - commit string - commitAllowed bool - }{ - { - allowlist: Allowlist{ - Commits: []string{"commitA"}, - }, - commit: "commitA", - commitAllowed: true, - }, - { - allowlist: Allowlist{ - Commits: []string{"commitB"}, - }, - commit: "commitA", - commitAllowed: false, - }, - { - allowlist: Allowlist{ - Commits: []string{"commitB"}, - }, - commit: "", - commitAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.commitAllowed, tt.allowlist.CommitAllowed(tt.commit)) - } -} - -func TestRegexAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - secret string - regexAllowed bool - }{ - { - allowlist: Allowlist{ - Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")}, - }, - secret: "a secret: matchthis, done", - regexAllowed: true, - }, - { - allowlist: Allowlist{ - Regexes: []*regexp.Regexp{regexp.MustCompile("matchthis")}, - }, - secret: "a secret", - regexAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.regexAllowed, tt.allowlist.RegexAllowed(tt.secret)) - } -} - -func TestPathAllowed(t *testing.T) { - tests := []struct { - allowlist Allowlist - path string - pathAllowed bool - }{ - { - allowlist: Allowlist{ - Paths: []*regexp.Regexp{regexp.MustCompile("path")}, - }, - path: "a path", - pathAllowed: true, - }, - { - allowlist: Allowlist{ - Paths: []*regexp.Regexp{regexp.MustCompile("path")}, - }, - path: "a ???", - pathAllowed: false, - }, - } - for _, tt := range tests { - assert.Equal(t, tt.pathAllowed, tt.allowlist.PathAllowed(tt.path)) - } -} diff --git a/cli/config/config.go b/cli/config/config.go deleted file mode 100644 index b1ce08e2b..000000000 --- a/cli/config/config.go +++ /dev/null @@ -1,279 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - _ "embed" - "fmt" - "regexp" - "strings" - - "github.com/rs/zerolog/log" - "github.com/spf13/viper" -) - -//go:embed infisical-scan.toml -var DefaultConfig string - -// use to keep track of how many configs we can extend -// yea I know, globals bad -var extendDepth int - -const maxExtendDepth = 2 - -const DefaultScanConfigFileName = ".infisical-scan.toml" -const DefaultScanConfigEnvName = "INFISICAL_SCAN_CONFIG" -const DefaultInfisicalIgnoreFineName = ".infisicalignore" - -// ViperConfig is the config struct used by the Viper config package -// to parse the config file. This struct does not include regular expressions. -// It is used as an intermediary to convert the Viper config to the Config struct. -type ViperConfig struct { - Description string - Extend Extend - Rules []struct { - ID string - Description string - Entropy float64 - SecretGroup int - Regex string - Keywords []string - Path string - Tags []string - - Allowlist struct { - RegexTarget string - Regexes []string - Paths []string - Commits []string - StopWords []string - } - } - Allowlist struct { - RegexTarget string - Regexes []string - Paths []string - Commits []string - StopWords []string - } -} - -// Config is a configuration struct that contains rules and an allowlist if present. -type Config struct { - Extend Extend - Path string - Description string - Rules map[string]Rule - Allowlist Allowlist - Keywords []string - - // used to keep sarif results consistent - orderedRules []string -} - -// Extend is a struct that allows users to define how they want their -// configuration extended by other configuration files. -type Extend struct { - Path string - URL string - UseDefault bool -} - -func (vc *ViperConfig) Translate() (Config, error) { - var ( - keywords []string - orderedRules []string - ) - rulesMap := make(map[string]Rule) - - for _, r := range vc.Rules { - var allowlistRegexes []*regexp.Regexp - for _, a := range r.Allowlist.Regexes { - allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) - } - var allowlistPaths []*regexp.Regexp - for _, a := range r.Allowlist.Paths { - allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) - } - - if r.Keywords == nil { - r.Keywords = []string{} - } else { - for _, k := range r.Keywords { - keywords = append(keywords, strings.ToLower(k)) - } - } - - if r.Tags == nil { - r.Tags = []string{} - } - - var configRegex *regexp.Regexp - var configPathRegex *regexp.Regexp - if r.Regex == "" { - configRegex = nil - } else { - configRegex = regexp.MustCompile(r.Regex) - } - if r.Path == "" { - configPathRegex = nil - } else { - configPathRegex = regexp.MustCompile(r.Path) - } - r := Rule{ - Description: r.Description, - RuleID: r.ID, - Regex: configRegex, - Path: configPathRegex, - SecretGroup: r.SecretGroup, - Entropy: r.Entropy, - Tags: r.Tags, - Keywords: r.Keywords, - Allowlist: Allowlist{ - RegexTarget: r.Allowlist.RegexTarget, - Regexes: allowlistRegexes, - Paths: allowlistPaths, - Commits: r.Allowlist.Commits, - StopWords: r.Allowlist.StopWords, - }, - } - orderedRules = append(orderedRules, r.RuleID) - - if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() { - return Config{}, fmt.Errorf("%s invalid regex secret group %d, max regex secret group %d", r.Description, r.SecretGroup, r.Regex.NumSubexp()) - } - rulesMap[r.RuleID] = r - } - var allowlistRegexes []*regexp.Regexp - for _, a := range vc.Allowlist.Regexes { - allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) - } - var allowlistPaths []*regexp.Regexp - for _, a := range vc.Allowlist.Paths { - allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) - } - c := Config{ - Description: vc.Description, - Extend: vc.Extend, - Rules: rulesMap, - Allowlist: Allowlist{ - RegexTarget: vc.Allowlist.RegexTarget, - Regexes: allowlistRegexes, - Paths: allowlistPaths, - Commits: vc.Allowlist.Commits, - StopWords: vc.Allowlist.StopWords, - }, - Keywords: keywords, - orderedRules: orderedRules, - } - - if maxExtendDepth != extendDepth { - // disallow both usedefault and path from being set - if c.Extend.Path != "" && c.Extend.UseDefault { - log.Fatal().Msg("unable to load config due to extend.path and extend.useDefault being set") - } - if c.Extend.UseDefault { - c.extendDefault() - } else if c.Extend.Path != "" { - c.extendPath() - } - - } - - return c, nil -} - -func (c *Config) OrderedRules() []Rule { - var orderedRules []Rule - for _, id := range c.orderedRules { - if _, ok := c.Rules[id]; ok { - orderedRules = append(orderedRules, c.Rules[id]) - } - } - return orderedRules -} - -func (c *Config) extendDefault() { - extendDepth++ - viper.SetConfigType("toml") - if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - defaultViperConfig := ViperConfig{} - if err := viper.Unmarshal(&defaultViperConfig); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - cfg, err := defaultViperConfig.Translate() - if err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - log.Debug().Msg("extending config with default config") - c.extend(cfg) - -} - -func (c *Config) extendPath() { - extendDepth++ - viper.SetConfigFile(c.Extend.Path) - if err := viper.ReadInConfig(); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - extensionViperConfig := ViperConfig{} - if err := viper.Unmarshal(&extensionViperConfig); err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - cfg, err := extensionViperConfig.Translate() - if err != nil { - log.Fatal().Msgf("failed to load extended config, err: %s", err) - return - } - log.Debug().Msgf("extending config with %s", c.Extend.Path) - c.extend(cfg) -} - -func (c *Config) extendURL() { - // TODO -} - -func (c *Config) extend(extensionConfig Config) { - for ruleID, rule := range extensionConfig.Rules { - if _, ok := c.Rules[ruleID]; !ok { - log.Trace().Msgf("adding %s to base config", ruleID) - c.Rules[ruleID] = rule - c.Keywords = append(c.Keywords, rule.Keywords...) - } - } - - // append allowlists, not attempting to merge - c.Allowlist.Commits = append(c.Allowlist.Commits, - extensionConfig.Allowlist.Commits...) - c.Allowlist.Paths = append(c.Allowlist.Paths, - extensionConfig.Allowlist.Paths...) - c.Allowlist.Regexes = append(c.Allowlist.Regexes, - extensionConfig.Allowlist.Regexes...) -} diff --git a/cli/config/config_test.go b/cli/config/config_test.go deleted file mode 100644 index e8f4d47b1..000000000 --- a/cli/config/config_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package config - -import ( - "fmt" - "regexp" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" -) - -const configPath = "../testdata/config/" - -func TestTranslate(t *testing.T) { - tests := []struct { - cfgName string - cfg Config - wantError error - }{ - { - cfgName: "allow_aws_re", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Regexes: []*regexp.Regexp{ - regexp.MustCompile("AKIALALEMEL33243OLIA"), - }, - }, - }, - }, - }, - }, - { - cfgName: "allow_commit", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Commits: []string{"allowthiscommit"}, - }, - }, - }, - }, - }, - { - cfgName: "allow_path", - cfg: Config{ - Rules: map[string]Rule{"aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - Allowlist: Allowlist{ - Paths: []*regexp.Regexp{ - regexp.MustCompile(".go"), - }, - }, - }, - }, - }, - }, - { - cfgName: "entropy_group", - cfg: Config{ - Rules: map[string]Rule{"discord-api-key": { - Description: "Discord API key", - Regex: regexp.MustCompile(`(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]`), - RuleID: "discord-api-key", - Allowlist: Allowlist{}, - Entropy: 3.5, - SecretGroup: 3, - Tags: []string{}, - Keywords: []string{}, - }, - }, - }, - }, - { - cfgName: "bad_entropy_group", - cfg: Config{}, - wantError: fmt.Errorf("Discord API key invalid regex secret group 5, max regex secret group 3"), - }, - { - cfgName: "base", - cfg: Config{ - Rules: map[string]Rule{ - "aws-access-key": { - Description: "AWS Access Key", - Regex: regexp.MustCompile("(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-access-key", - }, - "aws-secret-key": { - Description: "AWS Secret Key", - Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-secret-key", - }, - "aws-secret-key-again": { - Description: "AWS Secret Key", - Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`), - Tags: []string{"key", "AWS"}, - Keywords: []string{}, - RuleID: "aws-secret-key-again", - }, - }, - }, - }, - } - - for _, tt := range tests { - viper.Reset() - viper.AddConfigPath(configPath) - viper.SetConfigName(tt.cfgName) - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if tt.wantError != nil { - if err == nil { - t.Errorf("expected error") - } - assert.Equal(t, tt.wantError, err) - } - - assert.Equal(t, cfg.Rules, tt.cfg.Rules) - } -} diff --git a/cli/config/example-infisical-relay.yaml b/cli/config/example-infisical-relay.yaml deleted file mode 100644 index c913ed757..000000000 --- a/cli/config/example-infisical-relay.yaml +++ /dev/null @@ -1,8 +0,0 @@ -public_ip: 127.0.0.1 -auth_secret: example-auth-secret -realm: infisical.org -# set port 5349 for tls -# port: 5349 -# tls_private_key_path: /full-path -# tls_ca_path: /full-path -# tls_cert_path: /full-path diff --git a/cli/config/infisical-relay.yaml b/cli/config/infisical-relay.yaml deleted file mode 100644 index 89c6b5e45..000000000 --- a/cli/config/infisical-relay.yaml +++ /dev/null @@ -1,8 +0,0 @@ -public_ip: 127.0.0.1 -auth_secret: changeThisOnProduction -realm: infisical.org -# set port 5349 for tls -# port: 5349 -# tls_private_key_path: /full-path -# tls_ca_path: /full-path -# tls_cert_path: /full-path diff --git a/cli/config/infisical-scan.toml b/cli/config/infisical-scan.toml deleted file mode 100644 index 193883444..000000000 --- a/cli/config/infisical-scan.toml +++ /dev/null @@ -1,2803 +0,0 @@ - -# This file has been auto-generated. Do not edit manually. -# If you would like to contribute new rules, please use -# cmd/generate/config/main.go and follow the contributing guidelines -# at https://github.com/zricethezav/gitleaks/blob/master/CONTRIBUTING.md - -# This is the default gitleaks configuration file. -# Rules and allowlists are defined within this file. -# Rules instruct gitleaks on what should be considered a secret. -# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. - -title = "gitleaks config" - -[allowlist] -description = "global allow lists" -paths = [ - '''infisical-scan.toml''', - '''(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket)$''', - '''(go.mod|go.sum)$''', - '''gradle.lockfile''', - '''node_modules''', - '''package-lock.json''', - '''pnpm-lock.yaml''', - '''Database.refactorlog''', - '''vendor''', -] - -[[rules]] -description = "Adafruit API Key" -id = "adafruit-api-key" -regex = '''(?i)(?:adafruit)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "adafruit", -] - -[[rules]] -description = "Adobe Client ID (OAuth Web)" -id = "adobe-client-id" -regex = '''(?i)(?:adobe)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "adobe", -] - -[[rules]] -description = "Adobe Client Secret" -id = "adobe-client-secret" -regex = '''(?i)\b((p8e-)(?i)[a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "p8e-", -] - -[[rules]] -description = "Age secret key" -id = "age secret key" -regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' -keywords = [ - "age-secret-key-1", -] - -[[rules]] -description = "Airtable API Key" -id = "airtable-api-key" -regex = '''(?i)(?:airtable)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{17})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "airtable", -] - -[[rules]] -description = "Algolia API Key" -id = "algolia-api-key" -regex = '''(?i)(?:algolia)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "algolia", -] - -[[rules]] -description = "Alibaba AccessKey ID" -id = "alibaba-access-key-id" -regex = '''(?i)\b((LTAI)(?i)[a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "ltai", -] - -[[rules]] -description = "Alibaba Secret Key" -id = "alibaba-secret-key" -regex = '''(?i)(?:alibaba)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "alibaba", -] - -[[rules]] -description = "Asana Client ID" -id = "asana-client-id" -regex = '''(?i)(?:asana)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "asana", -] - -[[rules]] -description = "Asana Client Secret" -id = "asana-client-secret" -regex = '''(?i)(?:asana)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "asana", -] - -[[rules]] -description = "Atlassian API token" -id = "atlassian-api-token" -regex = '''(?i)(?:atlassian|confluence|jira)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "atlassian","confluence","jira", -] - -[[rules]] -description = "Authress Service Client Access Key" -id = "authress-service-client-access-key" -regex = '''(?i)\b((?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc_[a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sc_","ext_","scauth_","authress_", -] - -[[rules]] -description = "AWS" -id = "aws-access-token" -regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' -keywords = [ - "akia","agpa","aida","aroa","aipa","anpa","anva","asia", -] - -[[rules]] -description = "Beamer API token" -id = "beamer-api-token" -regex = '''(?i)(?:beamer)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(b_[a-z0-9=_\-]{44})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "beamer", -] - -[[rules]] -description = "Bitbucket Client ID" -id = "bitbucket-client-id" -regex = '''(?i)(?:bitbucket)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bitbucket", -] - -[[rules]] -description = "Bitbucket Client Secret" -id = "bitbucket-client-secret" -regex = '''(?i)(?:bitbucket)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bitbucket", -] - -[[rules]] -description = "Bittrex Access Key" -id = "bittrex-access-key" -regex = '''(?i)(?:bittrex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bittrex", -] - -[[rules]] -description = "Bittrex Secret Key" -id = "bittrex-secret-key" -regex = '''(?i)(?:bittrex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bittrex", -] - -[[rules]] -description = "Clojars API token" -id = "clojars-api-token" -regex = '''(?i)(CLOJARS_)[a-z0-9]{60}''' -keywords = [ - "clojars", -] - -[[rules]] -description = "Codecov Access Token" -id = "codecov-access-token" -regex = '''(?i)(?:codecov)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "codecov", -] - -[[rules]] -description = "Coinbase Access Token" -id = "coinbase-access-token" -regex = '''(?i)(?:coinbase)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "coinbase", -] - -[[rules]] -description = "Confluent Access Token" -id = "confluent-access-token" -regex = '''(?i)(?:confluent)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "confluent", -] - -[[rules]] -description = "Confluent Secret Key" -id = "confluent-secret-key" -regex = '''(?i)(?:confluent)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "confluent", -] - -[[rules]] -description = "Contentful delivery API token" -id = "contentful-delivery-api-token" -regex = '''(?i)(?:contentful)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{43})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "contentful", -] - -[[rules]] -description = "Databricks API token" -id = "databricks-api-token" -regex = '''(?i)\b(dapi[a-h0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dapi", -] - -[[rules]] -description = "Datadog Access Token" -id = "datadog-access-token" -regex = '''(?i)(?:datadog)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "datadog", -] - -[[rules]] -description = "Defined Networking API token" -id = "defined-networking-api-token" -regex = '''(?i)(?:dnkey)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dnkey", -] - -[[rules]] -description = "DigitalOcean OAuth Access Token" -id = "digitalocean-access-token" -regex = '''(?i)\b(doo_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "doo_v1_", -] - -[[rules]] -description = "DigitalOcean Personal Access Token" -id = "digitalocean-pat" -regex = '''(?i)\b(dop_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dop_v1_", -] - -[[rules]] -description = "DigitalOcean OAuth Refresh Token" -id = "digitalocean-refresh-token" -regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dor_v1_", -] - -[[rules]] -description = "Discord API key" -id = "discord-api-token" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Discord client ID" -id = "discord-client-id" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{18})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Discord client secret" -id = "discord-client-secret" -regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "discord", -] - -[[rules]] -description = "Doppler API token" -id = "doppler-api-token" -regex = '''(dp\.pt\.)(?i)[a-z0-9]{43}''' -keywords = [ - "doppler", -] - -[[rules]] -description = "Droneci Access Token" -id = "droneci-access-token" -regex = '''(?i)(?:droneci)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "droneci", -] - -[[rules]] -description = "Dropbox API secret" -id = "dropbox-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{15})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "dropbox", -] - -[[rules]] -description = "Dropbox long lived API token" -id = "dropbox-long-lived-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dropbox", -] - -[[rules]] -description = "Dropbox short lived API token" -id = "dropbox-short-lived-api-token" -regex = '''(?i)(?:dropbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(sl\.[a-z0-9\-=_]{135})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "dropbox", -] - -[[rules]] -description = "Duffel API token" -id = "duffel-api-token" -regex = '''duffel_(test|live)_(?i)[a-z0-9_\-=]{43}''' -keywords = [ - "duffel", -] - -[[rules]] -description = "Dynatrace API token" -id = "dynatrace-api-token" -regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' -keywords = [ - "dynatrace", -] - -[[rules]] -description = "EasyPost API token" -id = "easypost-api-token" -regex = '''\bEZAK(?i)[a-z0-9]{54}''' -keywords = [ - "ezak", -] - -[[rules]] -description = "EasyPost test API token" -id = "easypost-test-api-token" -regex = '''\bEZTK(?i)[a-z0-9]{54}''' -keywords = [ - "eztk", -] - -[[rules]] -description = "Etsy Access Token" -id = "etsy-access-token" -regex = '''(?i)(?:etsy)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "etsy", -] - -[[rules]] -description = "Facebook Access Token" -id = "facebook" -regex = '''(?i)(?:facebook)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "facebook", -] - -[[rules]] -description = "Fastly API key" -id = "fastly-api-token" -regex = '''(?i)(?:fastly)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "fastly", -] - -[[rules]] -description = "Finicity API token" -id = "finicity-api-token" -regex = '''(?i)(?:finicity)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finicity", -] - -[[rules]] -description = "Finicity Client Secret" -id = "finicity-client-secret" -regex = '''(?i)(?:finicity)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finicity", -] - -[[rules]] -description = "Finnhub Access Token" -id = "finnhub-access-token" -regex = '''(?i)(?:finnhub)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{20})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "finnhub", -] - -[[rules]] -description = "Flickr Access Token" -id = "flickr-access-token" -regex = '''(?i)(?:flickr)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "flickr", -] - -[[rules]] -description = "Flutterwave Encryption Key" -id = "flutterwave-encryption-key" -regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' -keywords = [ - "flwseck_test", -] - -[[rules]] -description = "Finicity Public Key" -id = "flutterwave-public-key" -regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' -keywords = [ - "flwpubk_test", -] - -[[rules]] -description = "Flutterwave Secret Key" -id = "flutterwave-secret-key" -regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' -keywords = [ - "flwseck_test", -] - -[[rules]] -description = "Frame.io API token" -id = "frameio-api-token" -regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' -keywords = [ - "fio-u-", -] - -[[rules]] -description = "Freshbooks Access Token" -id = "freshbooks-access-token" -regex = '''(?i)(?:freshbooks)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "freshbooks", -] - -[[rules]] -description = "GCP API key" -id = "gcp-api-key" -regex = '''(?i)\b(AIza[0-9A-Za-z\\-_]{35})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "aiza", -] - -[[rules]] -description = "Generic API Key" -id = "generic-api-key" -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -entropy = 3.5 -keywords = [ - "key","api","token","secret","client","passwd","password","auth","access", -] -[rules.allowlist] -stopwords= [ - "client", - "endpoint", - "vpn", - "_ec2_", - "aws_", - "authorize", - "author", - "define", - "config", - "credential", - "setting", - "sample", - "xxxxxx", - "000000", - "buffer", - "delete", - "aaaaaa", - "fewfwef", - "getenv", - "env_", - "system", - "example", - "ecdsa", - "sha256", - "sha1", - "sha2", - "md5", - "alert", - "wizard", - "target", - "onboard", - "welcome", - "page", - "exploit", - "experiment", - "expire", - "rabbitmq", - "scraper", - "widget", - "music", - "dns_", - "dns-", - "yahoo", - "want", - "json", - "action", - "script", - "fix_", - "fix-", - "develop", - "compas", - "stripe", - "service", - "master", - "metric", - "tech", - "gitignore", - "rich", - "open", - "stack", - "irc_", - "irc-", - "sublime", - "kohana", - "has_", - "has-", - "fabric", - "wordpres", - "role", - "osx_", - "osx-", - "boost", - "addres", - "queue", - "working", - "sandbox", - "internet", - "print", - "vision", - "tracking", - "being", - "generator", - "traffic", - "world", - "pull", - "rust", - "watcher", - "small", - "auth", - "full", - "hash", - "more", - "install", - "auto", - "complete", - "learn", - "paper", - "installer", - "research", - "acces", - "last", - "binding", - "spine", - "into", - "chat", - "algorithm", - "resource", - "uploader", - "video", - "maker", - "next", - "proc", - "lock", - "robot", - "snake", - "patch", - "matrix", - "drill", - "terminal", - "term", - "stuff", - "genetic", - "generic", - "identity", - "audit", - "pattern", - "audio", - "web_", - "web-", - "crud", - "problem", - "statu", - "cms-", - "cms_", - "arch", - "coffee", - "workflow", - "changelog", - "another", - "uiview", - "content", - "kitchen", - "gnu_", - "gnu-", - "gnu.", - "conf", - "couchdb", - "client", - "opencv", - "rendering", - "update", - "concept", - "varnish", - "gui_", - "gui-", - "gui.", - "version", - "shared", - "extra", - "product", - "still", - "not_", - "not-", - "not.", - "drop", - "ring", - "png_", - "png-", - "png.", - "actively", - "import", - "output", - "backup", - "start", - "embedded", - "registry", - "pool", - "semantic", - "instagram", - "bash", - "system", - "ninja", - "drupal", - "jquery", - "polyfill", - "physic", - "league", - "guide", - "pack", - "synopsi", - "sketch", - "injection", - "svg_", - "svg-", - "svg.", - "friendly", - "wave", - "convert", - "manage", - "camera", - "link", - "slide", - "timer", - "wrapper", - "gallery", - "url_", - "url-", - "url.", - "todomvc", - "requirej", - "party", - "http", - "payment", - "async", - "library", - "home", - "coco", - "gaia", - "display", - "universal", - "func", - "metadata", - "hipchat", - "under", - "room", - "config", - "personal", - "realtime", - "resume", - "database", - "testing", - "tiny", - "basic", - "forum", - "meetup", - "yet_", - "yet-", - "yet.", - "cento", - "dead", - "fluentd", - "editor", - "utilitie", - "run_", - "run-", - "run.", - "box_", - "box-", - "box.", - "bot_", - "bot-", - "bot.", - "making", - "sample", - "group", - "monitor", - "ajax", - "parallel", - "cassandra", - "ultimate", - "site", - "get_", - "get-", - "get.", - "gen_", - "gen-", - "gen.", - "gem_", - "gem-", - "gem.", - "extended", - "image", - "knife", - "asset", - "nested", - "zero", - "plugin", - "bracket", - "mule", - "mozilla", - "number", - "act_", - "act-", - "act.", - "map_", - "map-", - "map.", - "micro", - "debug", - "openshift", - "chart", - "expres", - "backend", - "task", - "source", - "translate", - "jbos", - "composer", - "sqlite", - "profile", - "mustache", - "mqtt", - "yeoman", - "have", - "builder", - "smart", - "like", - "oauth", - "school", - "guideline", - "captcha", - "filter", - "bitcoin", - "bridge", - "color", - "toolbox", - "discovery", - "new_", - "new-", - "new.", - "dashboard", - "when", - "setting", - "level", - "post", - "standard", - "port", - "platform", - "yui_", - "yui-", - "yui.", - "grunt", - "animation", - "haskell", - "icon", - "latex", - "cheat", - "lua_", - "lua-", - "lua.", - "gulp", - "case", - "author", - "without", - "simulator", - "wifi", - "directory", - "lisp", - "list", - "flat", - "adventure", - "story", - "storm", - "gpu_", - "gpu-", - "gpu.", - "store", - "caching", - "attention", - "solr", - "logger", - "demo", - "shortener", - "hadoop", - "finder", - "phone", - "pipeline", - "range", - "textmate", - "showcase", - "app_", - "app-", - "app.", - "idiomatic", - "edit", - "our_", - "our-", - "our.", - "out_", - "out-", - "out.", - "sentiment", - "linked", - "why_", - "why-", - "why.", - "local", - "cube", - "gmail", - "job_", - "job-", - "job.", - "rpc_", - "rpc-", - "rpc.", - "contest", - "tcp_", - "tcp-", - "tcp.", - "usage", - "buildout", - "weather", - "transfer", - "automated", - "sphinx", - "issue", - "sas_", - "sas-", - "sas.", - "parallax", - "jasmine", - "addon", - "machine", - "solution", - "dsl_", - "dsl-", - "dsl.", - "episode", - "menu", - "theme", - "best", - "adapter", - "debugger", - "chrome", - "tutorial", - "life", - "step", - "people", - "joomla", - "paypal", - "developer", - "solver", - "team", - "current", - "love", - "visual", - "date", - "data", - "canva", - "container", - "future", - "xml_", - "xml-", - "xml.", - "twig", - "nagio", - "spatial", - "original", - "sync", - "archived", - "refinery", - "science", - "mapping", - "gitlab", - "play", - "ext_", - "ext-", - "ext.", - "session", - "impact", - "set_", - "set-", - "set.", - "see_", - "see-", - "see.", - "migration", - "commit", - "community", - "shopify", - "what'", - "cucumber", - "statamic", - "mysql", - "location", - "tower", - "line", - "code", - "amqp", - "hello", - "send", - "index", - "high", - "notebook", - "alloy", - "python", - "field", - "document", - "soap", - "edition", - "email", - "php_", - "php-", - "php.", - "command", - "transport", - "official", - "upload", - "study", - "secure", - "angularj", - "akka", - "scalable", - "package", - "request", - "con_", - "con-", - "con.", - "flexible", - "security", - "comment", - "module", - "flask", - "graph", - "flash", - "apache", - "change", - "window", - "space", - "lambda", - "sheet", - "bookmark", - "carousel", - "friend", - "objective", - "jekyll", - "bootstrap", - "first", - "article", - "gwt_", - "gwt-", - "gwt.", - "classic", - "media", - "websocket", - "touch", - "desktop", - "real", - "read", - "recorder", - "moved", - "storage", - "validator", - "add-on", - "pusher", - "scs_", - "scs-", - "scs.", - "inline", - "asp_", - "asp-", - "asp.", - "timeline", - "base", - "encoding", - "ffmpeg", - "kindle", - "tinymce", - "pretty", - "jpa_", - "jpa-", - "jpa.", - "used", - "user", - "required", - "webhook", - "download", - "resque", - "espresso", - "cloud", - "mongo", - "benchmark", - "pure", - "cakephp", - "modx", - "mode", - "reactive", - "fuel", - "written", - "flickr", - "mail", - "brunch", - "meteor", - "dynamic", - "neo_", - "neo-", - "neo.", - "new_", - "new-", - "new.", - "net_", - "net-", - "net.", - "typo", - "type", - "keyboard", - "erlang", - "adobe", - "logging", - "ckeditor", - "message", - "iso_", - "iso-", - "iso.", - "hook", - "ldap", - "folder", - "reference", - "railscast", - "www_", - "www-", - "www.", - "tracker", - "azure", - "fork", - "form", - "digital", - "exporter", - "skin", - "string", - "template", - "designer", - "gollum", - "fluent", - "entity", - "language", - "alfred", - "summary", - "wiki", - "kernel", - "calendar", - "plupload", - "symfony", - "foundry", - "remote", - "talk", - "search", - "dev_", - "dev-", - "dev.", - "del_", - "del-", - "del.", - "token", - "idea", - "sencha", - "selector", - "interface", - "create", - "fun_", - "fun-", - "fun.", - "groovy", - "query", - "grail", - "red_", - "red-", - "red.", - "laravel", - "monkey", - "slack", - "supported", - "instant", - "value", - "center", - "latest", - "work", - "but_", - "but-", - "but.", - "bug_", - "bug-", - "bug.", - "virtual", - "tweet", - "statsd", - "studio", - "path", - "real-time", - "frontend", - "notifier", - "coding", - "tool", - "firmware", - "flow", - "random", - "mediawiki", - "bosh", - "been", - "beer", - "lightbox", - "theory", - "origin", - "redmine", - "hub_", - "hub-", - "hub.", - "require", - "pro_", - "pro-", - "pro.", - "ant_", - "ant-", - "ant.", - "any_", - "any-", - "any.", - "recipe", - "closure", - "mapper", - "event", - "todo", - "model", - "redi", - "provider", - "rvm_", - "rvm-", - "rvm.", - "program", - "memcached", - "rail", - "silex", - "foreman", - "activity", - "license", - "strategy", - "batch", - "streaming", - "fast", - "use_", - "use-", - "use.", - "usb_", - "usb-", - "usb.", - "impres", - "academy", - "slider", - "please", - "layer", - "cros", - "now_", - "now-", - "now.", - "miner", - "extension", - "own_", - "own-", - "own.", - "app_", - "app-", - "app.", - "debian", - "symphony", - "example", - "feature", - "serie", - "tree", - "project", - "runner", - "entry", - "leetcode", - "layout", - "webrtc", - "logic", - "login", - "worker", - "toolkit", - "mocha", - "support", - "back", - "inside", - "device", - "jenkin", - "contact", - "fake", - "awesome", - "ocaml", - "bit_", - "bit-", - "bit.", - "drive", - "screen", - "prototype", - "gist", - "binary", - "nosql", - "rest", - "overview", - "dart", - "dark", - "emac", - "mongoid", - "solarized", - "homepage", - "emulator", - "commander", - "django", - "yandex", - "gradle", - "xcode", - "writer", - "crm_", - "crm-", - "crm.", - "jade", - "startup", - "error", - "using", - "format", - "name", - "spring", - "parser", - "scratch", - "magic", - "try_", - "try-", - "try.", - "rack", - "directive", - "challenge", - "slim", - "counter", - "element", - "chosen", - "doc_", - "doc-", - "doc.", - "meta", - "should", - "button", - "packet", - "stream", - "hardware", - "android", - "infinite", - "password", - "software", - "ghost", - "xamarin", - "spec", - "chef", - "interview", - "hubot", - "mvc_", - "mvc-", - "mvc.", - "exercise", - "leaflet", - "launcher", - "air_", - "air-", - "air.", - "photo", - "board", - "boxen", - "way_", - "way-", - "way.", - "computing", - "welcome", - "notepad", - "portfolio", - "cat_", - "cat-", - "cat.", - "can_", - "can-", - "can.", - "magento", - "yaml", - "domain", - "card", - "yii_", - "yii-", - "yii.", - "checker", - "browser", - "upgrade", - "only", - "progres", - "aura", - "ruby_", - "ruby-", - "ruby.", - "polymer", - "util", - "lite", - "hackathon", - "rule", - "log_", - "log-", - "log.", - "opengl", - "stanford", - "skeleton", - "history", - "inspector", - "help", - "soon", - "selenium", - "lab_", - "lab-", - "lab.", - "scheme", - "schema", - "look", - "ready", - "leveldb", - "docker", - "game", - "minimal", - "logstash", - "messaging", - "within", - "heroku", - "mongodb", - "kata", - "suite", - "picker", - "win_", - "win-", - "win.", - "wip_", - "wip-", - "wip.", - "panel", - "started", - "starter", - "front-end", - "detector", - "deploy", - "editing", - "based", - "admin", - "capture", - "spree", - "page", - "bundle", - "goal", - "rpg_", - "rpg-", - "rpg.", - "setup", - "side", - "mean", - "reader", - "cookbook", - "mini", - "modern", - "seed", - "dom_", - "dom-", - "dom.", - "doc_", - "doc-", - "doc.", - "dot_", - "dot-", - "dot.", - "syntax", - "sugar", - "loader", - "website", - "make", - "kit_", - "kit-", - "kit.", - "protocol", - "human", - "daemon", - "golang", - "manager", - "countdown", - "connector", - "swagger", - "map_", - "map-", - "map.", - "mac_", - "mac-", - "mac.", - "man_", - "man-", - "man.", - "orm_", - "orm-", - "orm.", - "org_", - "org-", - "org.", - "little", - "zsh_", - "zsh-", - "zsh.", - "shop", - "show", - "workshop", - "money", - "grid", - "server", - "octopres", - "svn_", - "svn-", - "svn.", - "ember", - "embed", - "general", - "file", - "important", - "dropbox", - "portable", - "public", - "docpad", - "fish", - "sbt_", - "sbt-", - "sbt.", - "done", - "para", - "network", - "common", - "readme", - "popup", - "simple", - "purpose", - "mirror", - "single", - "cordova", - "exchange", - "object", - "design", - "gateway", - "account", - "lamp", - "intellij", - "math", - "mit_", - "mit-", - "mit.", - "control", - "enhanced", - "emitter", - "multi", - "add_", - "add-", - "add.", - "about", - "socket", - "preview", - "vagrant", - "cli_", - "cli-", - "cli.", - "powerful", - "top_", - "top-", - "top.", - "radio", - "watch", - "fluid", - "amazon", - "report", - "couchbase", - "automatic", - "detection", - "sprite", - "pyramid", - "portal", - "advanced", - "plu_", - "plu-", - "plu.", - "runtime", - "git_", - "git-", - "git.", - "uri_", - "uri-", - "uri.", - "haml", - "node", - "sql_", - "sql-", - "sql.", - "cool", - "core", - "obsolete", - "handler", - "iphone", - "extractor", - "array", - "copy", - "nlp_", - "nlp-", - "nlp.", - "reveal", - "pop_", - "pop-", - "pop.", - "engine", - "parse", - "check", - "html", - "nest", - "all_", - "all-", - "all.", - "chinese", - "buildpack", - "what", - "tag_", - "tag-", - "tag.", - "proxy", - "style", - "cookie", - "feed", - "restful", - "compiler", - "creating", - "prelude", - "context", - "java", - "rspec", - "mock", - "backbone", - "light", - "spotify", - "flex", - "related", - "shell", - "which", - "clas", - "webapp", - "swift", - "ansible", - "unity", - "console", - "tumblr", - "export", - "campfire", - "conway'", - "made", - "riak", - "hero", - "here", - "unix", - "unit", - "glas", - "smtp", - "how_", - "how-", - "how.", - "hot_", - "hot-", - "hot.", - "debug", - "release", - "diff", - "player", - "easy", - "right", - "old_", - "old-", - "old.", - "animate", - "time", - "push", - "explorer", - "course", - "training", - "nette", - "router", - "draft", - "structure", - "note", - "salt", - "where", - "spark", - "trello", - "power", - "method", - "social", - "via_", - "via-", - "via.", - "vim_", - "vim-", - "vim.", - "select", - "webkit", - "github", - "ftp_", - "ftp-", - "ftp.", - "creator", - "mongoose", - "led_", - "led-", - "led.", - "movie", - "currently", - "pdf_", - "pdf-", - "pdf.", - "load", - "markdown", - "phalcon", - "input", - "custom", - "atom", - "oracle", - "phonegap", - "ubuntu", - "great", - "rdf_", - "rdf-", - "rdf.", - "popcorn", - "firefox", - "zip_", - "zip-", - "zip.", - "cuda", - "dotfile", - "static", - "openwrt", - "viewer", - "powered", - "graphic", - "les_", - "les-", - "les.", - "doe_", - "doe-", - "doe.", - "maven", - "word", - "eclipse", - "lab_", - "lab-", - "lab.", - "hacking", - "steam", - "analytic", - "option", - "abstract", - "archive", - "reality", - "switcher", - "club", - "write", - "kafka", - "arduino", - "angular", - "online", - "title", - "don't", - "contao", - "notice", - "analyzer", - "learning", - "zend", - "external", - "staging", - "busines", - "tdd_", - "tdd-", - "tdd.", - "scanner", - "building", - "snippet", - "modular", - "bower", - "stm_", - "stm-", - "stm.", - "lib_", - "lib-", - "lib.", - "alpha", - "mobile", - "clean", - "linux", - "nginx", - "manifest", - "some", - "raspberry", - "gnome", - "ide_", - "ide-", - "ide.", - "block", - "statistic", - "info", - "drag", - "youtube", - "koan", - "facebook", - "paperclip", - "art_", - "art-", - "art.", - "quality", - "tab_", - "tab-", - "tab.", - "need", - "dojo", - "shield", - "computer", - "stat", - "state", - "twitter", - "utility", - "converter", - "hosting", - "devise", - "liferay", - "updated", - "force", - "tip_", - "tip-", - "tip.", - "behavior", - "active", - "call", - "answer", - "deck", - "better", - "principle", - "ches", - "bar_", - "bar-", - "bar.", - "reddit", - "three", - "haxe", - "just", - "plug-in", - "agile", - "manual", - "tetri", - "super", - "beta", - "parsing", - "doctrine", - "minecraft", - "useful", - "perl", - "sharing", - "agent", - "switch", - "view", - "dash", - "channel", - "repo", - "pebble", - "profiler", - "warning", - "cluster", - "running", - "markup", - "evented", - "mod_", - "mod-", - "mod.", - "share", - "csv_", - "csv-", - "csv.", - "response", - "good", - "house", - "connect", - "built", - "build", - "find", - "ipython", - "webgl", - "big_", - "big-", - "big.", - "google", - "scala", - "sdl_", - "sdl-", - "sdl.", - "sdk_", - "sdk-", - "sdk.", - "native", - "day_", - "day-", - "day.", - "puppet", - "text", - "routing", - "helper", - "linkedin", - "crawler", - "host", - "guard", - "merchant", - "poker", - "over", - "writing", - "free", - "classe", - "component", - "craft", - "nodej", - "phoenix", - "longer", - "quick", - "lazy", - "memory", - "clone", - "hacker", - "middleman", - "factory", - "motion", - "multiple", - "tornado", - "hack", - "ssh_", - "ssh-", - "ssh.", - "review", - "vimrc", - "driver", - "driven", - "blog", - "particle", - "table", - "intro", - "importer", - "thrift", - "xmpp", - "framework", - "refresh", - "react", - "font", - "librarie", - "variou", - "formatter", - "analysi", - "karma", - "scroll", - "tut_", - "tut-", - "tut.", - "apple", - "tag_", - "tag-", - "tag.", - "tab_", - "tab-", - "tab.", - "category", - "ionic", - "cache", - "homebrew", - "reverse", - "english", - "getting", - "shipping", - "clojure", - "boot", - "book", - "branch", - "combination", - "combo", -] -[[rules]] -description = "GitHub App Token" -id = "github-app-token" -regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' -keywords = [ - "ghu_","ghs_", -] - -[[rules]] -description = "GitHub Fine-Grained Personal Access Token" -id = "github-fine-grained-pat" -regex = '''github_pat_[0-9a-zA-Z_]{82}''' -keywords = [ - "github_pat_", -] - -[[rules]] -description = "GitHub OAuth Access Token" -id = "github-oauth" -regex = '''gho_[0-9a-zA-Z]{36}''' -keywords = [ - "gho_", -] - -[[rules]] -description = "GitHub Personal Access Token" -id = "github-pat" -regex = '''ghp_[0-9a-zA-Z]{36}''' -keywords = [ - "ghp_", -] - -[[rules]] -description = "GitHub Refresh Token" -id = "github-refresh-token" -regex = '''ghr_[0-9a-zA-Z]{36}''' -keywords = [ - "ghr_", -] - -[[rules]] -description = "GitLab Personal Access Token" -id = "gitlab-pat" -regex = '''glpat-[0-9a-zA-Z\-\_]{20}''' -keywords = [ - "glpat-", -] - -[[rules]] -description = "GitLab Pipeline Trigger Token" -id = "gitlab-ptt" -regex = '''glptt-[0-9a-f]{40}''' -keywords = [ - "glptt-", -] - -[[rules]] -description = "GitLab Runner Registration Token" -id = "gitlab-rrt" -regex = '''GR1348941[0-9a-zA-Z\-\_]{20}''' -keywords = [ - "gr1348941", -] - -[[rules]] -description = "Gitter Access Token" -id = "gitter-access-token" -regex = '''(?i)(?:gitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "gitter", -] - -[[rules]] -description = "GoCardless API token" -id = "gocardless-api-token" -regex = '''(?i)(?:gocardless)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(live_(?i)[a-z0-9\-_=]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "live_","gocardless", -] - -[[rules]] -description = "Grafana api key (or Grafana cloud api key)" -id = "grafana-api-key" -regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "eyjrijoi", -] - -[[rules]] -description = "Grafana cloud api token" -id = "grafana-cloud-api-token" -regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "glc_", -] - -[[rules]] -description = "Grafana service account token" -id = "grafana-service-account-token" -regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "glsa_", -] - -[[rules]] -description = "HashiCorp Terraform user/org API token" -id = "hashicorp-tf-api-token" -regex = '''(?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}''' -keywords = [ - "atlasv1", -] - -[[rules]] -description = "Heroku API Key" -id = "heroku-api-key" -regex = '''(?i)(?:heroku)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "heroku", -] - -[[rules]] -description = "HubSpot API Token" -id = "hubspot-api-key" -regex = '''(?i)(?:hubspot)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "hubspot", -] - -[[rules]] -description = "Intercom API Token" -id = "intercom-api-key" -regex = '''(?i)(?:intercom)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{60})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "intercom", -] - -[[rules]] -description = "JSON Web Token" -id = "jwt" -regex = '''(?i)\b(ey[0-9a-z]{30,34}\.ey[0-9a-z-\/_]{30,500}\.[0-9a-zA-Z-\/_]{10,200}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "ey", -] - -[[rules]] -description = "Kraken Access Token" -id = "kraken-access-token" -regex = '''(?i)(?:kraken)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9\/=_\+\-]{80,90})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kraken", -] - -[[rules]] -description = "Kucoin Access Token" -id = "kucoin-access-token" -regex = '''(?i)(?:kucoin)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kucoin", -] - -[[rules]] -description = "Kucoin Secret Key" -id = "kucoin-secret-key" -regex = '''(?i)(?:kucoin)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "kucoin", -] - -[[rules]] -description = "Launchdarkly Access Token" -id = "launchdarkly-access-token" -regex = '''(?i)(?:launchdarkly)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "launchdarkly", -] - -[[rules]] -description = "Linear API Token" -id = "linear-api-key" -regex = '''lin_api_(?i)[a-z0-9]{40}''' -keywords = [ - "lin_api_", -] - -[[rules]] -description = "Linear Client Secret" -id = "linear-client-secret" -regex = '''(?i)(?:linear)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linear", -] - -[[rules]] -description = "LinkedIn Client ID" -id = "linkedin-client-id" -regex = '''(?i)(?:linkedin|linked-in)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{14})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linkedin","linked-in", -] - -[[rules]] -description = "LinkedIn Client secret" -id = "linkedin-client-secret" -regex = '''(?i)(?:linkedin|linked-in)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "linkedin","linked-in", -] - -[[rules]] -description = "Lob API Key" -id = "lob-api-key" -regex = '''(?i)(?:lob)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}((live|test)_[a-f0-9]{35})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "test_","live_", -] - -[[rules]] -description = "Lob Publishable API Key" -id = "lob-pub-api-key" -regex = '''(?i)(?:lob)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}((test|live)_pub_[a-f0-9]{31})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "test_pub","live_pub","_pub", -] - -[[rules]] -description = "Mailchimp API key" -id = "mailchimp-api-key" -regex = '''(?i)(?:mailchimp)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{32}-us20)(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailchimp", -] - -[[rules]] -description = "Mailgun private API token" -id = "mailgun-private-api-token" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(key-[a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "Mailgun public validation key" -id = "mailgun-pub-key" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(pubkey-[a-f0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "Mailgun webhook signing key" -id = "mailgun-signing-key" -regex = '''(?i)(?:mailgun)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mailgun", -] - -[[rules]] -description = "MapBox API token" -id = "mapbox-api-token" -regex = '''(?i)(?:mapbox)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mapbox", -] - -[[rules]] -description = "Mattermost Access Token" -id = "mattermost-access-token" -regex = '''(?i)(?:mattermost)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{26})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "mattermost", -] - -[[rules]] -description = "MessageBird API token" -id = "messagebird-api-token" -regex = '''(?i)(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{25})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "messagebird","message-bird","message_bird", -] - -[[rules]] -description = "MessageBird client ID" -id = "messagebird-client-id" -regex = '''(?i)(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "messagebird","message-bird","message_bird", -] - -[[rules]] -description = "Microsoft Teams Webhook" -id = "microsoft-teams-webhook" -regex = '''https:\/\/[a-z0-9]+\.webhook\.office\.com\/webhookb2\/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\/IncomingWebhook\/[a-z0-9]{32}\/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' -keywords = [ - "webhook.office.com","webhookb2","incomingwebhook", -] - -[[rules]] -description = "Netlify Access Token" -id = "netlify-access-token" -regex = '''(?i)(?:netlify)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{40,46})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "netlify", -] - -[[rules]] -description = "New Relic ingest browser API token" -id = "new-relic-browser-api-token" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(NRJS-[a-f0-9]{19})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nrjs-", -] - -[[rules]] -description = "New Relic user API ID" -id = "new-relic-user-api-id" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "new-relic","newrelic","new_relic", -] - -[[rules]] -description = "New Relic user API Key" -id = "new-relic-user-api-key" -regex = '''(?i)(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(NRAK-[a-z0-9]{27})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nrak", -] - -[[rules]] -description = "npm access token" -id = "npm-access-token" -regex = '''(?i)\b(npm_[a-z0-9]{36})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "npm_", -] - -[[rules]] -description = "Nytimes Access Token" -id = "nytimes-access-token" -regex = '''(?i)(?:nytimes|new-york-times,|newyorktimes)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "nytimes","new-york-times","newyorktimes", -] - -[[rules]] -description = "Okta Access Token" -id = "okta-access-token" -regex = '''(?i)(?:okta)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{42})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "okta", -] - -[[rules]] -description = "Plaid API Token" -id = "plaid-api-token" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "Plaid Client ID" -id = "plaid-client-id" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{24})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "Plaid Secret key" -id = "plaid-secret-key" -regex = '''(?i)(?:plaid)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "plaid", -] - -[[rules]] -description = "PlanetScale API token" -id = "planetscale-api-token" -regex = '''(?i)\b(pscale_tkn_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_tkn_", -] - -[[rules]] -description = "PlanetScale OAuth token" -id = "planetscale-oauth-token" -regex = '''(?i)\b(pscale_oauth_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_oauth_", -] - -[[rules]] -description = "PlanetScale password" -id = "planetscale-password" -regex = '''(?i)\b(pscale_pw_(?i)[a-z0-9=\-_\.]{32,64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pscale_pw_", -] - -[[rules]] -description = "Postman API token" -id = "postman-api-token" -regex = '''(?i)\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pmak-", -] - -[[rules]] -description = "Prefect API token" -id = "prefect-api-token" -regex = '''(?i)\b(pnu_[a-z0-9]{36})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pnu_", -] - -[[rules]] -description = "Private Key" -id = "private-key" -regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY( BLOCK)?-----[\s\S-]*KEY( BLOCK)?----''' -keywords = [ - "-----begin", -] - -[[rules]] -description = "Pulumi API token" -id = "pulumi-api-token" -regex = '''(?i)\b(pul-[a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "pul-", -] - -[[rules]] -description = "PyPI upload token" -id = "pypi-upload-token" -regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}''' -keywords = [ - "pypi-ageichlwas5vcmc", -] - -[[rules]] -description = "RapidAPI Access Token" -id = "rapidapi-access-token" -regex = '''(?i)(?:rapidapi)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9_-]{50})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rapidapi", -] - -[[rules]] -description = "Readme API token" -id = "readme-api-token" -regex = '''(?i)\b(rdme_[a-z0-9]{70})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rdme_", -] - -[[rules]] -description = "Rubygem API token" -id = "rubygems-api-token" -regex = '''(?i)\b(rubygems_[a-f0-9]{48})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "rubygems_", -] - -[[rules]] -description = "Sendbird Access ID" -id = "sendbird-access-id" -regex = '''(?i)(?:sendbird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sendbird", -] - -[[rules]] -description = "Sendbird Access Token" -id = "sendbird-access-token" -regex = '''(?i)(?:sendbird)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sendbird", -] - -[[rules]] -description = "SendGrid API token" -id = "sendgrid-api-token" -regex = '''(?i)\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sg.", -] - -[[rules]] -description = "Sendinblue API token" -id = "sendinblue-api-token" -regex = '''(?i)\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "xkeysib-", -] - -[[rules]] -description = "Sentry Access Token" -id = "sentry-access-token" -regex = '''(?i)(?:sentry)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sentry", -] - -[[rules]] -description = "Shippo API token" -id = "shippo-api-token" -regex = '''(?i)\b(shippo_(live|test)_[a-f0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "shippo_", -] - -[[rules]] -description = "Shopify access token" -id = "shopify-access-token" -regex = '''shpat_[a-fA-F0-9]{32}''' -keywords = [ - "shpat_", -] - -[[rules]] -description = "Shopify custom access token" -id = "shopify-custom-access-token" -regex = '''shpca_[a-fA-F0-9]{32}''' -keywords = [ - "shpca_", -] - -[[rules]] -description = "Shopify private app access token" -id = "shopify-private-app-access-token" -regex = '''shppa_[a-fA-F0-9]{32}''' -keywords = [ - "shppa_", -] - -[[rules]] -description = "Shopify shared secret" -id = "shopify-shared-secret" -regex = '''shpss_[a-fA-F0-9]{32}''' -keywords = [ - "shpss_", -] - -[[rules]] -description = "Sidekiq Secret" -id = "sidekiq-secret" -regex = '''(?i)(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "bundle_enterprise__contribsys__com","bundle_gems__contribsys__com", -] - -[[rules]] -description = "Sidekiq Sensitive URL" -id = "sidekiq-sensitive-url" -regex = '''(?i)\b(http(?:s??):\/\/)([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' -secretGroup = 2 -keywords = [ - "gems.contribsys.com","enterprise.contribsys.com", -] - -[[rules]] -description = "Slack token" -id = "slack-access-token" -regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})''' -keywords = [ - "xoxb","xoxa","xoxp","xoxr","xoxs", -] - -[[rules]] -description = "Slack Webhook" -id = "slack-web-hook" -regex = '''https:\/\/hooks.slack.com\/(services|workflows)\/[A-Za-z0-9+\/]{44,46}''' -keywords = [ - "hooks.slack.com", -] - -[[rules]] -description = "Square Access Token" -id = "square-access-token" -regex = '''(?i)\b(sq0atp-[0-9A-Za-z\-_]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "sq0atp-", -] - -[[rules]] -description = "Squarespace Access Token" -id = "squarespace-access-token" -regex = '''(?i)(?:squarespace)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "squarespace", -] - -[[rules]] -description = "Stripe Access Token" -id = "stripe-access-token" -regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}''' -keywords = [ - "sk_test","pk_test","sk_live","pk_live", -] - -[[rules]] -description = "SumoLogic Access ID" -id = "sumologic-access-id" -regex = '''(?i)(?:sumo)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{14})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sumo", -] - -[[rules]] -description = "SumoLogic Access Token" -id = "sumologic-access-token" -regex = '''(?i)(?:sumo)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{64})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "sumo", -] - -[[rules]] -description = "Telegram Bot API Token" -id = "telegram-bot-api-token" -regex = '''(?i)(?:^|[^0-9])([0-9]{5,16}:A[a-zA-Z0-9_\-]{34})(?:$|[^a-zA-Z0-9_\-])''' -secretGroup = 1 -keywords = [ - "telegram","api","bot","token","url", -] - -[[rules]] -description = "Travis CI Access Token" -id = "travisci-access-token" -regex = '''(?i)(?:travis)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{22})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "travis", -] - -[[rules]] -description = "Twilio API Key" -id = "twilio-api-key" -regex = '''SK[0-9a-fA-F]{32}''' -keywords = [ - "twilio", -] - -[[rules]] -description = "Twitch API token" -id = "twitch-api-token" -regex = '''(?i)(?:twitch)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{30})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitch", -] - -[[rules]] -description = "Twitter Access Secret" -id = "twitter-access-secret" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{45})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter Access Token" -id = "twitter-access-token" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter API Key" -id = "twitter-api-key" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{25})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter API Secret" -id = "twitter-api-secret" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{50})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Twitter Bearer Token" -id = "twitter-bearer-token" -regex = '''(?i)(?:twitter)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "twitter", -] - -[[rules]] -description = "Typeform API token" -id = "typeform-api-token" -regex = '''(?i)(?:typeform)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(tfp_[a-z0-9\-_\.=]{59})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "tfp_", -] - -[[rules]] -description = "Vault Batch Token" -id = "vault-batch-token" -regex = '''(?i)\b(hvb\.[a-z0-9_-]{138,212})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "hvb", -] - -[[rules]] -description = "Vault Service Token" -id = "vault-service-token" -regex = '''(?i)\b(hvs\.[a-z0-9_-]{90,100})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -keywords = [ - "hvs", -] - -[[rules]] -description = "Yandex Access Token" -id = "yandex-access-token" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Yandex API Key" -id = "yandex-api-key" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Yandex AWS Access Token" -id = "yandex-aws-access-token" -regex = '''(?i)(?:yandex)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}(YC[a-zA-Z0-9_\-]{38})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "yandex", -] - -[[rules]] -description = "Zendesk Secret Key" -id = "zendesk-secret-key" -regex = '''(?i)(?:zendesk)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)''' -secretGroup = 1 -keywords = [ - "zendesk", -] - - diff --git a/cli/config/rule.go b/cli/config/rule.go deleted file mode 100644 index b7c8c1518..000000000 --- a/cli/config/rule.go +++ /dev/null @@ -1,43 +0,0 @@ -package config - -import ( - "regexp" -) - -// Rules contain information that define details on how to detect secrets -type Rule struct { - // Description is the description of the rule. - Description string - - // RuleID is a unique identifier for this rule - RuleID string - - // Entropy is a float representing the minimum shannon - // entropy a regex group must have to be considered a secret. - Entropy float64 - - // SecretGroup is an int used to extract secret from regex - // match and used as the group that will have its entropy - // checked if `entropy` is set. - SecretGroup int - - // Regex is a golang regular expression used to detect secrets. - Regex *regexp.Regexp - - // Path is a golang regular expression used to - // filter secrets by path - Path *regexp.Regexp - - // Tags is an array of strings used for metadata - // and reporting purposes. - Tags []string - - // Keywords are used for pre-regex check filtering. Rules that contain - // keywords will perform a quick string compare check to make sure the - // keyword(s) are in the content being scanned. - Keywords []string - - // Allowlist allows a rule to be ignored for specific - // regexes, paths, and/or commits - Allowlist Allowlist -} diff --git a/cli/config/utils.go b/cli/config/utils.go deleted file mode 100644 index ada6ff0fe..000000000 --- a/cli/config/utils.go +++ /dev/null @@ -1,24 +0,0 @@ -package config - -import ( - "regexp" -) - -func anyRegexMatch(f string, res []*regexp.Regexp) bool { - for _, re := range res { - if regexMatched(f, re) { - return true - } - } - return false -} - -func regexMatched(f string, re *regexp.Regexp) bool { - if re == nil { - return false - } - if re.FindString(f) != "" { - return true - } - return false -} diff --git a/cli/detect/baseline.go b/cli/detect/baseline.go index bd4c25665..eeaa2a73a 100644 --- a/cli/detect/baseline.go +++ b/cli/detect/baseline.go @@ -25,35 +25,31 @@ package detect import ( "encoding/json" "fmt" - "io" "os" + "path/filepath" - "github.com/rs/zerolog/log" - - "github.com/Infisical/infisical-merge/report" + "github.com/Infisical/infisical-merge/detect/report" ) -func IsNew(finding report.Finding, baseline []report.Finding) bool { +func IsNew(finding report.Finding, redact uint, baseline []report.Finding) bool { // Explicitly testing each property as it gives significantly better performance in comparison to cmp.Equal(). Drawback is that - // the code requires maintanance if/when the Finding struct changes + // the code requires maintenance if/when the Finding struct changes for _, b := range baseline { - - if finding.Author == b.Author && - finding.Commit == b.Commit && - finding.Date == b.Date && + if finding.RuleID == b.RuleID && finding.Description == b.Description && - finding.Email == b.Email && - finding.EndColumn == b.EndColumn && + finding.StartLine == b.StartLine && finding.EndLine == b.EndLine && - finding.Entropy == b.Entropy && - finding.File == b.File && - // Omit checking finding.Fingerprint - if the format of the fingerprint changes, the users will see unexpected behaviour - finding.Match == b.Match && - finding.Message == b.Message && - finding.RuleID == b.RuleID && - finding.Secret == b.Secret && finding.StartColumn == b.StartColumn && - finding.StartLine == b.StartLine { + finding.EndColumn == b.EndColumn && + (redact > 0 || (finding.Match == b.Match && finding.Secret == b.Secret)) && + finding.File == b.File && + finding.Commit == b.Commit && + finding.Author == b.Author && + finding.Email == b.Email && + finding.Date == b.Date && + finding.Message == b.Message && + // Omit checking finding.Fingerprint - if the format of the fingerprint changes, the users will see unexpected behaviour + finding.Entropy == b.Entropy { return false } } @@ -61,23 +57,12 @@ func IsNew(finding report.Finding, baseline []report.Finding) bool { } func LoadBaseline(baselinePath string) ([]report.Finding, error) { - var previousFindings []report.Finding - jsonFile, err := os.Open(baselinePath) + bytes, err := os.ReadFile(baselinePath) if err != nil { return nil, fmt.Errorf("could not open %s", baselinePath) } - defer func() { - if cerr := jsonFile.Close(); cerr != nil { - log.Warn().Err(cerr).Msg("problem closing jsonFile handle") - } - }() - - bytes, err := io.ReadAll(jsonFile) - if err != nil { - return nil, fmt.Errorf("could not read data from the file %s", baselinePath) - } - + var previousFindings []report.Finding err = json.Unmarshal(bytes, &previousFindings) if err != nil { return nil, fmt.Errorf("the format of the file %s is not supported", baselinePath) @@ -85,3 +70,34 @@ func LoadBaseline(baselinePath string) ([]report.Finding, error) { return previousFindings, nil } + +func (d *Detector) AddBaseline(baselinePath string, source string) error { + if baselinePath != "" { + absoluteSource, err := filepath.Abs(source) + if err != nil { + return err + } + + absoluteBaseline, err := filepath.Abs(baselinePath) + if err != nil { + return err + } + + relativeBaseline, err := filepath.Rel(absoluteSource, absoluteBaseline) + if err != nil { + return err + } + + baseline, err := LoadBaseline(baselinePath) + if err != nil { + return err + } + + d.baseline = baseline + baselinePath = relativeBaseline + + } + + d.baselinePath = baselinePath + return nil +} diff --git a/cli/detect/baseline_test.go b/cli/detect/baseline_test.go deleted file mode 100644 index 91d2eb72e..000000000 --- a/cli/detect/baseline_test.go +++ /dev/null @@ -1,160 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/Infisical/infisical-merge/report" -) - -func TestIsNew(t *testing.T) { - tests := []struct { - findings report.Finding - baseline []report.Finding - expect bool - }{ - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0000", - }, - }, - expect: false, - }, - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0002", - }, - }, - expect: true, - }, - { - findings: report.Finding{ - Author: "a", - Commit: "0000", - Tags: []string{"a", "b"}, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "0000", - Tags: []string{"a", "c"}, - }, - }, - expect: false, // Updated tags doesn't make it a new finding - }, - } - for _, test := range tests { - assert.Equal(t, test.expect, IsNew(test.findings, test.baseline)) - } -} - -func TestFileLoadBaseline(t *testing.T) { - tests := []struct { - Filename string - ExpectedError error - }{ - { - Filename: "../testdata/baseline/baseline.csv", - ExpectedError: errors.New("the format of the file ../testdata/baseline/baseline.csv is not supported"), - }, - { - Filename: "../testdata/baseline/baseline.sarif", - ExpectedError: errors.New("the format of the file ../testdata/baseline/baseline.sarif is not supported"), - }, - { - Filename: "../testdata/baseline/notfound.json", - ExpectedError: errors.New("could not open ../testdata/baseline/notfound.json"), - }, - } - - for _, test := range tests { - _, err := LoadBaseline(test.Filename) - assert.Equal(t, test.ExpectedError.Error(), err.Error()) - } -} - -func TestIgnoreIssuesInBaseline(t *testing.T) { - tests := []struct { - findings []report.Finding - baseline []report.Finding - expectCount int - }{ - { - findings: []report.Finding{ - { - Author: "a", - Commit: "5", - }, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "5", - }, - }, - expectCount: 0, - }, - { - findings: []report.Finding{ - { - Author: "a", - Commit: "5", - Fingerprint: "a", - }, - }, - baseline: []report.Finding{ - { - Author: "a", - Commit: "5", - Fingerprint: "b", - }, - }, - expectCount: 0, - }, - } - - for _, test := range tests { - d, _ := NewDetectorDefaultConfig() - d.baseline = test.baseline - for _, finding := range test.findings { - d.addFinding(finding) - } - assert.Equal(t, test.expectCount, len(d.findings)) - } -} diff --git a/cli/detect/cmd/scm/scm.go b/cli/detect/cmd/scm/scm.go new file mode 100644 index 000000000..66868aadc --- /dev/null +++ b/cli/detect/cmd/scm/scm.go @@ -0,0 +1,66 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package scm + +import ( + "fmt" + "strings" +) + +type Platform int + +const ( + UnknownPlatform Platform = iota + NoPlatform // Explicitly disable the feature + GitHubPlatform + GitLabPlatform + AzureDevOpsPlatform + // TODO: Add others. +) + +func (p Platform) String() string { + return [...]string{ + "unknown", + "none", + "github", + "gitlab", + "azuredevops", + }[p] +} + +func PlatformFromString(s string) (Platform, error) { + switch strings.ToLower(s) { + case "", "unknown": + return UnknownPlatform, nil + case "none": + return NoPlatform, nil + case "github": + return GitHubPlatform, nil + case "gitlab": + return GitLabPlatform, nil + case "azuredevops": + return AzureDevOpsPlatform, nil + default: + return UnknownPlatform, fmt.Errorf("invalid scm platform value: %s", s) + } +} diff --git a/cli/config/allowlist.go b/cli/detect/config/allowlist.go similarity index 53% rename from cli/config/allowlist.go rename to cli/detect/config/allowlist.go index 373325758..d91188f68 100644 --- a/cli/config/allowlist.go +++ b/cli/detect/config/allowlist.go @@ -23,63 +23,137 @@ package config import ( - "regexp" + "fmt" "strings" + + "golang.org/x/exp/maps" + + "github.com/Infisical/infisical-merge/detect/regexp" ) +type AllowlistMatchCondition int + +const ( + AllowlistMatchOr AllowlistMatchCondition = iota + AllowlistMatchAnd +) + +func (a AllowlistMatchCondition) String() string { + return [...]string{ + "OR", + "AND", + }[a] +} + // Allowlist allows a rule to be ignored for specific // regexes, paths, and/or commits type Allowlist struct { // Short human readable description of the allowlist. Description string - // Regexes is slice of content regular expressions that are allowed to be ignored. - Regexes []*regexp.Regexp + // MatchCondition determines whether all criteria must match. + MatchCondition AllowlistMatchCondition - // RegexTarget - RegexTarget string + // Commits is a slice of commit SHAs that are allowed to be ignored. Defaults to "OR". + Commits []string // Paths is a slice of path regular expressions that are allowed to be ignored. Paths []*regexp.Regexp - // Commits is a slice of commit SHAs that are allowed to be ignored. - Commits []string + // Can be `match` or `line`. + // + // If `match` the _Regexes_ will be tested against the match of the _Rule.Regex_. + // + // If `line` the _Regexes_ will be tested against the entire line. + // + // If RegexTarget is empty, it will be tested against the found secret. + RegexTarget string + + // Regexes is slice of content regular expressions that are allowed to be ignored. + Regexes []*regexp.Regexp // StopWords is a slice of stop words that are allowed to be ignored. // This targets the _secret_, not the content of the regex match like the // Regexes slice. StopWords []string + + // validated is an internal flag to track whether `Validate()` has been called. + validated bool +} + +func (a *Allowlist) Validate() error { + if a.validated { + return nil + } + + // Disallow empty allowlists. + if len(a.Commits) == 0 && + len(a.Paths) == 0 && + len(a.Regexes) == 0 && + len(a.StopWords) == 0 { + return fmt.Errorf("must contain at least one check for: commits, paths, regexes, or stopwords") + } + + // Deduplicate commits and stopwords. + if len(a.Commits) > 0 { + uniqueCommits := make(map[string]struct{}) + for _, commit := range a.Commits { + uniqueCommits[commit] = struct{}{} + } + a.Commits = maps.Keys(uniqueCommits) + } + if len(a.StopWords) > 0 { + uniqueStopwords := make(map[string]struct{}) + for _, stopWord := range a.StopWords { + uniqueStopwords[stopWord] = struct{}{} + } + a.StopWords = maps.Keys(uniqueStopwords) + } + + a.validated = true + return nil } // CommitAllowed returns true if the commit is allowed to be ignored. -func (a *Allowlist) CommitAllowed(c string) bool { - if c == "" { - return false +func (a *Allowlist) CommitAllowed(c string) (bool, string) { + if a == nil || c == "" { + return false, "" } + for _, commit := range a.Commits { if commit == c { - return true + return true, c } } - return false + return false, "" } // PathAllowed returns true if the path is allowed to be ignored. func (a *Allowlist) PathAllowed(path string) bool { + if a == nil || path == "" { + return false + } return anyRegexMatch(path, a.Paths) } // RegexAllowed returns true if the regex is allowed to be ignored. -func (a *Allowlist) RegexAllowed(s string) bool { - return anyRegexMatch(s, a.Regexes) +func (a *Allowlist) RegexAllowed(secret string) bool { + if a == nil || secret == "" { + return false + } + return anyRegexMatch(secret, a.Regexes) } -func (a *Allowlist) ContainsStopWord(s string) bool { +func (a *Allowlist) ContainsStopWord(s string) (bool, string) { + if a == nil || s == "" { + return false, "" + } + s = strings.ToLower(s) for _, stopWord := range a.StopWords { if strings.Contains(s, strings.ToLower(stopWord)) { - return true + return true, stopWord } } - return false + return false, "" } diff --git a/cli/detect/config/config.go b/cli/detect/config/config.go new file mode 100644 index 000000000..10c6db7e0 --- /dev/null +++ b/cli/detect/config/config.go @@ -0,0 +1,426 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package config + +import ( + _ "embed" + "errors" + "fmt" + "sort" + "strings" + + "github.com/spf13/viper" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/regexp" +) + +const DefaultScanConfigFileName = ".infisical-scan.toml" +const DefaultScanConfigEnvName = "INFISICAL_SCAN_CONFIG" +const DefaultInfisicalIgnoreFineName = ".infisicalignore" + +var ( + //go:embed gitleaks.toml + DefaultConfig string + + // use to keep track of how many configs we can extend + // yea I know, globals bad + extendDepth int +) + +const maxExtendDepth = 2 + +// ViperConfig is the config struct used by the Viper config package +// to parse the config file. This struct does not include regular expressions. +// It is used as an intermediary to convert the Viper config to the Config struct. +type ViperConfig struct { + Title string + Description string + Extend Extend + Rules []struct { + ID string + Description string + Path string + Regex string + SecretGroup int + Entropy float64 + Keywords []string + Tags []string + + // Deprecated: this is a shim for backwards-compatibility. + // TODO: Remove this in 9.x. + AllowList *viperRuleAllowlist + Allowlists []*viperRuleAllowlist + } + // Deprecated: this is a shim for backwards-compatibility. + // TODO: Remove this in 9.x. + AllowList *viperGlobalAllowlist + Allowlists []*viperGlobalAllowlist +} + +type viperRuleAllowlist struct { + Description string + Condition string + Commits []string + Paths []string + RegexTarget string + Regexes []string + StopWords []string +} + +type viperGlobalAllowlist struct { + TargetRules []string + viperRuleAllowlist `mapstructure:",squash"` +} + +// Config is a configuration struct that contains rules and an allowlist if present. +type Config struct { + Title string + Extend Extend + Path string + Description string + Rules map[string]Rule + Keywords map[string]struct{} + // used to keep sarif results consistent + OrderedRules []string + Allowlists []*Allowlist +} + +// Extend is a struct that allows users to define how they want their +// configuration extended by other configuration files. +type Extend struct { + Path string + URL string + UseDefault bool + DisabledRules []string +} + +func (vc *ViperConfig) Translate() (Config, error) { + var ( + keywords = make(map[string]struct{}) + orderedRules []string + rulesMap = make(map[string]Rule) + ruleAllowlists = make(map[string][]*Allowlist) + ) + + // Validate individual rules. + for _, vr := range vc.Rules { + var ( + pathPat *regexp.Regexp + regexPat *regexp.Regexp + ) + if vr.Path != "" { + pathPat = regexp.MustCompile(vr.Path) + } + if vr.Regex != "" { + regexPat = regexp.MustCompile(vr.Regex) + } + if vr.Keywords == nil { + vr.Keywords = []string{} + } else { + for i, k := range vr.Keywords { + keyword := strings.ToLower(k) + keywords[keyword] = struct{}{} + vr.Keywords[i] = keyword + } + } + if vr.Tags == nil { + vr.Tags = []string{} + } + cr := Rule{ + RuleID: vr.ID, + Description: vr.Description, + Regex: regexPat, + SecretGroup: vr.SecretGroup, + Entropy: vr.Entropy, + Path: pathPat, + Keywords: vr.Keywords, + Tags: vr.Tags, + } + + // Parse the rule allowlists, including the older format for backwards compatibility. + if vr.AllowList != nil { + // TODO: Remove this in v9. + if len(vr.Allowlists) > 0 { + return Config{}, fmt.Errorf("%s: [rules.allowlist] is deprecated, it cannot be used alongside [[rules.allowlist]]", cr.RuleID) + } + vr.Allowlists = append(vr.Allowlists, vr.AllowList) + } + for _, a := range vr.Allowlists { + allowlist, err := parseAllowlist(a) + if err != nil { + return Config{}, fmt.Errorf("%s: [[rules.allowlists]] %w", cr.RuleID, err) + } + cr.Allowlists = append(cr.Allowlists, allowlist) + } + orderedRules = append(orderedRules, cr.RuleID) + rulesMap[cr.RuleID] = cr + } + + // Assemble the config. + c := Config{ + Title: vc.Title, + Description: vc.Description, + Extend: vc.Extend, + Rules: rulesMap, + Keywords: keywords, + OrderedRules: orderedRules, + } + // Parse the config allowlists, including the older format for backwards compatibility. + if vc.AllowList != nil { + // TODO: Remove this in v9. + if len(vc.Allowlists) > 0 { + return Config{}, errors.New("[allowlist] is deprecated, it cannot be used alongside [[allowlists]]") + } + vc.Allowlists = append(vc.Allowlists, vc.AllowList) + } + for _, a := range vc.Allowlists { + allowlist, err := parseAllowlist(&a.viperRuleAllowlist) + if err != nil { + return Config{}, fmt.Errorf("[[allowlists]] %w", err) + } + // Allowlists with |targetRules| aren't added to the global list. + if len(a.TargetRules) > 0 { + for _, ruleID := range a.TargetRules { + // It's not possible to validate |ruleID| until after extend. + ruleAllowlists[ruleID] = append(ruleAllowlists[ruleID], allowlist) + } + } else { + c.Allowlists = append(c.Allowlists, allowlist) + } + } + + if maxExtendDepth != extendDepth { + // disallow both usedefault and path from being set + if c.Extend.Path != "" && c.Extend.UseDefault { + return Config{}, errors.New("unable to load config due to extend.path and extend.useDefault being set") + } + if c.Extend.UseDefault { + if err := c.extendDefault(); err != nil { + return Config{}, err + } + } else if c.Extend.Path != "" { + if err := c.extendPath(); err != nil { + return Config{}, err + } + } + } + + // Validate the rules after everything has been assembled (including extended configs). + if extendDepth == 0 { + for _, rule := range c.Rules { + if err := rule.Validate(); err != nil { + return Config{}, err + } + } + + // Populate targeted configs. + for ruleID, allowlists := range ruleAllowlists { + rule, ok := c.Rules[ruleID] + if !ok { + return Config{}, fmt.Errorf("[[allowlists]] target rule ID '%s' does not exist", ruleID) + } + rule.Allowlists = append(rule.Allowlists, allowlists...) + c.Rules[ruleID] = rule + } + } + + return c, nil +} + +func parseAllowlist(a *viperRuleAllowlist) (*Allowlist, error) { + var matchCondition AllowlistMatchCondition + switch strings.ToUpper(a.Condition) { + case "AND", "&&": + matchCondition = AllowlistMatchAnd + case "", "OR", "||": + matchCondition = AllowlistMatchOr + default: + return nil, fmt.Errorf("unknown allowlist |condition| '%s' (expected 'and', 'or')", a.Condition) + } + + // Validate the target. + regexTarget := a.RegexTarget + if regexTarget != "" { + switch regexTarget { + case "secret": + regexTarget = "" + case "match", "line": + // do nothing + default: + return nil, fmt.Errorf("unknown allowlist |regexTarget| '%s' (expected 'match', 'line')", regexTarget) + } + } + var allowlistRegexes []*regexp.Regexp + for _, a := range a.Regexes { + allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) + } + var allowlistPaths []*regexp.Regexp + for _, a := range a.Paths { + allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) + } + + allowlist := &Allowlist{ + Description: a.Description, + MatchCondition: matchCondition, + Commits: a.Commits, + Paths: allowlistPaths, + RegexTarget: regexTarget, + Regexes: allowlistRegexes, + StopWords: a.StopWords, + } + if err := allowlist.Validate(); err != nil { + return nil, err + } + return allowlist, nil +} + +func (c *Config) GetOrderedRules() []Rule { + var orderedRules []Rule + for _, id := range c.OrderedRules { + if _, ok := c.Rules[id]; ok { + orderedRules = append(orderedRules, c.Rules[id]) + } + } + return orderedRules +} + +func (c *Config) extendDefault() error { + extendDepth++ + viper.SetConfigType("toml") + if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + } + defaultViperConfig := ViperConfig{} + if err := viper.Unmarshal(&defaultViperConfig); err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + } + cfg, err := defaultViperConfig.Translate() + if err != nil { + return fmt.Errorf("failed to load extended default config, err: %w", err) + + } + logging.Debug().Msg("extending config with default config") + c.extend(cfg) + return nil +} + +func (c *Config) extendPath() error { + extendDepth++ + viper.SetConfigFile(c.Extend.Path) + if err := viper.ReadInConfig(); err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + extensionViperConfig := ViperConfig{} + if err := viper.Unmarshal(&extensionViperConfig); err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + cfg, err := extensionViperConfig.Translate() + if err != nil { + return fmt.Errorf("failed to load extended config, err: %w", err) + } + logging.Debug().Msgf("extending config with %s", c.Extend.Path) + c.extend(cfg) + return nil +} + +func (c *Config) extendURL() { + // TODO +} + +func (c *Config) extend(extensionConfig Config) { + // Get config name for helpful log messages. + var configName string + if c.Extend.Path != "" { + configName = c.Extend.Path + } else { + configName = "default" + } + // Convert |Config.DisabledRules| into a map for ease of access. + disabledRuleIDs := map[string]struct{}{} + for _, id := range c.Extend.DisabledRules { + if _, ok := extensionConfig.Rules[id]; !ok { + logging.Warn(). + Str("rule-id", id). + Str("config", configName). + Msg("Disabled rule doesn't exist in extended config.") + } + disabledRuleIDs[id] = struct{}{} + } + + for ruleID, baseRule := range extensionConfig.Rules { + // Skip the rule. + if _, ok := disabledRuleIDs[ruleID]; ok { + logging.Debug(). + Str("rule-id", ruleID). + Str("config", configName). + Msg("Ignoring rule from extended config.") + continue + } + + currentRule, ok := c.Rules[ruleID] + if !ok { + // Rule doesn't exist, add it to the config. + c.Rules[ruleID] = baseRule + for _, k := range baseRule.Keywords { + c.Keywords[k] = struct{}{} + } + c.OrderedRules = append(c.OrderedRules, ruleID) + } else { + // Rule exists, merge our changes into the base. + if currentRule.Description != "" { + baseRule.Description = currentRule.Description + } + if currentRule.Entropy != 0 { + baseRule.Entropy = currentRule.Entropy + } + if currentRule.SecretGroup != 0 { + baseRule.SecretGroup = currentRule.SecretGroup + } + if currentRule.Regex != nil { + baseRule.Regex = currentRule.Regex + } + if currentRule.Path != nil { + baseRule.Path = currentRule.Path + } + baseRule.Tags = append(baseRule.Tags, currentRule.Tags...) + baseRule.Keywords = append(baseRule.Keywords, currentRule.Keywords...) + for _, a := range currentRule.Allowlists { + baseRule.Allowlists = append(baseRule.Allowlists, a) + } + // The keywords from the base rule and the extended rule must be merged into the global keywords list + for _, k := range baseRule.Keywords { + c.Keywords[k] = struct{}{} + } + c.Rules[ruleID] = baseRule + } + } + + // append allowlists, not attempting to merge + for _, a := range extensionConfig.Allowlists { + c.Allowlists = append(c.Allowlists, a) + } + + // sort to keep extended rules in order + sort.Strings(c.OrderedRules) +} diff --git a/cli/detect/config/gitleaks.toml b/cli/detect/config/gitleaks.toml new file mode 100644 index 000000000..92a06a319 --- /dev/null +++ b/cli/detect/config/gitleaks.toml @@ -0,0 +1,3130 @@ +# This file has been auto-generated. Do not edit manually. +# If you would like to contribute new rules, please use +# cmd/generate/config/main.go and follow the contributing guidelines +# at https://github.com/gitleaks/gitleaks/blob/master/CONTRIBUTING.md +# +# How the hell does secret scanning work? Read this: +# https://lookingatcomputer.substack.com/p/regex-is-almost-all-you-need +# +# This is the default gitleaks configuration file. +# Rules and allowlists are defined within this file. +# Rules instruct gitleaks on what should be considered a secret. +# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. + +title = "gitleaks config" + +# TODO: change to [[allowlists]] +[allowlist] +description = "global allow lists" +paths = [ + '''gitleaks\.toml''', + '''(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$''', + '''(?i)\.(?:eot|[ot]tf|woff2?)$''', + '''(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf|zip)$''', + '''go\.(?:mod|sum|work(?:\.sum)?)$''', + '''(?:^|/)vendor/modules\.txt$''', + '''(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$''', + '''(?:^|/)gradlew(?:\.bat)?$''', + '''(?:^|/)gradle\.lockfile$''', + '''(?:^|/)mvnw(?:\.cmd)?$''', + '''(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''', + '''(?:^|/)node_modules(?:/.*)?$''', + '''(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''', + '''(?:^|/)bower_components(?:/.*)?$''', + '''(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$''', + '''(?:^|/)javascript\.json$''', + '''(?:^|/)(?:Pipfile|poetry)\.lock$''', + '''(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$''', + '''(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$''', + '''(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$''', + '''(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$''', + '''\.gem$''', + '''verification-metadata\.xml''', + '''Database.refactorlog''', +] +regexes = [ + '''(?i)^true|false|null$''', + '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''', + '''^\$(?:\d+|{\d+})$''', + '''^\$(?:[A-Z_]+|[a-z_]+)$''', + '''^\${(?:[A-Z_]+|[a-z_]+)}$''', + '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''', + '''^\$\{\{[ \t]*(?:(?:env|github|secrets|vars)(?:\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''', + '''^%(?:[A-Z_]+|[a-z_]+)%$''', + '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''', + '''^\{\d{0,2}}$''', + '''^@(?:[A-Z_]+|[a-z_]+)@$''', + '''^/Users/(?i)[a-z0-9]+/[\w .-/]+$''', + '''^/(?:bin|etc|home|opt|tmp|usr|var)/[\w ./-]+$''', +] +stopwords = [ + "abcdefghijklmnopqrstuvwxyz", + "014df517-39d1-4453-b7b3-9930c563627c", +] + +[[rules]] +id = "1password-secret-key" +description = "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults." +regex = '''\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b''' +entropy = 3.8 +keywords = ["a3-"] + +[[rules]] +id = "1password-service-account-token" +description = "Uncovered a possible 1Password service account token, potentially compromising access to secrets in vaults." +regex = '''ops_eyJ[a-zA-Z0-9+/]{250,}={0,3}''' +entropy = 4 +keywords = ["ops_"] + +[[rules]] +id = "adafruit-api-key" +description = "Identified a potential Adafruit API Key, which could lead to unauthorized access to Adafruit services and sensitive data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:adafruit)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["adafruit"] + +[[rules]] +id = "adobe-client-id" +description = "Detected a pattern that resembles an Adobe OAuth Web Client ID, posing a risk of compromised Adobe integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:adobe)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["adobe"] + +[[rules]] +id = "adobe-client-secret" +description = "Discovered a potential Adobe Client Secret, which, if exposed, could allow unauthorized Adobe service access and data manipulation." +regex = '''\b(p8e-(?i)[a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["p8e-"] + +[[rules]] +id = "age-secret-key" +description = "Discovered a potential Age encryption tool secret key, risking data decryption and unauthorized access to sensitive information." +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' +keywords = ["age-secret-key-1"] + +[[rules]] +id = "airtable-api-key" +description = "Uncovered a possible Airtable API Key, potentially compromising database access and leading to data leakage or alteration." +regex = '''(?i)[\w.-]{0,50}?(?:airtable)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{17})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["airtable"] + +[[rules]] +id = "algolia-api-key" +description = "Identified an Algolia API Key, which could result in unauthorized search operations and data exposure on Algolia-managed platforms." +regex = '''(?i)[\w.-]{0,50}?(?:algolia)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["algolia"] + +[[rules]] +id = "alibaba-access-key-id" +description = "Detected an Alibaba Cloud AccessKey ID, posing a risk of unauthorized cloud resource access and potential data compromise." +regex = '''\b(LTAI(?i)[a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["ltai"] + +[[rules]] +id = "alibaba-secret-key" +description = "Discovered a potential Alibaba Cloud Secret Key, potentially allowing unauthorized operations and data access within Alibaba Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:alibaba)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["alibaba"] + +[[rules]] +id = "asana-client-id" +description = "Discovered a potential Asana Client ID, risking unauthorized access to Asana projects and sensitive task information." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "asana-client-secret" +description = "Identified an Asana Client Secret, which could lead to compromised project management integrity and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "atlassian-api-token" +description = "Detected an Atlassian API token, posing a threat to project management and collaboration tool security and data confidentiality." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:atlassian|confluence|jira)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)|\b(ATATT3[A-Za-z0-9_\-=]{186})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "atlassian", + "confluence", + "jira", + "atatt3", +] + +[[rules]] +id = "authress-service-client-access-key" +description = "Uncovered a possible Authress Service Client Access Key, which may compromise access control services and sensitive data." +regex = '''\b((?:sc|ext|scauth|authress)_(?i)[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.(?-i:acc)[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sc_", + "ext_", + "scauth_", + "authress_", +] + +[[rules]] +id = "aws-access-token" +description = "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms." +regex = '''\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16})\b''' +entropy = 3 +keywords = [ + "a3t", + "akia", + "asia", + "abia", + "acca", +] +[[rules.allowlists]] +regexes = [ + '''.+EXAMPLE$''', +] + +[[rules]] +id = "azure-ad-client-secret" +description = "Azure AD Client Secret" +regex = '''(?:^|[\\'"\x60\s>=:(,)])([a-zA-Z0-9_~.]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:$|[\\'"\x60\s<),])''' +entropy = 3 +keywords = ["q~"] + +[[rules]] +id = "beamer-api-token" +description = "Detected a Beamer API token, potentially compromising content management and exposing sensitive notifications and updates." +regex = '''(?i)[\w.-]{0,50}?(?:beamer)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(b_[a-z0-9=_\-]{44})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["beamer"] + +[[rules]] +id = "bitbucket-client-id" +description = "Discovered a potential Bitbucket Client ID, risking unauthorized repository access and potential codebase exposure." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bitbucket-client-secret" +description = "Discovered a potential Bitbucket Client Secret, posing a risk of compromised code repositories and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bittrex-access-key" +description = "Identified a Bittrex Access Key, which could lead to unauthorized access to cryptocurrency trading accounts and financial loss." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "bittrex-secret-key" +description = "Detected a Bittrex Secret Key, potentially compromising cryptocurrency transactions and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "cisco-meraki-api-key" +description = "Cisco Meraki is a cloud-managed IT solution that provides networking, security, and device management through an easy-to-use interface." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Mm]eraki|MERAKI))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["meraki"] + +[[rules]] +id = "clickhouse-cloud-api-secret-key" +description = "Identified a pattern that may indicate clickhouse cloud API secret key, risking unauthorized clickhouse cloud api access and data breaches on ClickHouse Cloud platforms." +regex = '''\b(4b1d[A-Za-z0-9]{38})\b''' +entropy = 3 +keywords = ["4b1d"] + +[[rules]] +id = "clojars-api-token" +description = "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation." +regex = '''(?i)CLOJARS_[a-z0-9]{60}''' +entropy = 2 +keywords = ["clojars_"] + +[[rules]] +id = "cloudflare-api-key" +description = "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-global-api-key" +description = "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{37})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-origin-ca-key" +description = "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security." +regex = '''\b(v1\.0-[a-f0-9]{24}-[a-f0-9]{146})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "cloudflare", + "v1.0-", +] + +[[rules]] +id = "codecov-access-token" +description = "Found a pattern resembling a Codecov Access Token, posing a risk of unauthorized access to code coverage reports and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:codecov)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["codecov"] + +[[rules]] +id = "cohere-api-token" +description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:cohere|CO_API_KEY)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "cohere", + "co_api_key", +] + +[[rules]] +id = "coinbase-access-token" +description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions." +regex = '''(?i)[\w.-]{0,50}?(?:coinbase)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["coinbase"] + +[[rules]] +id = "confluent-access-token" +description = "Identified a Confluent Access Token, which could compromise access to streaming data platforms and sensitive data flow." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "confluent-secret-key" +description = "Found a Confluent Secret Key, potentially risking unauthorized operations and data access within Confluent services." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "contentful-delivery-api-token" +description = "Discovered a Contentful delivery API token, posing a risk to content management systems and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:contentful)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["contentful"] + +[[rules]] +id = "curl-auth-header" +description = "Discovered a potential authorization token provided in a curl command header, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*?|.*?(?:[\r\n]{1,2}.*?){1,5})[ \t\n\r](?:-H|--header)(?:=|[ \t]{0,5})(?:"(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))"|'(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))')(?:\B|\s|\z)''' +entropy = 2.75 +keywords = ["curl"] + +[[rules]] +id = "curl-auth-user" +description = "Discovered a potential basic authorization token provided in a curl command, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*|.*(?:[\r\n]{1,2}.*){1,5})[ \t\n\r](?:-u|--user)(?:=|[ \t]{0,5})("(:[^"]{3,}|[^:"]{3,}:|[^:"]{3,}:[^"]{3,})"|'([^:']{3,}:[^']{3,})'|((?:"[^"]{3,}"|'[^']{3,}'|[\w$@.-]+):(?:"[^"]{3,}"|'[^']{3,}'|[\w${}@.-]+)))(?:\s|\z)''' +entropy = 2 +keywords = ["curl"] +[[rules.allowlists]] +regexes = [ + '''[^:]+:(?:change(?:it|me)|pass(?:word)?|pwd|test|token|\*+|x+)''', + '''['"]?<[^>]+>['"]?:['"]?<[^>]+>|<[^:]+:[^>]+>['"]?''', + '''[^:]+:\[[^]]+]''', + '''['"]?[^:]+['"]?:['"]?\$(?:\d|\w+|\{(?:\d|\w+)})['"]?''', + '''\$\([^)]+\):\$\([^)]+\)''', + '''['"]?\$?{{[^}]+}}['"]?:['"]?\$?{{[^}]+}}['"]?''', +] + +[[rules]] +id = "databricks-api-token" +description = "Uncovered a Databricks API token, which may compromise big data analytics platforms and sensitive data processing." +regex = '''\b(dapi[a-f0-9]{32}(?:-\d)?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dapi"] + +[[rules]] +id = "datadog-access-token" +description = "Detected a Datadog Access Token, potentially risking monitoring and analytics data exposure and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:datadog)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["datadog"] + +[[rules]] +id = "defined-networking-api-token" +description = "Identified a Defined Networking API token, which could lead to unauthorized network operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:dnkey)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dnkey"] + +[[rules]] +id = "digitalocean-access-token" +description = "Found a DigitalOcean OAuth Access Token, risking unauthorized cloud resource access and data compromise." +regex = '''\b(doo_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["doo_v1_"] + +[[rules]] +id = "digitalocean-pat" +description = "Discovered a DigitalOcean Personal Access Token, posing a threat to cloud infrastructure security and data privacy." +regex = '''\b(dop_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dop_v1_"] + +[[rules]] +id = "digitalocean-refresh-token" +description = "Uncovered a DigitalOcean OAuth Refresh Token, which could allow prolonged unauthorized access and resource manipulation." +regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dor_v1_"] + +[[rules]] +id = "discord-api-token" +description = "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["discord"] + +[[rules]] +id = "discord-client-id" +description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{18})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "discord-client-secret" +description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "doppler-api-token" +description = "Discovered a Doppler API token, posing a risk to environment and secrets management security." +regex = '''dp\.pt\.(?i)[a-z0-9]{43}''' +entropy = 2 +keywords = ["dp.pt."] + +[[rules]] +id = "droneci-access-token" +description = "Detected a Droneci Access Token, potentially compromising continuous integration and deployment workflows." +regex = '''(?i)[\w.-]{0,50}?(?:droneci)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["droneci"] + +[[rules]] +id = "dropbox-api-token" +description = "Identified a Dropbox API secret, which could lead to unauthorized file access and data breaches in Dropbox storage." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{15})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Found a Dropbox long-lived API token, risking prolonged unauthorized access to cloud storage and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Discovered a Dropbox short-lived API token, posing a risk of temporary but potentially harmful data access and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(sl\.[a-z0-9\-=_]{135})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "duffel-api-token" +description = "Uncovered a Duffel API token, which may compromise travel platform integrations and sensitive customer data." +regex = '''duffel_(?:test|live)_(?i)[a-z0-9_\-=]{43}''' +entropy = 2 +keywords = ["duffel_"] + +[[rules]] +id = "dynatrace-api-token" +description = "Detected a Dynatrace API token, potentially risking application performance monitoring and data exposure." +regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' +entropy = 4 +keywords = ["dt0c01."] + +[[rules]] +id = "easypost-api-token" +description = "Identified an EasyPost API token, which could lead to unauthorized postal and shipment service access and data exposure." +regex = '''\bEZAK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["ezak"] + +[[rules]] +id = "easypost-test-api-token" +description = "Detected an EasyPost test API token, risking exposure of test environments and potentially sensitive shipment data." +regex = '''\bEZTK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["eztk"] + +[[rules]] +id = "etsy-access-token" +description = "Found an Etsy Access Token, potentially compromising Etsy shop management and customer data." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ETSY|[Ee]tsy))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["etsy"] + +[[rules]] +id = "facebook-access-token" +description = "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)\b(\d{15,16}(\||%)[0-9a-z\-_]{27,40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "facebook-page-access-token" +description = "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''\b(EAA[MC](?i)[a-z0-9]{100,})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "eaam", + "eaac", +] + +[[rules]] +id = "facebook-secret" +description = "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:facebook)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "fastly-api-token" +description = "Uncovered a Fastly API key, which may compromise CDN and edge cloud services, leading to content delivery and security issues." +regex = '''(?i)[\w.-]{0,50}?(?:fastly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["fastly"] + +[[rules]] +id = "finicity-api-token" +description = "Detected a Finicity API token, potentially risking financial data access and unauthorized financial operations." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finicity-client-secret" +description = "Identified a Finicity Client Secret, which could lead to compromised financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finnhub-access-token" +description = "Found a Finnhub Access Token, risking unauthorized access to financial market data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:finnhub)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finnhub"] + +[[rules]] +id = "flickr-access-token" +description = "Discovered a Flickr Access Token, posing a risk of unauthorized photo management and potential data leakage." +regex = '''(?i)[\w.-]{0,50}?(?:flickr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["flickr"] + +[[rules]] +id = "flutterwave-encryption-key" +description = "Uncovered a Flutterwave Encryption Key, which may compromise payment processing and sensitive financial information." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flutterwave-public-key" +description = "Detected a Finicity Public Key, potentially exposing public cryptographic operations and integrations." +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwpubk_test"] + +[[rules]] +id = "flutterwave-secret-key" +description = "Identified a Flutterwave Secret Key, risking unauthorized financial transactions and data breaches." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flyio-access-token" +description = "Uncovered a Fly.io API key" +regex = '''\b((?:fo1_[\w-]{43}|fm1[ar]_[a-zA-Z0-9+\/]{100,}={0,3}|fm2_[a-zA-Z0-9+\/]{100,}={0,3}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "fo1_", + "fm1", + "fm2_", +] + +[[rules]] +id = "frameio-api-token" +description = "Found a Frame.io API token, potentially compromising video collaboration and project management." +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' +keywords = ["fio-u-"] + +[[rules]] +id = "freemius-secret-key" +description = "Detected a Freemius secret key, potentially exposing sensitive information." +regex = '''(?i)["']secret_key["']\s*=>\s*["'](sk_[\S]{29})["']''' +path = '''(?i)\.php$''' +keywords = ["secret_key"] + +[[rules]] +id = "freshbooks-access-token" +description = "Discovered a Freshbooks Access Token, posing a risk to accounting software access and sensitive financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:freshbooks)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["freshbooks"] + +[[rules]] +id = "gcp-api-key" +description = "Uncovered a GCP API key, which could lead to unauthorized access to Google Cloud services and data breaches." +regex = '''\b(AIza[\w-]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["aiza"] +[[rules.allowlists]] +regexes = [ + '''AIzaSyabcdefghijklmnopqrstuvwxyz1234567''', + '''AIzaSyAnLA7NfeLquW1tJFpx_eQCxoX-oo6YyIs''', + '''AIzaSyCkEhVjf3pduRDt6d1yKOMitrUEke8agEM''', + '''AIzaSyDMAScliyLx7F0NPDEJi1QmyCgHIAODrlU''', + '''AIzaSyD3asb-2pEZVqMkmL6M9N6nHZRR_znhrh0''', + '''AIzayDNSXIbFmlXbIE6mCzDLQAqITYefhixbX4A''', + '''AIzaSyAdOS2zB6NCsk1pCdZ4-P6GBdi_UUPwX7c''', + '''AIzaSyASWm6HmTMdYWpgMnjRBjxcQ9CKctWmLd4''', + '''AIzaSyANUvH9H9BsUccjsu2pCmEkOPjjaXeDQgY''', + '''AIzaSyA5_iVawFQ8ABuTZNUdcwERLJv_a_p4wtM''', + '''AIzaSyA4UrcGxgwQFTfaI3no3t7Lt1sjmdnP5sQ''', + '''AIzaSyDSb51JiIcB6OJpwwMicseKRhhrOq1cS7g''', + '''AIzaSyBF2RrAIm4a0mO64EShQfqfd2AFnzAvvuU''', + '''AIzaSyBcE-OOIbhjyR83gm4r2MFCu4MJmprNXsw''', + '''AIzaSyB8qGxt4ec15vitgn44duC5ucxaOi4FmqE''', + '''AIzaSyA8vmApnrHNFE0bApF4hoZ11srVL_n0nvY''', +] + +[[rules]] +id = "generic-api-key" +description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." +regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "access", + "api", + "auth", + "key", + "credential", + "creds", + "passwd", + "password", + "secret", + "token", +] +[[rules.allowlists]] +regexes = [ + '''^[a-zA-Z_.-]+$''', +] +[[rules.allowlists]] +description = "Allowlist for Generic API Keys" +regexTarget = "match" +regexes = [ + '''(?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))''', +] +stopwords = [ + "000000", + "6fe4476ee5a1832882e326b506d14126", + "_ec2_", + "aaaaaa", + "about", + "abstract", + "academy", + "acces", + "account", + "act-", + "act.", + "act_", + "action", + "active", + "actively", + "activity", + "adapter", + "add-", + "add-on", + "add.", + "add_", + "addon", + "addres", + "admin", + "adobe", + "advanced", + "adventure", + "agent", + "agile", + "air-", + "air.", + "air_", + "ajax", + "akka", + "alert", + "alfred", + "algorithm", + "all-", + "all.", + "all_", + "alloy", + "alpha", + "amazon", + "amqp", + "analysi", + "analytic", + "analyzer", + "android", + "angular", + "angularj", + "animate", + "animation", + "another", + "ansible", + "answer", + "ant-", + "ant.", + "ant_", + "any-", + "any.", + "any_", + "apache", + "app-", + "app.", + "app_", + "apple", + "arch", + "archive", + "archived", + "arduino", + "array", + "art-", + "art.", + "art_", + "article", + "asp-", + "asp.", + "asp_", + "asset", + "async", + "atom", + "attention", + "audio", + "audit", + "aura", + "auth", + "author", + "authorize", + "auto", + "automated", + "automatic", + "awesome", + "aws_", + "azure", + "back", + "backbone", + "backend", + "backup", + "bar-", + "bar.", + "bar_", + "base", + "based", + "bash", + "basic", + "batch", + "been", + "beer", + "behavior", + "being", + "benchmark", + "best", + "beta", + "better", + "big-", + "big.", + "big_", + "binary", + "binding", + "bit-", + "bit.", + "bit_", + "bitcoin", + "block", + "blog", + "board", + "book", + "bookmark", + "boost", + "boot", + "bootstrap", + "bosh", + "bot-", + "bot.", + "bot_", + "bower", + "box-", + "box.", + "box_", + "boxen", + "bracket", + "branch", + "bridge", + "browser", + "brunch", + "buffer", + "bug-", + "bug.", + "bug_", + "build", + "builder", + "building", + "buildout", + "buildpack", + "built", + "bundle", + "busines", + "but-", + "but.", + "but_", + "button", + "cache", + "caching", + "cakephp", + "calendar", + "call", + "camera", + "campfire", + "can-", + "can.", + "can_", + "canva", + "captcha", + "capture", + "card", + "carousel", + "case", + "cassandra", + "cat-", + "cat.", + "cat_", + "category", + "center", + "cento", + "challenge", + "change", + "changelog", + "channel", + "chart", + "chat", + "cheat", + "check", + "checker", + "chef", + "ches", + "chinese", + "chosen", + "chrome", + "ckeditor", + "clas", + "classe", + "classic", + "clean", + "cli-", + "cli.", + "cli_", + "client", + "clojure", + "clone", + "closure", + "cloud", + "club", + "cluster", + "cms-", + "cms_", + "coco", + "code", + "coding", + "coffee", + "color", + "combination", + "combo", + "command", + "commander", + "comment", + "commit", + "common", + "community", + "compas", + "compiler", + "complete", + "component", + "composer", + "computer", + "computing", + "con-", + "con.", + "con_", + "concept", + "conf", + "config", + "connect", + "connector", + "console", + "contact", + "container", + "contao", + "content", + "contest", + "context", + "control", + "convert", + "converter", + "conway'", + "cookbook", + "cookie", + "cool", + "copy", + "cordova", + "core", + "couchbase", + "couchdb", + "countdown", + "counter", + "course", + "craft", + "crawler", + "create", + "creating", + "creator", + "credential", + "crm-", + "crm.", + "crm_", + "cros", + "crud", + "csv-", + "csv.", + "csv_", + "cube", + "cucumber", + "cuda", + "current", + "currently", + "custom", + "daemon", + "dark", + "dart", + "dash", + "dashboard", + "data", + "database", + "date", + "day-", + "day.", + "day_", + "dead", + "debian", + "debug", + "debugger", + "deck", + "define", + "del-", + "del.", + "del_", + "delete", + "demo", + "deploy", + "design", + "designer", + "desktop", + "detection", + "detector", + "dev-", + "dev.", + "dev_", + "develop", + "developer", + "device", + "devise", + "diff", + "digital", + "directive", + "directory", + "discovery", + "display", + "django", + "dns-", + "dns_", + "doc-", + "doc.", + "doc_", + "docker", + "docpad", + "doctrine", + "document", + "doe-", + "doe.", + "doe_", + "dojo", + "dom-", + "dom.", + "dom_", + "domain", + "don't", + "done", + "dot-", + "dot.", + "dot_", + "dotfile", + "download", + "draft", + "drag", + "drill", + "drive", + "driven", + "driver", + "drop", + "dropbox", + "drupal", + "dsl-", + "dsl.", + "dsl_", + "dynamic", + "easy", + "ecdsa", + "eclipse", + "edit", + "editing", + "edition", + "editor", + "element", + "emac", + "email", + "embed", + "embedded", + "ember", + "emitter", + "emulator", + "encoding", + "endpoint", + "engine", + "english", + "enhanced", + "entity", + "entry", + "env_", + "episode", + "erlang", + "error", + "espresso", + "event", + "evented", + "example", + "exchange", + "exercise", + "experiment", + "expire", + "exploit", + "explorer", + "export", + "exporter", + "expres", + "ext-", + "ext.", + "ext_", + "extended", + "extension", + "external", + "extra", + "extractor", + "fabric", + "facebook", + "factory", + "fake", + "fast", + "feature", + "feed", + "fewfwef", + "ffmpeg", + "field", + "file", + "filter", + "find", + "finder", + "firefox", + "firmware", + "first", + "fish", + "fix-", + "fix_", + "flash", + "flask", + "flat", + "flex", + "flexible", + "flickr", + "flow", + "fluent", + "fluentd", + "fluid", + "folder", + "font", + "force", + "foreman", + "fork", + "form", + "format", + "formatter", + "forum", + "foundry", + "framework", + "free", + "friend", + "friendly", + "front-end", + "frontend", + "ftp-", + "ftp.", + "ftp_", + "fuel", + "full", + "fun-", + "fun.", + "fun_", + "func", + "future", + "gaia", + "gallery", + "game", + "gateway", + "gem-", + "gem.", + "gem_", + "gen-", + "gen.", + "gen_", + "general", + "generator", + "generic", + "genetic", + "get-", + "get.", + "get_", + "getenv", + "getting", + "ghost", + "gist", + "git-", + "git.", + "git_", + "github", + "gitignore", + "gitlab", + "glas", + "gmail", + "gnome", + "gnu-", + "gnu.", + "gnu_", + "goal", + "golang", + "gollum", + "good", + "google", + "gpu-", + "gpu.", + "gpu_", + "gradle", + "grail", + "graph", + "graphic", + "great", + "grid", + "groovy", + "group", + "grunt", + "guard", + "gui-", + "gui.", + "gui_", + "guide", + "guideline", + "gulp", + "gwt-", + "gwt.", + "gwt_", + "hack", + "hackathon", + "hacker", + "hacking", + "hadoop", + "haml", + "handler", + "hardware", + "has-", + "has_", + "hash", + "haskell", + "have", + "haxe", + "hello", + "help", + "helper", + "here", + "hero", + "heroku", + "high", + "hipchat", + "history", + "home", + "homebrew", + "homepage", + "hook", + "host", + "hosting", + "hot-", + "hot.", + "hot_", + "house", + "how-", + "how.", + "how_", + "html", + "http", + "hub-", + "hub.", + "hub_", + "hubot", + "human", + "icon", + "ide-", + "ide.", + "ide_", + "idea", + "identity", + "idiomatic", + "image", + "impact", + "import", + "important", + "importer", + "impres", + "index", + "infinite", + "info", + "injection", + "inline", + "input", + "inside", + "inspector", + "instagram", + "install", + "installer", + "instant", + "intellij", + "interface", + "internet", + "interview", + "into", + "intro", + "ionic", + "iphone", + "ipython", + "irc-", + "irc_", + "iso-", + "iso.", + "iso_", + "issue", + "jade", + "jasmine", + "java", + "jbos", + "jekyll", + "jenkin", + "jetbrains", + "job-", + "job.", + "job_", + "joomla", + "jpa-", + "jpa.", + "jpa_", + "jquery", + "json", + "just", + "kafka", + "karma", + "kata", + "kernel", + "keyboard", + "kindle", + "kit-", + "kit.", + "kit_", + "kitchen", + "knife", + "koan", + "kohana", + "lab-", + "lab.", + "lab_", + "lambda", + "lamp", + "language", + "laravel", + "last", + "latest", + "latex", + "launcher", + "layer", + "layout", + "lazy", + "ldap", + "leaflet", + "league", + "learn", + "learning", + "led-", + "led.", + "led_", + "leetcode", + "les-", + "les.", + "les_", + "level", + "leveldb", + "lib-", + "lib.", + "lib_", + "librarie", + "library", + "license", + "life", + "liferay", + "light", + "lightbox", + "like", + "line", + "link", + "linked", + "linkedin", + "linux", + "lisp", + "list", + "lite", + "little", + "load", + "loader", + "local", + "location", + "lock", + "log-", + "log.", + "log_", + "logger", + "logging", + "logic", + "login", + "logstash", + "longer", + "look", + "love", + "lua-", + "lua.", + "lua_", + "mac-", + "mac.", + "mac_", + "machine", + "made", + "magento", + "magic", + "mail", + "make", + "maker", + "making", + "man-", + "man.", + "man_", + "manage", + "manager", + "manifest", + "manual", + "map-", + "map.", + "map_", + "mapper", + "mapping", + "markdown", + "markup", + "master", + "math", + "matrix", + "maven", + "md5", + "mean", + "media", + "mediawiki", + "meetup", + "memcached", + "memory", + "menu", + "merchant", + "message", + "messaging", + "meta", + "metadata", + "meteor", + "method", + "metric", + "micro", + "middleman", + "migration", + "minecraft", + "miner", + "mini", + "minimal", + "mirror", + "mit-", + "mit.", + "mit_", + "mobile", + "mocha", + "mock", + "mod-", + "mod.", + "mod_", + "mode", + "model", + "modern", + "modular", + "module", + "modx", + "money", + "mongo", + "mongodb", + "mongoid", + "mongoose", + "monitor", + "monkey", + "more", + "motion", + "moved", + "movie", + "mozilla", + "mqtt", + "mule", + "multi", + "multiple", + "music", + "mustache", + "mvc-", + "mvc.", + "mvc_", + "mysql", + "nagio", + "name", + "native", + "need", + "neo-", + "neo.", + "neo_", + "nest", + "nested", + "net-", + "net.", + "net_", + "nette", + "network", + "new-", + "new.", + "new_", + "next", + "nginx", + "ninja", + "nlp-", + "nlp.", + "nlp_", + "node", + "nodej", + "nosql", + "not-", + "not.", + "not_", + "note", + "notebook", + "notepad", + "notice", + "notifier", + "now-", + "now.", + "now_", + "number", + "oauth", + "object", + "objective", + "obsolete", + "ocaml", + "octopres", + "official", + "old-", + "old.", + "old_", + "onboard", + "online", + "only", + "open", + "opencv", + "opengl", + "openshift", + "openwrt", + "option", + "oracle", + "org-", + "org.", + "org_", + "origin", + "original", + "orm-", + "orm.", + "orm_", + "osx-", + "osx_", + "our-", + "our.", + "our_", + "out-", + "out.", + "out_", + "output", + "over", + "overview", + "own-", + "own.", + "own_", + "pack", + "package", + "packet", + "page", + "panel", + "paper", + "paperclip", + "para", + "parallax", + "parallel", + "parse", + "parser", + "parsing", + "particle", + "party", + "password", + "patch", + "path", + "pattern", + "payment", + "paypal", + "pdf-", + "pdf.", + "pdf_", + "pebble", + "people", + "perl", + "personal", + "phalcon", + "phoenix", + "phone", + "phonegap", + "photo", + "php-", + "php.", + "php_", + "physic", + "picker", + "pipeline", + "platform", + "play", + "player", + "please", + "plu-", + "plu.", + "plu_", + "plug-in", + "plugin", + "plupload", + "png-", + "png.", + "png_", + "poker", + "polyfill", + "polymer", + "pool", + "pop-", + "pop.", + "pop_", + "popcorn", + "popup", + "port", + "portable", + "portal", + "portfolio", + "post", + "power", + "powered", + "powerful", + "prelude", + "pretty", + "preview", + "principle", + "print", + "pro-", + "pro.", + "pro_", + "problem", + "proc", + "product", + "profile", + "profiler", + "program", + "progres", + "project", + "protocol", + "prototype", + "provider", + "proxy", + "public", + "pull", + "puppet", + "pure", + "purpose", + "push", + "pusher", + "pyramid", + "python", + "quality", + "query", + "queue", + "quick", + "rabbitmq", + "rack", + "radio", + "rail", + "railscast", + "random", + "range", + "raspberry", + "rdf-", + "rdf.", + "rdf_", + "react", + "reactive", + "read", + "reader", + "readme", + "ready", + "real", + "real-time", + "reality", + "realtime", + "recipe", + "recorder", + "red-", + "red.", + "red_", + "reddit", + "redi", + "redmine", + "reference", + "refinery", + "refresh", + "registry", + "related", + "release", + "remote", + "rendering", + "repo", + "report", + "request", + "require", + "required", + "requirej", + "research", + "resource", + "response", + "resque", + "rest", + "restful", + "resume", + "reveal", + "reverse", + "review", + "riak", + "rich", + "right", + "ring", + "robot", + "role", + "room", + "router", + "routing", + "rpc-", + "rpc.", + "rpc_", + "rpg-", + "rpg.", + "rpg_", + "rspec", + "ruby-", + "ruby.", + "ruby_", + "rule", + "run-", + "run.", + "run_", + "runner", + "running", + "runtime", + "rust", + "rvm-", + "rvm.", + "rvm_", + "salt", + "sample", + "sandbox", + "sas-", + "sas.", + "sas_", + "sbt-", + "sbt.", + "sbt_", + "scala", + "scalable", + "scanner", + "schema", + "scheme", + "school", + "science", + "scraper", + "scratch", + "screen", + "script", + "scroll", + "scs-", + "scs.", + "scs_", + "sdk-", + "sdk.", + "sdk_", + "sdl-", + "sdl.", + "sdl_", + "search", + "secure", + "security", + "see-", + "see.", + "see_", + "seed", + "select", + "selector", + "selenium", + "semantic", + "sencha", + "send", + "sentiment", + "serie", + "server", + "service", + "session", + "set-", + "set.", + "set_", + "setting", + "setup", + "sha1", + "sha2", + "sha256", + "share", + "shared", + "sharing", + "sheet", + "shell", + "shield", + "shipping", + "shop", + "shopify", + "shortener", + "should", + "show", + "showcase", + "side", + "silex", + "simple", + "simulator", + "single", + "site", + "skeleton", + "sketch", + "skin", + "slack", + "slide", + "slider", + "slim", + "small", + "smart", + "smtp", + "snake", + "snapshot", + "snippet", + "soap", + "social", + "socket", + "software", + "solarized", + "solr", + "solution", + "solver", + "some", + "soon", + "source", + "space", + "spark", + "spatial", + "spec", + "sphinx", + "spine", + "spotify", + "spree", + "spring", + "sprite", + "sql-", + "sql.", + "sql_", + "sqlite", + "ssh-", + "ssh.", + "ssh_", + "stack", + "staging", + "standard", + "stanford", + "start", + "started", + "starter", + "startup", + "stat", + "statamic", + "state", + "static", + "statistic", + "statsd", + "statu", + "steam", + "step", + "still", + "stm-", + "stm.", + "stm_", + "storage", + "store", + "storm", + "story", + "strategy", + "stream", + "streaming", + "string", + "stripe", + "structure", + "studio", + "study", + "stuff", + "style", + "sublime", + "sugar", + "suite", + "summary", + "super", + "support", + "supported", + "svg-", + "svg.", + "svg_", + "svn-", + "svn.", + "svn_", + "swagger", + "swift", + "switch", + "switcher", + "symfony", + "symphony", + "sync", + "synopsi", + "syntax", + "system", + "tab-", + "tab.", + "tab_", + "table", + "tag-", + "tag.", + "tag_", + "talk", + "target", + "task", + "tcp-", + "tcp.", + "tcp_", + "tdd-", + "tdd.", + "tdd_", + "team", + "tech", + "template", + "term", + "terminal", + "testing", + "tetri", + "text", + "textmate", + "theme", + "theory", + "three", + "thrift", + "time", + "timeline", + "timer", + "tiny", + "tinymce", + "tip-", + "tip.", + "tip_", + "title", + "todo", + "todomvc", + "token", + "tool", + "toolbox", + "toolkit", + "top-", + "top.", + "top_", + "tornado", + "touch", + "tower", + "tracker", + "tracking", + "traffic", + "training", + "transfer", + "translate", + "transport", + "tree", + "trello", + "try-", + "try.", + "try_", + "tumblr", + "tut-", + "tut.", + "tut_", + "tutorial", + "tweet", + "twig", + "twitter", + "type", + "typo", + "ubuntu", + "uiview", + "ultimate", + "under", + "unit", + "unity", + "universal", + "unix", + "update", + "updated", + "upgrade", + "upload", + "uploader", + "uri-", + "uri.", + "uri_", + "url-", + "url.", + "url_", + "usage", + "usb-", + "usb.", + "usb_", + "use-", + "use.", + "use_", + "used", + "useful", + "user", + "using", + "util", + "utilitie", + "utility", + "vagrant", + "validator", + "value", + "variou", + "varnish", + "version", + "via-", + "via.", + "via_", + "video", + "view", + "viewer", + "vim-", + "vim.", + "vim_", + "vimrc", + "virtual", + "vision", + "visual", + "vpn", + "want", + "warning", + "watch", + "watcher", + "wave", + "way-", + "way.", + "way_", + "weather", + "web-", + "web_", + "webapp", + "webgl", + "webhook", + "webkit", + "webrtc", + "website", + "websocket", + "welcome", + "what", + "what'", + "when", + "where", + "which", + "why-", + "why.", + "why_", + "widget", + "wifi", + "wiki", + "win-", + "win.", + "win_", + "window", + "wip-", + "wip.", + "wip_", + "within", + "without", + "wizard", + "word", + "wordpres", + "work", + "worker", + "workflow", + "working", + "workshop", + "world", + "wrapper", + "write", + "writer", + "writing", + "written", + "www-", + "www.", + "www_", + "xamarin", + "xcode", + "xml-", + "xml.", + "xml_", + "xmpp", + "xxxxxx", + "yahoo", + "yaml", + "yandex", + "yeoman", + "yet-", + "yet.", + "yet_", + "yii-", + "yii.", + "yii_", + "youtube", + "yui-", + "yui.", + "yui_", + "zend", + "zero", + "zip-", + "zip.", + "zip_", + "zsh-", + "zsh.", + "zsh_", +] +[[rules.allowlists]] +regexTarget = "line" +regexes = [ + '''--mount=type=secret,''', + '''import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]''', +] +[[rules.allowlists]] +condition = "AND" +paths = [ + '''\.bb$''','''\.bbappend$''','''\.bbclass$''','''\.inc$''', +] +regexTarget = "line" +regexes = [ + '''LICENSE[^=]*=\s*"[^"]+''', + '''LIC_FILES_CHKSUM[^=]*=\s*"[^"]+''', + '''SRC[^=]*=\s*"[a-zA-Z0-9]+''', +] + +[[rules]] +id = "github-app-token" +description = "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security." +regex = '''(?:ghu|ghs)_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = [ + "ghu_", + "ghs_", +] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-fine-grained-pat" +description = "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation." +regex = '''github_pat_\w{82}''' +entropy = 3 +keywords = ["github_pat_"] + +[[rules]] +id = "github-oauth" +description = "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks." +regex = '''gho_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["gho_"] + +[[rules]] +id = "github-pat" +description = "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure." +regex = '''ghp_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghp_"] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-refresh-token" +description = "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services." +regex = '''ghr_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghr_"] + +[[rules]] +id = "gitlab-cicd-job-token" +description = "Identified a GitLab CI/CD Job Token, potential access to projects and some APIs on behalf of a user while the CI job is running." +regex = '''glcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}''' +entropy = 3 +keywords = ["glcbt-"] + +[[rules]] +id = "gitlab-deploy-token" +description = "Identified a GitLab Deploy Token, risking access to repositories, packages and containers with write access." +regex = '''gldt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["gldt-"] + +[[rules]] +id = "gitlab-feature-flag-client-token" +description = "Identified a GitLab feature flag client token, risks exposing user lists and features flags used by an application." +regex = '''glffct-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glffct-"] + +[[rules]] +id = "gitlab-feed-token" +description = "Identified a GitLab feed token, risking exposure of user data." +regex = '''glft-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glft-"] + +[[rules]] +id = "gitlab-incoming-mail-token" +description = "Identified a GitLab incoming mail token, risking manipulation of data sent by mail." +regex = '''glimt-[0-9a-zA-Z_\-]{25}''' +entropy = 3 +keywords = ["glimt-"] + +[[rules]] +id = "gitlab-kubernetes-agent-token" +description = "Identified a GitLab Kubernetes Agent token, risking access to repos and registry of projects connected via agent." +regex = '''glagent-[0-9a-zA-Z_\-]{50}''' +entropy = 3 +keywords = ["glagent-"] + +[[rules]] +id = "gitlab-oauth-app-secret" +description = "Identified a GitLab OIDC Application Secret, risking access to apps using GitLab as authentication provider." +regex = '''gloas-[0-9a-zA-Z_\-]{64}''' +entropy = 3 +keywords = ["gloas-"] + +[[rules]] +id = "gitlab-pat" +description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''glpat-[\w-]{20}''' +entropy = 3 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-pat-routable" +description = "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-ptt" +description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security." +regex = '''glptt-[0-9a-f]{40}''' +entropy = 3 +keywords = ["glptt-"] + +[[rules]] +id = "gitlab-rrt" +description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''GR1348941[\w-]{20}''' +entropy = 3 +keywords = ["gr1348941"] + +[[rules]] +id = "gitlab-runner-authentication-token" +description = "Discovered a GitLab Runner Authentication Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''glrt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-runner-authentication-token-routable" +description = "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-scim-token" +description = "Discovered a GitLab SCIM Token, posing a risk to unauthorized access for a organization or instance." +regex = '''glsoat-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glsoat-"] + +[[rules]] +id = "gitlab-session-cookie" +description = "Discovered a GitLab Session Cookie, posing a risk to unauthorized access to a user account." +regex = '''_gitlab_session=[0-9a-z]{32}''' +entropy = 3 +keywords = ["_gitlab_session="] + +[[rules]] +id = "gitter-access-token" +description = "Uncovered a Gitter Access Token, which may lead to unauthorized access to chat and communication services." +regex = '''(?i)[\w.-]{0,50}?(?:gitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["gitter"] + +[[rules]] +id = "gocardless-api-token" +description = "Detected a GoCardless API token, potentially risking unauthorized direct debit payment operations and financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:gocardless)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(live_(?i)[a-z0-9\-_=]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "live_", + "gocardless", +] + +[[rules]] +id = "grafana-api-key" +description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics." +regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["eyjrijoi"] + +[[rules]] +id = "grafana-cloud-api-token" +description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure." +regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glc_"] + +[[rules]] +id = "grafana-service-account-token" +description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity." +regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glsa_"] + +[[rules]] +id = "harness-api-key" +description = "Identified a Harness Access Token (PAT or SAT), risking unauthorized access to a Harness account." +regex = '''(?:pat|sat)\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9]{24}\.[a-zA-Z0-9]{20}''' +keywords = [ + "pat.", + "sat.", +] + +[[rules]] +id = "hashicorp-tf-api-token" +description = "Uncovered a HashiCorp Terraform user/org API token, which may lead to unauthorized infrastructure management and security breaches." +regex = '''(?i)[a-z0-9]{14}\.(?-i:atlasv1)\.[a-z0-9\-_=]{60,70}''' +entropy = 3.5 +keywords = ["atlasv1"] + +[[rules]] +id = "hashicorp-tf-password" +description = "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches." +regex = '''(?i)[\w.-]{0,50}?(?:administrator_login_password|password)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}("[a-z0-9=_\-]{8,20}")(?:[\x60'"\s;]|\\[nr]|$)''' +path = '''(?i)\.(?:tf|hcl)$''' +entropy = 2 +keywords = [ + "administrator_login_password", + "password", +] + +[[rules]] +id = "heroku-api-key" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:heroku)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["heroku"] + +[[rules]] +id = "hubspot-api-key" +description = "Found a HubSpot API Token, posing a risk to CRM data integrity and unauthorized marketing operations." +regex = '''(?i)[\w.-]{0,50}?(?:hubspot)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["hubspot"] + +[[rules]] +id = "huggingface-access-token" +description = "Discovered a Hugging Face Access token, which could lead to unauthorized access to AI models and sensitive data." +regex = '''\b(hf_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["hf_"] + +[[rules]] +id = "huggingface-organization-api-token" +description = "Uncovered a Hugging Face Organization API token, potentially compromising AI organization accounts and associated data." +regex = '''\b(api_org_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["api_org_"] + +[[rules]] +id = "infracost-api-token" +description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data." +regex = '''\b(ico-[a-zA-Z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ico-"] + +[[rules]] +id = "intercom-api-key" +description = "Identified an Intercom API Token, which could compromise customer communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:intercom)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{60})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["intercom"] + +[[rules]] +id = "intra42-client-secret" +description = "Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data." +regex = '''\b(s-s4t2(?:ud|af)-(?i)[abcdef0123456789]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "intra", + "s-s4t2ud-", + "s-s4t2af-", +] + +[[rules]] +id = "jfrog-api-key" +description = "Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{73})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jfrog-identity-token" +description = "Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jwt" +description = "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data." +regex = '''\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ey"] + +[[rules]] +id = "jwt-base64" +description = "Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information." +regex = '''\bZXlK(?:(?PaGJHY2lPaU)|(?PaGNIVWlPaU)|(?PaGNIWWlPaU)|(?PaGRXUWlPaU)|(?PaU5qUWlP)|(?PamNtbDBJanBi)|(?PamRIa2lPaU)|(?PbGNHc2lPbn)|(?PbGJtTWlPaU)|(?PcWEzVWlPaU)|(?PcWQyc2lPb)|(?PcGMzTWlPaU)|(?PcGRpSTZJ)|(?PcmFXUWlP)|(?PclpYbGZiM0J6SWpwY)|(?PcmRIa2lPaUp)|(?PdWIyNWpaU0k2)|(?Pd01tTWlP)|(?Pd01uTWlPaU)|(?Pd2NIUWlPaU)|(?PemRXSWlPaU)|(?PemRuUWlP)|(?PMFlXY2lPaU)|(?PMGVYQWlPaUp)|(?PMWNtd2l)|(?PMWMyVWlPaUp)|(?PMlpYSWlPaU)|(?PMlpYSnphVzl1SWpv)|(?PNElqb2)|(?PNE5XTWlP)|(?PNE5YUWlPaU)|(?PNE5YUWpVekkxTmlJNkl)|(?PNE5YVWlPaU)|(?PNmFYQWlPaU))[a-zA-Z0-9\/\\_+\-\r\n]{40,}={0,2}''' +entropy = 2 +keywords = ["zxlk"] + +[[rules]] +id = "kraken-access-token" +description = "Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:kraken)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9\/=_\+\-]{80,90})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kraken"] + +[[rules]] +id = "kubernetes-secret-yaml" +description = "Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments" +regex = '''(?i)(?:\bkind:[ \t]*["']?\bsecret\b["']?(?s:.){0,200}?\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))|\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))(?s:.){0,200}?\bkind:[ \t]*["']?\bsecret\b["']?)''' +path = '''(?i)\.ya?ml$''' +keywords = ["secret"] +[[rules.allowlists]] +regexes = [ + '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''', +] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''(kind:(?s:.)+\n---\n(?s:.)+\bdata:|data:(?s:.)+\n---\n(?s:.)+\bkind:)''', +] + +[[rules]] +id = "kucoin-access-token" +description = "Found a Kucoin Access Token, risking unauthorized access to cryptocurrency exchange services and transactions." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "kucoin-secret-key" +description = "Discovered a Kucoin Secret Key, which could lead to compromised cryptocurrency operations and financial data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "launchdarkly-access-token" +description = "Uncovered a Launchdarkly Access Token, potentially compromising feature flag management and application functionality." +regex = '''(?i)[\w.-]{0,50}?(?:launchdarkly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["launchdarkly"] + +[[rules]] +id = "linear-api-key" +description = "Detected a Linear API Token, posing a risk to project management tools and sensitive task data." +regex = '''lin_api_(?i)[a-z0-9]{40}''' +entropy = 2 +keywords = ["lin_api_"] + +[[rules]] +id = "linear-client-secret" +description = "Identified a Linear Client Secret, which may compromise secure integrations and sensitive project management data." +regex = '''(?i)[\w.-]{0,50}?(?:linear)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["linear"] + +[[rules]] +id = "linkedin-client-id" +description = "Found a LinkedIn Client ID, risking unauthorized access to LinkedIn integrations and professional data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{14})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "linkedin-client-secret" +description = "Discovered a LinkedIn Client secret, potentially compromising LinkedIn application integrations and user data." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "lob-api-key" +description = "Uncovered a Lob API Key, which could lead to unauthorized access to mailing and address verification services." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((live|test)_[a-f0-9]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_", + "live_", +] + +[[rules]] +id = "lob-pub-api-key" +description = "Detected a Lob Publishable API Key, posing a risk of exposing mail and print service integrations." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((test|live)_pub_[a-f0-9]{31})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_pub", + "live_pub", + "_pub", +] + +[[rules]] +id = "mailchimp-api-key" +description = "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data." +regex = '''(?i)[\w.-]{0,50}?(?:MailchimpSDK.initialize|mailchimp)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32}-us\d\d)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailchimp"] + +[[rules]] +id = "mailgun-private-api-token" +description = "Found a Mailgun private API token, risking unauthorized email service operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(key-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-pub-key" +description = "Discovered a Mailgun public validation key, which could expose email verification processes and associated data." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pubkey-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-signing-key" +description = "Uncovered a Mailgun webhook signing key, potentially compromising email automation and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mapbox-api-token" +description = "Detected a MapBox API token, posing a risk to geospatial services and sensitive location data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:mapbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mapbox"] + +[[rules]] +id = "mattermost-access-token" +description = "Identified a Mattermost Access Token, which may compromise team communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:mattermost)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mattermost"] + +[[rules]] +id = "maxmind-license-key" +description = "Discovered a potential MaxMind license key." +regex = '''\b([A-Za-z0-9]{6}_[A-Za-z0-9]{29}_mmk)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["_mmk"] + +[[rules]] +id = "messagebird-api-token" +description = "Found a MessageBird API token, risking unauthorized access to communication platforms and message data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "messagebird-client-id" +description = "Discovered a MessageBird client ID, potentially compromising API integrations and sensitive communication data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "microsoft-teams-webhook" +description = "Uncovered a Microsoft Teams Webhook, which could lead to unauthorized access to team collaboration tools and data leaks." +regex = '''https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}/IncomingWebhook/[a-z0-9]{32}/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' +keywords = [ + "webhook.office.com", + "webhookb2", + "incomingwebhook", +] + +[[rules]] +id = "netlify-access-token" +description = "Detected a Netlify Access Token, potentially compromising web hosting services and site management." +regex = '''(?i)[\w.-]{0,50}?(?:netlify)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40,46})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["netlify"] + +[[rules]] +id = "new-relic-browser-api-token" +description = "Identified a New Relic ingest browser API token, risking unauthorized access to application performance data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRJS-[a-f0-9]{19})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrjs-"] + +[[rules]] +id = "new-relic-insert-key" +description = "Discovered a New Relic insight insert key, compromising data injection into the platform." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRII-[a-z0-9-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrii-"] + +[[rules]] +id = "new-relic-user-api-id" +description = "Found a New Relic user API ID, posing a risk to application monitoring services and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "new-relic", + "newrelic", + "new_relic", +] + +[[rules]] +id = "new-relic-user-api-key" +description = "Discovered a New Relic user API Key, which could lead to compromised application insights and performance monitoring." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRAK-[a-z0-9]{27})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrak"] + +[[rules]] +id = "npm-access-token" +description = "Uncovered an npm access token, potentially compromising package management and code repository access." +regex = '''(?i)\b(npm_[a-z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["npm_"] + +[[rules]] +id = "nuget-config-password" +description = "Identified a password within a Nuget config file, potentially compromising package management access." +regex = '''(?i)''' +path = '''(?i)nuget\.config$''' +entropy = 1 +keywords = ["|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "nytimes", + "new-york-times", + "newyorktimes", +] + +[[rules]] +id = "octopus-deploy-api-key" +description = "Discovered a potential Octopus Deploy API key, risking application deployments and operational security." +regex = '''\b(API-[A-Z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["api-"] + +[[rules]] +id = "okta-access-token" +description = "Identified an Okta Access Token, which may compromise identity management services and user authentication data." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Oo]kta|OKTA))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(00[\w=\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["okta"] + +[[rules]] +id = "openai-api-key" +description = "Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''\b(sk-(?:proj|svcacct|admin)-(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})T3BlbkFJ(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})\b|sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["t3blbkfj"] + +[[rules]] +id = "openshift-user-token" +description = "Found an OpenShift user token, potentially compromising an OpenShift/Kubernetes cluster." +regex = '''\b(sha256~[\w-]{43})(?:[^\w-]|\z)''' +entropy = 3.5 +keywords = ["sha256~"] + +[[rules]] +id = "perplexity-api-key" +description = "Detected a Perplexity API key, which could lead to unauthorized access to Perplexity AI services and data exposure." +regex = '''\b(pplx-[a-zA-Z0-9]{48})(?:[\x60'"\s;]|\\[nr]|$|\b)''' +entropy = 4 +keywords = ["pplx-"] + +[[rules]] +id = "pkcs12-file" +description = "Found a PKCS #12 file, which commonly contain bundled private keys." +path = '''(?i)(?:^|\/)[^\/]+\.p(?:12|fx)$''' + +[[rules]] +id = "plaid-api-token" +description = "Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["plaid"] + +[[rules]] +id = "plaid-client-id" +description = "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "plaid-secret-key" +description = "Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "planetscale-api-token" +description = "Identified a PlanetScale API token, potentially compromising database management and operations." +regex = '''\b(pscale_tkn_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_tkn_"] + +[[rules]] +id = "planetscale-oauth-token" +description = "Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity." +regex = '''\b(pscale_oauth_[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_oauth_"] + +[[rules]] +id = "planetscale-password" +description = "Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches." +regex = '''(?i)\b(pscale_pw_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_pw_"] + +[[rules]] +id = "postman-api-token" +description = "Uncovered a Postman API token, potentially compromising API testing and development workflows." +regex = '''\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pmak-"] + +[[rules]] +id = "prefect-api-token" +description = "Detected a Prefect API token, risking unauthorized access to workflow management and automation services." +regex = '''\b(pnu_[a-zA-Z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pnu_"] + +[[rules]] +id = "private-key" +description = "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption." +regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----''' +keywords = ["-----begin"] + +[[rules]] +id = "privateai-api-token" +description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:private[_-]?ai)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "privateai", + "private_ai", + "private-ai", +] + +[[rules]] +id = "pulumi-api-token" +description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management." +regex = '''\b(pul-[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pul-"] + +[[rules]] +id = "pypi-upload-token" +description = "Discovered a PyPI upload token, potentially compromising Python package distribution and repository integrity." +regex = '''pypi-AgEIcHlwaS5vcmc[\w-]{50,1000}''' +entropy = 3 +keywords = ["pypi-ageichlwas5vcmc"] + +[[rules]] +id = "rapidapi-access-token" +description = "Uncovered a RapidAPI Access Token, which could lead to unauthorized access to various APIs and data services." +regex = '''(?i)[\w.-]{0,50}?(?:rapidapi)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["rapidapi"] + +[[rules]] +id = "readme-api-token" +description = "Detected a Readme API token, risking unauthorized documentation management and content exposure." +regex = '''\b(rdme_[a-z0-9]{70})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rdme_"] + +[[rules]] +id = "rubygems-api-token" +description = "Identified a Rubygem API token, potentially compromising Ruby library distribution and package management." +regex = '''\b(rubygems_[a-f0-9]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rubygems_"] + +[[rules]] +id = "scalingo-api-token" +description = "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security." +regex = '''\b(tk-us-[\w-]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["tk-us-"] + +[[rules]] +id = "sendbird-access-id" +description = "Discovered a Sendbird Access ID, which could compromise chat and messaging platform integrations." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendbird-access-token" +description = "Uncovered a Sendbird Access Token, potentially risking unauthorized access to communication services and user data." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendgrid-api-token" +description = "Detected a SendGrid API token, posing a risk of unauthorized email service operations and data exposure." +regex = '''\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["sg."] + +[[rules]] +id = "sendinblue-api-token" +description = "Identified a Sendinblue API token, which may compromise email marketing services and subscriber data privacy." +regex = '''\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["xkeysib-"] + +[[rules]] +id = "sentry-access-token" +description = "Found a Sentry.io Access Token (old format), risking unauthorized access to error tracking services and sensitive application data." +regex = '''(?i)[\w.-]{0,50}?(?:sentry)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sentry"] + +[[rules]] +id = "sentry-org-token" +description = "Found a Sentry.io Organization Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\bsntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}(?:[^a-zA-Z0-9+/]|\z)''' +entropy = 4.5 +keywords = ["sntrys_eyjpyxqio"] + +[[rules]] +id = "sentry-user-token" +description = "Found a Sentry.io User Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\b(sntryu_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["sntryu_"] + +[[rules]] +id = "settlemint-application-access-token" +description = "Found a Settlemint Application Access Token." +regex = '''\b(sm_aat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_aat"] + +[[rules]] +id = "settlemint-personal-access-token" +description = "Found a Settlemint Personal Access Token." +regex = '''\b(sm_pat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_pat"] + +[[rules]] +id = "settlemint-service-access-token" +description = "Found a Settlemint Service Access Token." +regex = '''\b(sm_sat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_sat"] + +[[rules]] +id = "shippo-api-token" +description = "Discovered a Shippo API token, potentially compromising shipping services and customer order data." +regex = '''\b(shippo_(?:live|test)_[a-fA-F0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["shippo_"] + +[[rules]] +id = "shopify-access-token" +description = "Uncovered a Shopify access token, which could lead to unauthorized e-commerce platform access and data breaches." +regex = '''shpat_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpat_"] + +[[rules]] +id = "shopify-custom-access-token" +description = "Detected a Shopify custom access token, potentially compromising custom app integrations and e-commerce data security." +regex = '''shpca_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpca_"] + +[[rules]] +id = "shopify-private-app-access-token" +description = "Identified a Shopify private app access token, risking unauthorized access to private app data and store operations." +regex = '''shppa_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shppa_"] + +[[rules]] +id = "shopify-shared-secret" +description = "Found a Shopify shared secret, posing a risk to application authentication and e-commerce platform security." +regex = '''shpss_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpss_"] + +[[rules]] +id = "sidekiq-secret" +description = "Discovered a Sidekiq Secret, which could lead to compromised background job processing and application data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "bundle_enterprise__contribsys__com", + "bundle_gems__contribsys__com", +] + +[[rules]] +id = "sidekiq-sensitive-url" +description = "Uncovered a Sidekiq Sensitive URL, potentially exposing internal job queues and sensitive operation details." +regex = '''(?i)\bhttps?://([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' +keywords = [ + "gems.contribsys.com", + "enterprise.contribsys.com", +] + +[[rules]] +id = "slack-app-token" +description = "Detected a Slack App-level token, risking unauthorized access to Slack applications and workspace data." +regex = '''(?i)xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+''' +entropy = 2 +keywords = ["xapp"] + +[[rules]] +id = "slack-bot-token" +description = "Identified a Slack Bot token, which may compromise bot integrations and communication channel security." +regex = '''xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*''' +entropy = 3 +keywords = ["xoxb"] + +[[rules]] +id = "slack-config-access-token" +description = "Found a Slack Configuration access token, posing a risk to workspace configuration and sensitive data access." +regex = '''(?i)xoxe.xox[bp]-\d-[A-Z0-9]{163,166}''' +entropy = 2 +keywords = [ + "xoxe.xoxb-", + "xoxe.xoxp-", +] + +[[rules]] +id = "slack-config-refresh-token" +description = "Discovered a Slack Configuration refresh token, potentially allowing prolonged unauthorized access to configuration settings." +regex = '''(?i)xoxe-\d-[A-Z0-9]{146}''' +entropy = 2 +keywords = ["xoxe-"] + +[[rules]] +id = "slack-legacy-bot-token" +description = "Uncovered a Slack Legacy bot token, which could lead to compromised legacy bot operations and data exposure." +regex = '''xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26}''' +entropy = 2 +keywords = ["xoxb"] + +[[rules]] +id = "slack-legacy-token" +description = "Detected a Slack Legacy token, risking unauthorized access to older Slack integrations and user data." +regex = '''xox[os]-\d+-\d+-\d+-[a-fA-F\d]+''' +entropy = 2 +keywords = [ + "xoxo", + "xoxs", +] + +[[rules]] +id = "slack-legacy-workspace-token" +description = "Identified a Slack Legacy Workspace token, potentially compromising access to workspace data and legacy features." +regex = '''xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48}''' +entropy = 2 +keywords = [ + "xoxa", + "xoxr", +] + +[[rules]] +id = "slack-user-token" +description = "Found a Slack User token, posing a risk of unauthorized user impersonation and data access within Slack workspaces." +regex = '''xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34}''' +entropy = 2 +keywords = [ + "xoxp-", + "xoxe-", +] + +[[rules]] +id = "slack-webhook-url" +description = "Discovered a Slack Webhook, which could lead to unauthorized message posting and data leakage in Slack channels." +regex = '''(?:https?://)?hooks.slack.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}''' +keywords = ["hooks.slack.com"] + +[[rules]] +id = "snyk-api-token" +description = "Uncovered a Snyk API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["snyk"] + +[[rules]] +id = "sonar-api-token" +description = "Uncovered a Sonar API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:sonar[_.-]?(login|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sonar"] + +[[rules]] +id = "sourcegraph-access-token" +description = "Sourcegraph is a code search and navigation engine." +regex = '''(?i)\b(\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "sgp_", + "sourcegraph", +] + +[[rules]] +id = "square-access-token" +description = "Detected a Square Access Token, risking unauthorized payment processing and financial transaction exposure." +regex = '''\b((?:EAAA|sq0atp-)[\w-]{22,60})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sq0atp-", + "eaaa", +] + +[[rules]] +id = "squarespace-access-token" +description = "Identified a Squarespace Access Token, which may compromise website management and content control on Squarespace." +regex = '''(?i)[\w.-]{0,50}?(?:squarespace)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["squarespace"] + +[[rules]] +id = "stripe-access-token" +description = "Found a Stripe Access Token, posing a risk to payment processing services and sensitive financial data." +regex = '''\b((?:sk|rk)_(?:test|live|prod)_[a-zA-Z0-9]{10,99})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sk_test", + "sk_live", + "sk_prod", + "rk_test", + "rk_live", + "rk_prod", +] + +[[rules]] +id = "sumologic-access-id" +description = "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(su[a-zA-Z0-9]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "sumologic-access-token" +description = "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "telegram-bot-api-token" +description = "Detected a Telegram Bot API Token, risking unauthorized bot operations and message interception on Telegram." +regex = '''(?i)[\w.-]{0,50}?(?:telegr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{5,16}:(?-i:A)[a-z0-9_\-]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["telegr"] + +[[rules]] +id = "travisci-access-token" +description = "Identified a Travis CI Access Token, potentially compromising continuous integration services and codebase security." +regex = '''(?i)[\w.-]{0,50}?(?:travis)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["travis"] + +[[rules]] +id = "twilio-api-key" +description = "Found a Twilio API Key, posing a risk to communication services and sensitive customer interaction data." +regex = '''SK[0-9a-fA-F]{32}''' +entropy = 3 +keywords = ["sk"] + +[[rules]] +id = "twitch-api-token" +description = "Discovered a Twitch API token, which could compromise streaming services and account integrations." +regex = '''(?i)[\w.-]{0,50}?(?:twitch)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitch"] + +[[rules]] +id = "twitter-access-secret" +description = "Uncovered a Twitter Access Secret, potentially risking unauthorized Twitter integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{45})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-access-token" +description = "Detected a Twitter Access Token, posing a risk of unauthorized account operations and social media data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-key" +description = "Identified a Twitter API Key, which may compromise Twitter application integrations and user data security." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-secret" +description = "Found a Twitter API Secret, risking the security of Twitter app integrations and sensitive data access." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-bearer-token" +description = "Discovered a Twitter Bearer Token, potentially compromising API access and data retrieval from Twitter." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "typeform-api-token" +description = "Uncovered a Typeform API token, which could lead to unauthorized survey management and data collection." +regex = '''(?i)[\w.-]{0,50}?(?:typeform)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(tfp_[a-z0-9\-_\.=]{59})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["tfp_"] + +[[rules]] +id = "vault-batch-token" +description = "Detected a Vault Batch Token, risking unauthorized access to secret management services and sensitive data." +regex = '''\b(hvb\.[\w-]{138,300})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hvb."] + +[[rules]] +id = "vault-service-token" +description = "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials." +regex = '''\b((?:hvs\.[\w-]{90,120}|s\.(?i:[a-z0-9]{24})))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "hvs.", + "s.", +] +[[rules.allowlists]] +regexes = [ + '''s\.[A-Za-z]{24}''', +] + +[[rules]] +id = "yandex-access-token" +description = "Found a Yandex Access Token, posing a risk to Yandex service integrations and user data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-api-key" +description = "Discovered a Yandex API Key, which could lead to unauthorized access to Yandex services and data manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-aws-access-token" +description = "Uncovered a Yandex AWS Access Token, potentially compromising cloud resource access and data security on Yandex Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(YC[a-zA-Z0-9_\-]{38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "zendesk-secret-key" +description = "Detected a Zendesk Secret Key, risking unauthorized access to customer support services and sensitive ticketing data." +regex = '''(?i)[\w.-]{0,50}?(?:zendesk)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["zendesk"] + diff --git a/cli/detect/config/rule.go b/cli/detect/config/rule.go new file mode 100644 index 000000000..6d2b61326 --- /dev/null +++ b/cli/detect/config/rule.go @@ -0,0 +1,114 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package config + +import ( + "fmt" + "strings" + + "github.com/Infisical/infisical-merge/detect/regexp" +) + +// Rules contain information that define details on how to detect secrets +type Rule struct { + // RuleID is a unique identifier for this rule + RuleID string + + // Description is the description of the rule. + Description string + + // Entropy is a float representing the minimum shannon + // entropy a regex group must have to be considered a secret. + Entropy float64 + + // SecretGroup is an int used to extract secret from regex + // match and used as the group that will have its entropy + // checked if `entropy` is set. + SecretGroup int + + // Regex is a golang regular expression used to detect secrets. + Regex *regexp.Regexp + + // Path is a golang regular expression used to + // filter secrets by path + Path *regexp.Regexp + + // Tags is an array of strings used for metadata + // and reporting purposes. + Tags []string + + // Keywords are used for pre-regex check filtering. Rules that contain + // keywords will perform a quick string compare check to make sure the + // keyword(s) are in the content being scanned. + Keywords []string + + // Allowlists allows a rule to be ignored for specific commits, paths, regexes, and/or stopwords. + Allowlists []*Allowlist + + // validated is an internal flag to track whether `Validate()` has been called. + validated bool +} + +// Validate guards against common misconfigurations. +func (r *Rule) Validate() error { + if r.validated { + return nil + } + + // Ensure |id| is present. + if strings.TrimSpace(r.RuleID) == "" { + // Try to provide helpful context, since |id| is empty. + var context string + if r.Regex != nil { + context = ", regex: " + r.Regex.String() + } else if r.Path != nil { + context = ", path: " + r.Path.String() + } else if r.Description != "" { + context = ", description: " + r.Description + } + return fmt.Errorf("rule |id| is missing or empty" + context) + } + + // Ensure the rule actually matches something. + if r.Regex == nil && r.Path == nil { + return fmt.Errorf("%s: both |regex| and |path| are empty, this rule will have no effect", r.RuleID) + } + + // Ensure |secretGroup| works. + if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() { + return fmt.Errorf("%s: invalid regex secret group %d, max regex secret group %d", r.RuleID, r.SecretGroup, r.Regex.NumSubexp()) + } + + for _, allowlist := range r.Allowlists { + // This will probably never happen. + if allowlist == nil { + continue + } + if err := allowlist.Validate(); err != nil { + return fmt.Errorf("%s: %w", r.RuleID, err) + } + } + + r.validated = true + return nil +} diff --git a/cli/report/report.go b/cli/detect/config/utils.go similarity index 65% rename from cli/report/report.go rename to cli/detect/config/utils.go index 1191a4f33..e28a5cb37 100644 --- a/cli/report/report.go +++ b/cli/detect/config/utils.go @@ -20,35 +20,27 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -package report +package config import ( - "os" - "strings" - - "github.com/Infisical/infisical-merge/config" + "github.com/Infisical/infisical-merge/detect/regexp" ) -const ( - // https://cwe.mitre.org/data/definitions/798.html - CWE = "CWE-798" - CWE_DESCRIPTION = "Use of Hard-coded Credentials" -) - -func Write(findings []Finding, cfg config.Config, ext string, reportPath string) error { - file, err := os.Create(reportPath) - if err != nil { - return err +func anyRegexMatch(f string, res []*regexp.Regexp) bool { + for _, re := range res { + if regexMatched(f, re) { + return true + } } - ext = strings.ToLower(ext) - switch ext { - case ".json", "json": - err = writeJson(findings, file) - case ".csv", "csv": - err = writeCsv(findings, file) - case ".sarif", "sarif": - err = writeSarif(cfg, findings, file) + return false +} + +func regexMatched(f string, re *regexp.Regexp) bool { + if re == nil { + return false + } + if re.FindString(f) != "" { + return true } - - return err + return false } diff --git a/cli/detect/decoder.go b/cli/detect/decoder.go new file mode 100644 index 000000000..6ec509757 --- /dev/null +++ b/cli/detect/decoder.go @@ -0,0 +1,328 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bytes" + "encoding/base64" + "fmt" + "regexp" + "unicode" + + "github.com/Infisical/infisical-merge/detect/logging" +) + +var b64LikelyChars [128]byte +var b64Regexp = regexp.MustCompile(`[\w/+-]{16,}={0,3}`) +var decoders = []func(string) ([]byte, error){ + base64.StdEncoding.DecodeString, + base64.RawURLEncoding.DecodeString, +} + +func init() { + // Basically look for anything that isn't just letters + for _, c := range `0123456789+/-_` { + b64LikelyChars[c] = 1 + } +} + +// EncodedSegment represents a portion of text that is encoded in some way. +// `decode` supports recusive decoding and can result in "segment trees". +// There can be multiple segments in the original text, so each can be thought +// of as its own tree with the root being the original segment. +type EncodedSegment struct { + // The parent segment in a segment tree. If nil, it is a root segment + parent *EncodedSegment + + // Relative start/end are the bounds of the encoded value in the current pass. + relativeStart int + relativeEnd int + + // Absolute start/end refer to the bounds of the root segment in this segment + // tree + absoluteStart int + absoluteEnd int + + // Decoded start/end refer to the bounds of the decoded value in the current + // pass. These can differ from relative values because decoding can shrink + // or grow the size of the segment. + decodedStart int + decodedEnd int + + // This is the actual decoded content in the segment + decodedValue string + + // This is the type of encoding + encoding string +} + +// isChildOf inspects the bounds of two segments to determine +// if one should be the child of another +func (s EncodedSegment) isChildOf(parent EncodedSegment) bool { + return parent.decodedStart <= s.relativeStart && parent.decodedEnd >= s.relativeEnd +} + +// decodedOverlaps checks if the decoded bounds of the segment overlaps a range +func (s EncodedSegment) decodedOverlaps(start, end int) bool { + return start <= s.decodedEnd && end >= s.decodedStart +} + +// adjustMatchIndex takes the matchIndex from the current decoding pass and +// updates it to match the absolute matchIndex in the original text. +func (s EncodedSegment) adjustMatchIndex(matchIndex []int) []int { + // The match is within the bounds of the segment so we just return + // the absolute start and end of the root segment. + if s.decodedStart <= matchIndex[0] && matchIndex[1] <= s.decodedEnd { + return []int{ + s.absoluteStart, + s.absoluteEnd, + } + } + + // Since it overlaps one side and/or the other, we're going to have to adjust + // and climb parents until we're either at the root or we've determined + // we're fully inside one of the parent segments. + adjustedMatchIndex := make([]int, 2) + + if matchIndex[0] < s.decodedStart { + // It starts before the encoded segment so adjust the start to match + // the location before it was decoded + matchStartDelta := s.decodedStart - matchIndex[0] + adjustedMatchIndex[0] = s.relativeStart - matchStartDelta + } else { + // It starts within the encoded segment so set the bound to the + // relative start + adjustedMatchIndex[0] = s.relativeStart + } + + if matchIndex[1] > s.decodedEnd { + // It ends after the encoded segment so adjust the end to match + // the location before it was decoded + matchEndDelta := matchIndex[1] - s.decodedEnd + adjustedMatchIndex[1] = s.relativeEnd + matchEndDelta + } else { + // It ends within the encoded segment so set the bound to the relative end + adjustedMatchIndex[1] = s.relativeEnd + } + + // We're still not at a root segment so we'll need to keep on adjusting + if s.parent != nil { + return s.parent.adjustMatchIndex(adjustedMatchIndex) + } + + return adjustedMatchIndex +} + +// depth reports how many levels of decoding needed to be done (default is 1) +func (s EncodedSegment) depth() int { + depth := 1 + + // Climb the tree and increment the depth + for current := &s; current.parent != nil; current = current.parent { + depth++ + } + + return depth +} + +// tags returns additional meta data tags related to the types of segments +func (s EncodedSegment) tags() []string { + return []string{ + fmt.Sprintf("decoded:%s", s.encoding), + fmt.Sprintf("decode-depth:%d", s.depth()), + } +} + +// Decoder decodes various types of data in place +type Decoder struct { + decodedMap map[string]string +} + +// NewDecoder creates a default decoder struct +func NewDecoder() *Decoder { + return &Decoder{ + decodedMap: make(map[string]string), + } +} + +// decode returns the data with the values decoded in-place +func (d *Decoder) decode(data string, parentSegments []EncodedSegment) (string, []EncodedSegment) { + segments := d.findEncodedSegments(data, parentSegments) + + if len(segments) > 0 { + result := bytes.NewBuffer(make([]byte, 0, len(data))) + + relativeStart := 0 + for _, segment := range segments { + result.WriteString(data[relativeStart:segment.relativeStart]) + result.WriteString(segment.decodedValue) + relativeStart = segment.relativeEnd + } + result.WriteString(data[relativeStart:]) + + return result.String(), segments + } + + return data, segments +} + +// findEncodedSegments finds the encoded segments in the data and updates the +// segment tree for this pass +func (d *Decoder) findEncodedSegments(data string, parentSegments []EncodedSegment) []EncodedSegment { + if len(data) == 0 { + return []EncodedSegment{} + } + + matchIndices := b64Regexp.FindAllStringIndex(data, -1) + if matchIndices == nil { + return []EncodedSegment{} + } + + segments := make([]EncodedSegment, 0, len(matchIndices)) + + // Keeps up with offsets from the text changing size as things are decoded + decodedShift := 0 + + for _, matchIndex := range matchIndices { + encodedValue := data[matchIndex[0]:matchIndex[1]] + + if !isLikelyB64(encodedValue) { + d.decodedMap[encodedValue] = "" + continue + } + + decodedValue, alreadyDecoded := d.decodedMap[encodedValue] + + // We haven't decoded this yet, so go ahead and decode it + if !alreadyDecoded { + decodedValue = decodeValue(encodedValue) + d.decodedMap[encodedValue] = decodedValue + } + + // Skip this segment because there was nothing to check + if len(decodedValue) == 0 { + continue + } + + // Create a segment for the encoded data + segment := EncodedSegment{ + relativeStart: matchIndex[0], + relativeEnd: matchIndex[1], + absoluteStart: matchIndex[0], + absoluteEnd: matchIndex[1], + decodedStart: matchIndex[0] + decodedShift, + decodedEnd: matchIndex[0] + decodedShift + len(decodedValue), + decodedValue: decodedValue, + encoding: "base64", + } + + // Shift decoded start and ends based on size changes + decodedShift += len(decodedValue) - len(encodedValue) + + // Adjust the absolute position of segments contained in parent segments + for _, parentSegment := range parentSegments { + if segment.isChildOf(parentSegment) { + segment.absoluteStart = parentSegment.absoluteStart + segment.absoluteEnd = parentSegment.absoluteEnd + segment.parent = &parentSegment + break + } + } + + logging.Debug().Msgf("segment found: %#v", segment) + segments = append(segments, segment) + } + + return segments +} + +// decoders tries a list of decoders and returns the first successful one +func decodeValue(encodedValue string) string { + for _, decoder := range decoders { + decodedValue, err := decoder(encodedValue) + + if err == nil && len(decodedValue) > 0 && isASCII(decodedValue) { + return string(decodedValue) + } + } + + return "" +} + +func isASCII(b []byte) bool { + for i := 0; i < len(b); i++ { + if b[i] > unicode.MaxASCII || b[i] < '\t' { + return false + } + } + + return true +} + +// Skip a lot of method signatures and things at the risk of missing about +// 1% of base64 +func isLikelyB64(s string) bool { + for _, c := range s { + if b64LikelyChars[c] != 0 { + return true + } + } + + return false +} + +// Find a segment where the decoded bounds overlaps a range +func segmentWithDecodedOverlap(encodedSegments []EncodedSegment, start, end int) *EncodedSegment { + for _, segment := range encodedSegments { + if segment.decodedOverlaps(start, end) { + return &segment + } + } + + return nil +} + +func (s EncodedSegment) currentLine(currentRaw string) string { + start := 0 + end := len(currentRaw) + + // Find the start of the range + for i := s.decodedStart; i > -1; i-- { + c := currentRaw[i] + if c == '\n' { + start = i + break + } + } + + // Find the end of the range + for i := s.decodedEnd; i < end; i++ { + c := currentRaw[i] + if c == '\n' { + end = i + break + } + } + + return currentRaw[start:end] +} diff --git a/cli/detect/detect.go b/cli/detect/detect.go index 84f058d4b..f2e42cccc 100644 --- a/cli/detect/detect.go +++ b/cli/detect/detect.go @@ -26,39 +26,37 @@ import ( "bufio" "context" "fmt" - "io" - "io/fs" "os" - "path/filepath" - "regexp" + "runtime" "strings" "sync" + "sync/atomic" + "time" - "github.com/h2non/filetype" - - "github.com/Infisical/infisical-merge/config" - "github.com/Infisical/infisical-merge/detect/git" - "github.com/Infisical/infisical-merge/report" + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/regexp" + "github.com/Infisical/infisical-merge/detect/report" + ahocorasick "github.com/BobuSumisu/aho-corasick" "github.com/fatih/semgroup" - "github.com/gitleaks/go-gitdiff/gitdiff" - ahocorasick "github.com/petar-dambovaliev/aho-corasick" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" "github.com/spf13/viper" + "golang.org/x/exp/maps" ) -// Type used to differentiate between git scan types: -// $ gitleaks detect -// $ gitleaks protect -// $ gitleaks protect staged -type GitScanType int - const ( - DetectType GitScanType = iota - ProtectType - ProtectStagedType + gitleaksAllowSignature = "gitleaks:allow" + chunkSize = 100 * 1_000 // 100kb - gitleaksAllowSignature = "infisical-scan:ignore" + // SlowWarningThreshold is the amount of time to wait before logging that a file is slow. + // This is useful for identifying problematic files and tuning the allowlist. + SlowWarningThreshold = 5 * time.Second +) + +var ( + newLineRegexp = regexp.MustCompile("\n") + isWindows = runtime.GOOS == "windows" ) // Detector is the main detector struct @@ -69,11 +67,14 @@ type Detector struct { // Redact is a flag to redact findings. This is exported // so users using gitleaks as a library can set this flag // without calling `detector.Start(cmd *cobra.Command)` - Redact bool + Redact uint // verbose is a flag to print findings Verbose bool + // MaxDecodeDepths limits how many recursive decoding passes are allowed + MaxDecodeDepth int + // files larger than this will be skipped MaxTargetMegaBytes int @@ -83,6 +84,9 @@ type Detector struct { // NoColor is a flag to disable color output NoColor bool + // IgnoreGitleaksAllow is a flag to ignore gitleaks:allow comments. + IgnoreGitleaksAllow bool + // commitMap is used to keep track of commits that have been scanned. // This is only used for logging purposes and git scans. commitMap map[string]bool @@ -98,7 +102,7 @@ type Detector struct { // prefilter is a ahocorasick struct used for doing efficient string // matching given a set of words (keywords from the rules in the config) - prefilter ahocorasick.AhoCorasick + prefilter ahocorasick.Trie // a list of known findings that should be ignored baseline []report.Finding @@ -107,7 +111,16 @@ type Detector struct { baselinePath string // gitleaksIgnore - gitleaksIgnore map[string]bool + gitleaksIgnore map[string]struct{} + + // Sema (https://github.com/fatih/semgroup) controls the concurrency + Sema *semgroup.Group + + // report-related settings. + ReportPath string + Reporter report.Reporter + + TotalBytes atomic.Uint64 } // Fragment contains the data to be scanned @@ -115,9 +128,15 @@ type Fragment struct { // Raw is the raw content of the fragment Raw string - // FilePath is the path to the file if applicable + Bytes []byte + + // FilePath is the path to the file, if applicable. + // The path separator MUST be normalized to `/`. FilePath string SymlinkFile string + // WindowsFilePath is the path with the original separator. + // This provides a backwards-compatible solution to https://github.com/gitleaks/gitleaks/issues/1565. + WindowsFilePath string `json:"-"` // TODO: remove this in v9. // CommitSHA is the SHA of the commit if applicable CommitSHA string @@ -125,28 +144,18 @@ type Fragment struct { // newlineIndices is a list of indices of newlines in the raw content. // This is used to calculate the line location of a finding newlineIndices [][]int - - // keywords is a map of all the keywords contain within the contents - // of this fragment - keywords map[string]bool } // NewDetector creates a new detector with the given config func NewDetector(cfg config.Config) *Detector { - builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{ - AsciiCaseInsensitive: true, - MatchOnlyWholeWords: false, - MatchKind: ahocorasick.LeftMostLongestMatch, - DFA: true, - }) - return &Detector{ commitMap: make(map[string]bool), - gitleaksIgnore: make(map[string]bool), + gitleaksIgnore: make(map[string]struct{}), findingMutex: &sync.Mutex{}, findings: make([]report.Finding, 0), Config: cfg, - prefilter: builder.Build(cfg.Keywords), + prefilter: *ahocorasick.NewTrieBuilder().AddStrings(maps.Keys(cfg.Keywords)).Build(), + Sema: semgroup.NewGroup(context.Background(), 40), } } @@ -170,58 +179,47 @@ func NewDetectorDefaultConfig() (*Detector, error) { } func (d *Detector) AddGitleaksIgnore(gitleaksIgnorePath string) error { - log.Debug().Msg("found .gitleaksignore file") + logging.Debug().Msgf("found .gitleaksignore file: %s", gitleaksIgnorePath) file, err := os.Open(gitleaksIgnorePath) - if err != nil { return err } - - // https://github.com/securego/gosec/issues/512 defer func() { + // https://github.com/securego/gosec/issues/512 if err := file.Close(); err != nil { - log.Warn().Msgf("Error closing .gitleaksignore file: %s\n", err) + logging.Warn().Msgf("Error closing .gitleaksignore file: %s\n", err) } }() + scanner := bufio.NewScanner(file) - + replacer := strings.NewReplacer("\\", "/") for scanner.Scan() { - d.gitleaksIgnore[scanner.Text()] = true + line := strings.TrimSpace(scanner.Text()) + // Skip lines that start with a comment + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Normalize the path. + // TODO: Make this a breaking change in v9. + s := strings.Split(line, ":") + switch len(s) { + case 3: + // Global fingerprint. + // `file:rule-id:start-line` + s[0] = replacer.Replace(s[0]) + case 4: + // Commit fingerprint. + // `commit:file:rule-id:start-line` + s[1] = replacer.Replace(s[1]) + default: + logging.Warn().Str("fingerprint", line).Msg("Invalid .gitleaksignore entry") + } + d.gitleaksIgnore[strings.Join(s, ":")] = struct{}{} } return nil } -func (d *Detector) AddBaseline(baselinePath string, source string) error { - if baselinePath != "" { - absoluteSource, err := filepath.Abs(source) - if err != nil { - return err - } - - absoluteBaseline, err := filepath.Abs(baselinePath) - if err != nil { - return err - } - - relativeBaseline, err := filepath.Rel(absoluteSource, absoluteBaseline) - if err != nil { - return err - } - - baseline, err := LoadBaseline(baselinePath) - if err != nil { - return err - } - - d.baseline = baseline - baselinePath = relativeBaseline - - } - - d.baselinePath = baselinePath - return nil -} - // DetectBytes scans the given bytes and returns a list of findings func (d *Detector) DetectBytes(content []byte) []report.Finding { return d.DetectString(string(content)) @@ -234,56 +232,179 @@ func (d *Detector) DetectString(content string) []report.Finding { }) } -// detectRule scans the given fragment for the given rule and returns a list of findings -func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Finding { - var findings []report.Finding +// Detect scans the given fragment and returns a list of findings +func (d *Detector) Detect(fragment Fragment) []report.Finding { + if fragment.Bytes == nil { + d.TotalBytes.Add(uint64(len(fragment.Raw))) + } + d.TotalBytes.Add(uint64(len(fragment.Bytes))) - // check if filepath or commit is allowed for this rule - if rule.Allowlist.CommitAllowed(fragment.CommitSHA) || - rule.Allowlist.PathAllowed(fragment.FilePath) { + var ( + findings []report.Finding + logger = func() zerolog.Logger { + l := logging.With().Str("path", fragment.FilePath) + if fragment.CommitSHA != "" { + l = l.Str("commit", fragment.CommitSHA) + } + return l.Logger() + }() + ) + + // check if filepath is allowed + if fragment.FilePath != "" { + // is the path our config or baseline file? + if fragment.FilePath == d.Config.Path || (d.baselinePath != "" && fragment.FilePath == d.baselinePath) { + logging.Trace().Msg("skipping file: matches config or baseline path") + return findings + } + } + // check if commit or filepath is allowed. + if isAllowed, event := checkCommitOrPathAllowed(logger, fragment, d.Config.Allowlists); isAllowed { + event.Msg("skipping file: global allowlist") return findings } - if rule.Path != nil && rule.Regex == nil { - // Path _only_ rule - if rule.Path.Match([]byte(fragment.FilePath)) { - finding := report.Finding{ - Description: rule.Description, - File: fragment.FilePath, - SymlinkFile: fragment.SymlinkFile, - RuleID: rule.RuleID, - Match: fmt.Sprintf("file detected: %s", fragment.FilePath), - Tags: rule.Tags, - } - return append(findings, finding) + // add newline indices for location calculation in detectRule + fragment.newlineIndices = newLineRegexp.FindAllStringIndex(fragment.Raw, -1) + + // setup variables to handle different decoding passes + currentRaw := fragment.Raw + encodedSegments := []EncodedSegment{} + currentDecodeDepth := 0 + decoder := NewDecoder() + + for { + // build keyword map for prefiltering rules + keywords := make(map[string]bool) + normalizedRaw := strings.ToLower(currentRaw) + matches := d.prefilter.MatchString(normalizedRaw) + for _, m := range matches { + keywords[normalizedRaw[m.Pos():int(m.Pos())+len(m.Match())]] = true } - } else if rule.Path != nil { - // if path is set _and_ a regex is set, then we need to check both - // so if the path does not match, then we should return early and not - // consider the regex - if !rule.Path.Match([]byte(fragment.FilePath)) { - return findings + + for _, rule := range d.Config.Rules { + if len(rule.Keywords) == 0 { + // if no keywords are associated with the rule always scan the + // fragment using the rule + findings = append(findings, d.detectRule(fragment, currentRaw, rule, encodedSegments)...) + continue + } + + // check if keywords are in the fragment + for _, k := range rule.Keywords { + if _, ok := keywords[strings.ToLower(k)]; ok { + findings = append(findings, d.detectRule(fragment, currentRaw, rule, encodedSegments)...) + break + } + } + } + + // increment the depth by 1 as we start our decoding pass + currentDecodeDepth++ + + // stop the loop if we've hit our max decoding depth + if currentDecodeDepth > d.MaxDecodeDepth { + break + } + + // decode the currentRaw for the next pass + currentRaw, encodedSegments = decoder.decode(currentRaw, encodedSegments) + + // stop the loop when there's nothing else to decode + if len(encodedSegments) == 0 { + break + } + } + + return filter(findings, d.Redact) +} + +// detectRule scans the given fragment for the given rule and returns a list of findings +func (d *Detector) detectRule(fragment Fragment, currentRaw string, r config.Rule, encodedSegments []EncodedSegment) []report.Finding { + var ( + findings []report.Finding + logger = func() zerolog.Logger { + l := logging.With().Str("rule-id", r.RuleID).Str("path", fragment.FilePath) + if fragment.CommitSHA != "" { + l = l.Str("commit", fragment.CommitSHA) + } + return l.Logger() + }() + ) + + // check if commit or file is allowed for this rule. + if isAllowed, event := checkCommitOrPathAllowed(logger, fragment, r.Allowlists); isAllowed { + event.Msg("skipping file: rule allowlist") + return findings + } + + if r.Path != nil { + if r.Regex == nil && len(encodedSegments) == 0 { + // Path _only_ rule + if r.Path.MatchString(fragment.FilePath) || (fragment.WindowsFilePath != "" && r.Path.MatchString(fragment.WindowsFilePath)) { + finding := report.Finding{ + RuleID: r.RuleID, + Description: r.Description, + File: fragment.FilePath, + SymlinkFile: fragment.SymlinkFile, + Match: fmt.Sprintf("file detected: %s", fragment.FilePath), + Tags: r.Tags, + } + return append(findings, finding) + } + } else { + // if path is set _and_ a regex is set, then we need to check both + // so if the path does not match, then we should return early and not + // consider the regex + if !(r.Path.MatchString(fragment.FilePath) || (fragment.WindowsFilePath != "" && r.Path.MatchString(fragment.WindowsFilePath))) { + return findings + } } } // if path only rule, skip content checks - if rule.Regex == nil { + if r.Regex == nil { return findings } - // If flag configure and raw data size bigger then the flag + // if flag configure and raw data size bigger then the flag if d.MaxTargetMegaBytes > 0 { - rawLength := len(fragment.Raw) / 1000000 + rawLength := len(currentRaw) / 1000000 if rawLength > d.MaxTargetMegaBytes { - log.Debug().Msgf("skipping file: %s scan due to size: %d", fragment.FilePath, rawLength) + logger.Debug(). + Int("size", rawLength). + Int("max-size", d.MaxTargetMegaBytes). + Msg("skipping fragment: size") return findings } } - matchIndices := rule.Regex.FindAllStringIndex(fragment.Raw, -1) - for _, matchIndex := range matchIndices { - // extract secret from match - secret := strings.Trim(fragment.Raw[matchIndex[0]:matchIndex[1]], "\n") + // use currentRaw instead of fragment.Raw since this represents the current + // decoding pass on the text + for _, matchIndex := range r.Regex.FindAllStringIndex(currentRaw, -1) { + // Extract secret from match + secret := strings.Trim(currentRaw[matchIndex[0]:matchIndex[1]], "\n") + + // For any meta data from decoding + var metaTags []string + currentLine := "" + + // Check if the decoded portions of the segment overlap with the match + // to see if its potentially a new match + if len(encodedSegments) > 0 { + if segment := segmentWithDecodedOverlap(encodedSegments, matchIndex[0], matchIndex[1]); segment != nil { + matchIndex = segment.adjustMatchIndex(matchIndex) + metaTags = append(metaTags, segment.tags()...) + currentLine = segment.currentLine(currentRaw) + } else { + // This item has already been added to a finding + continue + } + } else { + // Fixes: https://github.com/gitleaks/gitleaks/issues/1352 + // removes the incorrectly following line that was detected by regex expression '\n' + matchIndex[1] = matchIndex[0] + len(secret) + } // determine location of match. Note that the location // in the finding will be the line/column numbers of the _match_ @@ -296,345 +417,112 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find } finding := report.Finding{ - Description: rule.Description, - File: fragment.FilePath, - SymlinkFile: fragment.SymlinkFile, - RuleID: rule.RuleID, + RuleID: r.RuleID, + Description: r.Description, StartLine: loc.startLine, EndLine: loc.endLine, StartColumn: loc.startColumn, EndColumn: loc.endColumn, - Secret: secret, - Match: secret, - Tags: rule.Tags, Line: fragment.Raw[loc.startLineIndex:loc.endLineIndex], + Match: secret, + Secret: secret, + File: fragment.FilePath, + SymlinkFile: fragment.SymlinkFile, + Tags: append(r.Tags, metaTags...), } - if strings.Contains(fragment.Raw[loc.startLineIndex:loc.endLineIndex], - gitleaksAllowSignature) { + if !d.IgnoreGitleaksAllow && strings.Contains(finding.Line, gitleaksAllowSignature) { + logger.Trace(). + Str("finding", finding.Secret). + Msg("skipping finding: 'gitleaks:allow' signature") continue } - // extract secret from secret group if set - if rule.SecretGroup != 0 { - groups := rule.Regex.FindStringSubmatch(secret) - if len(groups) <= rule.SecretGroup || len(groups) == 0 { - // Config validation should prevent this - continue + if currentLine == "" { + currentLine = finding.Line + } + + // Set the value of |secret|, if the pattern contains at least one capture group. + // (The first element is the full match, hence we check >= 2.) + groups := r.Regex.FindStringSubmatch(finding.Secret) + if len(groups) >= 2 { + if r.SecretGroup > 0 { + if len(groups) <= r.SecretGroup { + // Config validation should prevent this + continue + } + finding.Secret = groups[r.SecretGroup] + } else { + // If |secretGroup| is not set, we will use the first suitable capture group. + for _, s := range groups[1:] { + if len(s) > 0 { + finding.Secret = s + break + } + } } - secret = groups[rule.SecretGroup] - finding.Secret = secret - } - - // check if the regexTarget is defined in the allowlist "regexes" entry - allowlistTarget := finding.Secret - switch rule.Allowlist.RegexTarget { - case "match": - allowlistTarget = finding.Match - case "line": - allowlistTarget = finding.Line - } - - globalAllowlistTarget := finding.Secret - switch d.Config.Allowlist.RegexTarget { - case "match": - globalAllowlistTarget = finding.Match - case "line": - globalAllowlistTarget = finding.Line - } - if rule.Allowlist.RegexAllowed(allowlistTarget) || - d.Config.Allowlist.RegexAllowed(globalAllowlistTarget) { - continue - } - - // check if the secret is in the list of stopwords - if rule.Allowlist.ContainsStopWord(finding.Secret) || - d.Config.Allowlist.ContainsStopWord(finding.Secret) { - continue } // check entropy entropy := shannonEntropy(finding.Secret) finding.Entropy = float32(entropy) - if rule.Entropy != 0.0 { - if entropy <= rule.Entropy { - // entropy is too low, skip this finding + if r.Entropy != 0.0 { + // entropy is too low, skip this finding + if entropy <= r.Entropy { + logger.Trace(). + Str("finding", finding.Secret). + Float32("entropy", finding.Entropy). + Msg("skipping finding: low entropy") continue } - // NOTE: this is a goofy hack to get around the fact there golang's regex engine - // does not support positive lookaheads. Ideally we would want to add a - // restriction on generic rules regex that requires the secret match group - // contains both numbers and alphabetical characters, not just alphabetical characters. - // What this bit of code does is check if the ruleid is prepended with "generic" and enforces the - // secret contains both digits and alphabetical characters. - // TODO: this should be replaced with stop words - if strings.HasPrefix(rule.RuleID, "generic") { - if !containsDigit(secret) { - continue - } - } } + // check if the result matches any of the global allowlists. + if isAllowed, event := checkFindingAllowed(logger, finding, fragment, currentLine, d.Config.Allowlists); isAllowed { + event.Msg("skipping finding: global allowlist") + continue + } + + // check if the result matches any of the rule allowlists. + if isAllowed, event := checkFindingAllowed(logger, finding, fragment, currentLine, r.Allowlists); isAllowed { + event.Msg("skipping finding: rule allowlist") + continue + } findings = append(findings, finding) } return findings } -// GitScan accepts a *gitdiff.File channel which contents a git history generated from -// the output of `git log -p ...`. startGitScan will look at each file (patch) in the history -// and determine if the patch contains any findings. -func (d *Detector) DetectGit(source string, logOpts string, gitScanType GitScanType) ([]report.Finding, error) { - var ( - gitdiffFiles <-chan *gitdiff.File - err error - ) - switch gitScanType { - case DetectType: - gitdiffFiles, err = git.GitLog(source, logOpts) - if err != nil { - return d.findings, err - } - case ProtectType: - gitdiffFiles, err = git.GitDiff(source, false) - if err != nil { - return d.findings, err - } - case ProtectStagedType: - gitdiffFiles, err = git.GitDiff(source, true) - if err != nil { - return d.findings, err - } - } - - s := semgroup.NewGroup(context.Background(), 4) - - for gitdiffFile := range gitdiffFiles { - gitdiffFile := gitdiffFile - - // skip binary files - if gitdiffFile.IsBinary || gitdiffFile.IsDelete { - continue - } - - // Check if commit is allowed - commitSHA := "" - if gitdiffFile.PatchHeader != nil { - commitSHA = gitdiffFile.PatchHeader.SHA - if d.Config.Allowlist.CommitAllowed(gitdiffFile.PatchHeader.SHA) { - continue - } - } - d.addCommit(commitSHA) - - s.Go(func() error { - for _, textFragment := range gitdiffFile.TextFragments { - if textFragment == nil { - return nil - } - - fragment := Fragment{ - Raw: textFragment.Raw(gitdiff.OpAdd), - CommitSHA: commitSHA, - FilePath: gitdiffFile.NewName, - } - - for _, finding := range d.Detect(fragment) { - d.addFinding(augmentGitFinding(finding, textFragment, gitdiffFile)) - } - } - return nil - }) - } - - if err := s.Wait(); err != nil { - return d.findings, err - } - log.Info().Msgf("%d commits scanned.", len(d.commitMap)) - log.Debug().Msg("Note: this number might be smaller than expected due to commits with no additions") - if git.ErrEncountered { - return d.findings, fmt.Errorf("%s", "git error encountered, see logs") - } - return d.findings, nil -} - -type scanTarget struct { - Path string - Symlink string -} - -// DetectFiles accepts a path to a source directory or file and begins a scan of the -// file or directory. -func (d *Detector) DetectFiles(source string) ([]report.Finding, error) { - s := semgroup.NewGroup(context.Background(), 4) - paths := make(chan scanTarget) - s.Go(func() error { - defer close(paths) - return filepath.Walk(source, - func(path string, fInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fInfo.Name() == ".git" && fInfo.IsDir() { - return filepath.SkipDir - } - if fInfo.Size() == 0 { - return nil - } - if fInfo.Mode().IsRegular() { - paths <- scanTarget{ - Path: path, - Symlink: "", - } - } - if fInfo.Mode().Type() == fs.ModeSymlink && d.FollowSymlinks { - realPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - realPathFileInfo, _ := os.Stat(realPath) - if realPathFileInfo.IsDir() { - log.Debug().Msgf("found symlinked directory: %s -> %s [skipping]", path, realPath) - return nil - } - paths <- scanTarget{ - Path: realPath, - Symlink: path, - } - } - return nil - }) - }) - for pa := range paths { - p := pa - s.Go(func() error { - b, err := os.ReadFile(p.Path) - if err != nil { - return err - } - - mimetype, err := filetype.Match(b) - if err != nil { - return err - } - if mimetype.MIME.Type == "application" { - return nil // skip binary files - } - - fragment := Fragment{ - Raw: string(b), - FilePath: p.Path, - } - if p.Symlink != "" { - fragment.SymlinkFile = p.Symlink - } - for _, finding := range d.Detect(fragment) { - // need to add 1 since line counting starts at 1 - finding.EndLine++ - finding.StartLine++ - d.addFinding(finding) - } - - return nil - }) - } - - if err := s.Wait(); err != nil { - return d.findings, err - } - - return d.findings, nil -} - -// DetectReader accepts an io.Reader and a buffer size for the reader in KB -func (d *Detector) DetectReader(r io.Reader, bufSize int) ([]report.Finding, error) { - reader := bufio.NewReader(r) - buf := make([]byte, 0, 1000*bufSize) - findings := []report.Finding{} - - for { - n, err := reader.Read(buf[:cap(buf)]) - buf = buf[:n] - if err != nil { - if err != io.EOF { - return findings, err - } - break - } - - fragment := Fragment{ - Raw: string(buf), - } - for _, finding := range d.Detect(fragment) { - findings = append(findings, finding) - if d.Verbose { - printFinding(finding, d.NoColor) - } - } - } - - return findings, nil -} - -// Detect scans the given fragment and returns a list of findings -func (d *Detector) Detect(fragment Fragment) []report.Finding { - var findings []report.Finding - - // initiate fragment keywords - fragment.keywords = make(map[string]bool) - - // check if filepath is allowed - if fragment.FilePath != "" && (d.Config.Allowlist.PathAllowed(fragment.FilePath) || - fragment.FilePath == d.Config.Path || (d.baselinePath != "" && fragment.FilePath == d.baselinePath)) { - return findings - } - - // add newline indices for location calculation in detectRule - fragment.newlineIndices = regexp.MustCompile("\n").FindAllStringIndex(fragment.Raw, -1) - - // build keyword map for prefiltering rules - normalizedRaw := strings.ToLower(fragment.Raw) - matches := d.prefilter.FindAll(normalizedRaw) - for _, m := range matches { - fragment.keywords[normalizedRaw[m.Start():m.End()]] = true - } - - for _, rule := range d.Config.Rules { - if len(rule.Keywords) == 0 { - // if not keywords are associated with the rule always scan the - // fragment using the rule - findings = append(findings, d.detectRule(fragment, rule)...) - continue - } - fragmentContainsKeyword := false - // check if keywords are in the fragment - for _, k := range rule.Keywords { - if _, ok := fragment.keywords[strings.ToLower(k)]; ok { - fragmentContainsKeyword = true - } - } - if fragmentContainsKeyword { - findings = append(findings, d.detectRule(fragment, rule)...) - } - } - return filter(findings, d.Redact) -} - -// addFinding synchronously adds a finding to the findings slice -func (d *Detector) addFinding(finding report.Finding) { - if finding.Commit == "" { - finding.Fingerprint = fmt.Sprintf("%s:%s:%d", finding.File, finding.RuleID, finding.StartLine) - } else { +// AddFinding synchronously adds a finding to the findings slice +func (d *Detector) AddFinding(finding report.Finding) { + globalFingerprint := fmt.Sprintf("%s:%s:%d", finding.File, finding.RuleID, finding.StartLine) + if finding.Commit != "" { finding.Fingerprint = fmt.Sprintf("%s:%s:%s:%d", finding.Commit, finding.File, finding.RuleID, finding.StartLine) - } - // check if we should ignore this finding - if _, ok := d.gitleaksIgnore[finding.Fingerprint]; ok { - log.Debug().Msgf("ignoring finding with Fingerprint %s", - finding.Fingerprint) - return + } else { + finding.Fingerprint = globalFingerprint } - if d.baseline != nil && !IsNew(finding, d.baseline) { - log.Debug().Msgf("baseline duplicate -- ignoring finding with Fingerprint %s", finding.Fingerprint) + // check if we should ignore this finding + logger := logging.With().Str("finding", finding.Secret).Logger() + if _, ok := d.gitleaksIgnore[globalFingerprint]; ok { + logger.Debug(). + Str("fingerprint", globalFingerprint). + Msg("skipping finding: global fingerprint") + return + } else if finding.Commit != "" { + // Awkward nested if because I'm not sure how to chain these two conditions. + if _, ok := d.gitleaksIgnore[finding.Fingerprint]; ok { + logger.Debug(). + Str("fingerprint", finding.Fingerprint). + Msgf("skipping finding: fingerprint") + return + } + } + + if d.baseline != nil && !IsNew(finding, d.Redact, d.baseline) { + logger.Debug(). + Str("fingerprint", finding.Fingerprint). + Msgf("skipping finding: baseline") return } @@ -646,7 +534,166 @@ func (d *Detector) addFinding(finding report.Finding) { d.findingMutex.Unlock() } -// addCommit synchronously adds a commit to the commit slice +// Findings returns the findings added to the detector +func (d *Detector) Findings() []report.Finding { + return d.findings +} + +// AddCommit synchronously adds a commit to the commit slice func (d *Detector) addCommit(commit string) { d.commitMap[commit] = true } + +// checkCommitOrPathAllowed evaluates |fragment| against all provided |allowlists|. +// +// If the match condition is "OR", only commit and path are checked. +// Otherwise, if regexes or stopwords are defined this will fail. +func checkCommitOrPathAllowed( + logger zerolog.Logger, + fragment Fragment, + allowlists []*config.Allowlist, +) (bool, *zerolog.Event) { + if fragment.FilePath == "" && fragment.CommitSHA == "" { + return false, nil + } + + for _, a := range allowlists { + var ( + isAllowed bool + allowlistChecks []bool + commitAllowed, _ = a.CommitAllowed(fragment.CommitSHA) + pathAllowed = a.PathAllowed(fragment.FilePath) || (fragment.WindowsFilePath != "" && a.PathAllowed(fragment.WindowsFilePath)) + ) + // If the condition is "AND" we need to check all conditions. + if a.MatchCondition == config.AllowlistMatchAnd { + if len(a.Commits) > 0 { + allowlistChecks = append(allowlistChecks, commitAllowed) + } + if len(a.Paths) > 0 { + allowlistChecks = append(allowlistChecks, pathAllowed) + } + // These will be checked later. + if len(a.Regexes) > 0 { + continue + } + if len(a.StopWords) > 0 { + continue + } + + isAllowed = allTrue(allowlistChecks) + } else { + isAllowed = commitAllowed || pathAllowed + } + if isAllowed { + event := logger.Trace().Str("condition", a.MatchCondition.String()) + if commitAllowed { + event.Bool("allowed-commit", commitAllowed) + } + if pathAllowed { + event.Bool("allowed-path", pathAllowed) + } + return true, event + } + } + return false, nil +} + +// checkFindingAllowed evaluates |finding| against all provided |allowlists|. +// +// If the match condition is "OR", only regex and stopwords are run. (Commit and path should be handled separately). +// Otherwise, all conditions are checked. +// +// TODO: The method signature is awkward. I can't think of a better way to log helpful info. +func checkFindingAllowed( + logger zerolog.Logger, + finding report.Finding, + fragment Fragment, + currentLine string, + allowlists []*config.Allowlist, +) (bool, *zerolog.Event) { + for _, a := range allowlists { + allowlistTarget := finding.Secret + switch a.RegexTarget { + case "match": + allowlistTarget = finding.Match + case "line": + allowlistTarget = currentLine + } + + var ( + checks []bool + isAllowed bool + commitAllowed bool + commit string + pathAllowed bool + regexAllowed = a.RegexAllowed(allowlistTarget) + containsStopword, word = a.ContainsStopWord(finding.Secret) + ) + // If the condition is "AND" we need to check all conditions. + if a.MatchCondition == config.AllowlistMatchAnd { + // Determine applicable checks. + if len(a.Commits) > 0 { + commitAllowed, commit = a.CommitAllowed(fragment.CommitSHA) + checks = append(checks, commitAllowed) + } + if len(a.Paths) > 0 { + pathAllowed = a.PathAllowed(fragment.FilePath) || (fragment.WindowsFilePath != "" && a.PathAllowed(fragment.WindowsFilePath)) + checks = append(checks, pathAllowed) + } + if len(a.Regexes) > 0 { + checks = append(checks, regexAllowed) + } + if len(a.StopWords) > 0 { + checks = append(checks, containsStopword) + } + + isAllowed = allTrue(checks) + } else { + isAllowed = regexAllowed || containsStopword + } + + if isAllowed { + event := logger.Trace(). + Str("finding", finding.Secret). + Str("condition", a.MatchCondition.String()) + if commitAllowed { + event.Str("allowed-commit", commit) + } + if pathAllowed { + event.Bool("allowed-path", pathAllowed) + } + if regexAllowed { + event.Bool("allowed-regex", regexAllowed) + } + if containsStopword { + event.Str("allowed-stopword", word) + } + return true, event + } + } + return false, nil +} + +func allTrue(bools []bool) bool { + for _, check := range bools { + if !check { + return false + } + } + return true +} + +func fileExists(fileName string) bool { + // check for a .infisicalignore file + info, err := os.Stat(fileName) + if err != nil && !os.IsNotExist(err) { + return false + } + + if info != nil && err == nil { + if !info.IsDir() { + return true + } + } + return false +} diff --git a/cli/detect/detect_test.go b/cli/detect/detect_test.go deleted file mode 100644 index 5a0f50828..000000000 --- a/cli/detect/detect_test.go +++ /dev/null @@ -1,754 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - - "github.com/Infisical/infisical-merge/config" - "github.com/Infisical/infisical-merge/report" -) - -const configPath = "../testdata/config/" -const repoBasePath = "../testdata/repos/" - -func TestDetect(t *testing.T) { - tests := []struct { - cfgName string - baselinePath string - fragment Fragment - // NOTE: for expected findings, all line numbers will be 0 - // because line deltas are added _after_ the finding is created. - // I.e, if the finding is from a --no-git file, the line number will be - // increase by 1 in DetectFromFiles(). If the finding is from git, - // the line number will be increased by the patch delta. - expectedFindings []report.Finding - wantError error - }{ - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OKIA\ // infisical-scan:ignore"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \ - - \"AKIALALEMEL33243OKIA\ // infisical-scan:ignore" - - `, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OKIA\" - - // infisical-scan:ignore" - - `, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - Secret: "AKIALALEMEL33243OKIA", - Match: "AKIALALEMEL33243OKIA", - File: "tmp.go", - Line: `awsToken := \"AKIALALEMEL33243OKIA\"`, - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - StartLine: 0, - EndLine: 0, - StartColumn: 15, - EndColumn: 34, - Entropy: 3.1464393, - }, - }, - }, - { - cfgName: "escaped_character_group", - fragment: Fragment{ - Raw: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "PyPI upload token", - Secret: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB", - Match: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB", - Line: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`, - File: "tmp.go", - RuleID: "pypi-upload-token", - Tags: []string{"key", "pypi"}, - StartLine: 0, - EndLine: 0, - StartColumn: 1, - EndColumn: 86, - Entropy: 1.9606875, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - Line: `awsToken := \"AKIALALEMEL33243OLIA\"`, - File: "tmp.go", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - StartLine: 0, - EndLine: 0, - StartColumn: 15, - EndColumn: 34, - Entropy: 3.0841837, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Secret", - Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;", - Secret: "cafebabe:deadbeef", - Line: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`, - File: "tmp.sh", - RuleID: "sidekiq-secret", - Tags: []string{}, - Entropy: 2.6098502, - StartLine: 0, - EndLine: 0, - StartColumn: 8, - EndColumn: 60, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Secret", - Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=\"cafebabe:deadbeef\"", - Secret: "cafebabe:deadbeef", - File: "tmp.sh", - Line: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`, - RuleID: "sidekiq-secret", - Tags: []string{}, - Entropy: 2.6098502, - StartLine: 0, - EndLine: 0, - StartColumn: 21, - EndColumn: 74, - }, - }, - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `url = "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true¶m2=false#heading1"`, - FilePath: "tmp.sh", - }, - expectedFindings: []report.Finding{ - { - Description: "Sidekiq Sensitive URL", - Match: "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:", - Secret: "cafeb4b3:d3adb33f", - File: "tmp.sh", - Line: `url = "http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true¶m2=false#heading1"`, - RuleID: "sidekiq-sensitive-url", - Tags: []string{}, - Entropy: 2.984234, - StartLine: 0, - EndLine: 0, - StartColumn: 8, - EndColumn: 58, - }, - }, - }, - { - cfgName: "allow_aws_re", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_path", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_commit", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - CommitSHA: "allowthiscommit", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "entropy_group", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{ - { - Description: "Discord API key", - Match: "Discord_Public_Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"", - Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5", - Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - File: "tmp.go", - RuleID: "discord-api-key", - Tags: []string{}, - Entropy: 3.7906237, - StartLine: 0, - EndLine: 0, - StartColumn: 7, - EndColumn: 93, - }, - }, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{ - { - Description: "Generic API Key", - Match: "Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"", - Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5", - Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - File: "tmp.py", - RuleID: "generic-api-key", - Tags: []string{}, - Entropy: 3.7906237, - StartLine: 0, - EndLine: 0, - StartColumn: 22, - EndColumn: 93, - }, - }, - }, - { - cfgName: "path_only", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{ - { - Description: "Python Files", - Match: "file detected: tmp.py", - File: "tmp.py", - RuleID: "python-files-only", - Tags: []string{}, - }, - }, - }, - { - cfgName: "bad_entropy_group", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - wantError: fmt.Errorf("Discord API key invalid regex secret group 5, max regex secret group 3"), - }, - { - cfgName: "simple", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: filepath.Join(configPath, "simple.toml"), - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "allow_global_aws_re", - fragment: Fragment{ - Raw: `awsToken := \"AKIALALEMEL33243OLIA\"`, - FilePath: "tmp.go", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "generic_with_py_path", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "load2523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: "tmp.py", - }, - expectedFindings: []report.Finding{}, - }, - { - cfgName: "path_only", - baselinePath: ".baseline.json", - fragment: Fragment{ - Raw: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`, - FilePath: ".baseline.json", - }, - expectedFindings: []report.Finding{}, - }, - } - - for _, tt := range tests { - viper.Reset() - viper.AddConfigPath(configPath) - viper.SetConfigName(tt.cfgName) - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - cfg.Path = filepath.Join(configPath, tt.cfgName+".toml") - if tt.wantError != nil { - if err == nil { - t.Errorf("expected error") - } - assert.Equal(t, tt.wantError, err) - } - d := NewDetector(cfg) - d.baselinePath = tt.baselinePath - - findings := d.Detect(tt.fragment) - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -// TestFromGit tests the FromGit function -func TestFromGit(t *testing.T) { - tests := []struct { - cfgName string - source string - logOpts string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "small"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 19, - EndColumn: 38, - Line: "\n awsToken := \"AKIALALEMEL33243OLIA\"", - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - File: "main.go", - Date: "2021-11-02T23:37:53Z", - Commit: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587", - Author: "Zachary Rice", - Email: "zricer@protonmail.com", - Message: "Accidentally add a secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587:main.go:aws-access-key:20", - }, - { - Description: "AWS Access Key", - StartLine: 9, - EndLine: 9, - StartColumn: 17, - EndColumn: 36, - Secret: "AKIALALEMEL33243OLIA", - Match: "AKIALALEMEL33243OLIA", - Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"", - File: "foo/foo.go", - Date: "2021-11-02T23:48:06Z", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "491504d5a31946ce75e22554cc34203d8e5ff3ca:foo/foo.go:aws-access-key:9", - }, - }, - }, - { - source: filepath.Join(repoBasePath, "small"), - logOpts: "--all foo...", - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 9, - EndLine: 9, - StartColumn: 17, - EndColumn: 36, - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"", - Match: "AKIALALEMEL33243OLIA", - Date: "2021-11-02T23:48:06Z", - File: "foo/foo.go", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "491504d5a31946ce75e22554cc34203d8e5ff3ca:foo/foo.go:aws-access-key:9", - }, - }, - }, - } - - err := moveDotGit("dotGit", ".git") - if err != nil { - t.Fatal(err) - } - defer func() { - if err := moveDotGit(".git", "dotGit"); err != nil { - t.Error(err) - } - }() - - for _, tt := range tests { - - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err = viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if err != nil { - t.Error(err) - } - detector := NewDetector(cfg) - findings, err := detector.DetectGit(tt.source, tt.logOpts, DetectType) - if err != nil { - t.Error(err) - } - - for _, f := range findings { - f.Match = "" // remove lines cause copying and pasting them has some wack formatting - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} -func TestFromGitStaged(t *testing.T) { - tests := []struct { - cfgName string - source string - logOpts string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "staged"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 7, - EndLine: 7, - StartColumn: 18, - EndColumn: 37, - Line: "\n\taws_token2 := \"AKIALALEMEL33243OLIA\" // this one is not", - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - File: "api/api.go", - SymlinkFile: "", - Commit: "", - Entropy: 3.0841837, - Author: "", - Email: "", - Date: "0001-01-01T00:00:00Z", - Message: "", - Tags: []string{ - "key", - "AWS", - }, - RuleID: "aws-access-key", - Fingerprint: "api/api.go:aws-access-key:7", - }, - }, - }, - } - - err := moveDotGit("dotGit", ".git") - if err != nil { - t.Fatal(err) - } - defer func() { - if err := moveDotGit(".git", "dotGit"); err != nil { - t.Error(err) - } - }() - - for _, tt := range tests { - - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err = viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, err := vc.Translate() - if err != nil { - t.Error(err) - } - detector := NewDetector(cfg) - detector.AddGitleaksIgnore(filepath.Join(tt.source, ".gitleaksignore")) - findings, err := detector.DetectGit(tt.source, tt.logOpts, ProtectStagedType) - if err != nil { - t.Error(err) - } - - for _, f := range findings { - f.Match = "" // remove lines cause copying and pasting them has some wack formatting - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -// TestFromFiles tests the FromFiles function -func TestFromFiles(t *testing.T) { - tests := []struct { - cfgName string - source string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "nogit"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 16, - EndColumn: 35, - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"", - File: "../testdata/repos/nogit/main.go", - SymlinkFile: "", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "../testdata/repos/nogit/main.go:aws-access-key:20", - }, - }, - }, - { - source: filepath.Join(repoBasePath, "nogit", "main.go"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "AWS Access Key", - StartLine: 20, - EndLine: 20, - StartColumn: 16, - EndColumn: 35, - Match: "AKIALALEMEL33243OLIA", - Secret: "AKIALALEMEL33243OLIA", - Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"", - File: "../testdata/repos/nogit/main.go", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, - Entropy: 3.0841837, - Fingerprint: "../testdata/repos/nogit/main.go:aws-access-key:20", - }, - }, - }, - } - - for _, tt := range tests { - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, _ := vc.Translate() - detector := NewDetector(cfg) - detector.FollowSymlinks = true - findings, err := detector.DetectFiles(tt.source) - if err != nil { - t.Error(err) - } - - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -func TestDetectWithSymlinks(t *testing.T) { - tests := []struct { - cfgName string - source string - expectedFindings []report.Finding - }{ - { - source: filepath.Join(repoBasePath, "symlinks/file_symlink"), - cfgName: "simple", - expectedFindings: []report.Finding{ - { - Description: "Asymmetric Private Key", - StartLine: 1, - EndLine: 1, - StartColumn: 1, - EndColumn: 35, - Match: "-----BEGIN OPENSSH PRIVATE KEY-----", - Secret: "-----BEGIN OPENSSH PRIVATE KEY-----", - Line: "-----BEGIN OPENSSH PRIVATE KEY-----", - File: "../testdata/repos/symlinks/source_file/id_ed25519", - SymlinkFile: "../testdata/repos/symlinks/file_symlink/symlinked_id_ed25519", - RuleID: "apkey", - Tags: []string{"key", "AsymmetricPrivateKey"}, - Entropy: 3.587164, - Fingerprint: "../testdata/repos/symlinks/source_file/id_ed25519:apkey:1", - }, - }, - }, - } - - for _, tt := range tests { - viper.AddConfigPath(configPath) - viper.SetConfigName("simple") - viper.SetConfigType("toml") - err := viper.ReadInConfig() - if err != nil { - t.Error(err) - } - - var vc config.ViperConfig - err = viper.Unmarshal(&vc) - if err != nil { - t.Error(err) - } - cfg, _ := vc.Translate() - detector := NewDetector(cfg) - detector.FollowSymlinks = true - findings, err := detector.DetectFiles(tt.source) - if err != nil { - t.Error(err) - } - assert.ElementsMatch(t, tt.expectedFindings, findings) - } -} - -func moveDotGit(from, to string) error { - repoDirs, err := os.ReadDir("../testdata/repos") - if err != nil { - return err - } - for _, dir := range repoDirs { - if to == ".git" { - _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), "dotGit")) - if os.IsNotExist(err) { - // dont want to delete the only copy of .git accidentally - continue - } - os.RemoveAll(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), ".git")) - } - if !dir.IsDir() { - continue - } - _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from)) - if os.IsNotExist(err) { - continue - } - - err = os.Rename(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from), - fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), to)) - if err != nil { - return err - } - } - return nil -} diff --git a/cli/detect/directory.go b/cli/detect/directory.go new file mode 100644 index 000000000..56f4999f2 --- /dev/null +++ b/cli/detect/directory.go @@ -0,0 +1,225 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bufio" + "bytes" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/h2non/filetype" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" +) + +const maxPeekSize = 25 * 1_000 // 10kb + +func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Finding, error) { + for pa := range paths { + d.Sema.Go(func() error { + logger := logging.With().Str("path", pa.Path).Logger() + logger.Trace().Msg("Scanning path") + + f, err := os.Open(pa.Path) + if err != nil { + if os.IsPermission(err) { + logger.Warn().Msg("Skipping file: permission denied") + return nil + } + return err + } + defer func() { + _ = f.Close() + }() + + // Get file size + fileInfo, err := f.Stat() + if err != nil { + return err + } + fileSize := fileInfo.Size() + if d.MaxTargetMegaBytes > 0 { + rawLength := fileSize / 1000000 + if rawLength > int64(d.MaxTargetMegaBytes) { + logger.Debug(). + Int64("size", rawLength). + Msg("Skipping file: exceeds --max-target-megabytes") + return nil + } + } + + var ( + // Buffer to hold file chunks + reader = bufio.NewReaderSize(f, chunkSize) + buf = make([]byte, chunkSize) + totalLines = 0 + ) + for { + n, err := reader.Read(buf) + + // "Callers should always process the n > 0 bytes returned before considering the error err." + // https://pkg.go.dev/io#Reader + if n > 0 { + // Only check the filetype at the start of file. + if totalLines == 0 { + // TODO: could other optimizations be introduced here? + if mimetype, err := filetype.Match(buf[:n]); err != nil { + return nil + } else if mimetype.MIME.Type == "application" { + return nil // skip binary files + } + } + + // Try to split chunks across large areas of whitespace, if possible. + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + return readErr + } + + // Count the number of newlines in this chunk + chunk := peekBuf.String() + linesInChunk := strings.Count(chunk, "\n") + totalLines += linesInChunk + fragment := Fragment{ + Raw: chunk, + Bytes: peekBuf.Bytes(), + } + if pa.Symlink != "" { + fragment.SymlinkFile = pa.Symlink + } + + if isWindows { + fragment.FilePath = filepath.ToSlash(pa.Path) + fragment.SymlinkFile = filepath.ToSlash(fragment.SymlinkFile) + fragment.WindowsFilePath = pa.Path + } else { + fragment.FilePath = pa.Path + } + + timer := time.AfterFunc(SlowWarningThreshold, func() { + logger.Debug().Msgf("Taking longer than %s to inspect fragment", SlowWarningThreshold.String()) + }) + for _, finding := range d.Detect(fragment) { + // need to add 1 since line counting starts at 1 + finding.StartLine += (totalLines - linesInChunk) + 1 + finding.EndLine += (totalLines - linesInChunk) + 1 + d.AddFinding(finding) + } + if timer != nil { + timer.Stop() + timer = nil + } + } + + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } + }) + } + + if err := d.Sema.Wait(); err != nil { + return d.findings, err + } + + return d.findings, nil +} + +// readUntilSafeBoundary consumes |f| until it finds two consecutive `\n` characters, up to |maxPeekSize|. +// This hopefully avoids splitting. (https://github.com/gitleaks/gitleaks/issues/1651) +func readUntilSafeBoundary(r *bufio.Reader, n int, maxPeekSize int, peekBuf *bytes.Buffer) error { + if peekBuf.Len() == 0 { + return nil + } + + // Does the buffer end in consecutive newlines? + var ( + data = peekBuf.Bytes() + lastChar = data[len(data)-1] + newlineCount = 0 // Tracks consecutive newlines + ) + if isWhitespace(lastChar) { + for i := len(data) - 1; i >= 0; i-- { + lastChar = data[i] + if lastChar == '\n' { + newlineCount++ + + // Stop if two consecutive newlines are found + if newlineCount >= 2 { + return nil + } + } else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' { + // The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count. + // (Intentionally do nothing.) + } else { + break + } + } + } + + // If not, read ahead until we (hopefully) find some. + newlineCount = 0 + for { + data = peekBuf.Bytes() + // Check if the last character is a newline. + lastChar = data[len(data)-1] + if lastChar == '\n' { + newlineCount++ + + // Stop if two consecutive newlines are found + if newlineCount >= 2 { + break + } + } else if lastChar == '\r' || lastChar == ' ' || lastChar == '\t' { + // The presence of other whitespace characters (`\r`, ` `, `\t`) shouldn't reset the count. + // (Intentionally do nothing.) + } else { + newlineCount = 0 // Reset if a non-newline character is found + } + + // Stop growing the buffer if it reaches maxSize + if (peekBuf.Len() - n) >= maxPeekSize { + break + } + + // Read additional data into a temporary buffer + b, err := r.ReadByte() + if err != nil { + if err == io.EOF { + break + } + return err + } + peekBuf.WriteByte(b) + } + return nil +} diff --git a/cli/detect/git.go b/cli/detect/git.go new file mode 100644 index 000000000..ddde0757d --- /dev/null +++ b/cli/detect/git.go @@ -0,0 +1,214 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bytes" + "errors" + "fmt" + "net/url" + "os/exec" + "regexp" + "strings" + "time" + + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/gitleaks/go-gitdiff/gitdiff" + + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" +) + +func (d *Detector) DetectGit(cmd *sources.GitCmd, remote *RemoteInfo) ([]report.Finding, error) { + defer cmd.Wait() + var ( + diffFilesCh = cmd.DiffFilesCh() + errCh = cmd.ErrCh() + ) + + // loop to range over both DiffFiles (stdout) and ErrCh (stderr) + for diffFilesCh != nil || errCh != nil { + select { + case gitdiffFile, open := <-diffFilesCh: + if !open { + diffFilesCh = nil + break + } + + // skip binary files + if gitdiffFile.IsBinary || gitdiffFile.IsDelete { + continue + } + + // Check if commit is allowed + commitSHA := "" + if gitdiffFile.PatchHeader != nil { + commitSHA = gitdiffFile.PatchHeader.SHA + for _, a := range d.Config.Allowlists { + if ok, c := a.CommitAllowed(gitdiffFile.PatchHeader.SHA); ok { + logging.Trace().Str("allowed-commit", c).Msg("skipping commit: global allowlist") + continue + } + } + } + d.addCommit(commitSHA) + + d.Sema.Go(func() error { + for _, textFragment := range gitdiffFile.TextFragments { + if textFragment == nil { + return nil + } + + fragment := Fragment{ + Raw: textFragment.Raw(gitdiff.OpAdd), + CommitSHA: commitSHA, + FilePath: gitdiffFile.NewName, + } + + timer := time.AfterFunc(SlowWarningThreshold, func() { + logging.Debug(). + Str("commit", commitSHA[:7]). + Str("path", fragment.FilePath). + Msgf("Taking longer than %s to inspect fragment", SlowWarningThreshold.String()) + }) + for _, finding := range d.Detect(fragment) { + d.AddFinding(augmentGitFinding(remote, finding, textFragment, gitdiffFile)) + } + if timer != nil { + timer.Stop() + timer = nil + } + } + return nil + }) + case err, open := <-errCh: + if !open { + errCh = nil + break + } + + return d.findings, err + } + } + + if err := d.Sema.Wait(); err != nil { + return d.findings, err + } + logging.Info().Msgf("%d commits scanned.", len(d.commitMap)) + logging.Debug().Msg("Note: this number might be smaller than expected due to commits with no additions") + return d.findings, nil +} + +type RemoteInfo struct { + Platform scm.Platform + Url string +} + +func NewRemoteInfo(platform scm.Platform, source string) *RemoteInfo { + if platform == scm.NoPlatform { + return &RemoteInfo{Platform: platform} + } + + remoteUrl, err := getRemoteUrl(source) + if err != nil { + if strings.Contains(err.Error(), "No remote configured") { + logging.Debug().Msg("skipping finding links: repository has no configured remote.") + platform = scm.NoPlatform + } else { + logging.Error().Err(err).Msg("skipping finding links: unable to parse remote URL") + } + goto End + } + + if platform == scm.UnknownPlatform { + platform = platformFromHost(remoteUrl) + if platform == scm.UnknownPlatform { + logging.Info(). + Str("host", remoteUrl.Hostname()). + Msg("Unknown SCM platform. Use --platform to include links in findings.") + } else { + logging.Debug(). + Str("host", remoteUrl.Hostname()). + Str("platform", platform.String()). + Msg("SCM platform parsed from host") + } + } + +End: + var rUrl string + if remoteUrl != nil { + rUrl = remoteUrl.String() + } + return &RemoteInfo{ + Platform: platform, + Url: rUrl, + } +} + +var sshUrlpat = regexp.MustCompile(`^git@([a-zA-Z0-9.-]+):([\w/.-]+?)(?:\.git)?$`) + +func getRemoteUrl(source string) (*url.URL, error) { + // This will return the first remote — typically, "origin". + cmd := exec.Command("git", "ls-remote", "--quiet", "--get-url") + if source != "." { + cmd.Dir = source + } + + stdout, err := cmd.Output() + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return nil, fmt.Errorf("command failed (%d): %w, stderr: %s", exitError.ExitCode(), err, string(bytes.TrimSpace(exitError.Stderr))) + } + return nil, err + } + + remoteUrl := string(bytes.TrimSpace(stdout)) + if matches := sshUrlpat.FindStringSubmatch(remoteUrl); matches != nil { + remoteUrl = fmt.Sprintf("https://%s/%s", matches[1], matches[2]) + } + remoteUrl = strings.TrimSuffix(remoteUrl, ".git") + + parsedUrl, err := url.Parse(remoteUrl) + if err != nil { + return nil, fmt.Errorf("unable to parse remote URL: %w", err) + } + + // Remove any user info. + parsedUrl.User = nil + return parsedUrl, nil +} + +func platformFromHost(u *url.URL) scm.Platform { + switch strings.ToLower(u.Hostname()) { + case "github.com": + return scm.GitHubPlatform + case "gitlab.com": + return scm.GitLabPlatform + case "dev.azure.com", "visualstudio.com": + return scm.AzureDevOpsPlatform + default: + return scm.UnknownPlatform + } +} diff --git a/cli/detect/git/git.go b/cli/detect/git/git.go deleted file mode 100644 index 6eb5e9d6d..000000000 --- a/cli/detect/git/git.go +++ /dev/null @@ -1,143 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package git - -import ( - "bufio" - "io" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/gitleaks/go-gitdiff/gitdiff" - "github.com/rs/zerolog/log" -) - -var ErrEncountered bool - -// GitLog returns a channel of gitdiff.File objects from the -// git log -p command for the given source. -func GitLog(source string, logOpts string) (<-chan *gitdiff.File, error) { - sourceClean := filepath.Clean(source) - var cmd *exec.Cmd - if logOpts != "" { - args := []string{"-C", sourceClean, "log", "-p", "-U0"} - args = append(args, strings.Split(logOpts, " ")...) - cmd = exec.Command("git", args...) - } else { - cmd = exec.Command("git", "-C", sourceClean, "log", "-p", "-U0", - "--full-history", "--all") - } - - log.Debug().Msgf("executing: %s", cmd.String()) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - go listenForStdErr(stderr) - - if err := cmd.Start(); err != nil { - return nil, err - } - // HACK: to avoid https://github.com/zricethezav/gitleaks/issues/722 - time.Sleep(50 * time.Millisecond) - - return gitdiff.Parse(cmd, stdout) -} - -// GitDiff returns a channel of gitdiff.File objects from -// the git diff command for the given source. -func GitDiff(source string, staged bool) (<-chan *gitdiff.File, error) { - sourceClean := filepath.Clean(source) - var cmd *exec.Cmd - cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", ".") - if staged { - cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", - "--staged", ".") - } - log.Debug().Msgf("executing: %s", cmd.String()) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - go listenForStdErr(stderr) - - if err := cmd.Start(); err != nil { - return nil, err - } - // HACK: to avoid https://github.com/zricethezav/gitleaks/issues/722 - time.Sleep(50 * time.Millisecond) - - return gitdiff.Parse(cmd, stdout) -} - -// listenForStdErr listens for stderr output from git and prints it to stdout -// then exits with exit code 1 -func listenForStdErr(stderr io.ReadCloser) { - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - // if git throws one of the following errors: - // - // exhaustive rename detection was skipped due to too many files. - // you may want to set your diff.renameLimit variable to at least - // (some large number) and retry the command. - // - // inexact rename detection was skipped due to too many files. - // you may want to set your diff.renameLimit variable to at least - // (some large number) and retry the command. - // - // we skip exiting the program as git log -p/git diff will continue - // to send data to stdout and finish executing. This next bit of - // code prevents gitleaks from stopping mid scan if this error is - // encountered - if strings.Contains(scanner.Text(), - "exhaustive rename detection was skipped") || - strings.Contains(scanner.Text(), - "inexact rename detection was skipped") || - strings.Contains(scanner.Text(), - "you may want to set your diff.renameLimit") { - log.Warn().Msg(scanner.Text()) - } else { - log.Error().Msgf("[git] %s", scanner.Text()) - - // asynchronously set this error flag to true so that we can - // capture a log message and exit with a non-zero exit code - // This value should get set before the `git` command exits so it's - // safe-ish, although I know I know, bad practice. - ErrEncountered = true - } - } -} diff --git a/cli/detect/git/git_test.go b/cli/detect/git/git_test.go deleted file mode 100644 index 3a2ea9c35..000000000 --- a/cli/detect/git/git_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package git_test - -// TODO: commenting out this test for now because it's flaky. Alternatives to consider to get this working: -// -- use `git stash` instead of `restore()` - -// const repoBasePath = "../../testdata/repos/" - -// const expectPath = "../../testdata/expected/" - -// func TestGitLog(t *testing.T) { -// tests := []struct { -// source string -// logOpts string -// expected string -// }{ -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: filepath.Join(expectPath, "git", "small.txt"), -// }, -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: filepath.Join(expectPath, "git", "small-branch-foo.txt"), -// logOpts: "--all foo...", -// }, -// } - -// err := moveDotGit("dotGit", ".git") -// if err != nil { -// t.Fatal(err) -// } -// defer func() { -// if err = moveDotGit(".git", "dotGit"); err != nil { -// t.Fatal(err) -// } -// }() - -// for _, tt := range tests { -// files, err := git.GitLog(tt.source, tt.logOpts) -// if err != nil { -// t.Error(err) -// } - -// var diffSb strings.Builder -// for f := range files { -// for _, tf := range f.TextFragments { -// diffSb.WriteString(tf.Raw(gitdiff.OpAdd)) -// } -// } - -// expectedBytes, err := os.ReadFile(tt.expected) -// if err != nil { -// t.Error(err) -// } -// expected := string(expectedBytes) -// if expected != diffSb.String() { -// // write string builder to .got file using os.Create -// err = os.WriteFile(strings.Replace(tt.expected, ".txt", ".got.txt", 1), []byte(diffSb.String()), 0644) -// if err != nil { -// t.Error(err) -// } -// t.Error("expected: ", expected, "got: ", diffSb.String()) -// } -// } -// } - -// func TestGitDiff(t *testing.T) { -// tests := []struct { -// source string -// expected string -// additions string -// target string -// }{ -// { -// source: filepath.Join(repoBasePath, "small"), -// expected: "this line is added\nand another one", -// additions: "this line is added\nand another one", -// target: filepath.Join(repoBasePath, "small", "main.go"), -// }, -// } - -// err := moveDotGit("dotGit", ".git") -// if err != nil { -// t.Fatal(err) -// } -// defer func() { -// if err = moveDotGit(".git", "dotGit"); err != nil { -// t.Fatal(err) -// } -// }() - -// for _, tt := range tests { -// noChanges, err := os.ReadFile(tt.target) -// if err != nil { -// t.Error(err) -// } -// err = os.WriteFile(tt.target, []byte(tt.additions), 0644) -// if err != nil { -// restore(tt.target, noChanges, t) -// t.Error(err) -// } - -// files, err := git.GitDiff(tt.source, false) -// if err != nil { -// restore(tt.target, noChanges, t) -// t.Error(err) -// } - -// for f := range files { -// sb := strings.Builder{} -// for _, tf := range f.TextFragments { -// sb.WriteString(tf.Raw(gitdiff.OpAdd)) -// } -// if sb.String() != tt.expected { -// restore(tt.target, noChanges, t) -// t.Error("expected: ", tt.expected, "got: ", sb.String()) -// } -// } -// restore(tt.target, noChanges, t) -// } -// } - -// func restore(path string, data []byte, t *testing.T) { -// err := os.WriteFile(path, data, 0644) -// if err != nil { -// t.Fatal(err) -// } -// } - -// func moveDotGit(from, to string) error { -// repoDirs, err := os.ReadDir("../../testdata/repos") -// if err != nil { -// return err -// } -// for _, dir := range repoDirs { -// if to == ".git" { -// _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), "dotGit")) -// if os.IsNotExist(err) { -// // dont want to delete the only copy of .git accidentally -// continue -// } -// os.RemoveAll(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), ".git")) -// } -// if !dir.IsDir() { -// continue -// } -// _, err := os.Stat(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from)) -// if os.IsNotExist(err) { -// continue -// } - -// err = os.Rename(fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), from), -// fmt.Sprintf("%s/%s/%s", repoBasePath, dir.Name(), to)) -// if err != nil { -// return err -// } -// } -// return nil -// } diff --git a/cli/detect/location.go b/cli/detect/location.go index 418af83f6..81419511c 100644 --- a/cli/detect/location.go +++ b/cli/detect/location.go @@ -72,6 +72,7 @@ func location(fragment Fragment, matchIndex []int) Location { location.endColumn = (end - prevNewLine) location.endLineIndex = newLineByteIndex } + prevNewLine = pair[0] } diff --git a/cli/detect/location_test.go b/cli/detect/location_test.go deleted file mode 100644 index f76a6f814..000000000 --- a/cli/detect/location_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package detect - -import ( - "testing" -) - -// TestGetLocation tests the getLocation function. -func TestGetLocation(t *testing.T) { - tests := []struct { - linePairs [][]int - start int - end int - wantLocation Location - }{ - { - linePairs: [][]int{ - {0, 39}, - {40, 55}, - {56, 57}, - }, - start: 35, - end: 38, - wantLocation: Location{ - startLine: 1, - startColumn: 36, - endLine: 1, - endColumn: 38, - startLineIndex: 0, - endLineIndex: 40, - }, - }, - { - linePairs: [][]int{ - {0, 39}, - {40, 55}, - {56, 57}, - }, - start: 40, - end: 44, - wantLocation: Location{ - startLine: 2, - startColumn: 1, - endLine: 2, - endColumn: 4, - startLineIndex: 40, - endLineIndex: 56, - }, - }, - } - - for _, test := range tests { - loc := location(Fragment{newlineIndices: test.linePairs}, []int{test.start, test.end}) - if loc != test.wantLocation { - t.Errorf("\nstartLine %d\nstartColumn: %d\nendLine: %d\nendColumn: %d\nstartLineIndex: %d\nendlineIndex %d", - loc.startLine, loc.startColumn, loc.endLine, loc.endColumn, loc.startLineIndex, loc.endLineIndex) - - t.Error("got", loc, "want", test.wantLocation) - } - } -} diff --git a/cli/detect/logging/log.go b/cli/detect/logging/log.go new file mode 100644 index 000000000..efac01725 --- /dev/null +++ b/cli/detect/logging/log.go @@ -0,0 +1,72 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package logging + +import ( + "os" + + "github.com/rs/zerolog" +) + +var Logger zerolog.Logger + +func init() { + // send all logs to stdout + Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). + Level(zerolog.InfoLevel). + With().Timestamp().Logger() +} + +func With() zerolog.Context { + return Logger.With() +} + +func Trace() *zerolog.Event { + return Logger.Trace() +} + +func Debug() *zerolog.Event { + return Logger.Debug() +} +func Info() *zerolog.Event { + return Logger.Info() +} +func Warn() *zerolog.Event { + return Logger.Warn() +} + +func Error() *zerolog.Event { + return Logger.Error() +} + +func Err(err error) *zerolog.Event { + return Logger.Err(err) +} + +func Fatal() *zerolog.Event { + return Logger.Fatal() +} + +func Panic() *zerolog.Event { + return Logger.Panic() +} diff --git a/cli/detect/reader.go b/cli/detect/reader.go new file mode 100644 index 000000000..d3559b68a --- /dev/null +++ b/cli/detect/reader.go @@ -0,0 +1,149 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package detect + +import ( + "bufio" + "bytes" + "errors" + "io" + + "github.com/Infisical/infisical-merge/detect/report" +) + +// DetectReader accepts an io.Reader and a buffer size for the reader in KB +func (d *Detector) DetectReader(r io.Reader, bufSize int) ([]report.Finding, error) { + reader := bufio.NewReader(r) + buf := make([]byte, 1000*bufSize) + findings := []report.Finding{} + + for { + n, err := reader.Read(buf) + + // "Callers should always process the n > 0 bytes returned before considering the error err." + // https://pkg.go.dev/io#Reader + if n > 0 { + // Try to split chunks across large areas of whitespace, if possible. + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + return findings, readErr + } + + fragment := Fragment{ + Raw: peekBuf.String(), + } + for _, finding := range d.Detect(fragment) { + findings = append(findings, finding) + if d.Verbose { + printFinding(finding, d.NoColor) + } + } + } + + if err != nil { + if err == io.EOF { + break + } + return findings, err + } + } + + return findings, nil +} + +// StreamDetectReader streams the detection results from the provided io.Reader. +// It reads data using the specified buffer size (in KB) and processes each chunk through +// the existing detection logic. Findings are sent down the returned findings channel as soon as +// they are detected, while a separate error channel signals a terminal error (or nil upon successful completion). +// The function returns two channels: +// - findingsCh: a receive-only channel that emits report.Finding objects as they are found. +// - errCh: a receive-only channel that emits a single final error (or nil if no error occurred) +// once the stream ends. +// +// Recommended Usage: +// +// Since there will only ever be a single value on the errCh, it is recommended to consume the findingsCh +// first. Once findingsCh is closed, the consumer should then read from errCh to determine +// if the stream completed successfully or if an error occurred. +// +// This design avoids the need for a select loop, keeping client code simple. +// +// Example: +// +// // Assume detector is an instance of *Detector and myReader implements io.Reader. +// findingsCh, errCh := detector.StreamDetectReader(myReader, 64) // using 64 KB buffer size +// +// // Process findings as they arrive. +// for finding := range findingsCh { +// fmt.Printf("Found secret: %+v\n", finding) +// } +// +// // After the findings channel is closed, check the final error. +// if err := <-errCh; err != nil { +// log.Fatalf("StreamDetectReader encountered an error: %v", err) +// } else { +// fmt.Println("Scanning completed successfully.") +// } +func (d *Detector) StreamDetectReader(r io.Reader, bufSize int) (<-chan report.Finding, <-chan error) { + findingsCh := make(chan report.Finding, 1) + errCh := make(chan error, 1) + + go func() { + defer close(findingsCh) + defer close(errCh) + + reader := bufio.NewReader(r) + buf := make([]byte, 1000*bufSize) + + for { + n, err := reader.Read(buf) + + if n > 0 { + peekBuf := bytes.NewBuffer(buf[:n]) + if readErr := readUntilSafeBoundary(reader, n, maxPeekSize, peekBuf); readErr != nil { + errCh <- readErr + return + } + + fragment := Fragment{Raw: peekBuf.String()} + for _, finding := range d.Detect(fragment) { + findingsCh <- finding + if d.Verbose { + printFinding(finding, d.NoColor) + } + } + } + + if err != nil { + if errors.Is(err, io.EOF) { + errCh <- nil + return + } + errCh <- err + return + } + } + }() + + return findingsCh, errCh +} diff --git a/cli/detect/regexp/stdlib_regex.go b/cli/detect/regexp/stdlib_regex.go new file mode 100644 index 000000000..81e2089b7 --- /dev/null +++ b/cli/detect/regexp/stdlib_regex.go @@ -0,0 +1,37 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +//go:build !gore2regex + +package regexp + +import ( + re "regexp" +) + +const Version = "stdlib" + +type Regexp = re.Regexp + +func MustCompile(str string) *re.Regexp { + return re.MustCompile(str) +} diff --git a/cli/detect/regexp/wasilibs_regex.go b/cli/detect/regexp/wasilibs_regex.go new file mode 100644 index 000000000..bc64fb14b --- /dev/null +++ b/cli/detect/regexp/wasilibs_regex.go @@ -0,0 +1,37 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +//go:build gore2regex + +package regexp + +import ( + re "github.com/wasilibs/go-re2" +) + +const Version = "github.com/wasilibs/go-re2" + +type Regexp = re.Regexp + +func MustCompile(str string) *re.Regexp { + return re.MustCompile(str) +} diff --git a/cli/report/constants.go b/cli/detect/report/constants.go similarity index 99% rename from cli/report/constants.go rename to cli/detect/report/constants.go index 8bad495cb..c4f06a9a3 100644 --- a/cli/report/constants.go +++ b/cli/detect/report/constants.go @@ -19,6 +19,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. + package report const version = "v8.0.0" diff --git a/cli/report/csv.go b/cli/detect/report/csv.go similarity index 72% rename from cli/report/csv.go rename to cli/detect/report/csv.go index 0a30c9fd5..1f8812f97 100644 --- a/cli/report/csv.go +++ b/cli/detect/report/csv.go @@ -26,16 +26,24 @@ import ( "encoding/csv" "io" "strconv" + "strings" ) -// writeCsv writes the list of findings to a writeCloser. -func writeCsv(f []Finding, w io.WriteCloser) error { - if len(f) == 0 { +type CsvReporter struct { +} + +var _ Reporter = (*CsvReporter)(nil) + +func (r *CsvReporter) Write(w io.WriteCloser, findings []Finding) error { + if len(findings) == 0 { return nil } - defer w.Close() - cw := csv.NewWriter(w) - err := cw.Write([]string{"RuleID", + + var ( + cw = csv.NewWriter(w) + err error + ) + columns := []string{"RuleID", "Commit", "File", "SymlinkFile", @@ -50,12 +58,18 @@ func writeCsv(f []Finding, w io.WriteCloser) error { "Date", "Email", "Fingerprint", - }) - if err != nil { + "Tags", + } + // A miserable attempt at "omitempty" so tests don't yell at me. + if findings[0].Link != "" { + columns = append(columns, "Link") + } + + if err = cw.Write(columns); err != nil { return err } - for _, f := range f { - err = cw.Write([]string{f.RuleID, + for _, f := range findings { + row := []string{f.RuleID, f.Commit, f.File, f.SymlinkFile, @@ -70,8 +84,13 @@ func writeCsv(f []Finding, w io.WriteCloser) error { f.Date, f.Email, f.Fingerprint, - }) - if err != nil { + strings.Join(f.Tags, " "), + } + if findings[0].Link != "" { + row = append(row, f.Link) + } + + if err = cw.Write(row); err != nil { return err } } diff --git a/cli/report/finding.go b/cli/detect/report/finding.go similarity index 75% rename from cli/report/finding.go rename to cli/detect/report/finding.go index be461072b..c53f16ee7 100644 --- a/cli/report/finding.go +++ b/cli/detect/report/finding.go @@ -23,13 +23,17 @@ package report import ( + "math" "strings" ) // Finding contains information about strings that // have been captured by a tree-sitter query. type Finding struct { + // Rule is the name of the rule that was matched + RuleID string Description string + StartLine int EndLine int StartColumn int @@ -47,6 +51,7 @@ type Finding struct { File string SymlinkFile string Commit string + Link string `json:",omitempty"` // Entropy is the shannon entropy of Value Entropy float32 @@ -57,16 +62,31 @@ type Finding struct { Message string Tags []string - // Rule is the name of the rule that was matched - RuleID string - - // unique identifer + // unique identifier Fingerprint string } // Redact removes sensitive information from a finding. -func (f *Finding) Redact() { - f.Line = strings.Replace(f.Line, f.Secret, "REDACTED", -1) - f.Match = strings.Replace(f.Match, f.Secret, "REDACTED", -1) - f.Secret = "REDACTED" +func (f *Finding) Redact(percent uint) { + secret := maskSecret(f.Secret, percent) + if percent >= 100 { + secret = "REDACTED" + } + f.Line = strings.Replace(f.Line, f.Secret, secret, -1) + f.Match = strings.Replace(f.Match, f.Secret, secret, -1) + f.Secret = secret +} + +func maskSecret(secret string, percent uint) string { + if percent > 100 { + percent = 100 + } + len := float64(len(secret)) + if len <= 0 { + return secret + } + prc := float64(100 - percent) + lth := int64(math.RoundToEven(len * prc / float64(100))) + + return secret[:lth] + "..." } diff --git a/cli/report/json.go b/cli/detect/report/json.go similarity index 89% rename from cli/report/json.go rename to cli/detect/report/json.go index d091ac3c5..f47b7eee0 100644 --- a/cli/report/json.go +++ b/cli/detect/report/json.go @@ -27,10 +27,12 @@ import ( "io" ) -func writeJson(findings []Finding, w io.WriteCloser) error { - if len(findings) == 0 { - findings = []Finding{} - } +type JsonReporter struct { +} + +var _ Reporter = (*JsonReporter)(nil) + +func (t *JsonReporter) Write(w io.WriteCloser, findings []Finding) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") return encoder.Encode(findings) diff --git a/cli/detect/report/junit.go b/cli/detect/report/junit.go new file mode 100644 index 000000000..0862a45f1 --- /dev/null +++ b/cli/detect/report/junit.go @@ -0,0 +1,129 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package report + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "strconv" +) + +type JunitReporter struct { +} + +var _ Reporter = (*JunitReporter)(nil) + +func (r *JunitReporter) Write(w io.WriteCloser, findings []Finding) error { + testSuites := TestSuites{ + TestSuites: getTestSuites(findings), + } + + io.WriteString(w, xml.Header) + encoder := xml.NewEncoder(w) + encoder.Indent("", "\t") + return encoder.Encode(testSuites) +} + +func getTestSuites(findings []Finding) []TestSuite { + return []TestSuite{ + { + Failures: strconv.Itoa(len(findings)), + Name: "gitleaks", + Tests: strconv.Itoa(len(findings)), + TestCases: getTestCases(findings), + Time: "", + }, + } +} + +func getTestCases(findings []Finding) []TestCase { + testCases := []TestCase{} + for _, f := range findings { + testCase := TestCase{ + Classname: f.Description, + Failure: getFailure(f), + File: f.File, + Name: getMessage(f), + Time: "", + } + testCases = append(testCases, testCase) + } + return testCases +} + +func getFailure(f Finding) Failure { + return Failure{ + Data: getData(f), + Message: getMessage(f), + Type: f.Description, + } +} + +func getData(f Finding) string { + data, err := json.MarshalIndent(f, "", "\t") + if err != nil { + fmt.Println(err) + return "" + } + return string(data) +} + +func getMessage(f Finding) string { + if f.Commit == "" { + return fmt.Sprintf("%s has detected a secret in file %s, line %s.", f.RuleID, f.File, strconv.Itoa(f.StartLine)) + } + + return fmt.Sprintf("%s has detected a secret in file %s, line %s, at commit %s.", f.RuleID, f.File, strconv.Itoa(f.StartLine), f.Commit) +} + +type TestSuites struct { + XMLName xml.Name `xml:"testsuites"` + TestSuites []TestSuite +} + +type TestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Failures string `xml:"failures,attr"` + Name string `xml:"name,attr"` + Tests string `xml:"tests,attr"` + TestCases []TestCase `xml:"testcase"` + Time string `xml:"time,attr"` +} + +type TestCase struct { + XMLName xml.Name `xml:"testcase"` + Classname string `xml:"classname,attr"` + Failure Failure `xml:"failure"` + File string `xml:"file,attr"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` +} + +type Failure struct { + XMLName xml.Name `xml:"failure"` + Data string `xml:",chardata"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` +} diff --git a/cli/report/finding_test.go b/cli/detect/report/report.go similarity index 73% rename from cli/report/finding_test.go rename to cli/detect/report/report.go index cdb74a329..120841bb8 100644 --- a/cli/report/finding_test.go +++ b/cli/detect/report/report.go @@ -19,30 +19,20 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. + package report -import "testing" +import ( + "io" +) -func TestRedact(t *testing.T) { - tests := []struct { - findings []Finding - redact bool - }{ - { - redact: true, - findings: []Finding{ - { - Secret: "line containing secret", - Match: "secret", - }, - }}, - } - for _, test := range tests { - for _, f := range test.findings { - f.Redact() - if f.Secret != "REDACTED" { - t.Error("redact not redacting: ", f.Secret) - } - } - } +const ( + // https://cwe.mitre.org/data/definitions/798.html + CWE = "CWE-798" + CWE_DESCRIPTION = "Use of Hard-coded Credentials" + StdoutReportPath = "-" +) + +type Reporter interface { + Write(w io.WriteCloser, findings []Finding) error } diff --git a/cli/report/sarif.go b/cli/detect/report/sarif.go similarity index 84% rename from cli/report/sarif.go rename to cli/detect/report/sarif.go index e5120887a..f7457eb57 100644 --- a/cli/report/sarif.go +++ b/cli/detect/report/sarif.go @@ -27,14 +27,20 @@ import ( "fmt" "io" - "github.com/Infisical/infisical-merge/config" + "github.com/Infisical/infisical-merge/detect/config" ) -func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error { +type SarifReporter struct { + OrderedRules []config.Rule +} + +var _ Reporter = (*SarifReporter)(nil) + +func (r *SarifReporter) Write(w io.WriteCloser, findings []Finding) error { sarif := Sarif{ Schema: "https://json.schemastore.org/sarif-2.1.0.json", Version: "2.1.0", - Runs: getRuns(cfg, findings), + Runs: r.getRuns(findings), } encoder := json.NewEncoder(w) @@ -42,22 +48,22 @@ func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error { return encoder.Encode(sarif) } -func getRuns(cfg config.Config, findings []Finding) []Runs { +func (r *SarifReporter) getRuns(findings []Finding) []Runs { return []Runs{ { - Tool: getTool(cfg), + Tool: r.getTool(), Results: getResults(findings), }, } } -func getTool(cfg config.Config) Tool { +func (r *SarifReporter) getTool() Tool { tool := Tool{ Driver: Driver{ Name: driver, SemanticVersion: version, - InformationUri: "https://github.com/Infisical/infisical", - Rules: getRules(cfg), + InformationUri: "https://github.com/gitleaks/gitleaks", + Rules: r.getRules(), }, } @@ -73,26 +79,15 @@ func hasEmptyRules(tool Tool) bool { return len(tool.Driver.Rules) == 0 } -func getRules(cfg config.Config) []Rules { +func (r *SarifReporter) getRules() []Rules { // TODO for _, rule := range cfg.Rules { var rules []Rules - for _, rule := range cfg.OrderedRules() { - shortDescription := ShortDescription{ - Text: rule.Description, - } - if rule.Regex != nil { - shortDescription = ShortDescription{ - Text: rule.Regex.String(), - } - } else if rule.Path != nil { - shortDescription = ShortDescription{ - Text: rule.Path.String(), - } - } + for _, rule := range r.OrderedRules { rules = append(rules, Rules{ - ID: rule.RuleID, - Name: rule.Description, - Description: shortDescription, + ID: rule.RuleID, + Description: ShortDescription{ + Text: rule.Description, + }, }) } return rules @@ -125,6 +120,9 @@ func getResults(findings []Finding) []Results { Date: f.Date, Author: f.Author, }, + Properties: Properties{ + Tags: f.Tags, + }, } results = append(results, r) } @@ -180,7 +178,6 @@ type FullDescription struct { type Rules struct { ID string `json:"id"` - Name string `json:"name"` Description ShortDescription `json:"shortDescription"` } @@ -224,11 +221,16 @@ type Locations struct { PhysicalLocation PhysicalLocation `json:"physicalLocation"` } +type Properties struct { + Tags []string `json:"tags"` +} + type Results struct { Message Message `json:"message"` RuleId string `json:"ruleId"` Locations []Locations `json:"locations"` PartialFingerPrints `json:"partialFingerprints"` + Properties Properties `json:"properties"` } type Runs struct { diff --git a/cli/detect/report/template.go b/cli/detect/report/template.go new file mode 100644 index 000000000..094aaaea9 --- /dev/null +++ b/cli/detect/report/template.go @@ -0,0 +1,68 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package report + +import ( + "fmt" + "io" + "os" + "text/template" + + "github.com/Masterminds/sprig/v3" +) + +type TemplateReporter struct { + template *template.Template +} + +var _ Reporter = (*TemplateReporter)(nil) + +func NewTemplateReporter(templatePath string) (*TemplateReporter, error) { + if templatePath == "" { + return nil, fmt.Errorf("template path cannot be empty") + } + + file, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + templateText := string(file) + + // TODO: Add helper functions like escaping for JSON, XML, etc. + t := template.New("custom") + t = t.Funcs(sprig.TxtFuncMap()) + t, err = t.Parse(templateText) + if err != nil { + return nil, fmt.Errorf("error parsing file: %w", err) + } + return &TemplateReporter{template: t}, nil +} + +// writeTemplate renders the findings using the user-provided template. +// https://www.digitalocean.com/community/tutorials/how-to-use-templates-in-go +func (t *TemplateReporter) Write(w io.WriteCloser, findings []Finding) error { + if err := t.template.Execute(w, findings); err != nil { + return err + } + return nil +} diff --git a/cli/detect/sources/directory.go b/cli/detect/sources/directory.go new file mode 100644 index 000000000..0ad46c3d8 --- /dev/null +++ b/cli/detect/sources/directory.go @@ -0,0 +1,127 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package sources + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + + "github.com/fatih/semgroup" + + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" +) + +type ScanTarget struct { + Path string + Symlink string +} + +var isWindows = runtime.GOOS == "windows" + +func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool, allowlists []*config.Allowlist) (<-chan ScanTarget, error) { + paths := make(chan ScanTarget) + s.Go(func() error { + defer close(paths) + return filepath.Walk(source, + func(path string, fInfo os.FileInfo, err error) error { + logger := logging.With().Str("path", path).Logger() + + if err != nil { + if os.IsPermission(err) { + // This seems to only fail on directories at this stage. + logger.Warn().Msg("Skipping directory: permission denied") + return filepath.SkipDir + } + return err + } + + // Empty; nothing to do here. + if fInfo.Size() == 0 { + return nil + } + + // Unwrap symlinks, if |followSymlinks| is set. + scanTarget := ScanTarget{ + Path: path, + } + if fInfo.Mode().Type() == fs.ModeSymlink { + if !followSymlinks { + logger.Debug().Msg("Skipping symlink") + return nil + } + + realPath, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + + realPathFileInfo, _ := os.Stat(realPath) + if realPathFileInfo.IsDir() { + logger.Warn().Str("target", realPath).Msg("Skipping symlinked directory") + return nil + } + + scanTarget.Path = realPath + scanTarget.Symlink = path + } + + // TODO: Also run this check against the resolved symlink? + var skip bool + for _, a := range allowlists { + skip = a.PathAllowed(path) || + // TODO: Remove this in v9. + // This is an awkward hack to mitigate https://github.com/gitleaks/gitleaks/issues/1641. + (isWindows && a.PathAllowed(filepath.ToSlash(path))) + if skip { + break + } + } + if fInfo.IsDir() { + // Directory + if skip { + logger.Debug().Msg("Skipping directory due to global allowlist") + return filepath.SkipDir + } + + if fInfo.Name() == ".git" { + // Don't scan .git directories. + // TODO: Add this to the config allowlist, instead of hard-coding it. + return filepath.SkipDir + } + } else { + // File + if skip { + logger.Debug().Msg("Skipping file due to global allowlist") + return nil + } + + paths <- scanTarget + } + return nil + }) + }) + return paths, nil +} diff --git a/cli/detect/sources/git.go b/cli/detect/sources/git.go new file mode 100644 index 000000000..95b829a9a --- /dev/null +++ b/cli/detect/sources/git.go @@ -0,0 +1,211 @@ +// MIT License + +// Copyright (c) 2019 Zachary Rice + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package sources + +import ( + "bufio" + "errors" + "io" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/gitleaks/go-gitdiff/gitdiff" + + "github.com/Infisical/infisical-merge/detect/logging" +) + +var quotedOptPattern = regexp.MustCompile(`^(?:"[^"]+"|'[^']+')$`) + +// GitCmd helps to work with Git's output. +type GitCmd struct { + cmd *exec.Cmd + diffFilesCh <-chan *gitdiff.File + errCh <-chan error +} + +// NewGitLogCmd returns `*DiffFilesCmd` with two channels: `<-chan *gitdiff.File` and `<-chan error`. +// Caller should read everything from channels until receiving a signal about their closure and call +// the `func (*DiffFilesCmd) Wait()` error in order to release resources. +func NewGitLogCmd(source string, logOpts string) (*GitCmd, error) { + sourceClean := filepath.Clean(source) + var cmd *exec.Cmd + if logOpts != "" { + args := []string{"-C", sourceClean, "log", "-p", "-U0"} + + // Ensure that the user-provided |logOpts| aren't wrapped in quotes. + // https://github.com/gitleaks/gitleaks/issues/1153 + userArgs := strings.Split(logOpts, " ") + var quotedOpts []string + for _, element := range userArgs { + if quotedOptPattern.MatchString(element) { + quotedOpts = append(quotedOpts, element) + } + } + if len(quotedOpts) > 0 { + logging.Warn().Msgf("the following `--log-opts` values may not work as expected: %v\n\tsee https://github.com/gitleaks/gitleaks/issues/1153 for more information", quotedOpts) + } + + args = append(args, userArgs...) + cmd = exec.Command("git", args...) + } else { + cmd = exec.Command("git", "-C", sourceClean, "log", "-p", "-U0", + "--full-history", "--all") + } + + logging.Debug().Msgf("executing: %s", cmd.String()) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + errCh := make(chan error) + go listenForStdErr(stderr, errCh) + + gitdiffFiles, err := gitdiff.Parse(stdout) + if err != nil { + return nil, err + } + + return &GitCmd{ + cmd: cmd, + diffFilesCh: gitdiffFiles, + errCh: errCh, + }, nil +} + +// NewGitDiffCmd returns `*DiffFilesCmd` with two channels: `<-chan *gitdiff.File` and `<-chan error`. +// Caller should read everything from channels until receiving a signal about their closure and call +// the `func (*DiffFilesCmd) Wait()` error in order to release resources. +func NewGitDiffCmd(source string, staged bool) (*GitCmd, error) { + sourceClean := filepath.Clean(source) + var cmd *exec.Cmd + cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", "--no-ext-diff", ".") + if staged { + cmd = exec.Command("git", "-C", sourceClean, "diff", "-U0", "--no-ext-diff", + "--staged", ".") + } + logging.Debug().Msgf("executing: %s", cmd.String()) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + errCh := make(chan error) + go listenForStdErr(stderr, errCh) + + gitdiffFiles, err := gitdiff.Parse(stdout) + if err != nil { + return nil, err + } + + return &GitCmd{ + cmd: cmd, + diffFilesCh: gitdiffFiles, + errCh: errCh, + }, nil +} + +// DiffFilesCh returns a channel with *gitdiff.File. +func (c *GitCmd) DiffFilesCh() <-chan *gitdiff.File { + return c.diffFilesCh +} + +// ErrCh returns a channel that could produce an error if there is something in stderr. +func (c *GitCmd) ErrCh() <-chan error { + return c.errCh +} + +// Wait waits for the command to exit and waits for any copying to +// stdin or copying from stdout or stderr to complete. +// +// Wait also closes underlying stdout and stderr. +func (c *GitCmd) Wait() (err error) { + return c.cmd.Wait() +} + +// listenForStdErr listens for stderr output from git, prints it to stdout, +// sends to errCh and closes it. +func listenForStdErr(stderr io.ReadCloser, errCh chan<- error) { + defer close(errCh) + + var errEncountered bool + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + // if git throws one of the following errors: + // + // exhaustive rename detection was skipped due to too many files. + // you may want to set your diff.renameLimit variable to at least + // (some large number) and retry the command. + // + // inexact rename detection was skipped due to too many files. + // you may want to set your diff.renameLimit variable to at least + // (some large number) and retry the command. + // + // Auto packing the repository in background for optimum performance. + // See "git help gc" for manual housekeeping. + // + // we skip exiting the program as git log -p/git diff will continue + // to send data to stdout and finish executing. This next bit of + // code prevents gitleaks from stopping mid scan if this error is + // encountered + if strings.Contains(scanner.Text(), + "exhaustive rename detection was skipped") || + strings.Contains(scanner.Text(), + "inexact rename detection was skipped") || + strings.Contains(scanner.Text(), + "you may want to set your diff.renameLimit") || + strings.Contains(scanner.Text(), + "See \"git help gc\" for manual housekeeping") || + strings.Contains(scanner.Text(), + "Auto packing the repository in background for optimum performance") { + logging.Warn().Msg(scanner.Text()) + } else { + logging.Error().Msgf("[git] %s", scanner.Text()) + errEncountered = true + } + } + + if errEncountered { + errCh <- errors.New("stderr is not empty") + return + } +} diff --git a/cli/detect/utils.go b/cli/detect/utils.go index 462716239..255d01fbe 100644 --- a/cli/detect/utils.go +++ b/cli/detect/utils.go @@ -26,20 +26,21 @@ import ( // "encoding/json" "fmt" "math" + "path/filepath" "strings" "time" + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/charmbracelet/lipgloss" - - "github.com/Infisical/infisical-merge/report" - "github.com/gitleaks/go-gitdiff/gitdiff" - "github.com/rs/zerolog/log" ) // augmentGitFinding updates the start and end line numbers of a finding to include the // delta from the git diff -func augmentGitFinding(finding report.Finding, textFragment *gitdiff.TextFragment, f *gitdiff.File) report.Finding { +func augmentGitFinding(remote *RemoteInfo, finding report.Finding, textFragment *gitdiff.TextFragment, f *gitdiff.File) report.Finding { if !strings.HasPrefix(finding.Match, "file detected") { finding.StartLine += int(textFragment.NewPosition) finding.EndLine += int(textFragment.NewPosition) @@ -47,16 +48,76 @@ func augmentGitFinding(finding report.Finding, textFragment *gitdiff.TextFragmen if f.PatchHeader != nil { finding.Commit = f.PatchHeader.SHA - finding.Message = f.PatchHeader.Message() if f.PatchHeader.Author != nil { finding.Author = f.PatchHeader.Author.Name finding.Email = f.PatchHeader.Author.Email } finding.Date = f.PatchHeader.AuthorDate.UTC().Format(time.RFC3339) + finding.Message = f.PatchHeader.Message() + // Results from `git diff` shouldn't have a link. + if finding.Commit != "" { + finding.Link = createScmLink(remote.Platform, remote.Url, finding) + } } return finding } +var linkCleaner = strings.NewReplacer( + " ", "%20", + "%", "%25", +) + +func createScmLink(scmPlatform scm.Platform, remoteUrl string, finding report.Finding) string { + if scmPlatform == scm.UnknownPlatform || scmPlatform == scm.NoPlatform { + return "" + } + + // Clean the path. + var ( + filePath = linkCleaner.Replace(finding.File) + ext = strings.ToLower(filepath.Ext(filePath)) + ) + + switch scmPlatform { + case scm.GitHubPlatform: + link := fmt.Sprintf("%s/blob/%s/%s", remoteUrl, finding.Commit, filePath) + if ext == ".ipynb" || ext == ".md" { + link += "?plain=1" + } + if finding.StartLine != 0 { + link += fmt.Sprintf("#L%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("-L%d", finding.EndLine) + } + return link + case scm.GitLabPlatform: + link := fmt.Sprintf("%s/blob/%s/%s", remoteUrl, finding.Commit, filePath) + if finding.StartLine != 0 { + link += fmt.Sprintf("#L%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("-%d", finding.EndLine) + } + return link + case scm.AzureDevOpsPlatform: + link := fmt.Sprintf("%s/commit/%s?path=/%s", remoteUrl, finding.Commit, filePath) + // Add line information if applicable + if finding.StartLine != 0 { + link += fmt.Sprintf("&line=%d", finding.StartLine) + } + if finding.EndLine != finding.StartLine { + link += fmt.Sprintf("&lineEnd=%d", finding.EndLine) + } + // This is a bit dirty, but Azure DevOps does not highlight the line when the lineStartColumn and lineEndColumn are not provided + link += "&lineStartColumn=1&lineEndColumn=10000000&type=2&lineStyle=plain&_a=files" + return link + default: + // This should never happen. + return "" + } +} + // shannonEntropy calculates the entropy of data using the formula defined here: // https://en.wiktionary.org/wiki/Shannon_entropy // Another way to think about what this is doing is calculating the number of bits @@ -82,7 +143,7 @@ func shannonEntropy(data string) (entropy float64) { } // filter will dedupe and redact findings -func filter(findings []report.Finding, redact bool) []report.Finding { +func filter(findings []report.Finding, redact uint) []report.Finding { var retFindings []report.Finding for _, f := range findings { include := true @@ -96,15 +157,15 @@ func filter(findings []report.Finding, redact bool) []report.Finding { genericMatch := strings.Replace(f.Match, f.Secret, "REDACTED", -1) betterMatch := strings.Replace(fPrime.Match, fPrime.Secret, "REDACTED", -1) - log.Trace().Msgf("skipping %s finding (%s), %s rule takes precendence (%s)", f.RuleID, genericMatch, fPrime.RuleID, betterMatch) + logging.Trace().Msgf("skipping %s finding (%s), %s rule takes precedence (%s)", f.RuleID, genericMatch, fPrime.RuleID, betterMatch) include = false break } } } - if redact { - f.Redact() + if redact > 0 { + f.Redact(redact) } if include { retFindings = append(retFindings, f) @@ -152,7 +213,7 @@ func printFinding(f report.Finding, noColor bool) { lineEndIdx := matchInLineIDX + len(f.Match) if len(f.Line)-1 <= lineEndIdx { - lineEndIdx = len(f.Line) - 1 + lineEndIdx = len(f.Line) } lineEnd := f.Line[lineEndIdx:] @@ -184,6 +245,9 @@ func printFinding(f report.Finding, noColor bool) { fmt.Println("") return } + if len(f.Tags) > 0 { + fmt.Printf("%-12s %s\n", "Tags:", f.Tags) + } fmt.Printf("%-12s %s\n", "File:", f.File) fmt.Printf("%-12s %d\n", "Line:", f.StartLine) if f.Commit == "" { @@ -196,16 +260,12 @@ func printFinding(f report.Finding, noColor bool) { fmt.Printf("%-12s %s\n", "Email:", f.Email) fmt.Printf("%-12s %s\n", "Date:", f.Date) fmt.Printf("%-12s %s\n", "Fingerprint:", f.Fingerprint) + if f.Link != "" { + fmt.Printf("%-12s %s\n", "Link:", f.Link) + } fmt.Println("") } -func containsDigit(s string) bool { - for _, c := range s { - switch c { - case '1', '2', '3', '4', '5', '6', '7', '8', '9': - return true - } - - } - return false +func isWhitespace(ch byte) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } diff --git a/cli/go.mod b/cli/go.mod index 52cb79f38..a2b256f8a 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -10,7 +10,7 @@ require ( github.com/creack/pty v1.1.21 github.com/denisbrodbeck/machineid v1.0.1 github.com/fatih/semgroup v1.2.0 - github.com/gitleaks/go-gitdiff v0.8.0 + github.com/gitleaks/go-gitdiff v0.9.1 github.com/h2non/filetype v1.1.3 github.com/infisical/go-sdk v0.5.92 github.com/infisical/infisical-kmip v0.3.5 @@ -42,6 +42,11 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.4.0 // indirect cloud.google.com/go/iam v1.1.11 // indirect + dario.cat/mergo v1.0.1 // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/aws/aws-sdk-go-v2 v1.27.2 // indirect @@ -74,17 +79,21 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/pprof v0.0.0-20250302191652-9094ed2288e7 // indirect github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/muesli/mango v0.1.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect @@ -98,8 +107,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cast v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/wlynxg/anet v0.0.5 // indirect diff --git a/cli/go.sum b/cli/go.sum index 49566f1cc..cb1b1c1cf 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -44,13 +44,23 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= +github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Infisical/go-keyring v1.0.2 h1:dWOkI/pB/7RocfSJgGXbXxLDcVYsdslgjEPmVhb+nl8= github.com/Infisical/go-keyring v1.0.2/go.mod h1:LWOnn/sw9FxDW/0VY+jHFAfOFEe03xmwBVSfJnBowto= github.com/Infisical/turn/v4 v4.0.1 h1:omdelNsnFfzS5cu86W5OBR68by68a8sva4ogR0lQQnw= github.com/Infisical/turn/v4 v4.0.1/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -142,6 +152,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= +github.com/gitleaks/go-gitdiff v0.9.1 h1:ni6z6/3i9ODT685OLCTf+s/ERlWUNWQF4x1pvoNICw0= +github.com/gitleaks/go-gitdiff v0.9.1/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -273,6 +285,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= @@ -315,6 +329,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -324,6 +340,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -393,6 +411,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -402,6 +422,8 @@ github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= diff --git a/cli/packages/cmd/scan.go b/cli/packages/cmd/scan.go index 1226e3319..42ff0f1e1 100644 --- a/cli/packages/cmd/scan.go +++ b/cli/packages/cmd/scan.go @@ -32,10 +32,13 @@ import ( "strings" "time" - "github.com/Infisical/infisical-merge/config" "github.com/Infisical/infisical-merge/detect" + "github.com/Infisical/infisical-merge/detect/cmd/scm" + "github.com/Infisical/infisical-merge/detect/config" + "github.com/Infisical/infisical-merge/detect/logging" + "github.com/Infisical/infisical-merge/detect/report" + "github.com/Infisical/infisical-merge/detect/sources" "github.com/Infisical/infisical-merge/packages/util" - "github.com/Infisical/infisical-merge/report" "github.com/manifoldco/promptui" "github.com/posthog/posthog-go" "github.com/rs/zerolog/log" @@ -240,9 +243,17 @@ var scanCmd = &cobra.Command{ log.Fatal().Err(err).Msg("") } // set redact flag - if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil { + + redactFlag, err := cmd.Flags().GetBool("redact") + if err != nil { log.Fatal().Err(err).Msg("") } + if redactFlag { + detector.Redact = 100 + } else { + detector.Redact = 0 + } + if detector.MaxTargetMegaBytes, err = cmd.Flags().GetInt("max-target-megabytes"); err != nil { log.Fatal().Err(err).Msg("") } @@ -293,31 +304,49 @@ var scanCmd = &cobra.Command{ // start the detector scan if noGit { - findings, err = detector.DetectFiles(source) + paths, err := sources.DirectoryTargets( + source, + detector.Sema, + detector.FollowSymlinks, + detector.Config.Allowlists, + ) if err != nil { + logging.Fatal().Err(err).Send() + } + + if findings, err = detector.DetectFiles(paths); err != nil { // don't exit on error, just log it - log.Error().Err(err).Msg("") + logging.Error().Err(err).Msg("failed scan directory") } } else if fromPipe { - findings, err = detector.DetectReader(os.Stdin, 10) - if err != nil { + if findings, err = detector.DetectReader(os.Stdin, 10); err != nil { // log fatal to exit, no need to continue since a report // will not be generated when scanning from a pipe...for now - log.Fatal().Err(err).Msg("") + logging.Fatal().Err(err).Msg("failed scan input from stdin") } } else { + var ( + gitCmd *sources.GitCmd + scmPlatform scm.Platform + remote *detect.RemoteInfo + ) + var logOpts string logOpts, err = cmd.Flags().GetString("log-opts") - if err != nil { - log.Fatal().Err(err).Msg("") + + if gitCmd, err = sources.NewGitLogCmd(source, logOpts); err != nil { + logging.Fatal().Err(err).Msg("could not create Git cmd") } - findings, err = detector.DetectGit(source, logOpts, detect.DetectType) - if err != nil { + if scmPlatform, err = scm.PlatformFromString("github"); err != nil { + logging.Fatal().Err(err).Send() + } + remote = detect.NewRemoteInfo(scmPlatform, source) + + if findings, err = detector.DetectGit(gitCmd, remote); err != nil { // don't exit on error, just log it - log.Error().Err(err).Msg("") + logging.Error().Err(err).Msg("failed to scan Git repository") } } - // log info about the scan if err == nil { log.Info().Msgf("scan completed in %s", FormatDuration(time.Since(start))) @@ -341,9 +370,7 @@ var scanCmd = &cobra.Command{ reportPath, _ := cmd.Flags().GetString("report-path") ext, _ := cmd.Flags().GetString("report-format") if reportPath != "" { - if err := report.Write(findings, cfg, ext, reportPath); err != nil { - log.Fatal().Err(err).Msg("could not write") - } + reportFindings(findings, reportPath, ext, &cfg) } if err != nil { @@ -375,7 +402,6 @@ var scanGitChangesCmd = &cobra.Command{ cfg.Path, _ = cmd.Flags().GetString("config") exitCode, _ := cmd.Flags().GetInt("exit-code") staged, _ := cmd.Flags().GetBool("staged") - start := time.Now() // Setup detector detector := detect.NewDetector(cfg) @@ -397,9 +423,17 @@ var scanGitChangesCmd = &cobra.Command{ log.Fatal().Err(err).Msg("") } // set redact flag - if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil { + + redactFlag, err := cmd.Flags().GetBool("redact") + if err != nil { log.Fatal().Err(err).Msg("") } + if redactFlag { + detector.Redact = 100 + } else { + detector.Redact = 0 + } + if detector.MaxTargetMegaBytes, err = cmd.Flags().GetInt("max-target-megabytes"); err != nil { log.Fatal().Err(err).Msg("") } @@ -414,32 +448,22 @@ var scanGitChangesCmd = &cobra.Command{ } } - // get log options for git scan - logOpts, err := cmd.Flags().GetString("log-opts") - if err != nil { - log.Fatal().Err(err).Msg("") - } - - log.Info().Msgf("scanning for exposed secrets...") - // start git scan - var findings []report.Finding - if staged { - findings, err = detector.DetectGit(source, logOpts, detect.ProtectStagedType) - } else { - findings, err = detector.DetectGit(source, logOpts, detect.ProtectType) - } - if err != nil { - // don't exit on error, just log it - log.Error().Err(err).Msg("") - } + var ( + findings []report.Finding - // log info about the scan - log.Info().Msgf("scan completed in %s", FormatDuration(time.Since(start))) - if len(findings) != 0 { - log.Warn().Msgf("leaks found: %d", len(findings)) - } else { - log.Info().Msg("no leaks found") + gitCmd *sources.GitCmd + remote *detect.RemoteInfo + ) + + if gitCmd, err = sources.NewGitDiffCmd(source, staged); err != nil { + logging.Fatal().Err(err).Msg("could not create Git diff cmd") + } + remote = &detect.RemoteInfo{Platform: scm.NoPlatform} + + if findings, err = detector.DetectGit(gitCmd, remote); err != nil { + // don't exit on error, just log it + logging.Error().Err(err).Msg("failed to scan Git repository") } Telemetry.CaptureEvent("cli-command:scan git-changes", posthog.NewProperties().Set("risks", len(findings)).Set("version", util.CLI_VERSION)) @@ -447,9 +471,7 @@ var scanGitChangesCmd = &cobra.Command{ reportPath, _ := cmd.Flags().GetString("report-path") ext, _ := cmd.Flags().GetString("report-format") if reportPath != "" { - if err = report.Write(findings, cfg, ext, reportPath); err != nil { - log.Fatal().Err(err).Msg("") - } + reportFindings(findings, reportPath, ext, &cfg) } if len(findings) != 0 { os.Exit(exitCode) @@ -457,6 +479,36 @@ var scanGitChangesCmd = &cobra.Command{ }, } +func reportFindings(findings []report.Finding, reportPath string, ext string, cfg *config.Config) { + + var reporter report.Reporter + + switch ext { + case "csv": + reporter = &report.CsvReporter{} + case "json": + reporter = &report.JsonReporter{} + case "junit": + reporter = &report.JunitReporter{} + case "sarif": + reporter = &report.SarifReporter{ + OrderedRules: cfg.GetOrderedRules(), + } + default: + logging.Fatal().Msgf("unknown report format %s", ext) + } + + file, err := os.Create(reportPath) + if err != nil { + log.Fatal().Err(err).Msg("could not create file") + } + + if err := reporter.Write(file, findings); err != nil { + log.Fatal().Err(err).Msg("could not write") + } + +} + func fileExists(fileName string) bool { // check for a .infisicalignore file info, err := os.Stat(fileName) diff --git a/cli/report/csv_test.go b/cli/report/csv_test.go deleted file mode 100644 index 967026519..000000000 --- a/cli/report/csv_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestWriteCSV(t *testing.T) { - tests := []struct { - findings []Finding - testReportName string - expected string - wantEmpty bool - }{ - { - testReportName: "simple", - expected: filepath.Join(expectPath, "report", "csv_simple.csv"), - findings: []Finding{ - { - RuleID: "test-rule", - Match: "line containing secret", - Secret: "a secret", - StartLine: 1, - EndLine: 2, - StartColumn: 1, - EndColumn: 2, - Message: "opps", - File: "auth.py", - SymlinkFile: "", - Commit: "0000000000000000", - Author: "John Doe", - Email: "johndoe@gmail.com", - Date: "10-19-2003", - Fingerprint: "fingerprint", - }, - }}, - { - - wantEmpty: true, - testReportName: "empty", - expected: filepath.Join(expectPath, "report", "this_should_not_exist.csv"), - findings: []Finding{}}, - } - - for _, test := range tests { - tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".csv")) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = writeCsv(test.findings, tmpfile) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - if test.wantEmpty { - if len(got) > 0 { - t.Errorf("Expected empty file, got %s", got) - } - os.Remove(tmpfile.Name()) - continue - } - want, err := os.ReadFile(test.expected) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - - if string(got) != string(want) { - err = os.WriteFile(strings.Replace(test.expected, ".csv", ".got.csv", 1), got, 0644) - if err != nil { - t.Error(err) - } - t.Errorf("got %s, want %s", string(got), string(want)) - } - - os.Remove(tmpfile.Name()) - } -} diff --git a/cli/report/json_test.go b/cli/report/json_test.go deleted file mode 100644 index e81aaa827..000000000 --- a/cli/report/json_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestWriteJSON(t *testing.T) { - tests := []struct { - findings []Finding - testReportName string - expected string - wantEmpty bool - }{ - { - testReportName: "simple", - expected: filepath.Join(expectPath, "report", "json_simple.json"), - findings: []Finding{ - { - - Description: "", - RuleID: "test-rule", - Match: "line containing secret", - Secret: "a secret", - StartLine: 1, - EndLine: 2, - StartColumn: 1, - EndColumn: 2, - Message: "opps", - File: "auth.py", - SymlinkFile: "", - Commit: "0000000000000000", - Author: "John Doe", - Email: "johndoe@gmail.com", - Date: "10-19-2003", - Tags: []string{}, - }, - }}, - { - - testReportName: "empty", - expected: filepath.Join(expectPath, "report", "empty.json"), - findings: []Finding{}}, - } - - for _, test := range tests { - // create tmp file using os.TempDir() - tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".json")) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = writeJson(test.findings, tmpfile) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - if test.wantEmpty { - if len(got) > 0 { - os.Remove(tmpfile.Name()) - t.Errorf("Expected empty file, got %s", got) - } - os.Remove(tmpfile.Name()) - continue - } - want, err := os.ReadFile(test.expected) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - - if string(got) != string(want) { - err = os.WriteFile(strings.Replace(test.expected, ".json", ".got.json", 1), got, 0644) - if err != nil { - t.Error(err) - } - t.Errorf("got %s, want %s", string(got), string(want)) - } - - os.Remove(tmpfile.Name()) - } -} diff --git a/cli/report/report_test.go b/cli/report/report_test.go deleted file mode 100644 index ef38f19c2..000000000 --- a/cli/report/report_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -import ( - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/Infisical/infisical-merge/config" -) - -const ( - expectPath = "../testdata/expected/" - tmpPath = "../testdata/tmp" -) - -func TestReport(t *testing.T) { - tests := []struct { - findings []Finding - ext string - wantEmpty bool - }{ - { - ext: "json", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: ".json", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: ".jsonj", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - wantEmpty: true, - }, - { - ext: ".csv", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: "csv", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - { - ext: "CSV", - findings: []Finding{ - { - RuleID: "test-rule", - }, - }, - }, - // { - // ext: "SARIF", - // findings: []Finding{ - // { - // RuleID: "test-rule", - // }, - // }, - // }, - } - - for i, test := range tests { - tmpfile, err := os.Create(filepath.Join(tmpPath, strconv.Itoa(i)+test.ext)) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - err = Write(test.findings, config.Config{}, test.ext, tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - got, err := os.ReadFile(tmpfile.Name()) - if err != nil { - os.Remove(tmpfile.Name()) - t.Error(err) - } - os.Remove(tmpfile.Name()) - - if len(got) == 0 && !test.wantEmpty { - t.Errorf("got empty file with extension " + test.ext) - } - - if test.wantEmpty { - if len(got) > 0 { - t.Errorf("Expected empty file, got %s", got) - } - continue - } - } -} diff --git a/cli/report/sarif_test.go b/cli/report/sarif_test.go deleted file mode 100644 index 9331060f1..000000000 --- a/cli/report/sarif_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// MIT License - -// Copyright (c) 2019 Zachary Rice - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package report - -const configPath = "../testdata/config/" - -// func TestWriteSarif(t *testing.T) { -// tests := []struct { -// findings []Finding -// testReportName string -// expected string -// wantEmpty bool -// cfgName string -// }{ -// { -// cfgName: "simple", -// testReportName: "simple", -// expected: filepath.Join(expectPath, "report", "sarif_simple.sarif"), -// findings: []Finding{ -// { - -// Description: "A test rule", -// RuleID: "test-rule", -// Match: "line containing secret", -// Secret: "a secret", -// StartLine: 1, -// EndLine: 2, -// StartColumn: 1, -// EndColumn: 2, -// Message: "opps", -// File: "auth.py", -// Commit: "0000000000000000", -// Author: "John Doe", -// Email: "johndoe@gmail.com", -// Date: "10-19-2003", -// Tags: []string{}, -// }, -// }}, -// } - -// for _, test := range tests { -// // create tmp file using os.TempDir() -// tmpfile, err := os.Create(filepath.Join(tmpPath, test.testReportName+".json")) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// viper.Reset() -// viper.AddConfigPath(configPath) -// viper.SetConfigName(test.cfgName) -// viper.SetConfigType("toml") -// err = viper.ReadInConfig() -// if err != nil { -// t.Error(err) -// } - -// var vc config.ViperConfig -// err = viper.Unmarshal(&vc) -// if err != nil { -// t.Error(err) -// } - -// cfg, err := vc.Translate() -// if err != nil { -// t.Error(err) -// } -// err = writeSarif(cfg, test.findings, tmpfile) -// fmt.Println(cfg) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// got, err := os.ReadFile(tmpfile.Name()) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } -// if test.wantEmpty { -// if len(got) > 0 { -// os.Remove(tmpfile.Name()) -// t.Errorf("Expected empty file, got %s", got) -// } -// os.Remove(tmpfile.Name()) -// continue -// } -// want, err := os.ReadFile(test.expected) -// if err != nil { -// os.Remove(tmpfile.Name()) -// t.Error(err) -// } - -// if string(got) != string(want) { -// err = os.WriteFile(strings.Replace(test.expected, ".sarif", ".got.sarif", 1), got, 0644) -// if err != nil { -// t.Error(err) -// } -// t.Errorf("got %s, want %s", string(got), string(want)) -// } - -// os.Remove(tmpfile.Name()) -// } -// } diff --git a/docs/api-reference/endpoints/certificates/bundle.mdx b/docs/api-reference/endpoints/certificates/bundle.mdx index 5fbda7d96..60d37a2d8 100644 --- a/docs/api-reference/endpoints/certificates/bundle.mdx +++ b/docs/api-reference/endpoints/certificates/bundle.mdx @@ -1,6 +1,6 @@ --- title: "Get Certificate Bundle" -openapi: "GET /api/v2/workspace/{slug}/bundle" +openapi: "GET /api/v1/pki/certificates/{serialNumber}/bundle" --- diff --git a/docs/api-reference/endpoints/certificates/private-key.mdx b/docs/api-reference/endpoints/certificates/private-key.mdx index 244aecea3..d0b93e65c 100644 --- a/docs/api-reference/endpoints/certificates/private-key.mdx +++ b/docs/api-reference/endpoints/certificates/private-key.mdx @@ -1,4 +1,4 @@ --- title: "Get Certificate Private Key" -openapi: "GET /api/v2/workspace/{slug}/private-key" +openapi: "GET /api/v1/pki/certificates/{serialNumber}/private-key" --- diff --git a/docs/api-reference/endpoints/ldap-auth/attach.mdx b/docs/api-reference/endpoints/ldap-auth/attach.mdx new file mode 100644 index 000000000..512878887 --- /dev/null +++ b/docs/api-reference/endpoints/ldap-auth/attach.mdx @@ -0,0 +1,4 @@ +--- +title: "Attach" +openapi: "POST /api/v1/auth/ldap-auth/identities/{identityId}" +--- diff --git a/docs/api-reference/endpoints/ldap-auth/login.mdx b/docs/api-reference/endpoints/ldap-auth/login.mdx new file mode 100644 index 000000000..737afb857 --- /dev/null +++ b/docs/api-reference/endpoints/ldap-auth/login.mdx @@ -0,0 +1,4 @@ +--- +title: "Login" +openapi: "POST /api/v1/auth/ldap-auth/login" +--- diff --git a/docs/api-reference/endpoints/ldap-auth/retrieve.mdx b/docs/api-reference/endpoints/ldap-auth/retrieve.mdx new file mode 100644 index 000000000..fe4974cde --- /dev/null +++ b/docs/api-reference/endpoints/ldap-auth/retrieve.mdx @@ -0,0 +1,4 @@ +--- +title: "Retrieve" +openapi: "GET /api/v1/auth/ldap-auth/identities/{identityId}" +--- diff --git a/docs/api-reference/endpoints/ldap-auth/revoke.mdx b/docs/api-reference/endpoints/ldap-auth/revoke.mdx new file mode 100644 index 000000000..2ef0996fd --- /dev/null +++ b/docs/api-reference/endpoints/ldap-auth/revoke.mdx @@ -0,0 +1,4 @@ +--- +title: "Revoke" +openapi: "DELETE /api/v1/auth/ldap-auth/identities/{identityId}" +--- diff --git a/docs/api-reference/endpoints/ldap-auth/update.mdx b/docs/api-reference/endpoints/ldap-auth/update.mdx new file mode 100644 index 000000000..74b54efd3 --- /dev/null +++ b/docs/api-reference/endpoints/ldap-auth/update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update" +openapi: "PATCH /api/v1/auth/ldap-auth/identities/{identityId}" +--- diff --git a/docs/documentation/platform/access-controls/abac/managing-user-metadata.mdx b/docs/documentation/platform/access-controls/abac/managing-user-metadata.mdx index 3f62a3b61..b6d1d691f 100644 --- a/docs/documentation/platform/access-controls/abac/managing-user-metadata.mdx +++ b/docs/documentation/platform/access-controls/abac/managing-user-metadata.mdx @@ -27,7 +27,7 @@ User identities can have metadata attributes assigned directly. These attributes #### Applying ABAC Policies with User Metadata -Attribute-based access controls are currently only available for polices defined on Secrets Manager projects. +Attribute-based access controls are currently only available for policies defined on Secrets Manager projects. You can set ABAC permissions to dynamically set access to environments, folders, secrets, and secret tags. diff --git a/docs/documentation/platform/identities/aws-auth.mdx b/docs/documentation/platform/identities/aws-auth.mdx index 494606ccd..1c853957b 100644 --- a/docs/documentation/platform/identities/aws-auth.mdx +++ b/docs/documentation/platform/identities/aws-auth.mdx @@ -62,7 +62,7 @@ access the Infisical API using the AWS Auth authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/azure-auth.mdx b/docs/documentation/platform/identities/azure-auth.mdx index 03d997ffb..9576c4d0f 100644 --- a/docs/documentation/platform/identities/azure-auth.mdx +++ b/docs/documentation/platform/identities/azure-auth.mdx @@ -62,7 +62,7 @@ access the Infisical API using the Azure Auth authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/gcp-auth.mdx b/docs/documentation/platform/identities/gcp-auth.mdx index 6573544de..17dc5acd9 100644 --- a/docs/documentation/platform/identities/gcp-auth.mdx +++ b/docs/documentation/platform/identities/gcp-auth.mdx @@ -68,7 +68,7 @@ access the Infisical API using the GCP ID Token authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) @@ -237,7 +237,7 @@ access the Infisical API using the GCP IAM authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/jwt-auth.mdx b/docs/documentation/platform/identities/jwt-auth.mdx index 3dcf12b29..339138881 100644 --- a/docs/documentation/platform/identities/jwt-auth.mdx +++ b/docs/documentation/platform/identities/jwt-auth.mdx @@ -57,7 +57,7 @@ In the following steps, we explore how to create and use identities to access th - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/kubernetes-auth.mdx b/docs/documentation/platform/identities/kubernetes-auth.mdx index 58069f09e..cfa0e861a 100644 --- a/docs/documentation/platform/identities/kubernetes-auth.mdx +++ b/docs/documentation/platform/identities/kubernetes-auth.mdx @@ -163,7 +163,7 @@ In the following steps, we explore how to create and use identities for your app - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/ldap-auth/general.mdx b/docs/documentation/platform/identities/ldap-auth/general.mdx new file mode 100644 index 000000000..7fb2798c7 --- /dev/null +++ b/docs/documentation/platform/identities/ldap-auth/general.mdx @@ -0,0 +1,87 @@ +--- +title: General +description: "Learn how to authenticate with Infisical using LDAP." +--- + +**LDAP Auth** is an LDAP based authentication method that allows you to authenticate with Infisical using a machine identity configured with an [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) directory. + +## Guide + + + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. + + ![Create identity](/images/platform/identities/ldap/identities-org-create-identity.png) + + When creating an identity, you specify an organization level role for it to assume; you can configure roles in Organization Settings > Access Control > Organization Roles. + + ![Create identity modal](/images/platform/identities/ldap/identities-org-create-identity-modal.png) + + Now input a few details for your new identity. Here's some guidance for each field: + + - Name (required): A friendly name for the identity. + - Role (required): A role from the Organization Roles tab for the identity to assume. The organization role assigned will determine what organization level resources this identity can have access to. + + Once you've created an identity, you'll be redirected to a page where you can manage the identity. + + + + To configure LDAP auth for your identity, press the **Add Auth Method** button on the identity's page. + + ![Add auth method](/images/platform/identities/ldap/identities-org-add-auth-method.png) + + Now select **LDAP Auth** from the list of available auth methods for the identity. + + ![Select LDAP auth](/images/platform/identities/ldap/identities-org-add-auth-method-modal.png) + + + After selecting **LDAP Auth**, you'll see the form you need to fill out to configure LDAP auth for your identity. The following fields are available: + + - `URL`: The LDAP server to connect to such as `ldap://ldap.your-org.com`, `ldaps://ldap.myorg.com:636` _(for connection over SSL/TLS)_, etc. + - `Bind DN`: The DN to bind to the LDAP server with. + - `Bind Pass`: The password to bind to the LDAP server with. + - `Search Base / DN`: Base DN under which to perform user search such as `ou=Users,dc=acme,dc=com`. + - `User Search Filter`: Template used to construct the LDAP user search filter such as `(uid={{username}})`; use literal `{{username}}` to have the given username used in the search. The default is `(uid={{username}})` which is compatible with several common directory schemas. + - `Required Attributes`: A key/value pair of attributes that must be present in the LDAP user entry for them to be authenticated. As an example, if you set key `uid` to value `user1,user2,user3`, then only users with `uid` of `user1`, `user2`, or `user3` will be able to login with this identity. Each value is a comma separated list of attributes. + - `CA Certificate`: The CA certificate to use when verifying the LDAP server certificate. This field is optional but recommended. + - `Access Token TTL` _(default is 2592000 equivalent to 30 days)_: The lifetime for an access token in seconds. This value will be referenced at renewal time. + - `Access Token Max TTL` _(default is 2592000 equivalent to 30 days)_: The maximum lifetime for an access token in seconds. This value will be referenced at renewal time. + - `Access Token Max Number of Uses` _(default is 0)_: The maximum number of times that an access token can be used; a value of 0 implies infinite number of uses. + - `Access Token Trusted IPs`: The IPs or CIDR ranges that access tokens can be used from. By default, each token is given the 0.0.0.0/0, allowing usage from any network address. + + Once you've filled out the form, press **Add** to save your changes. + + ![Configure LDAP auth](/images/platform/identities/ldap/identities-org-configure-ldap.png) + + + After configuring LDAP auth for your identity, you can authenticate with the identity and obtain an access token using your LDAP credentials. + + ```bash + curl --request POST \ + --url https://app.infisical.com/api/v1/auth/ldap-auth/login \ + --header 'Content-Type: application/json' \ + --data '{ + "identityId": "", + "username": "", + "password": "" + }' + ``` + + + For EU Cloud and Self-Hosted users, make sure to replace `https://app.infisical.com` with `https://eu.infisical.com` or your self-hosted instance's URL in the request URL. + + + If successful, you'll receive an access token in the response body. + + ```json + { + "accessToken": "your-access-token", + "expiresIn": 2592000, + "accessTokenMaxTTL": 2592000, + "tokenType": "Bearer" + } + ``` + + You can read more about the login API endpoint [here](/api-reference/endpoints/ldap-auth/login). + + + diff --git a/docs/documentation/platform/identities/ldap-auth/jumpcloud.mdx b/docs/documentation/platform/identities/ldap-auth/jumpcloud.mdx new file mode 100644 index 000000000..4bd497eac --- /dev/null +++ b/docs/documentation/platform/identities/ldap-auth/jumpcloud.mdx @@ -0,0 +1,97 @@ +--- +title: JumpCloud +description: "Learn how to authenticate with Infisical using LDAP with JumpCloud." +--- + +**LDAP Auth** is an LDAP based authentication method that allows you to authenticate with Infisical using a machine identity configured with an [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) directory. + +## Guide + + + + In JumpCloud, head to USER MANAGEMENT > Users and create a new user via the Manual user entry option. + This user will be used as a privileged service account to facilitate Infisical's ability to bind/search the LDAP directory. + + Next after creating the user, under User Security Settings and Permissions > Permission Settings, check the box next to Enable as LDAP Bind DN. + + ![User management](/images/platform/identities/ldap/jumpcloud-users-management.png) + + + + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. + + ![Create identity](/images/platform/identities/ldap/identities-org-create-identity.png) + + When creating an identity, you specify an organization level role for it to assume; you can configure roles in Organization Settings > Access Control > Organization Roles. + + ![Create identity modal](/images/platform/identities/ldap/identities-org-create-identity-modal.png) + + Now input a few details for your new identity. Here's some guidance for each field: + + - Name (required): A friendly name for the identity. + - Role (required): A role from the Organization Roles tab for the identity to assume. The organization role assigned will determine what organization level resources this identity can have access to. + + Once you've created an identity, you'll be redirected to a page where you can manage the identity. + + + + To configure LDAP auth for your identity, press the **Add Auth Method** button on the identity's page. + + ![Add auth method](/images/platform/identities/ldap/identities-org-add-auth-method.png) + + Now select **LDAP Auth** from the list of available auth methods for the identity. + + ![Select LDAP auth](/images/platform/identities/ldap/identities-org-add-auth-method-modal.png) + + + After selecting **LDAP Auth**, you'll see the form you need to fill out to configure LDAP auth for your identity. The following fields are available: + + - `URL`: The LDAP server to connect to (`ldaps://ldap.jumpcloud.com:636`). + - `Bind DN`: The distinguished name of object to bind when performing the user search (`uid=,ou=Users,o=,dc=jumpcloud,dc=com`). + - `Bind Pass`: The password to use along with Bind DN when performing the user search. This is the password for the user created in the previous step. + - `Search Base / DN`: Base DN under which to perform user search (`ou=Users,o=,dc=jumpcloud,dc=com`). + - `User Search Filter`: Template used to construct the LDAP user search filter (`(uid={{username}})`). + - `Required Attributes`: A key/value pair of attributes that must be present in the LDAP user entry for them to be authenticated. As an example, if you set key `uid` to value `user1,user2,user3`, then only users with `uid` of `user1`, `user2`, or `user3` will be able to login with this identity. Each value is a comma separated list of attributes. + - `CA Certificate`: The CA certificate to use when verifying the LDAP server certificate (instructions to obtain the certificate for JumpCloud [here](https://jumpcloud.com/support/connect-to-ldap-with-tls-ssl)). + - `Access Token TTL` _(default is 2592000 equivalent to 30 days)_: The lifetime for an access token in seconds. This value will be referenced at renewal time. + - `Access Token Max TTL` _(default is 2592000 equivalent to 30 days)_: The maximum lifetime for an access token in seconds. This value will be referenced at renewal time. + - `Access Token Max Number of Uses` _(default is 0)_: The maximum number of times that an access token can be used; a value of 0 implies infinite number of uses. + - `Access Token Trusted IPs`: The IPs or CIDR ranges that access tokens can be used from. By default, each token is given the 0.0.0.0/0, allowing usage from any network address. + + Once you've filled out the form, press **Add** to save your changes. + + ![Configure LDAP auth](/images/platform/identities/ldap/identities-org-configure-ldap.png) + + + After configuring LDAP auth for your identity, you can authenticate with the identity and obtain an access token using your LDAP credentials. + + ```bash + curl --request POST \ + --url https://app.infisical.com/api/v1/auth/ldap-auth/login \ + --header 'Content-Type: application/json' \ + --data '{ + "identityId": "", + "username": "", + "password": "" + }' + ``` + + + For EU Cloud and Self-Hosted users, make sure to replace `https://app.infisical.com` with `https://eu.infisical.com` or your self-hosted instance's URL in the request URL. + + + If successful, you'll receive an access token in the response body. + + ```json + { + "accessToken": "your-access-token", + "expiresIn": 2592000, + "accessTokenMaxTTL": 2592000, + "tokenType": "Bearer" + } + ``` + + You can read more about the login API endpoint [here](/api-reference/endpoints/ldap-auth/login). + + + diff --git a/docs/documentation/platform/identities/oidc-auth/circleci.mdx b/docs/documentation/platform/identities/oidc-auth/circleci.mdx index ddf74e3fa..6849b77f9 100644 --- a/docs/documentation/platform/identities/oidc-auth/circleci.mdx +++ b/docs/documentation/platform/identities/oidc-auth/circleci.mdx @@ -52,7 +52,7 @@ In the following steps, we explore how to create and use identities to access th - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/oidc-auth/general.mdx b/docs/documentation/platform/identities/oidc-auth/general.mdx index 776d175a4..9a39adba3 100644 --- a/docs/documentation/platform/identities/oidc-auth/general.mdx +++ b/docs/documentation/platform/identities/oidc-auth/general.mdx @@ -56,7 +56,7 @@ In the following steps, we explore how to create and use identities to access th - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/oidc-auth/github.mdx b/docs/documentation/platform/identities/oidc-auth/github.mdx index 47352a339..a377ac37c 100644 --- a/docs/documentation/platform/identities/oidc-auth/github.mdx +++ b/docs/documentation/platform/identities/oidc-auth/github.mdx @@ -55,7 +55,7 @@ In the following steps, we explore how to create and use identities to access th - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/oidc-auth/gitlab.mdx b/docs/documentation/platform/identities/oidc-auth/gitlab.mdx index 228392aa6..b52d2f894 100644 --- a/docs/documentation/platform/identities/oidc-auth/gitlab.mdx +++ b/docs/documentation/platform/identities/oidc-auth/gitlab.mdx @@ -55,7 +55,7 @@ In the following steps, we explore how to create and use identities to access th - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/token-auth.mdx b/docs/documentation/platform/identities/token-auth.mdx index 59c5f9abf..500adf509 100644 --- a/docs/documentation/platform/identities/token-auth.mdx +++ b/docs/documentation/platform/identities/token-auth.mdx @@ -38,7 +38,7 @@ using the Token Auth authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/identities/universal-auth.mdx b/docs/documentation/platform/identities/universal-auth.mdx index 4d66e30b4..30f1f10d2 100644 --- a/docs/documentation/platform/identities/universal-auth.mdx +++ b/docs/documentation/platform/identities/universal-auth.mdx @@ -42,7 +42,7 @@ using the Universal Auth authentication method. - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) diff --git a/docs/documentation/platform/project-templates.mdx b/docs/documentation/platform/project-templates.mdx index d84c6bdc1..7dd5ceb50 100644 --- a/docs/documentation/platform/project-templates.mdx +++ b/docs/documentation/platform/project-templates.mdx @@ -33,7 +33,7 @@ In the following steps, we'll explore how to set up a project template. - Navigate to the Project Templates tab on the Organization Settings page and tap on the **Add Template** button. + Navigate to the **Project Templates** tab on the Feature Settings page for the project type you want to create a template for and tap on the **Add Template** button. ![project template add button](/images/platform/project-templates/project-template-add-button.png) Specify your template details. Here's some guidance on each field: @@ -67,6 +67,7 @@ In the following steps, we'll explore how to set up a project template. --header 'Content-Type: application/json' \ --data '{ "name": "my-project-template", + "type": "secret-manager", "description": "...", "environments": "[...]", "roles": "[...]", diff --git a/docs/documentation/platform/secret-scanning.mdx b/docs/documentation/platform/secret-scanning.mdx index 4f030e882..da28bfa55 100644 --- a/docs/documentation/platform/secret-scanning.mdx +++ b/docs/documentation/platform/secret-scanning.mdx @@ -7,6 +7,113 @@ The Infisical Secret Scanner allows you to keep an overview and stay alert of ex To further enhance security, we recommend you also use our [CLI Secret Scanner](/cli/scanning-overview#automatically-scan-changes-before-you-commit) to scan for exposed secrets prior to pushing your changes. + + + + To setup secret scanning on your own instance of Infisical, you can follow the steps below. + + + + Create a new GitHub app in your GitHub organization or personal [Developer Settings](https://github.com/settings/apps). + + ![Create GitHub App](/images/platform/secret-scanning/github-create-app.png) + + ### Configure the GitHub App + To configure the GitHub app to work with Infisical, you'll need to modify the following settings: + - **Homepage URL**: Required to be set. Set it to the URL of your Infisical instance. (e.g. `https://app.infisical.com`) + - **Setup URL**: Set this to `https:///organization/secret-scanning` + - **Webhook URL**: Set this to `https:///api/v1/secret-scanning/webhook` + - **Webhook Secret**: Set this to a random string. This is used to verify the webhook request from Infisical. Use `openssl rand -base64 32` in your terminal to generate a random secret. + + + Remember to save the webhook secret as you will need it in the next step. + + + ![GitHub App Settings](/images/platform/secret-scanning/github-configure-app.png) + + ### Configure the GitHub App Permissions + The GitHub app needs the following permissions: + + Repository permissions: + - `Checks`: Read and Write + - `Contents`: Read-only + - `Issues`: Read and Write + - `Pull Requests`: Read and Write + - `Metadata`: Read-only (enabled by default) + + ![Github App Repository Permissions](/images/platform/secret-scanning/github-repo-permissions.png) + + Subscribed events: + - `Check run` + - `Pull request` + - `Push` + + ![Github App Subscribed Events](/images/platform/secret-scanning/github-subscribed-events.png) + + + ### Create the GitHub App + Now you can create the GitHub app by clicking on the "Create GitHub App" button. + + + If you want other Github users to be able to install the app, you need to tick the "Any account" option under "Where can this GitHub App be installed?" + + + ![Create GitHub App](/images/platform/secret-scanning/github-create-app-button.png) + + + + After clicking the "Create GitHub App" button, you will be redirected to the GitHub settings page. Here you can copy the "App ID" and save it for later when you need to configure your environment variables for your Infisical instance. + + ![Github App ID](/images/platform/secret-scanning/github-app-copy-app-id.png) + + + + The GitHub App slug is the name of the app you created in a slug friendly format. You can find the slug in the URL of the app you created. + + ![Github App Slug](/images/platform/secret-scanning/github-app-copy-slug.png) + + + + Create a new app private key by clicking on the "Generate a private key" button under the "Private keys" section. + + Once you click the "Generate a private key" button, the private key will be downloaded to your computer. Save this file for later as you will need the private key when configuring Infisical. + + ![Github App Private Key](/images/platform/secret-scanning/github-app-create-private-key.png) + + + Remember to save the private key as you will need it in the next step. + + + + + + + Now you can configure your Infisical instance by setting the following environment variables: + + - `SECRET_SCANNING_GIT_APP_ID`: The App ID of your GitHub App. + - `SECRET_SCANNING_GIT_APP_SLUG`: The slug of your GitHub App. + - `SECRET_SCANNING_PRIVATE_KEY`: The private key of your GitHub App that you created in a previous step. + - `SECRET_SCANNING_WEBHOOK_SECRET`: The webhook secret of your GitHub App that you created in a previous step. + + + + After restarting your Infisical instance, you should be able to use the secret scanning feature within your organization. Follow the steps below to add the GitHub App to your Infisical organization. + + +## Install the Infisical Radar GitHub App + +To install the GitHub App, press the "Integrate With GitHub" button in the top right corner of your Infisical Secret Scanning dashboard. + +![Integrate With GitHub](/images/platform/secret-scanning/infisical-connect-secret-scanner.png) + +Next, you'll be prompted to select which organization you'd like to install the app into. Select the organization you'd like to install the app into by clicking the organization in the menu. + +![Select Organization](/images/platform/secret-scanning/github-select-org-2.png) + +Select the repositories you'd like to scan for secrets and press the "Install" button. + +![Select Repositories](/images/platform/secret-scanning/github-select-repos.png) + ## Code Scanning ![Scanning Overview](/images/platform/secret-scanning/overview.png) diff --git a/docs/documentation/platform/ssh/overview.mdx b/docs/documentation/platform/ssh/overview.mdx index e71eeabe1..a252e1f71 100644 --- a/docs/documentation/platform/ssh/overview.mdx +++ b/docs/documentation/platform/ssh/overview.mdx @@ -31,16 +31,9 @@ we will register a remote host with Infisical through a [machine identity](/docu - 1.1. Start by creating a new Infisical SSH project in Infisical. + Start by creating a new Infisical SSH project in Infisical. ![ssh project create](/images/platform/ssh/v2/ssh-create-project.png) - - 1.2. Create a custom role in the project under Access Control > Project Roles to grant the machine identity that we will create in step 2 the ability to **Create** and **Issue Host Certificates** on the **SSH Host** resource; this will enable the linked machine identity to bootstrap a remote host with Infisical - and establish the necessary configuration on it. - - ![ssh custom role bootstrap 1](/images/platform/ssh/v2/ssh-add-bootstrap-role-1.png) - - ![ssh custom role bootstrap 2](/images/platform/ssh/v2/ssh-add-bootstrap-role-2.png) 2.1. Follow the instructions [here](/documentation/platform/identities/universal-auth) to configure a [machine identity](/documentation/platform/identities/machine-identities) in Infisical with Universal Auth. @@ -52,7 +45,14 @@ we will register a remote host with Infisical through a [machine identity](/docu You may use other authentication methods as suitable (e.g. [AWS Auth](/documentation/platform/identities/aws-auth), [Azure Auth](/documentation/platform/identities/azure-auth), [GCP Auth](/documentation/platform/identities/gcp-auth), etc.) as part of the machine identity configuration but, to keep this example simple, we will be using Universal Auth. - 2.2. Add the machine identity to the Infisical SSH project you created in the previous step and assign it the custom role you created in step 1.2. + 2.2. Add the machine identity to the Infisical SSH project you created in the previous step and assign it the **SSH Host Bootstrapper** role. + + This role grants the ability to **Create** and **Issue Host Certificates** on the **SSH Host** resource; this will enable the linked machine identity to bootstrap a remote host with Infisical + and establish the necessary configuration on it. + + + If you plan to use a custom role to bootstrap SSH hosts, ensure the role has the **Create** and **Issue Host Certificates** on the **SSH Host** resource. + ![ssh add identity to project](/images/platform/ssh/v2/ssh-add-identity-to-project.png) diff --git a/docs/images/platform/identities/ldap/identities-org-add-auth-method-modal.png b/docs/images/platform/identities/ldap/identities-org-add-auth-method-modal.png new file mode 100644 index 000000000..e9a5f276c Binary files /dev/null and b/docs/images/platform/identities/ldap/identities-org-add-auth-method-modal.png differ diff --git a/docs/images/platform/identities/ldap/identities-org-add-auth-method.png b/docs/images/platform/identities/ldap/identities-org-add-auth-method.png new file mode 100644 index 000000000..95d301010 Binary files /dev/null and b/docs/images/platform/identities/ldap/identities-org-add-auth-method.png differ diff --git a/docs/images/platform/identities/ldap/identities-org-configure-ldap.png b/docs/images/platform/identities/ldap/identities-org-configure-ldap.png new file mode 100644 index 000000000..c9dfb4950 Binary files /dev/null and b/docs/images/platform/identities/ldap/identities-org-configure-ldap.png differ diff --git a/docs/images/platform/identities/ldap/identities-org-create-identity-modal.png b/docs/images/platform/identities/ldap/identities-org-create-identity-modal.png new file mode 100644 index 000000000..3ac6555e4 Binary files /dev/null and b/docs/images/platform/identities/ldap/identities-org-create-identity-modal.png differ diff --git a/docs/images/platform/identities/ldap/identities-org-create-identity.png b/docs/images/platform/identities/ldap/identities-org-create-identity.png new file mode 100644 index 000000000..1086f6521 Binary files /dev/null and b/docs/images/platform/identities/ldap/identities-org-create-identity.png differ diff --git a/docs/images/platform/identities/ldap/jumpcloud-users-management.png b/docs/images/platform/identities/ldap/jumpcloud-users-management.png new file mode 100644 index 000000000..cc5dc13ca Binary files /dev/null and b/docs/images/platform/identities/ldap/jumpcloud-users-management.png differ diff --git a/docs/images/platform/project-templates/project-template-add-button.png b/docs/images/platform/project-templates/project-template-add-button.png index 965de1e9a..c71c5c938 100644 Binary files a/docs/images/platform/project-templates/project-template-add-button.png and b/docs/images/platform/project-templates/project-template-add-button.png differ diff --git a/docs/images/platform/project-templates/project-template-apply.png b/docs/images/platform/project-templates/project-template-apply.png index 1ec49cb43..0ed2d320c 100644 Binary files a/docs/images/platform/project-templates/project-template-apply.png and b/docs/images/platform/project-templates/project-template-apply.png differ diff --git a/docs/images/platform/project-templates/project-template-create.png b/docs/images/platform/project-templates/project-template-create.png index 6cd109049..6c485b4cc 100644 Binary files a/docs/images/platform/project-templates/project-template-create.png and b/docs/images/platform/project-templates/project-template-create.png differ diff --git a/docs/images/platform/project-templates/project-template-customized.png b/docs/images/platform/project-templates/project-template-customized.png index f21717326..182e669c2 100644 Binary files a/docs/images/platform/project-templates/project-template-customized.png and b/docs/images/platform/project-templates/project-template-customized.png differ diff --git a/docs/images/platform/project-templates/project-template-edit-form.png b/docs/images/platform/project-templates/project-template-edit-form.png index c4e29297f..72085468f 100644 Binary files a/docs/images/platform/project-templates/project-template-edit-form.png and b/docs/images/platform/project-templates/project-template-edit-form.png differ diff --git a/docs/images/platform/secret-scanning/github-app-copy-app-id.png b/docs/images/platform/secret-scanning/github-app-copy-app-id.png new file mode 100644 index 000000000..a94cb5ece Binary files /dev/null and b/docs/images/platform/secret-scanning/github-app-copy-app-id.png differ diff --git a/docs/images/platform/secret-scanning/github-app-copy-slug.png b/docs/images/platform/secret-scanning/github-app-copy-slug.png new file mode 100644 index 000000000..c555dcd41 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-app-copy-slug.png differ diff --git a/docs/images/platform/secret-scanning/github-app-create-private-key.png b/docs/images/platform/secret-scanning/github-app-create-private-key.png new file mode 100644 index 000000000..50f602a36 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-app-create-private-key.png differ diff --git a/docs/images/platform/secret-scanning/github-configure-app.png b/docs/images/platform/secret-scanning/github-configure-app.png new file mode 100644 index 000000000..df64eeb18 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-configure-app.png differ diff --git a/docs/images/platform/secret-scanning/github-create-app-button.png b/docs/images/platform/secret-scanning/github-create-app-button.png new file mode 100644 index 000000000..3ea4b2d38 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-create-app-button.png differ diff --git a/docs/images/platform/secret-scanning/github-create-app.png b/docs/images/platform/secret-scanning/github-create-app.png new file mode 100644 index 000000000..f4d1cdb8c Binary files /dev/null and b/docs/images/platform/secret-scanning/github-create-app.png differ diff --git a/docs/images/platform/secret-scanning/github-register-app.png b/docs/images/platform/secret-scanning/github-register-app.png new file mode 100644 index 000000000..904c07bf2 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-register-app.png differ diff --git a/docs/images/platform/secret-scanning/github-repo-permissions.png b/docs/images/platform/secret-scanning/github-repo-permissions.png new file mode 100644 index 000000000..53eae9a41 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-repo-permissions.png differ diff --git a/docs/images/platform/secret-scanning/github-select-org-2.png b/docs/images/platform/secret-scanning/github-select-org-2.png new file mode 100644 index 000000000..55b945c18 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-select-org-2.png differ diff --git a/docs/images/platform/secret-scanning/github-select-org.png b/docs/images/platform/secret-scanning/github-select-org.png new file mode 100644 index 000000000..7d6e5abc5 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-select-org.png differ diff --git a/docs/images/platform/secret-scanning/github-select-repos.png b/docs/images/platform/secret-scanning/github-select-repos.png new file mode 100644 index 000000000..51a6648d2 Binary files /dev/null and b/docs/images/platform/secret-scanning/github-select-repos.png differ diff --git a/docs/images/platform/secret-scanning/github-subscribed-events.png b/docs/images/platform/secret-scanning/github-subscribed-events.png new file mode 100644 index 000000000..7aa6b431f Binary files /dev/null and b/docs/images/platform/secret-scanning/github-subscribed-events.png differ diff --git a/docs/images/platform/secret-scanning/infisical-connect-secret-scanner.png b/docs/images/platform/secret-scanning/infisical-connect-secret-scanner.png new file mode 100644 index 000000000..11f24fd74 Binary files /dev/null and b/docs/images/platform/secret-scanning/infisical-connect-secret-scanner.png differ diff --git a/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-1.png b/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-1.png deleted file mode 100644 index 8acc1efe9..000000000 Binary files a/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-1.png and /dev/null differ diff --git a/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-2.png b/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-2.png deleted file mode 100644 index 2ad9804d4..000000000 Binary files a/docs/images/platform/ssh/v2/ssh-add-bootstrap-role-2.png and /dev/null differ diff --git a/docs/images/platform/ssh/v2/ssh-add-identity-to-project.png b/docs/images/platform/ssh/v2/ssh-add-identity-to-project.png index 83bd3c984..d921cc8d0 100644 Binary files a/docs/images/platform/ssh/v2/ssh-add-identity-to-project.png and b/docs/images/platform/ssh/v2/ssh-add-identity-to-project.png differ diff --git a/docs/integrations/platforms/kubernetes/infisical-dynamic-secret-crd.mdx b/docs/integrations/platforms/kubernetes/infisical-dynamic-secret-crd.mdx index 21f54994a..5962e4c10 100644 --- a/docs/integrations/platforms/kubernetes/infisical-dynamic-secret-crd.mdx +++ b/docs/integrations/platforms/kubernetes/infisical-dynamic-secret-crd.mdx @@ -165,7 +165,7 @@ spec: - Creation polices allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator. + Creation policies allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator. This is useful for tools such as ArgoCD, where every resource requires an owner reference; otherwise, it will be pruned automatically. #### Available options diff --git a/docs/integrations/platforms/kubernetes/infisical-push-secret-crd.mdx b/docs/integrations/platforms/kubernetes/infisical-push-secret-crd.mdx index 50f07bb76..d87648bbf 100644 --- a/docs/integrations/platforms/kubernetes/infisical-push-secret-crd.mdx +++ b/docs/integrations/platforms/kubernetes/infisical-push-secret-crd.mdx @@ -34,7 +34,7 @@ Before applying the InfisicalPushSecret CRD, you need to create a Kubernetes sec metadata: name: infisical-push-secret-demo spec: - resyncInterval: 1m + resyncInterval: 1m # Remove this field to disable automatic reconciliation of the InfisicalPushSecret CRD. hostAPI: https://app.infisical.com/api # Optional, defaults to no replacement. @@ -124,7 +124,9 @@ After applying the InfisicalPushSecret CRD, you should notice that the secrets y - The `resyncInterval` is a string-formatted duration that defines the time between each resync. + The `resyncInterval` is a string-formatted duration that defines the time between each resync. The field is optional, and will default to no automatic resync if not defined. + + If you don't want to automatically reconcile the InfisicalPushSecret CRD on an interval, you can remove the `resyncInterval` field entirely from your InfisicalPushSecret CRD. The format of the field is `[duration][unit]` where `duration` is a number and `unit` is a string representing the unit of time. @@ -239,7 +241,21 @@ After applying the InfisicalPushSecret CRD, you should notice that the secrets y DATABASE_URL: postgres://127.0.0.1:5432 ENCRYPTION_KEY: fabcc12-a22-facbaa4-11aa568aab ``` + + + The `generators[]` field is used to define the generators you want to use for your InfisicalPushSecret CRD. + You can follow the guide for [using generators to push secrets](#using-generators-to-push-secrets) for more information. + Example: + + ```yaml + push: + generators: + - destinationSecretName: password-generator-test + generatorRef: + kind: Password + name: password-generator + ``` @@ -459,6 +475,148 @@ Using Go templates, you can format, combine, and create new key-value pairs of s Please refer to the [templating functions documentation](/integrations/platforms/kubernetes/overview#available-helper-functions) for more information. +## Using generators to push secrets + +Generators allow secrets to be dynamically generated during each reconciliation cycle and then pushed to Infisical. They are useful for use cases where a new secret value is needed on every sync, such as ephemeral credentials or one-time-use tokens. + +A generator is defined as a custom resource (`ClusterGenerator`) within the cluster, which specifies the logic for generating secret values. Generators are stateless, each invocation triggers the creation of a new set of values, with no tracking or persistence of previously generated data. + +Because of this behavior, you may want to disable automatic syncing for the `InfisicalPushSecret` resource to avoid continuous regeneration of secrets. This can be done by omitting the `resyncInterval` field from the InfisicalPushSecret CRD. + +### Example usage +```yaml + push: + secret: + secretName: push-secret-source-secret + secretNamespace: dev + generators: + - destinationSecretName: password-generator # Name of the secret that will be created in Infisical + generatorRef: + kind: Password # Kind of the resource, must match the generator kind. + name: custom-generator # Name of the generator resource +``` + +To use a generator, you must specify at least one generator in the `push.generators[]` field. + + + + This field holds an array of the generators you want to use for your InfisicalPushSecret CRD. + + + + The name of the secret that will be created in Infisical. + + + + The reference to the generator resource. + + Valid fields: + - `kind`: The kind of the generator resource, must match the generator kind. + - `name`: The name of the generator resource. + + + + The kind of the generator resource, must match the generator kind. + + Valid values: + - `Password` + - `UUID` + + + + The name of the generator resource. + + +### Supported Generators +Below are the currently supported generators for the InfisicalPushSecret CRD. Each generator is a `ClusterGenerator` custom resource that can be used to customize the generated secret. + + + ### Password Generator + + The Password generator is a custom resource that is installed on the cluster that defines the logic for generating a password. + - `kind`: The kind of the generator resource, must match the generator kind. For the Password generator, the kind is `Password`. + - `generator.passwordSpec`: The spec of the password generator. + + + The `generator.kind` field must match the kind of the generator resource. For the Password generator, the kind should always be set to `Password`. + + + - `length`: The length of the password. + - `digits`: The number of digits in the password. + - `symbols`: The number of symbols in the password. + - `symbolCharacters`: The characters to use for the symbols in the password. + - `noUpper`: Whether to include uppercase letters in the password. + - `allowRepeat`: Whether to allow repeating characters in the password. + + + ```yaml password-cluster-generator.yaml + apiVersion: secrets.infisical.com/v1alpha1 + kind: ClusterGenerator + metadata: + name: password-generator + spec: + kind: Password + generator: + passwordSpec: + length: 10 + digits: 5 + symbols: 5 + symbolCharacters: "-_$@" + noUpper: false + allowRepeat: true + ``` + + Example InfisicalPushSecret CRD using the Password generator: + ```yaml infisical-push-secret-crd.yaml + push: + generators: + - destinationSecretName: password-generator-test + generatorRef: + kind: Password + name: password-generator + ``` + + + ### UUID Generator + + The UUID generator is a custom resource that is installed on the cluster that defines the logic for generating a UUID. + - `kind`: The kind of the generator resource, must match the generator kind. For the UUID generator, the kind is `UUID`. + - `generator.uuidSpec`: The spec of the UUID generator. For UUID's, this can be left empty. + + + The `generator.kind` field must match the kind of the generator resource. For the UUID generator, the kind should always be set to `UUID`. + + + + The spec of the UUID generator. For UUID's, this can be left empty. + + + ```yaml uuid-cluster-generator.yaml + apiVersion: secrets.infisical.com/v1alpha1 + kind: ClusterGenerator + metadata: + name: uuid-generator + spec: + kind: UUID + generator: + uuidSpec: + ``` + + Example InfisicalPushSecret CRD using the UUID generator: + + ```yaml infisical-push-secret-crd.yaml + push: + generators: + - destinationSecretName: uuid-generator-test + generatorRef: + kind: UUID + name: uuid-generator + ``` + + + + + ## Applying the InfisicalPushSecret CRD to your cluster Once you have configured the `InfisicalPushSecret` CRD with the required fields, you can apply it to your cluster. diff --git a/docs/integrations/platforms/kubernetes/infisical-secret-crd.mdx b/docs/integrations/platforms/kubernetes/infisical-secret-crd.mdx index a79a8d0a5..145737e96 100644 --- a/docs/integrations/platforms/kubernetes/infisical-secret-crd.mdx +++ b/docs/integrations/platforms/kubernetes/infisical-secret-crd.mdx @@ -232,7 +232,7 @@ spec: - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) @@ -407,7 +407,7 @@ spec: - To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. ![identities organization](/images/platform/identities/identities-org.png) @@ -832,7 +832,7 @@ The namespace of the managed Kubernetes secret to be created. Override the default Opaque type for managed secrets with this field. Useful for creating kubernetes.io/dockerconfigjson secrets. -Creation polices allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator. +Creation policies allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator. This is useful for tools such as ArgoCD, where every resource requires an owner reference; otherwise, it will be pruned automatically. #### Available options @@ -940,7 +940,7 @@ The Infisical operator will automatically create the Kubernetes config map in th The namespace of the managed Kubernetes config map that your Infisical data will be stored in. - Creation polices allow you to control whether or not owner references should be added to the managed Kubernetes config map that is generated by the Infisical operator. + Creation policies allow you to control whether or not owner references should be added to the managed Kubernetes config map that is generated by the Infisical operator. This is useful for tools such as ArgoCD, where every resource requires an owner reference; otherwise, it will be pruned automatically. #### Available options diff --git a/docs/internals/bug-bounty.mdx b/docs/internals/bug-bounty.mdx index b823e6246..e45de05bf 100644 --- a/docs/internals/bug-bounty.mdx +++ b/docs/internals/bug-bounty.mdx @@ -41,7 +41,7 @@ All final reward amounts are determined at Infisical's discretion based on impac ### Out of Scope -- Social engineering or phishing +- Social engineering or phishing (including email hyperlink injection without code execution) - Rate limiting issues on non-sensitive endpoints - Denial-of-service attacks that require authentication and don't impact core service availability - Findings based on outdated or forked code not maintained by the Infisical team @@ -57,4 +57,24 @@ We ask that researchers: - Use testing accounts where possible - Give us a reasonable window to investigate and patch before going public -Researchers can also spin up our [self-hosted version of Infisical](/self-hosting/overview) to test for vulnerabilities locally. \ No newline at end of file +Researchers can also spin up our [self-hosted version of Infisical](/self-hosting/overview) to test for vulnerabilities locally. + +### Program Conduct and Enforcement + +We value professional and collaborative interaction with security researchers. To maintain the integrity of our bug bounty program, we expect all participants to adhere to the following guidelines: + +- Maintain professional communication in all interactions +- Do not threaten public disclosure of vulnerabilities before we've had reasonable time to investigate and address the issue +- Do not attempt to extort or coerce compensation through threats +- Follow the responsible disclosure process outlined in this document +- Do not use automated scanning tools without prior permission + +Violations of these guidelines may result in: + +1. **Warning**: For minor violations, we may issue a warning explaining the violation and requesting compliance with program guidelines. +2. **Temporary Ban**: Repeated minor violations or more serious violations may result in a temporary suspension from the program. +3. **Permanent Ban**: Severe violations such as threats, extortion attempts, or unauthorized public disclosure will result in permanent removal from the Infisical Bug Bounty Program. + +We reserve the right to reject reports, withhold bounties, and remove participants from the program at our discretion for conduct that undermines the collaborative spirit of security research. + +Infisical is committed to working respectfully with security researchers who follow these guidelines, and we strive to recognize and reward valuable contributions that help protect our platform and users. diff --git a/docs/mint.json b/docs/mint.json index a25a70124..9e677843a 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -247,68 +247,88 @@ { "group": "Authentication Methods", "pages": [ - "documentation/platform/auth-methods/email-password", - "documentation/platform/token", - "documentation/platform/identities/token-auth", - "documentation/platform/identities/universal-auth", - "documentation/platform/identities/kubernetes-auth", - "documentation/platform/identities/gcp-auth", - "documentation/platform/identities/azure-auth", - "documentation/platform/identities/aws-auth", - "documentation/platform/identities/jwt-auth", { - "group": "OIDC Auth", + "group": "User Authentication", "pages": [ - "documentation/platform/identities/oidc-auth/general", - "documentation/platform/identities/oidc-auth/github", - "documentation/platform/identities/oidc-auth/circleci", - "documentation/platform/identities/oidc-auth/gitlab", - "documentation/platform/identities/oidc-auth/terraform-cloud" - ] - }, - "documentation/platform/mfa", - { - "group": "SSO", - "pages": [ - "documentation/platform/sso/overview", - "documentation/platform/sso/google", - "documentation/platform/sso/github", - "documentation/platform/sso/gitlab", - "documentation/platform/sso/okta", - "documentation/platform/sso/azure", - "documentation/platform/sso/jumpcloud", - "documentation/platform/sso/keycloak-saml", - "documentation/platform/sso/google-saml", - "documentation/platform/sso/auth0-saml", + "documentation/platform/auth-methods/email-password", { - "group": "Keycloak OIDC", + "group": "SSO", "pages": [ - "documentation/platform/sso/keycloak-oidc/overview", - "documentation/platform/sso/keycloak-oidc/group-membership-mapping" + "documentation/platform/sso/overview", + "documentation/platform/sso/google", + "documentation/platform/sso/github", + "documentation/platform/sso/gitlab", + "documentation/platform/sso/okta", + "documentation/platform/sso/azure", + "documentation/platform/sso/jumpcloud", + "documentation/platform/sso/keycloak-saml", + "documentation/platform/sso/google-saml", + "documentation/platform/sso/auth0-saml", + { + "group": "Keycloak OIDC", + "pages": [ + "documentation/platform/sso/keycloak-oidc/overview", + "documentation/platform/sso/keycloak-oidc/group-membership-mapping" + ] + }, + "documentation/platform/sso/auth0-oidc", + "documentation/platform/sso/general-oidc" ] }, - "documentation/platform/sso/auth0-oidc", - "documentation/platform/sso/general-oidc" + { + "group": "LDAP", + "pages": [ + "documentation/platform/ldap/overview", + "documentation/platform/ldap/jumpcloud", + "documentation/platform/ldap/general" + ] + }, + { + "group": "SCIM", + "pages": [ + "documentation/platform/scim/overview", + "documentation/platform/scim/okta", + "documentation/platform/scim/azure", + "documentation/platform/scim/jumpcloud", + "documentation/platform/scim/group-mappings" + ] + } ] }, + { - "group": "LDAP", + "group": "Machine Identities", "pages": [ - "documentation/platform/ldap/overview", - "documentation/platform/ldap/jumpcloud", - "documentation/platform/ldap/general" - ] - }, - { - "group": "SCIM", - "pages": [ - "documentation/platform/scim/overview", - "documentation/platform/scim/okta", - "documentation/platform/scim/azure", - "documentation/platform/scim/jumpcloud", - "documentation/platform/scim/group-mappings" + "documentation/platform/identities/token-auth", + "documentation/platform/identities/universal-auth", + "documentation/platform/identities/kubernetes-auth", + "documentation/platform/identities/gcp-auth", + "documentation/platform/identities/azure-auth", + "documentation/platform/identities/aws-auth", + "documentation/platform/identities/jwt-auth", + + { + "group": "OIDC Auth", + "pages": [ + "documentation/platform/identities/oidc-auth/general", + "documentation/platform/identities/oidc-auth/github", + "documentation/platform/identities/oidc-auth/circleci", + "documentation/platform/identities/oidc-auth/gitlab", + "documentation/platform/identities/oidc-auth/terraform-cloud" + ] + }, + + { + "group": "LDAP Auth", + "pages": [ + "documentation/platform/identities/ldap-auth/general", + "documentation/platform/identities/ldap-auth/jumpcloud" + ] + } ] }, + "documentation/platform/token", + "documentation/platform/mfa", "documentation/platform/github-org-sync" ] }, @@ -715,6 +735,16 @@ "api-reference/endpoints/jwt-auth/revoke" ] }, + { + "group": "LDAP Auth", + "pages": [ + "api-reference/endpoints/ldap-auth/login", + "api-reference/endpoints/ldap-auth/attach", + "api-reference/endpoints/ldap-auth/retrieve", + "api-reference/endpoints/ldap-auth/update", + "api-reference/endpoints/ldap-auth/revoke" + ] + }, { "group": "Groups", "pages": [ @@ -1454,6 +1484,8 @@ "api-reference/endpoints/certificates/revoke", "api-reference/endpoints/certificates/delete", "api-reference/endpoints/certificates/cert-body", + "api-reference/endpoints/certificates/bundle", + "api-reference/endpoints/certificates/private-key", "api-reference/endpoints/certificates/issue-certificate", "api-reference/endpoints/certificates/sign-certificate" ] diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index d9eef9cb0..b63c58d3a 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -29,6 +29,19 @@ Used to configure platform-specific security and operational settings Specifies the internal port on which the application listens. + + Specifies the network interface Infisical will bind to when accepting incoming connections. + + By default, Infisical binds to `localhost`, which restricts access to connections from the same machine. + + To make the application accessible externally (e.g., for self-hosted deployments), set this to `0.0.0.0`, which tells the server to listen on all network interfaces. + + Example values: + - `localhost` (default, same as `127.0.0.1`) + - `0.0.0.0` (all interfaces, accessible externally) + - `192.168.1.100` (specific interface IP) + + Telemetry helps us improve Infisical but if you want to disable it you may set this to `false`. @@ -612,6 +625,26 @@ To help you sync secrets from Infisical to services such as Github and Gitlab, I +## Secret Scanning + + + + The App ID of your GitHub App. + + + + The slug of your GitHub App. + + + + A private key for your GitHub App. + + + + The webhook secret of your GitHub App. + + + ## Observability You can configure Infisical to collect and expose telemetry data for analytics and monitoring. diff --git a/frontend/src/components/projects/NewProjectModal.tsx b/frontend/src/components/projects/NewProjectModal.tsx index dcd3040d8..c98118a96 100644 --- a/frontend/src/components/projects/NewProjectModal.tsx +++ b/frontend/src/components/projects/NewProjectModal.tsx @@ -72,7 +72,7 @@ const NewProjectForm = ({ onOpenChange, projectType }: NewProjectFormProps) => { OrgPermissionSubjects.ProjectTemplates ); - const { data: projectTemplates = [] } = useListProjectTemplates({ + const { data: projectTemplates = [] } = useListProjectTemplates(projectType, { enabled: Boolean(canReadProjectTemplates && subscription?.projectTemplates) }); diff --git a/frontend/src/components/projects/ProjectSettings/ProjectSettings.tsx b/frontend/src/components/projects/ProjectSettings/ProjectSettings.tsx new file mode 100644 index 000000000..3919ad86c --- /dev/null +++ b/frontend/src/components/projects/ProjectSettings/ProjectSettings.tsx @@ -0,0 +1,30 @@ +import { useState } from "react"; + +import { Tab, TabList, TabPanel, Tabs } from "@app/components/v2"; + +import { ProjectTemplatesTab } from "./components"; + +const tabs = [ + { name: "Project Templates", key: "project-templates", component: ProjectTemplatesTab } +]; + +export const ProjectSettings = () => { + const [selectedTab, setSelectedTab] = useState(tabs[0].key); + + return ( + + + {tabs.map((tab) => ( + + {tab.name} + + ))} + + {tabs.map(({ key, component: Component }) => ( + + + + ))} + + ); +}; diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/ProjectTemplatesTab.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/ProjectTemplatesTab.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/ProjectTemplatesTab.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/ProjectTemplatesTab.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/DeleteProjectTemplateModal.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/DeleteProjectTemplateModal.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/DeleteProjectTemplateModal.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/DeleteProjectTemplateModal.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/EditProjectTemplateSection.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/EditProjectTemplateSection.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/EditProjectTemplateSection.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/EditProjectTemplateSection.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx similarity index 92% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx index 98a675940..9b7164f80 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx +++ b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/EditProjectTemplate.tsx @@ -7,6 +7,7 @@ import { Button, DeleteActionModal } from "@app/components/v2"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/context"; import { usePopUp } from "@app/hooks"; import { TProjectTemplate, useDeleteProjectTemplate } from "@app/hooks/api/projectTemplates"; +import { ProjectType } from "@app/hooks/api/workspace/types"; import { ProjectTemplateDetailsModal } from "../../ProjectTemplateDetailsModal"; import { ProjectTemplateEnvironmentsForm } from "./ProjectTemplateEnvironmentsForm"; @@ -24,7 +25,7 @@ export const EditProjectTemplate = ({ isInfisicalTemplate, projectTemplate, onBa "editDetails" ] as const); - const { id: templateId, name, description } = projectTemplate; + const { id: templateId, name, description, type } = projectTemplate; const deleteProjectTemplate = useDeleteProjectTemplate(); @@ -94,10 +95,12 @@ export const EditProjectTemplate = ({ isInfisicalTemplate, projectTemplate, onBa )} - + {type === ProjectType.SecretManager && ( + + )} { - const { popUp, handlePopUpToggle } = usePopUp(["createPolicy"] as const); + const { popUp, handlePopUpToggle } = usePopUp(["addPolicy"] as const); const formMethods = useForm({ values: role ? { ...role, permissions: rolePermission2Form(role.permissions) } : undefined, @@ -119,34 +119,29 @@ export const ProjectTemplateEditRoleForm = ({ - handlePopUpToggle("createPolicy", isOpen)} + - - - handlePopUpToggle("createPolicy")} /> - - + Add Policies + + handlePopUpToggle("addPolicy", isOpen)} + /> )} diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx similarity index 93% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx index b72d12c73..ef2691d69 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx +++ b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateEnvironmentsForm.tsx @@ -19,7 +19,7 @@ import { THead, Tr } from "@app/components/v2"; -import { OrgPermissionActions, OrgPermissionSubjects } from "@app/context"; +import { OrgPermissionActions, OrgPermissionSubjects, useSubscription } from "@app/context"; import { TProjectTemplate, useUpdateProjectTemplate } from "@app/hooks/api/projectTemplates"; import { slugSchema } from "@app/lib/schemas"; @@ -35,6 +35,7 @@ const formSchema = z.object({ slug: slugSchema({ min: 1, max: 32 }) }) .array() + .nullish() }); type TFormSchema = z.infer; @@ -55,6 +56,8 @@ export const ProjectTemplateEnvironmentsForm = ({ resolver: zodResolver(formSchema) }); + const { subscription } = useSubscription(); + const { fields: environments, move, @@ -67,7 +70,7 @@ export const ProjectTemplateEnvironmentsForm = ({ const onFormSubmit = async (form: TFormSchema) => { try { const { environments: updatedEnvs } = await updateProjectTemplate.mutateAsync({ - environments: form.environments.map((env, index) => ({ + environments: form.environments?.map((env, index) => ({ ...env, position: index + 1 })), @@ -89,6 +92,9 @@ export const ProjectTemplateEnvironmentsForm = ({ } }; + const isEnvironmentLimitExceeded = + Boolean(subscription.environmentLimit) && environments.length >= subscription.environmentLimit; + return (
{(isAllowed) => ( diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateRolesSection.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateRolesSection.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateRolesSection.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/ProjectTemplateRolesSection.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/index.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/index.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/index.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/components/index.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/index.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/index.tsx similarity index 100% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/EditProjectTemplateSection/index.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/EditProjectTemplateSection/index.tsx diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx similarity index 91% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx index e601e0319..5b5728dac 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx +++ b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplateDetailsModal.tsx @@ -12,6 +12,7 @@ import { ModalContent, TextArea } from "@app/components/v2"; +import { useGetProjectTypeFromRoute } from "@app/hooks"; import { TProjectTemplate, useCreateProjectTemplate, @@ -41,6 +42,7 @@ type FormProps = { const ProjectTemplateForm = ({ onComplete, projectTemplate }: FormProps) => { const createProjectTemplate = useCreateProjectTemplate(); const updateProjectTemplate = useUpdateProjectTemplate(); + const projectType = useGetProjectTypeFromRoute(); const { handleSubmit, @@ -55,9 +57,17 @@ const ProjectTemplateForm = ({ onComplete, projectTemplate }: FormProps) => { }); const onFormSubmit = async (data: FormData) => { + if (!projectType) { + createNotification({ + text: "Failed to determine project type", + type: "error" + }); + return; + } + const mutation = projectTemplate ? updateProjectTemplate.mutateAsync({ templateId: projectTemplate.id, ...data }) - : createProjectTemplate.mutateAsync(data); + : createProjectTemplate.mutateAsync({ ...data, type: projectType }); try { const template = await mutation; createNotification({ diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx similarity index 98% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx index d4a076930..ec3f0aaff 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx +++ b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesSection.tsx @@ -50,7 +50,7 @@ export const ProjectTemplatesSection = () => { className="absolute min-h-[10rem] w-full" >
-

+

Create and configure templates with predefined roles and environments to streamline project setup

diff --git a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx similarity index 73% rename from frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx rename to frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx index b8dc00ab2..0465973e2 100644 --- a/frontend/src/pages/organization/SettingsPage/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx +++ b/frontend/src/components/projects/ProjectSettings/components/ProjectTemplatesTab/components/ProjectTemplatesTable.tsx @@ -23,8 +23,9 @@ import { Tr } from "@app/components/v2"; import { OrgPermissionActions, OrgPermissionSubjects, useSubscription } from "@app/context"; -import { usePopUp } from "@app/hooks"; +import { useGetProjectTypeFromRoute, usePopUp } from "@app/hooks"; import { TProjectTemplate, useListProjectTemplates } from "@app/hooks/api/projectTemplates"; +import { ProjectType } from "@app/hooks/api/workspace/types"; import { DeleteProjectTemplateModal } from "./DeleteProjectTemplateModal"; @@ -35,9 +36,12 @@ type Props = { export const ProjectTemplatesTable = ({ onEdit }: Props) => { const { subscription } = useSubscription(); - const { isPending, data: projectTemplates = [] } = useListProjectTemplates({ - enabled: subscription?.projectTemplates + const projectType = useGetProjectTypeFromRoute(); + + const { isPending, data: projectTemplates = [] } = useListProjectTemplates(projectType, { + enabled: subscription?.projectTemplates && Boolean(projectType) }); + const [search, setSearch] = useState(""); const { popUp, handlePopUpOpen, handlePopUpToggle } = usePopUp(["deleteTemplate"] as const); @@ -50,6 +54,10 @@ export const ProjectTemplatesTable = ({ onEdit }: Props) => { [search, projectTemplates] ); + const isSecretManagerTemplates = projectType === ProjectType.SecretManager; + + const colSpan = isSecretManagerTemplates ? 4 : 3; + return (
{ Name Roles - Environments + {isSecretManagerTemplates && Environments} - {isPending && ( + {subscription?.projectTemplates && isPending && ( )} {filteredTemplates.map((template) => { - const { id, name, roles, environments, description } = template; + const { id, name, roles, environments = [], description } = template; return ( onEdit(template)} @@ -116,28 +124,30 @@ export const ProjectTemplatesTable = ({ onEdit }: Props) => { )} - - {environments.length} - {environments.length > 0 && ( - - {environments - .sort((a, b) => (a.position > b.position ? 1 : -1)) - .map((env) => ( -
  • {env.name}
    • - ))} - - } - > - -     - )} - + {isSecretManagerTemplates && environments && ( + + {environments.length} + {environments.length > 0 && ( + + {environments + .sort((a, b) => (a.position > b.position ? 1 : -1)) + .map((env) => ( +
  • {env.name}
    • + ))} + + } + > + +     + )} + + )} {name !== "default" && ( { ); })} - {!isPending && filteredTemplates?.length === 0 && ( + {(!subscription?.projectTemplates || + (!isPending && filteredTemplates?.length === 0)) && ( - + { +export const NoticeBannerV2 = ({ title, children, className, titleClassName }: Props) => { return (
    { className )} > -
    +
    {title}
    diff --git a/frontend/src/helpers/roles.ts b/frontend/src/helpers/roles.ts index dcf80d452..8d7f5e81c 100644 --- a/frontend/src/helpers/roles.ts +++ b/frontend/src/helpers/roles.ts @@ -6,21 +6,9 @@ export enum OrgMembershipRole { NoAccess = "no-access" } -enum ProjectMemberRole { - Admin = "admin", - Member = "member", - Viewer = "viewer", - NoAccess = "no-access" -} - export const isCustomOrgRole = (slug: string) => !Object.values(OrgMembershipRole).includes(slug as OrgMembershipRole); -export const formatProjectRoleName = (name: string) => { - if (name === ProjectMemberRole.Member) return "developer"; - return name; -}; - export const isCustomProjectRole = (slug: string) => !Object.values(ProjectMembershipRole).includes(slug as ProjectMembershipRole); @@ -28,3 +16,24 @@ export const findOrgMembershipRole = (roles: TOrgRole[], roleIdOrSlug: string) = isCustomOrgRole(roleIdOrSlug) ? roles.find((r) => r.id === roleIdOrSlug) : roles.find((r) => r.slug === roleIdOrSlug); + +export const formatProjectRoleName = (role: string, customRoleName?: string) => { + switch (role) { + case ProjectMembershipRole.Admin: + return "Admin"; + case ProjectMembershipRole.Member: + return "Developer"; + case ProjectMembershipRole.Viewer: + return "Viewer"; + case ProjectMembershipRole.NoAccess: + return "No Access"; + case ProjectMembershipRole.Custom: + return customRoleName ?? role; + case ProjectMembershipRole.SshHostBootstrapper: + return "SSH Host Bootstrapper"; + case ProjectMembershipRole.KmsCryptographicOperator: + return "Cryptographic Operator"; + default: + return role; + } +}; diff --git a/frontend/src/hooks/api/admin/index.ts b/frontend/src/hooks/api/admin/index.ts index bc812e18e..5bedf1158 100644 --- a/frontend/src/hooks/api/admin/index.ts +++ b/frontend/src/hooks/api/admin/index.ts @@ -3,6 +3,7 @@ export { useAdminGrantServerAdminAccess, useAdminRemoveIdentitySuperAdminAccess, useCreateAdminUser, + useInvalidateCache, useRemoveUserServerAdminAccess, useUpdateServerConfig, useUpdateServerEncryptionStrategy diff --git a/frontend/src/hooks/api/admin/mutation.ts b/frontend/src/hooks/api/admin/mutation.ts index 2c88d4fd8..f220573c9 100644 --- a/frontend/src/hooks/api/admin/mutation.ts +++ b/frontend/src/hooks/api/admin/mutation.ts @@ -8,6 +8,7 @@ import { adminQueryKeys, adminStandaloneKeys } from "./queries"; import { RootKeyEncryptionStrategy, TCreateAdminUserDTO, + TInvalidateCacheDTO, TServerConfig, TUpdateServerConfigDTO } from "./types"; @@ -126,3 +127,15 @@ export const useUpdateServerEncryptionStrategy = () => { } }); }; + +export const useInvalidateCache = () => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: async (dto) => { + await apiRequest.post("/api/v1/admin/invalidate-cache", dto); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: adminQueryKeys.getInvalidateCache() }); + } + }); +}; diff --git a/frontend/src/hooks/api/admin/queries.ts b/frontend/src/hooks/api/admin/queries.ts index 1d44a93d0..85c6c153e 100644 --- a/frontend/src/hooks/api/admin/queries.ts +++ b/frontend/src/hooks/api/admin/queries.ts @@ -8,6 +8,7 @@ import { AdminGetIdentitiesFilters, AdminGetUsersFilters, AdminIntegrationsConfig, + TGetInvalidatingCacheStatus, TGetServerRootKmsEncryptionDetails, TServerConfig } from "./types"; @@ -22,8 +23,10 @@ export const adminQueryKeys = { getUsers: (filters: AdminGetUsersFilters) => [adminStandaloneKeys.getUsers, { filters }] as const, getIdentities: (filters: AdminGetIdentitiesFilters) => [adminStandaloneKeys.getIdentities, { filters }] as const, - getAdminIntegrationsConfig: () => ["admin-integrations-config"] as const, - getServerEncryptionStrategies: () => ["server-encryption-strategies"] as const + getAdminSlackConfig: () => ["admin-slack-config"] as const, + getServerEncryptionStrategies: () => ["server-encryption-strategies"] as const, + getInvalidateCache: () => ["admin-invalidate-cache"] as const, + getAdminIntegrationsConfig: () => ["admin-integrations-config"] as const }; export const fetchServerConfig = async () => { @@ -118,3 +121,18 @@ export const useGetServerRootKmsEncryptionDetails = () => { } }); }; + +export const useGetInvalidatingCacheStatus = (enabled = true) => { + return useQuery({ + queryKey: adminQueryKeys.getInvalidateCache(), + queryFn: async () => { + const { data } = await apiRequest.get( + "/api/v1/admin/invalidating-cache-status" + ); + + return data.invalidating; + }, + enabled, + refetchInterval: (data) => (data ? 3000 : false) + }); +}; diff --git a/frontend/src/hooks/api/admin/types.ts b/frontend/src/hooks/api/admin/types.ts index 4533b5963..8850b3375 100644 --- a/frontend/src/hooks/api/admin/types.ts +++ b/frontend/src/hooks/api/admin/types.ts @@ -24,6 +24,7 @@ export type TServerConfig = { enabledLoginMethods: LoginMethod[]; authConsentContent?: string; pageFrameContent?: string; + invalidatingCache: boolean; }; export type TUpdateServerConfigDTO = { @@ -84,3 +85,16 @@ export enum RootKeyEncryptionStrategy { Software = "SOFTWARE", HSM = "HSM" } + +export enum CacheType { + ALL = "all", + SECRETS = "secrets" +} + +export type TInvalidateCacheDTO = { + type: CacheType; +}; + +export type TGetInvalidatingCacheStatus = { + invalidating: boolean; +}; diff --git a/frontend/src/hooks/api/auditLogs/constants.tsx b/frontend/src/hooks/api/auditLogs/constants.tsx index 1bfe86d33..f84ddabe7 100644 --- a/frontend/src/hooks/api/auditLogs/constants.tsx +++ b/frontend/src/hooks/api/auditLogs/constants.tsx @@ -183,10 +183,17 @@ export const eventToNameMap: { [K in EventType]: string } = { "Microsoft Teams Workflow Integration Check Installation Status", [EventType.MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_GET_TEAMS]: "Get Microsoft Teams tenant teams", [EventType.MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_GET]: "Get Microsoft Teams Workflow Integration", - [EventType.MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_LIST]: "List Microsoft Teams Workflow Integration" + [EventType.MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_LIST]: + "List Microsoft Teams Workflow Integration", + + [EventType.LOGIN_IDENTITY_LDAP_AUTH]: "Identity login via LDAP Auth", + [EventType.ADD_IDENTITY_LDAP_AUTH]: "Attached LDAP Auth to identity", + [EventType.UPDATE_IDENTITY_LDAP_AUTH]: "Updated LDAP Auth for identity", + [EventType.GET_IDENTITY_LDAP_AUTH]: "Retrieved LDAP Auth for identity", + [EventType.REVOKE_IDENTITY_LDAP_AUTH]: "Revoked LDAP Auth for identity" }; -export const userAgentTTypeoNameMap: { [K in UserAgentType]: string } = { +export const userAgentTypeToNameMap: { [K in UserAgentType]: string } = { [UserAgentType.WEB]: "Web", [UserAgentType.CLI]: "CLI", [UserAgentType.K8_OPERATOR]: "K8s operator", diff --git a/frontend/src/hooks/api/auditLogs/enums.tsx b/frontend/src/hooks/api/auditLogs/enums.tsx index 9916619bd..82f07f385 100644 --- a/frontend/src/hooks/api/auditLogs/enums.tsx +++ b/frontend/src/hooks/api/auditLogs/enums.tsx @@ -47,6 +47,13 @@ export enum EventType { CREATE_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRET = "create-identity-universal-auth-client-secret", REVOKE_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRET = "revoke-identity-universal-auth-client-secret", GET_IDENTITY_UNIVERSAL_AUTH_CLIENT_SECRETS = "get-identity-universal-auth-client-secret", + + LOGIN_IDENTITY_LDAP_AUTH = "login-identity-ldap-auth", + ADD_IDENTITY_LDAP_AUTH = "add-identity-ldap-auth", + UPDATE_IDENTITY_LDAP_AUTH = "update-identity-ldap-auth", + GET_IDENTITY_LDAP_AUTH = "get-identity-ldap-auth", + REVOKE_IDENTITY_LDAP_AUTH = "revoke-identity-ldap-auth", + CREATE_ENVIRONMENT = "create-environment", UPDATE_ENVIRONMENT = "update-environment", DELETE_ENVIRONMENT = "delete-environment", diff --git a/frontend/src/hooks/api/identities/constants.tsx b/frontend/src/hooks/api/identities/constants.tsx index c11d7dc11..97acd6dfc 100644 --- a/frontend/src/hooks/api/identities/constants.tsx +++ b/frontend/src/hooks/api/identities/constants.tsx @@ -8,5 +8,6 @@ export const identityAuthToNameMap: { [I in IdentityAuthMethod]: string } = { [IdentityAuthMethod.AWS_AUTH]: "AWS Auth", [IdentityAuthMethod.AZURE_AUTH]: "Azure Auth", [IdentityAuthMethod.OIDC_AUTH]: "OIDC Auth", + [IdentityAuthMethod.LDAP_AUTH]: "LDAP Auth", [IdentityAuthMethod.JWT_AUTH]: "JWT Auth" }; diff --git a/frontend/src/hooks/api/identities/enums.tsx b/frontend/src/hooks/api/identities/enums.tsx index 415492e00..8a8d99fae 100644 --- a/frontend/src/hooks/api/identities/enums.tsx +++ b/frontend/src/hooks/api/identities/enums.tsx @@ -6,6 +6,7 @@ export enum IdentityAuthMethod { AWS_AUTH = "aws-auth", AZURE_AUTH = "azure-auth", OIDC_AUTH = "oidc-auth", + LDAP_AUTH = "ldap-auth", JWT_AUTH = "jwt-auth" } diff --git a/frontend/src/hooks/api/identities/index.tsx b/frontend/src/hooks/api/identities/index.tsx index f3b9fa012..bf49387ac 100644 --- a/frontend/src/hooks/api/identities/index.tsx +++ b/frontend/src/hooks/api/identities/index.tsx @@ -1,51 +1,4 @@ export { identityAuthToNameMap } from "./constants"; export { IdentityAuthMethod } from "./enums"; -export { - useAddIdentityAwsAuth, - useAddIdentityAzureAuth, - useAddIdentityGcpAuth, - useAddIdentityJwtAuth, - useAddIdentityKubernetesAuth, - useAddIdentityOidcAuth, - useAddIdentityTokenAuth, - useAddIdentityUniversalAuth, - useCreateIdentity, - useCreateIdentityUniversalAuthClientSecret, - useCreateTokenIdentityTokenAuth, - useDeleteIdentity, - useDeleteIdentityAwsAuth, - useDeleteIdentityAzureAuth, - useDeleteIdentityGcpAuth, - useDeleteIdentityJwtAuth, - useDeleteIdentityKubernetesAuth, - useDeleteIdentityOidcAuth, - useDeleteIdentityTokenAuth, - useDeleteIdentityUniversalAuth, - useRevokeIdentityTokenAuthToken, - useRevokeIdentityUniversalAuthClientSecret, - useUpdateIdentity, - useUpdateIdentityAwsAuth, - useUpdateIdentityAzureAuth, - useUpdateIdentityGcpAuth, - useUpdateIdentityJwtAuth, - useUpdateIdentityKubernetesAuth, - useUpdateIdentityOidcAuth, - useUpdateIdentityTokenAuth, - useUpdateIdentityTokenAuthToken, - useUpdateIdentityUniversalAuth -} from "./mutations"; -export { - useGetIdentityAwsAuth, - useGetIdentityAzureAuth, - useGetIdentityById, - useGetIdentityGcpAuth, - useGetIdentityJwtAuth, - useGetIdentityKubernetesAuth, - useGetIdentityOidcAuth, - useGetIdentityProjectMemberships, - useGetIdentityTokenAuth, - useGetIdentityTokensTokenAuth, - useGetIdentityUniversalAuth, - useGetIdentityUniversalAuthClientSecrets, - useSearchIdentities -} from "./queries"; +export * from "./mutations"; +export * from "./queries"; diff --git a/frontend/src/hooks/api/identities/mutations.tsx b/frontend/src/hooks/api/identities/mutations.tsx index d68595ad5..e0077527f 100644 --- a/frontend/src/hooks/api/identities/mutations.tsx +++ b/frontend/src/hooks/api/identities/mutations.tsx @@ -10,6 +10,7 @@ import { AddIdentityGcpAuthDTO, AddIdentityJwtAuthDTO, AddIdentityKubernetesAuthDTO, + AddIdentityLdapAuthDTO, AddIdentityOidcAuthDTO, AddIdentityTokenAuthDTO, AddIdentityUniversalAuthDTO, @@ -25,6 +26,7 @@ import { DeleteIdentityGcpAuthDTO, DeleteIdentityJwtAuthDTO, DeleteIdentityKubernetesAuthDTO, + DeleteIdentityLdapAuthDTO, DeleteIdentityOidcAuthDTO, DeleteIdentityTokenAuthDTO, DeleteIdentityUniversalAuthClientSecretDTO, @@ -36,6 +38,7 @@ import { IdentityGcpAuth, IdentityJwtAuth, IdentityKubernetesAuth, + IdentityLdapAuth, IdentityOidcAuth, IdentityTokenAuth, IdentityUniversalAuth, @@ -47,6 +50,7 @@ import { UpdateIdentityGcpAuthDTO, UpdateIdentityJwtAuthDTO, UpdateIdentityKubernetesAuthDTO, + UpdateIdentityLdapAuthDTO, UpdateIdentityOidcAuthDTO, UpdateIdentityTokenAuthDTO, UpdateIdentityUniversalAuthDTO, @@ -1049,3 +1053,116 @@ export const useRevokeIdentityTokenAuthToken = () => { } }); }; + +export const useAddIdentityLdapAuth = () => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: async ({ + identityId, + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps + }) => { + const { data } = await apiRequest.post<{ identityLdapAuth: IdentityLdapAuth }>( + `/api/v1/auth/ldap-auth/identities/${identityId}`, + { + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps + } + ); + return data.identityLdapAuth; + }, + onSuccess: (_, { identityId, organizationId }) => { + queryClient.invalidateQueries({ + queryKey: organizationKeys.getOrgIdentityMemberships(organizationId) + }); + queryClient.invalidateQueries({ queryKey: identitiesKeys.getIdentityById(identityId) }); + queryClient.invalidateQueries({ + queryKey: identitiesKeys.getIdentityLdapAuth(identityId) + }); + } + }); +}; + +export const useUpdateIdentityLdapAuth = () => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: async ({ + identityId, + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps + }) => { + const { data } = await apiRequest.patch<{ identityLdapAuth: IdentityLdapAuth }>( + `/api/v1/auth/ldap-auth/identities/${identityId}`, + { + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps + } + ); + return data.identityLdapAuth; + }, + onSuccess: (_, { identityId, organizationId }) => { + queryClient.invalidateQueries({ + queryKey: organizationKeys.getOrgIdentityMemberships(organizationId) + }); + queryClient.invalidateQueries({ queryKey: identitiesKeys.getIdentityById(identityId) }); + queryClient.invalidateQueries({ + queryKey: identitiesKeys.getIdentityLdapAuth(identityId) + }); + } + }); +}; + +export const useDeleteIdentityLdapAuth = () => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: async ({ identityId }) => { + const { data } = await apiRequest.delete(`/api/v1/auth/ldap-auth/identities/${identityId}`); + return data.identityLdapAuth; + }, + onSuccess: (_, { organizationId, identityId }) => { + queryClient.invalidateQueries({ + queryKey: organizationKeys.getOrgIdentityMemberships(organizationId) + }); + queryClient.invalidateQueries({ queryKey: identitiesKeys.getIdentityById(identityId) }); + queryClient.invalidateQueries({ + queryKey: identitiesKeys.getIdentityLdapAuth(identityId) + }); + } + }); +}; diff --git a/frontend/src/hooks/api/identities/queries.tsx b/frontend/src/hooks/api/identities/queries.tsx index 6b8a1d1cc..3bc94534c 100644 --- a/frontend/src/hooks/api/identities/queries.tsx +++ b/frontend/src/hooks/api/identities/queries.tsx @@ -11,6 +11,7 @@ import { IdentityGcpAuth, IdentityJwtAuth, IdentityKubernetesAuth, + IdentityLdapAuth, IdentityMembership, IdentityMembershipOrg, IdentityOidcAuth, @@ -34,6 +35,7 @@ export const identitiesKeys = { getIdentityAzureAuth: (identityId: string) => [{ identityId }, "identity-azure-auth"] as const, getIdentityTokenAuth: (identityId: string) => [{ identityId }, "identity-token-auth"] as const, getIdentityJwtAuth: (identityId: string) => [{ identityId }, "identity-jwt-auth"] as const, + getIdentityLdapAuth: (identityId: string) => [{ identityId }, "identity-ldap-auth"] as const, getIdentityTokensTokenAuth: (identityId: string) => [{ identityId }, "identity-tokens-token-auth"] as const, getIdentityProjectMemberships: (identityId: string) => @@ -231,6 +233,27 @@ export const useGetIdentityTokenAuth = ( }); }; +export const useGetIdentityLdapAuth = ( + identityId: string, + options?: TReactQueryOptions["options"] +) => { + return useQuery({ + queryKey: identitiesKeys.getIdentityLdapAuth(identityId), + queryFn: async () => { + const { + data: { identityLdapAuth } + } = await apiRequest.get<{ identityLdapAuth: IdentityLdapAuth }>( + `/api/v1/auth/ldap-auth/identities/${identityId}` + ); + return identityLdapAuth; + }, + staleTime: 0, + gcTime: 0, + ...options, + enabled: Boolean(identityId) && (options?.enabled ?? true) + }); +}; + export const useGetIdentityTokensTokenAuth = (identityId: string) => { return useQuery({ enabled: Boolean(identityId), diff --git a/frontend/src/hooks/api/identities/types.ts b/frontend/src/hooks/api/identities/types.ts index ca06219aa..c5f8cbc4a 100644 --- a/frontend/src/hooks/api/identities/types.ts +++ b/frontend/src/hooks/api/identities/types.ts @@ -425,6 +425,72 @@ export type IdentityTokenAuth = { accessTokenTrustedIps: IdentityTrustedIp[]; }; +export type AddIdentityLdapAuthDTO = { + organizationId: string; + identityId: string; + url: string; + bindDN: string; + bindPass: string; + searchBase: string; + searchFilter: string; + ldapCaCertificate?: string; + allowedFields?: { + key: string; + value: string; + }[]; + accessTokenTTL: number; + accessTokenMaxTTL: number; + accessTokenNumUsesLimit: number; + accessTokenTrustedIps: { + ipAddress: string; + }[]; +}; + +export type UpdateIdentityLdapAuthDTO = { + identityId: string; + organizationId: string; + url?: string; + bindDN?: string; + bindPass?: string; + searchBase?: string; + searchFilter?: string; + ldapCaCertificate?: string; + allowedFields?: { + key: string; + value: string; + }[]; + accessTokenTTL?: number; + accessTokenMaxTTL?: number; + accessTokenNumUsesLimit?: number; + accessTokenTrustedIps?: { + ipAddress: string; + }[]; +}; + +export type DeleteIdentityLdapAuthDTO = { + organizationId: string; + identityId: string; +}; + +export type IdentityLdapAuth = { + url: string; + bindDN: string; + bindPass: string; + searchBase: string; + searchFilter: string; + ldapCaCertificate?: string; + allowedFields?: { + key: string; + value: string; + }[]; + + identityId: string; + accessTokenTTL: number; + accessTokenMaxTTL: number; + accessTokenNumUsesLimit: number; + accessTokenTrustedIps: IdentityTrustedIp[]; +}; + export type AddIdentityTokenAuthDTO = { organizationId: string; identityId: string; diff --git a/frontend/src/hooks/api/projectTemplates/queries.tsx b/frontend/src/hooks/api/projectTemplates/queries.tsx index f5863915a..89bc96715 100644 --- a/frontend/src/hooks/api/projectTemplates/queries.tsx +++ b/frontend/src/hooks/api/projectTemplates/queries.tsx @@ -6,14 +6,17 @@ import { TProjectTemplate, TProjectTemplateResponse } from "@app/hooks/api/projectTemplates/types"; +import { ProjectType } from "@app/hooks/api/workspace/types"; export const projectTemplateKeys = { all: ["project-template"] as const, - list: () => [...projectTemplateKeys.all, "list"] as const, + list: (projectType?: ProjectType) => + [...projectTemplateKeys.all, "list", ...(projectType ? [projectType] : [])] as const, byId: (templateId: string) => [...projectTemplateKeys.all, templateId] as const }; export const useListProjectTemplates = ( + type?: ProjectType, options?: Omit< UseQueryOptions< TProjectTemplate[], @@ -25,9 +28,11 @@ export const useListProjectTemplates = ( > ) => { return useQuery({ - queryKey: projectTemplateKeys.list(), + queryKey: projectTemplateKeys.list(type), queryFn: async () => { - const { data } = await apiRequest.get("/api/v1/project-templates"); + const { data } = await apiRequest.get("/api/v1/project-templates", { + params: { type } + }); return data.projectTemplates; }, diff --git a/frontend/src/hooks/api/projectTemplates/types.ts b/frontend/src/hooks/api/projectTemplates/types.ts index 37c6cc902..9f6ea5ef3 100644 --- a/frontend/src/hooks/api/projectTemplates/types.ts +++ b/frontend/src/hooks/api/projectTemplates/types.ts @@ -1,11 +1,13 @@ import { TProjectRole } from "@app/hooks/api/roles/types"; +import { ProjectType } from "@app/hooks/api/workspace/types"; export type TProjectTemplate = { id: string; name: string; + type: ProjectType; description?: string; roles: Pick[]; - environments: { name: string; slug: string; position: number }[]; + environments?: { name: string; slug: string; position: number }[] | null; createdAt: string; updatedAt: string; }; @@ -14,6 +16,7 @@ export type TListProjectTemplates = { projectTemplates: TProjectTemplate[] }; export type TProjectTemplateResponse = { projectTemplate: TProjectTemplate }; export type TCreateProjectTemplateDTO = { + type: ProjectType; name: string; description?: string; }; diff --git a/frontend/src/hooks/api/roles/types.ts b/frontend/src/hooks/api/roles/types.ts index 0a48c9f97..ee95e8c23 100644 --- a/frontend/src/hooks/api/roles/types.ts +++ b/frontend/src/hooks/api/roles/types.ts @@ -3,7 +3,9 @@ export enum ProjectMembershipRole { Member = "member", Custom = "custom", Viewer = "viewer", - NoAccess = "no-access" + NoAccess = "no-access", + SshHostBootstrapper = "ssh-host-bootstrapper", + KmsCryptographicOperator = "cryptographic-operator" } export type TGetProjectRolesDTO = { diff --git a/frontend/src/hooks/api/secretScanning/mutation.ts b/frontend/src/hooks/api/secretScanning/mutation.ts index 7298b9af1..5055da4aa 100644 --- a/frontend/src/hooks/api/secretScanning/mutation.ts +++ b/frontend/src/hooks/api/secretScanning/mutation.ts @@ -10,15 +10,17 @@ import { } from "./types"; export const useCreateNewInstallationSession = () => { - return useMutation<{ sessionId: string }, object, { organizationId: string }>({ - mutationFn: async (opt) => { - const { data } = await apiRequest.post( - "/api/v1/secret-scanning/create-installation-session/organization", - opt - ); - return data; + return useMutation<{ sessionId: string; gitAppSlug: string }, object, { organizationId: string }>( + { + mutationFn: async (opt) => { + const { data } = await apiRequest.post( + "/api/v1/secret-scanning/create-installation-session/organization", + opt + ); + return data; + } } - }); + ); }; export const useUpdateRiskStatus = () => { diff --git a/frontend/src/hooks/api/sshHost/types.ts b/frontend/src/hooks/api/sshHost/types.ts index e92ddeaa8..ff33664b1 100644 --- a/frontend/src/hooks/api/sshHost/types.ts +++ b/frontend/src/hooks/api/sshHost/types.ts @@ -6,7 +6,8 @@ export enum LoginMappingSource { export type TLoginMapping = { loginUser: string; allowedPrincipals: { - usernames: string[]; + usernames?: string[]; + groups?: string[]; }; source: LoginMappingSource; }; diff --git a/frontend/src/hooks/api/workspace/query-keys.tsx b/frontend/src/hooks/api/workspace/query-keys.tsx index 91fe95a04..335248a80 100644 --- a/frontend/src/hooks/api/workspace/query-keys.tsx +++ b/frontend/src/hooks/api/workspace/query-keys.tsx @@ -16,8 +16,11 @@ export const workspaceKeys = { type ? ["workspaces", { type }] : (["workspaces"] as const), getWorkspaceAuditLogs: (workspaceId: string) => [{ workspaceId }, "workspace-audit-logs"] as const, - getWorkspaceUsers: (workspaceId: string, includeGroupMembers?: boolean, roles?: string[]) => - [{ workspaceId, includeGroupMembers, roles }, "workspace-users"] as const, + getWorkspaceUsers: ( + workspaceId: string, + includeGroupMembers: boolean = false, + roles: string[] = [] + ) => [{ workspaceId, includeGroupMembers, roles }, "workspace-users"] as const, getWorkspaceUserDetails: (workspaceId: string, membershipId: string) => [{ workspaceId, membershipId }, "workspace-user-details"] as const, getWorkspaceIdentityMemberships: (workspaceId: string) => diff --git a/frontend/src/hooks/index.ts b/frontend/src/hooks/index.ts index eb4cdbdca..9e5eff713 100644 --- a/frontend/src/hooks/index.ts +++ b/frontend/src/hooks/index.ts @@ -1,4 +1,5 @@ export { useDebounce } from "./useDebounce"; +export * from "./useGetProjectTypeFromRoute"; export { usePagination } from "./usePagination"; export { usePersistentState } from "./usePersistentState"; export { usePopUp } from "./usePopUp"; diff --git a/frontend/src/hooks/useGetProjectTypeFromRoute.tsx b/frontend/src/hooks/useGetProjectTypeFromRoute.tsx new file mode 100644 index 000000000..b3b8f3d5e --- /dev/null +++ b/frontend/src/hooks/useGetProjectTypeFromRoute.tsx @@ -0,0 +1,22 @@ +import { useMemo } from "react"; +import { useRouterState } from "@tanstack/react-router"; + +import { ProjectType } from "@app/hooks/api/workspace/types"; + +export const useGetProjectTypeFromRoute = () => { + const { location } = useRouterState(); + + return useMemo(() => { + const segments = location.pathname.split("/"); + + let type: ProjectType | undefined; + + // location of project type can vary in router path, so we need to check all possible values + segments.forEach((segment) => { + if (Object.values(ProjectType).includes(segment as ProjectType)) + type = segment as ProjectType; + }); + + return type; + }, [location]); +}; diff --git a/frontend/src/layouts/OrganizationLayout/OrganizationLayout.tsx b/frontend/src/layouts/OrganizationLayout/OrganizationLayout.tsx index 7cfc21838..6a2382191 100644 --- a/frontend/src/layouts/OrganizationLayout/OrganizationLayout.tsx +++ b/frontend/src/layouts/OrganizationLayout/OrganizationLayout.tsx @@ -11,11 +11,12 @@ import { BreadcrumbContainer, TBreadcrumbFormat } from "@app/components/v2"; import { OrgPermissionSubjects, useOrgPermission, useServerConfig } from "@app/context"; import { OrgPermissionSecretShareAction } from "@app/context/OrgPermissionContext/types"; import { usePopUp } from "@app/hooks"; +import { ProjectType } from "@app/hooks/api/workspace/types"; import { InsecureConnectionBanner } from "./components/InsecureConnectionBanner"; import { MinimizedOrgSidebar } from "./components/MinimizedOrgSidebar"; import { SidebarHeader } from "./components/SidebarHeader"; -import { DefaultSideBar, SecretSharingSideBar } from "./ProductsSideBar"; +import { DefaultSideBar, ProjectOverviewSideBar, SecretSharingSideBar } from "./ProductsSideBar"; export const OrganizationLayout = () => { const matches = useRouterState({ select: (s) => s.matches.at(-1)?.context }); @@ -45,21 +46,38 @@ export const OrganizationLayout = () => { ] as string[] ).includes(location.pathname); + const isProjectOverviewOrSettingsPage = ( + [ + linkOptions({ to: "/organization/secret-manager/overview" }).to, + linkOptions({ to: "/organization/secret-manager/settings" }).to, + linkOptions({ to: "/organization/cert-manager/overview" }).to, + linkOptions({ to: "/organization/cert-manager/settings" }).to, + linkOptions({ to: "/organization/kms/overview" }).to, + linkOptions({ to: "/organization/kms/settings" }).to, + linkOptions({ to: "/organization/ssh/overview" }).to, + linkOptions({ to: "/organization/ssh/settings" }).to + ] as string[] + ).includes(location.pathname); + const shouldShowOrgSidebar = location.pathname.startsWith("/organization") && (!isSecretSharingPage || shouldShowProductsSidebar) && - !( - [ - linkOptions({ to: "/organization/secret-manager/overview" }).to, - linkOptions({ to: "/organization/cert-manager/overview" }).to, - linkOptions({ to: "/organization/ssh/overview" }).to, - linkOptions({ to: "/organization/kms/overview" }).to, - linkOptions({ to: "/organization/secret-scanning" }).to - ] as string[] - ).includes(location.pathname); + !([linkOptions({ to: "/organization/secret-scanning" }).to] as string[]).includes( + location.pathname + ); const containerHeight = config.pageFrameContent ? "h-[94vh]" : "h-screen"; + let SideBarComponent = ; + + if (isSecretSharingPage) { + SideBarComponent = ; + } else if (isProjectOverviewOrSettingsPage) { + SideBarComponent = ( + + ); + } + return ( <> @@ -80,10 +98,12 @@ export const OrganizationLayout = () => { className="dark w-60 overflow-hidden border-r border-mineshaft-600 bg-gradient-to-tr from-mineshaft-700 via-mineshaft-800 to-mineshaft-900" > )} diff --git a/frontend/src/layouts/OrganizationLayout/ProductsSideBar/ProjectOverviewSideBar.tsx b/frontend/src/layouts/OrganizationLayout/ProductsSideBar/ProjectOverviewSideBar.tsx new file mode 100644 index 000000000..4b46cdd0a --- /dev/null +++ b/frontend/src/layouts/OrganizationLayout/ProductsSideBar/ProjectOverviewSideBar.tsx @@ -0,0 +1,94 @@ +import { faArrowUpRightFromSquare } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Link, useMatchRoute } from "@tanstack/react-router"; + +import { Menu, MenuGroup, MenuItem } from "@app/components/v2"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +type TProjectOverviewSideBarProps = { + type: ProjectType; +}; + +export const ProjectOverviewSideBar = ({ type }: TProjectOverviewSideBarProps) => { + const matchRoute = useMatchRoute(); + + const isOverviewActive = !!matchRoute({ + to: `/organization/${type}/overview`, + fuzzy: false + }); + + let label: string; + let icon: string; + let link: string; + + switch (type) { + case ProjectType.CertificateManager: + label = "Cert Management"; + icon = "note"; + link = "https://infisical.com/docs/documentation/platform/pki/overview"; + break; + case ProjectType.SecretManager: + label = "Secret Management"; + icon = "sliding-carousel"; + link = "https://infisical.com/docs/documentation/getting-started/introduction"; + break; + case ProjectType.KMS: + label = "KMS"; + icon = "unlock"; + link = "https://infisical.com/docs/documentation/platform/kms/overview"; + break; + case ProjectType.SSH: + label = "SSH"; + icon = "verified"; + link = "https://infisical.com/docs/documentation/platform/ssh/overview"; + break; + default: + throw new Error("Unknown project type"); + } + + return ( + <> + + + + + + {label} + + + + + + {({ isActive }) => ( + + Settings + + )} + + + + + ); +}; diff --git a/frontend/src/layouts/OrganizationLayout/ProductsSideBar/index.ts b/frontend/src/layouts/OrganizationLayout/ProductsSideBar/index.ts index b2c79d873..057f5fe6b 100644 --- a/frontend/src/layouts/OrganizationLayout/ProductsSideBar/index.ts +++ b/frontend/src/layouts/OrganizationLayout/ProductsSideBar/index.ts @@ -1,2 +1,3 @@ export * from "./DefaultSideBar"; +export * from "./ProjectOverviewSideBar"; export * from "./SecretSharingSideBar"; diff --git a/frontend/src/layouts/OrganizationLayout/components/MinimizedOrgSidebar/MinimizedOrgSidebar.tsx b/frontend/src/layouts/OrganizationLayout/components/MinimizedOrgSidebar/MinimizedOrgSidebar.tsx index ba0b4f194..8cf807d56 100644 --- a/frontend/src/layouts/OrganizationLayout/components/MinimizedOrgSidebar/MinimizedOrgSidebar.tsx +++ b/frontend/src/layouts/OrganizationLayout/components/MinimizedOrgSidebar/MinimizedOrgSidebar.tsx @@ -269,7 +269,9 @@ export const MinimizedOrgSidebar = () => { @@ -282,7 +284,9 @@ export const MinimizedOrgSidebar = () => { @@ -294,7 +298,8 @@ export const MinimizedOrgSidebar = () => { {({ isActive }) => ( @@ -306,7 +311,8 @@ export const MinimizedOrgSidebar = () => { {({ isActive }) => ( diff --git a/frontend/src/pages/admin/OverviewPage/OverviewPage.tsx b/frontend/src/pages/admin/OverviewPage/OverviewPage.tsx index af93b3c2a..790adff50 100644 --- a/frontend/src/pages/admin/OverviewPage/OverviewPage.tsx +++ b/frontend/src/pages/admin/OverviewPage/OverviewPage.tsx @@ -31,6 +31,7 @@ import { import { IdentityPanel } from "@app/pages/admin/OverviewPage/components/IdentityPanel"; import { AuthPanel } from "./components/AuthPanel"; +import { CachingPanel } from "./components/CachingPanel"; import { EncryptionPanel } from "./components/EncryptionPanel"; import { IntegrationPanel } from "./components/IntegrationPanel"; import { UserPanel } from "./components/UserPanel"; @@ -42,7 +43,8 @@ enum TabSections { Integrations = "integrations", Users = "users", Identities = "identities", - Kmip = "kmip" + Kmip = "kmip", + Caching = "caching" } enum SignUpModes { @@ -164,6 +166,7 @@ export const OverviewPage = () => { Integrations User Identities Machine Identities + Caching
    @@ -408,6 +411,9 @@ export const OverviewPage = () => { + + +
    )} diff --git a/frontend/src/pages/admin/OverviewPage/components/CachingPanel.tsx b/frontend/src/pages/admin/OverviewPage/components/CachingPanel.tsx new file mode 100644 index 000000000..170a85c0d --- /dev/null +++ b/frontend/src/pages/admin/OverviewPage/components/CachingPanel.tsx @@ -0,0 +1,101 @@ +import { useEffect, useState } from "react"; +import { faRotate } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { createNotification } from "@app/components/notifications"; +import { Badge, Button, DeleteActionModal } from "@app/components/v2"; +import { useUser } from "@app/context"; +import { usePopUp } from "@app/hooks"; +import { useInvalidateCache } from "@app/hooks/api"; +import { useGetInvalidatingCacheStatus } from "@app/hooks/api/admin/queries"; +import { CacheType } from "@app/hooks/api/admin/types"; + +export const CachingPanel = () => { + const { mutateAsync: invalidateCache } = useInvalidateCache(); + const { user } = useUser(); + + const [type, setType] = useState(null); + const [shouldPoll, setShouldPoll] = useState(false); + + const { + data: invalidationStatus, + isFetching, + refetch + } = useGetInvalidatingCacheStatus(shouldPoll); + const isInvalidating = Boolean(shouldPoll && (isFetching || invalidationStatus)); + + const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ + "invalidateCache" + ] as const); + + const handleInvalidateCacheSubmit = async () => { + if (!type || isInvalidating) return; + + try { + await invalidateCache({ type }); + createNotification({ text: `Began invalidating ${type} cache`, type: "success" }); + setShouldPoll(true); + handlePopUpClose("invalidateCache"); + } catch (err) { + console.error(err); + createNotification({ text: `Failed to invalidate ${type} cache`, type: "error" }); + } + }; + + useEffect(() => { + if (isInvalidating) return; + + if (shouldPoll) { + setShouldPoll(false); + createNotification({ text: "Successfully invalidated cache", type: "success" }); + } + }, [isInvalidating, shouldPoll]); + + useEffect(() => { + refetch().then((v) => setShouldPoll(v.data || false)); + }, []); + + return ( + <> +
    +
    +
    + Secrets Cache + {isInvalidating && ( + + + Invalidating Cache + + )} +
    + + The encrypted secrets cache encompasses all secrets stored within the system and + provides a temporary, secure storage location for frequently accessed credentials. + +
    + + +
    + handlePopUpToggle("invalidateCache", isOpen)} + deleteKey="confirm" + onDeleteApproved={handleInvalidateCacheSubmit} + /> + + ); +}; diff --git a/frontend/src/pages/admin/OverviewPage/components/MicrosoftTeamsIntegrationForm.tsx b/frontend/src/pages/admin/OverviewPage/components/MicrosoftTeamsIntegrationForm.tsx index ecd8288f4..3e83879b1 100644 --- a/frontend/src/pages/admin/OverviewPage/components/MicrosoftTeamsIntegrationForm.tsx +++ b/frontend/src/pages/admin/OverviewPage/components/MicrosoftTeamsIntegrationForm.tsx @@ -52,7 +52,7 @@ export const MicrosoftTeamsIntegrationForm = ({ adminIntegrationsConfig }: Props }); createNotification({ - text: "Updated admin Microsoft Teams configuration", + text: "Updated admin Microsoft Teams configuration. It can take up to 5 minutes to take effect.", type: "success" }); }; diff --git a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityAuthMethodModalContent.tsx b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityAuthMethodModalContent.tsx index 1380ebb39..0444b3bd1 100644 --- a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityAuthMethodModalContent.tsx +++ b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityAuthMethodModalContent.tsx @@ -13,6 +13,7 @@ import { IdentityAzureAuthForm } from "./IdentityAzureAuthForm"; import { IdentityGcpAuthForm } from "./IdentityGcpAuthForm"; import { IdentityJwtAuthForm } from "./IdentityJwtAuthForm"; import { IdentityKubernetesAuthForm } from "./IdentityKubernetesAuthForm"; +import { IdentityLdapAuthForm } from "./IdentityLdapAuthForm"; import { IdentityOidcAuthForm } from "./IdentityOidcAuthForm"; import { IdentityTokenAuthForm } from "./IdentityTokenAuthForm"; import { IdentityUniversalAuthForm } from "./IdentityUniversalAuthForm"; @@ -46,6 +47,7 @@ const identityAuthMethods = [ { label: "AWS Auth", value: IdentityAuthMethod.AWS_AUTH }, { label: "Azure Auth", value: IdentityAuthMethod.AZURE_AUTH }, { label: "OIDC Auth", value: IdentityAuthMethod.OIDC_AUTH }, + { label: "LDAP Auth", value: IdentityAuthMethod.LDAP_AUTH }, { label: "JWT Auth", value: IdentityAuthMethod.JWT_AUTH @@ -186,6 +188,16 @@ export const IdentityAuthMethodModalContent = ({ handlePopUpToggle={handlePopUpToggle} /> ) + }, + + [IdentityAuthMethod.LDAP_AUTH]: { + render: () => ( + + ) } }; diff --git a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityLdapAuthForm.tsx b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityLdapAuthForm.tsx new file mode 100644 index 000000000..62080bfd1 --- /dev/null +++ b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityLdapAuthForm.tsx @@ -0,0 +1,608 @@ +import { useEffect, useState } from "react"; +import { Controller, useFieldArray, useForm } from "react-hook-form"; +import { faPlus, faQuestionCircle, faXmark } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { + Button, + FormControl, + IconButton, + Input, + Tab, + TabList, + TabPanel, + Tabs, + TextArea, + Tooltip +} from "@app/components/v2"; +import { useOrganization, useSubscription } from "@app/context"; +import { + useAddIdentityLdapAuth, + useGetIdentityLdapAuth, + useUpdateIdentityLdapAuth +} from "@app/hooks/api"; +import { IdentityTrustedIp } from "@app/hooks/api/identities/types"; +import { UsePopUpState } from "@app/hooks/usePopUp"; + +import { IdentityFormTab } from "./types"; + +const schema = z + .object({ + url: z.string().min(1), + bindDN: z.string(), + bindPass: z.string(), + searchBase: z.string(), + searchFilter: z.string(), // defaults to (uid={{username}}) + ldapCaCertificate: z + .string() + .optional() + .transform((val) => val || undefined), + allowedFields: z + .object({ + key: z.string().trim(), + value: z + .string() + .trim() + .transform((val) => val.replace(/\s/g, "")) + }) + .array() + .optional(), + + accessTokenTTL: z.string().refine((val) => Number(val) <= 315360000, { + message: "Access Token TTL cannot be greater than 315360000" + }), + accessTokenMaxTTL: z.string().refine((val) => Number(val) <= 315360000, { + message: "Access Token Max TTL cannot be greater than 315360000" + }), + accessTokenNumUsesLimit: z.string(), + accessTokenTrustedIps: z + .array( + z.object({ + ipAddress: z.string().max(50) + }) + ) + .min(1) + }) + .required(); + +export type FormData = z.infer; + +type Props = { + handlePopUpOpen: (popUpName: keyof UsePopUpState<["upgradePlan"]>) => void; + handlePopUpToggle: ( + popUpName: keyof UsePopUpState<["identityAuthMethod"]>, + state?: boolean + ) => void; + identityId?: string; + isUpdate?: boolean; +}; + +export const IdentityLdapAuthForm = ({ + handlePopUpOpen, + handlePopUpToggle, + identityId, + isUpdate +}: Props) => { + const { currentOrg } = useOrganization(); + const orgId = currentOrg?.id || ""; + const { subscription } = useSubscription(); + + const { mutateAsync: addMutateAsync } = useAddIdentityLdapAuth(); + const { mutateAsync: updateMutateAsync } = useUpdateIdentityLdapAuth(); + const [tabValue, setTabValue] = useState(IdentityFormTab.Configuration); + + const { data } = useGetIdentityLdapAuth(identityId ?? "", { + enabled: isUpdate + }); + + const { + control, + handleSubmit, + reset, + + formState: { isSubmitting } + } = useForm({ + resolver: zodResolver(schema), + defaultValues: { + url: "", + bindDN: "", + bindPass: "", + searchBase: "", + searchFilter: "(uid={{username}})", + accessTokenTTL: "2592000", + accessTokenMaxTTL: "2592000", + accessTokenNumUsesLimit: "0", + accessTokenTrustedIps: [{ ipAddress: "0.0.0.0/0" }, { ipAddress: "::/0" }] + } + }); + + const { + fields: accessTokenTrustedIpsFields, + append: appendAccessTokenTrustedIp, + remove: removeAccessTokenTrustedIp + } = useFieldArray({ control, name: "accessTokenTrustedIps" }); + + const { + fields: allowedFieldsFields, + append: appendAllowedField, + remove: removeAllowedField + } = useFieldArray({ control, name: "allowedFields" }); + + useEffect(() => { + if (data) { + reset({ + url: data.url, + bindDN: data.bindDN, + bindPass: data.bindPass, + searchBase: data.searchBase, + searchFilter: data.searchFilter, + ldapCaCertificate: data.ldapCaCertificate || undefined, + allowedFields: data.allowedFields, + accessTokenTTL: String(data.accessTokenTTL), + accessTokenMaxTTL: String(data.accessTokenMaxTTL), + accessTokenNumUsesLimit: String(data.accessTokenNumUsesLimit), + accessTokenTrustedIps: data.accessTokenTrustedIps.map( + ({ ipAddress, prefix }: IdentityTrustedIp) => { + return { + ipAddress: `${ipAddress}${prefix !== undefined ? `/${prefix}` : ""}` + }; + } + ) + }); + } else { + reset({ + url: "", + bindDN: "", + bindPass: "", + searchBase: "", + searchFilter: "(uid={{username}})", + ldapCaCertificate: undefined, + allowedFields: [], + accessTokenTTL: "2592000", + accessTokenMaxTTL: "2592000", + accessTokenNumUsesLimit: "0", + accessTokenTrustedIps: [{ ipAddress: "0.0.0.0/0" }, { ipAddress: "::/0" }] + }); + } + }, [data]); + + useEffect(() => { + if (!subscription?.ldap) { + handlePopUpOpen("upgradePlan"); + handlePopUpToggle("identityAuthMethod", false); + } + }, [subscription]); + + const onFormSubmit = async ({ + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL, + accessTokenMaxTTL, + accessTokenNumUsesLimit, + accessTokenTrustedIps + }: FormData) => { + try { + if (!identityId) return; + + if (data) { + await updateMutateAsync({ + organizationId: orgId, + identityId, + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL: Number(accessTokenTTL), + accessTokenMaxTTL: Number(accessTokenMaxTTL), + accessTokenNumUsesLimit: Number(accessTokenNumUsesLimit), + accessTokenTrustedIps + }); + } else { + await addMutateAsync({ + organizationId: orgId, + identityId, + url, + bindDN, + bindPass, + searchBase, + searchFilter, + ldapCaCertificate, + allowedFields, + accessTokenTTL: Number(accessTokenTTL), + accessTokenMaxTTL: Number(accessTokenMaxTTL), + accessTokenNumUsesLimit: Number(accessTokenNumUsesLimit), + accessTokenTrustedIps + }); + } + + handlePopUpToggle("identityAuthMethod", false); + + createNotification({ + text: `Successfully ${isUpdate ? "updated" : "configured"} auth method`, + type: "success" + }); + + reset(); + } catch { + createNotification({ + text: `Failed to ${isUpdate ? "update" : "configure"} identity`, + type: "error" + }); + } + }; + + return ( + { + setTabValue( + [ + "url", + "bindDN", + "bindPass", + "searchBase", + "searchFilter", + "accessTokenTTL", + "allowedFields", + "accessTokenMaxTTL", + "accessTokenNumUsesLimit" + ].includes(Object.keys(fields)[0]) + ? IdentityFormTab.Configuration + : IdentityFormTab.Advanced + ); + })} + > + setTabValue(value as IdentityFormTab)}> + + Configuration + Advanced + + + ( + + + + )} + /> + ( + + + + )} + /> + ( + + + + )} + /> + ( + + + + )} + /> + + ( + + + + )} + /> + + {allowedFieldsFields.map(({ id }, index) => ( +
    + { + const isFirstField = index === 0; + + return ( + +

    + Specify the fields that the user must contain in their LDAP entry + in order to authenticate with this identity. If nothing is + specified, all users in the configured LDAP directory will be able + to authenticate. +

    + You can specify multiple required attributes by separating them + with a comma. +

    +

    +
    +

    Example:

    +

    + 'uid' → 'user1,user2,user3' +
    + 'mail' → 'user@example.com' +

    +
    + +

    + The above example would allow users with the UID user1, user2, or + user3 to authenticate but only if their emails also match + user@example.com +

    +
    + } + > + + + ) : undefined + } + isError={Boolean(error)} + errorText={error?.message} + > + field.onChange(e)} + placeholder="uid" + /> + + ); + }} + /> + { + return ( + + field.onChange(e)} + placeholder="userid1,userid2,userid3" + /> + + ); + }} + /> + removeAllowedField(index)} + size="lg" + colorSchema="danger" + variant="plain" + ariaLabel="update" + className="p-3" + > + + +
    + ))} +
    + +
    + + ( + + + + )} + /> + ( + + + + )} + /> + ( + + + + )} + /> + + + ( + +