diff --git a/.env.example b/.env.example index be7e0a621..8bcb4039d 100644 --- a/.env.example +++ b/.env.example @@ -36,16 +36,22 @@ CLIENT_ID_HEROKU= CLIENT_ID_VERCEL= CLIENT_ID_NETLIFY= CLIENT_ID_GITHUB= +CLIENT_ID_GITHUB_APP= +CLIENT_SLUG_GITHUB_APP= CLIENT_ID_GITLAB= CLIENT_ID_BITBUCKET= CLIENT_SECRET_HEROKU= CLIENT_SECRET_VERCEL= CLIENT_SECRET_NETLIFY= CLIENT_SECRET_GITHUB= +CLIENT_SECRET_GITHUB_APP= CLIENT_SECRET_GITLAB= CLIENT_SECRET_BITBUCKET= CLIENT_SLUG_VERCEL= +CLIENT_PRIVATE_KEY_GITHUB_APP= +CLIENT_APP_ID_GITHUB_APP= + # Sentry (optional) for monitoring errors SENTRY_DSN= diff --git a/.env.migration.example b/.env.migration.example index 4d1c8f9ef..2c5f5b957 100644 --- a/.env.migration.example +++ b/.env.migration.example @@ -1 +1,2 @@ DB_CONNECTION_URI= +AUDIT_LOGS_DB_CONNECTION_URI= diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ec62ef24..16a828578 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,6 +6,7 @@ - [ ] Bug fix - [ ] New feature +- [ ] Improvement - [ ] Breaking change - [ ] Documentation diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index dba298a37..552eb56e4 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -7,7 +7,6 @@ on: description: "Version number" required: true type: string - defaults: run: working-directory: ./backend @@ -49,9 +48,9 @@ jobs: - name: Package into node binary run: | if [ "${{ matrix.os }}" != "linux" ]; then - pkg --no-bytecode --public-packages "*" --public --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-core-${{ matrix.os }}-${{ matrix.arch }} . + pkg --no-bytecode --public-packages "*" --public --compress GZip --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-core-${{ matrix.os }}-${{ matrix.arch }} . else - pkg --no-bytecode --public-packages "*" --public --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-core . + pkg --no-bytecode --public-packages "*" --public --compress GZip --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-core . fi # Set up .deb package structure (Debian/Ubuntu only) @@ -83,6 +82,86 @@ jobs: dpkg-deb --build infisical-core mv infisical-core.deb ./binary/infisical-core-${{matrix.arch}}.deb + ### RPM + + # Set up .rpm package structure + - name: Set up .rpm package structure + if: matrix.os == 'linux' + run: | + mkdir -p infisical-core-rpm/usr/local/bin + cp ./binary/infisical-core infisical-core-rpm/usr/local/bin/ + chmod +x infisical-core-rpm/usr/local/bin/infisical-core + + # Install RPM build tools + - name: Install RPM build tools + if: matrix.os == 'linux' + run: sudo apt-get update && sudo apt-get install -y rpm + + # Create .spec file for RPM + - name: Create .spec file for RPM + if: matrix.os == 'linux' + run: | + cat < infisical-core.spec + + %global _enable_debug_package 0 + %global debug_package %{nil} + %global __os_install_post /usr/lib/rpm/brp-compress %{nil} + + Name: infisical-core + Version: ${{ github.event.inputs.version }} + Release: 1%{?dist} + Summary: Infisical Core standalone executable + License: Proprietary + URL: https://app.infisical.com + + %description + Infisical Core standalone executable (app.infisical.com) + + %install + mkdir -p %{buildroot}/usr/local/bin + cp %{_sourcedir}/infisical-core %{buildroot}/usr/local/bin/ + + %files + /usr/local/bin/infisical-core + + %pre + + %post + + %preun + + %postun + EOF + + # Build .rpm file + - name: Build .rpm package + if: matrix.os == 'linux' + run: | + # Create necessary directories + mkdir -p rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + + # Copy the binary directly to SOURCES + cp ./binary/infisical-core rpmbuild/SOURCES/ + + # Run rpmbuild with verbose output + rpmbuild -vv -bb \ + --define "_topdir $(pwd)/rpmbuild" \ + --define "_sourcedir $(pwd)/rpmbuild/SOURCES" \ + --define "_rpmdir $(pwd)/rpmbuild/RPMS" \ + --target ${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }} \ + infisical-core.spec + + # Try to find the RPM file + find rpmbuild -name "*.rpm" + + # Move the RPM file if found + if [ -n "$(find rpmbuild -name '*.rpm')" ]; then + mv $(find rpmbuild -name '*.rpm') ./binary/infisical-core-${{matrix.arch}}.rpm + else + echo "RPM file not found!" + exit 1 + fi + - uses: actions/setup-python@v4 with: python-version: "3.x" # Specify the Python version you need @@ -97,6 +176,12 @@ jobs: working-directory: ./backend run: cloudsmith push deb --republish --no-wait-for-sync --api-key=${{ secrets.CLOUDSMITH_API_KEY }} infisical/infisical-core/any-distro/any-version ./binary/infisical-core-${{ matrix.arch }}.deb + # Publish .rpm file to Cloudsmith (Red Hat-based systems only) + - name: Publish .rpm to Cloudsmith + if: matrix.os == 'linux' + working-directory: ./backend + run: cloudsmith push rpm --republish --no-wait-for-sync --api-key=${{ secrets.CLOUDSMITH_API_KEY }} infisical/infisical-core/any-distro/any-version ./binary/infisical-core-${{ matrix.arch }}.rpm + # Publish .exe file to Cloudsmith (Windows only) - name: Publish to Cloudsmith (Windows) if: matrix.os == 'win' diff --git a/.github/workflows/build-staging-and-deploy-aws.yml b/.github/workflows/build-staging-and-deploy-aws.yml index 347b6f5ee..84f22e3cc 100644 --- a/.github/workflows/build-staging-and-deploy-aws.yml +++ b/.github/workflows/build-staging-and-deploy-aws.yml @@ -127,6 +127,7 @@ jobs: - name: Change directory to backend and install dependencies env: DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }} + AUDIT_LOGS_DB_CONNECTION_URI: ${{ secrets.AUDIT_LOGS_DB_CONNECTION_URI }} run: | cd backend npm install diff --git a/Dockerfile.standalone-infisical b/Dockerfile.standalone-infisical index 8ffe7e3de..269cbfcf9 100644 --- a/Dockerfile.standalone-infisical +++ b/Dockerfile.standalone-infisical @@ -95,6 +95,10 @@ RUN mkdir frontend-build # Production stage FROM base AS production RUN apk add --upgrade --no-cache ca-certificates +RUN apk add --no-cache bash curl && curl -1sLf \ + 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \ + && apk add infisical=0.31.1 && apk add --no-cache git + RUN addgroup --system --gid 1001 nodejs \ && adduser --system --uid 1001 non-root-user diff --git a/README.md b/README.md index 7e3f40d52..d68481428 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,7 @@ Lean about Infisical's code scanning feature [here](https://infisical.com/docs/c This repo available under the [MIT expat license](https://github.com/Infisical/infisical/blob/main/LICENSE), with the exception of the `ee` directory which will contain premium enterprise features requiring a Infisical license. -If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo): - -Schedule a meeting +If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo). ## Security @@ -163,4 +161,3 @@ Not sure where to get started? You can: - [Twitter](https://twitter.com/infisical) for fast news - [YouTube](https://www.youtube.com/@infisical_os) for videos on secret management - [Blog](https://infisical.com/blog) for secret management insights, articles, tutorials, and updates -- [Roadmap](https://www.notion.so/infisical/be2d2585a6694e40889b03aef96ea36b?v=5b19a8127d1a4060b54769567a8785fa) for planned features \ No newline at end of file diff --git a/backend/e2e-test/routes/v1/project-env.spec.ts b/backend/e2e-test/routes/v1/project-env.spec.ts index ec06d6474..0726f3a50 100644 --- a/backend/e2e-test/routes/v1/project-env.spec.ts +++ b/backend/e2e-test/routes/v1/project-env.spec.ts @@ -123,7 +123,7 @@ describe("Project Environment Router", async () => { id: deletedProjectEnvironment.id, name: mockProjectEnv.name, slug: mockProjectEnv.slug, - position: 4, + position: 5, createdAt: expect.any(String), updatedAt: expect.any(String) }) diff --git a/backend/package-lock.json b/backend/package-lock.json index 31521fb7d..dbd8c8073 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -21,12 +21,14 @@ "@fastify/etag": "^5.1.0", "@fastify/formbody": "^7.4.0", "@fastify/helmet": "^11.1.1", + "@fastify/multipart": "8.3.0", "@fastify/passport": "^2.4.0", "@fastify/rate-limit": "^9.0.0", "@fastify/session": "^10.7.0", "@fastify/swagger": "^8.14.0", "@fastify/swagger-ui": "^2.1.0", "@node-saml/passport-saml": "^4.0.4", + "@octokit/auth-app": "^7.1.1", "@octokit/plugin-retry": "^5.0.5", "@octokit/rest": "^20.0.2", "@octokit/webhooks-types": "^7.3.1", @@ -61,7 +63,7 @@ "jwks-rsa": "^3.1.0", "knex": "^3.0.1", "ldapjs": "^3.0.7", - "ldif": "^0.5.1", + "ldif": "0.5.1", "libsodium-wrappers": "^0.7.13", "lodash.isequal": "^4.5.0", "mongodb": "^6.8.1", @@ -4311,6 +4313,15 @@ "fast-uri": "^2.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@fastify/cookie": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz", @@ -4381,6 +4392,20 @@ "helmet": "^7.0.0" } }, + "node_modules/@fastify/multipart": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-8.3.0.tgz", + "integrity": "sha512-A8h80TTyqUzaMVH0Cr9Qcm6RxSkVqmhK/MVBYHYeRRSUbUYv08WecjWKSlG2aSnD4aGI841pVxAjC+G1GafUeQ==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.1.0", + "@fastify/deepmerge": "^1.0.0", + "@fastify/error": "^3.0.0", + "fastify-plugin": "^4.0.0", + "secure-json-parse": "^2.4.0", + "stream-wormhole": "^1.1.0" + } + }, "node_modules/@fastify/passport": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@fastify/passport/-/passport-2.4.0.tgz", @@ -4976,24 +5001,73 @@ } }, "node_modules/@octokit/auth-app": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-6.0.3.tgz", - "integrity": "sha512-9N7IlBAKEJR3tJgPSubCxIDYGXSdc+2xbkjYpk9nCyqREnH8qEMoMhiEB1WgoA9yTFp91El92XNXAi+AjuKnfw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-7.1.1.tgz", + "integrity": "sha512-kRAd6yelV9OgvlEJE88H0VLlQdZcag9UlLr7dV0YYP37X8PPDvhgiTy66QVhDXdyoT0AleFN2w/qXkPdrSzINg==", "dependencies": { - "@octokit/auth-oauth-app": "^7.0.0", - "@octokit/auth-oauth-user": "^4.0.0", - "@octokit/request": "^8.0.2", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", - "deprecation": "^2.3.1", + "@octokit/auth-oauth-app": "^8.1.0", + "@octokit/auth-oauth-user": "^5.1.0", + "@octokit/request": "^9.1.1", + "@octokit/request-error": "^6.1.1", + "@octokit/types": "^13.4.1", "lru-cache": "^10.0.0", - "universal-github-app-jwt": "^1.1.2", - "universal-user-agent": "^6.0.0" + "universal-github-app-jwt": "^2.2.0", + "universal-user-agent": "^7.0.0" }, "engines": { "node": ">= 18" } }, + "node_modules/@octokit/auth-app/node_modules/@octokit/endpoint": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.1.tgz", + "integrity": "sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==", + "dependencies": { + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/request": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.3.tgz", + "integrity": "sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/request-error": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.5.tgz", + "integrity": "sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==", + "dependencies": { + "@octokit/types": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, "node_modules/@octokit/auth-app/node_modules/lru-cache": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", @@ -5002,53 +5076,220 @@ "node": "14 || >=16.14" } }, + "node_modules/@octokit/auth-app/node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==" + }, "node_modules/@octokit/auth-oauth-app": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-7.0.1.tgz", - "integrity": "sha512-RE0KK0DCjCHXHlQBoubwlLijXEKfhMhKm9gO56xYvFmP1QTMb+vvwRPmQLLx0V+5AvV9N9I3lr1WyTzwL3rMDg==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-8.1.1.tgz", + "integrity": "sha512-5UtmxXAvU2wfcHIPPDWzVSAWXVJzG3NWsxb7zCFplCWEmMCArSZV0UQu5jw5goLQXbFyOr5onzEH37UJB3zQQg==", "dependencies": { - "@octokit/auth-oauth-device": "^6.0.0", - "@octokit/auth-oauth-user": "^4.0.0", - "@octokit/request": "^8.0.2", - "@octokit/types": "^12.0.0", - "@types/btoa-lite": "^1.0.0", - "btoa-lite": "^1.0.0", - "universal-user-agent": "^6.0.0" + "@octokit/auth-oauth-device": "^7.0.0", + "@octokit/auth-oauth-user": "^5.0.1", + "@octokit/request": "^9.0.0", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.0" }, "engines": { "node": ">= 18" } }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/endpoint": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.1.tgz", + "integrity": "sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==", + "dependencies": { + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/request": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.3.tgz", + "integrity": "sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/request-error": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.5.tgz", + "integrity": "sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==", + "dependencies": { + "@octokit/types": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/@octokit/auth-oauth-app/node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==" + }, "node_modules/@octokit/auth-oauth-device": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-6.0.1.tgz", - "integrity": "sha512-yxU0rkL65QkjbqQedgVx3gmW7YM5fF+r5uaSj9tM/cQGVqloXcqP2xK90eTyYvl29arFVCW8Vz4H/t47mL0ELw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-7.1.1.tgz", + "integrity": "sha512-HWl8lYueHonuyjrKKIup/1tiy0xcmQCdq5ikvMO1YwkNNkxb6DXfrPjrMYItNLyCP/o2H87WuijuE+SlBTT8eg==", "dependencies": { - "@octokit/oauth-methods": "^4.0.0", - "@octokit/request": "^8.0.0", - "@octokit/types": "^12.0.0", - "universal-user-agent": "^6.0.0" + "@octokit/oauth-methods": "^5.0.0", + "@octokit/request": "^9.0.0", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.0" }, "engines": { "node": ">= 18" } }, - "node_modules/@octokit/auth-oauth-user": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-4.0.1.tgz", - "integrity": "sha512-N94wWW09d0hleCnrO5wt5MxekatqEJ4zf+1vSe8MKMrhZ7gAXKFOKrDEZW2INltvBWJCyDUELgGRv8gfErH1Iw==", + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/endpoint": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.1.tgz", + "integrity": "sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==", "dependencies": { - "@octokit/auth-oauth-device": "^6.0.0", - "@octokit/oauth-methods": "^4.0.0", - "@octokit/request": "^8.0.2", - "@octokit/types": "^12.0.0", - "btoa-lite": "^1.0.0", - "universal-user-agent": "^6.0.0" + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.2" }, "engines": { "node": ">= 18" } }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/request": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.3.tgz", + "integrity": "sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/request-error": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.5.tgz", + "integrity": "sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==", + "dependencies": { + "@octokit/types": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/@octokit/auth-oauth-device/node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==" + }, + "node_modules/@octokit/auth-oauth-user": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-5.1.1.tgz", + "integrity": "sha512-rRkMz0ErOppdvEfnemHJXgZ9vTPhBuC6yASeFaB7I2yLMd7QpjfrL1mnvRPlyKo+M6eeLxrKanXJ9Qte29SRsw==", + "dependencies": { + "@octokit/auth-oauth-device": "^7.0.1", + "@octokit/oauth-methods": "^5.0.0", + "@octokit/request": "^9.0.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/endpoint": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.1.tgz", + "integrity": "sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==", + "dependencies": { + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/request": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.3.tgz", + "integrity": "sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/request-error": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.5.tgz", + "integrity": "sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==", + "dependencies": { + "@octokit/types": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==" + }, "node_modules/@octokit/auth-token": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", @@ -5112,28 +5353,82 @@ } }, "node_modules/@octokit/oauth-authorization-url": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-6.0.2.tgz", - "integrity": "sha512-CdoJukjXXxqLNK4y/VOiVzQVjibqoj/xHgInekviUJV73y/BSIcwvJ/4aNHPBPKcPWFnd4/lO9uqRV65jXhcLA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-7.1.1.tgz", + "integrity": "sha512-ooXV8GBSabSWyhLUowlMIVd9l1s2nsOGQdlP2SQ4LnkEsGXzeCvbSbCPdZThXhEFzleGPwbapT0Sb+YhXRyjCA==", "engines": { "node": ">= 18" } }, "node_modules/@octokit/oauth-methods": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@octokit/oauth-methods/-/oauth-methods-4.0.1.tgz", - "integrity": "sha512-1NdTGCoBHyD6J0n2WGXg9+yDLZrRNZ0moTEex/LSPr49m530WNKcCfXDghofYptr3st3eTii+EHoG5k/o+vbtw==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@octokit/oauth-methods/-/oauth-methods-5.1.2.tgz", + "integrity": "sha512-C5lglRD+sBlbrhCUTxgJAFjWgJlmTx5bQ7Ch0+2uqRjYv7Cfb5xpX4WuSC9UgQna3sqRGBL9EImX9PvTpMaQ7g==", "dependencies": { - "@octokit/oauth-authorization-url": "^6.0.2", - "@octokit/request": "^8.0.2", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", - "btoa-lite": "^1.0.0" + "@octokit/oauth-authorization-url": "^7.0.0", + "@octokit/request": "^9.1.0", + "@octokit/request-error": "^6.1.0", + "@octokit/types": "^13.0.0" }, "engines": { "node": ">= 18" } }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/endpoint": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.1.tgz", + "integrity": "sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==", + "dependencies": { + "@octokit/types": "^13.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/request": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.3.tgz", + "integrity": "sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/request-error": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.5.tgz", + "integrity": "sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==", + "dependencies": { + "@octokit/types": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/@octokit/oauth-methods/node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==" + }, "node_modules/@octokit/openapi-types": { "version": "19.1.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", @@ -5248,13 +5543,13 @@ } }, "node_modules/@octokit/request": { - "version": "8.1.6", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.6.tgz", - "integrity": "sha512-YhPaGml3ncZC1NfXpP3WZ7iliL1ap6tLkAp6MvbK2fTTPytzVUyUesBBogcdMm86uRYO5rHaM1xIWxigWZ17MQ==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.0.tgz", + "integrity": "sha512-9Bb014e+m2TgBeEJGEbdplMVWwPmL1FPtggHQRkV+WVsMggPtEkLKPlcVYm/o8xKLkpJ7B+6N8WfQMtDLX2Dpw==", "dependencies": { - "@octokit/endpoint": "^9.0.0", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", + "@octokit/endpoint": "^9.0.1", + "@octokit/request-error": "^5.1.0", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" }, "engines": { @@ -5262,11 +5557,11 @@ } }, "node_modules/@octokit/request-error": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz", - "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.0.tgz", + "integrity": "sha512-GETXfE05J0+7H2STzekpKObFe765O5dlAKUTLNGeH+x47z7JjXHfsHKo5z21D/o/IOZTUEI6nyWyR+bZVP/n5Q==", "dependencies": { - "@octokit/types": "^12.0.0", + "@octokit/types": "^13.1.0", "deprecation": "^2.0.0", "once": "^1.4.0" }, @@ -5274,6 +5569,32 @@ "node": ">= 18" } }, + "node_modules/@octokit/request-error/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/request-error/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/@octokit/request/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/request/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, "node_modules/@octokit/rest": { "version": "20.0.2", "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-20.0.2.tgz", @@ -14160,6 +14481,154 @@ "@octokit/core": ">=5" } }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-app": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-6.1.2.tgz", + "integrity": "sha512-fWjIOpxnL8/YFY3kqquciFQ4o99aCqHw5kMFoGPYbz/h5HNZ11dJlV9zag5wS2nt0X1wJ5cs9BUo+CsAPfW4jQ==", + "dependencies": { + "@octokit/auth-oauth-app": "^7.1.0", + "@octokit/auth-oauth-user": "^4.1.0", + "@octokit/request": "^8.3.1", + "@octokit/request-error": "^5.1.0", + "@octokit/types": "^13.1.0", + "deprecation": "^2.3.1", + "lru-cache": "^10.0.0", + "universal-github-app-jwt": "^1.1.2", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-app/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-app": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-7.1.0.tgz", + "integrity": "sha512-w+SyJN/b0l/HEb4EOPRudo7uUOSW51jcK1jwLa+4r7PA8FPFpoxEnHBHMITqCsc/3Vo2qqFjgQfz/xUUvsSQnA==", + "dependencies": { + "@octokit/auth-oauth-device": "^6.1.0", + "@octokit/auth-oauth-user": "^4.1.0", + "@octokit/request": "^8.3.1", + "@octokit/types": "^13.0.0", + "@types/btoa-lite": "^1.0.0", + "btoa-lite": "^1.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-app/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-device": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-6.1.0.tgz", + "integrity": "sha512-FNQ7cb8kASufd6Ej4gnJ3f1QB5vJitkoV1O0/g6e6lUsQ7+VsSNRHRmFScN2tV4IgKA12frrr/cegUs0t+0/Lw==", + "dependencies": { + "@octokit/oauth-methods": "^4.1.0", + "@octokit/request": "^8.3.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-device/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-user": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-4.1.0.tgz", + "integrity": "sha512-FrEp8mtFuS/BrJyjpur+4GARteUCrPeR/tZJzD8YourzoVhRics7u7we/aDcKv+yywRNwNi/P4fRi631rG/OyQ==", + "dependencies": { + "@octokit/auth-oauth-device": "^6.1.0", + "@octokit/oauth-methods": "^4.1.0", + "@octokit/request": "^8.3.1", + "@octokit/types": "^13.0.0", + "btoa-lite": "^1.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/auth-oauth-user/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/oauth-authorization-url": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-6.0.2.tgz", + "integrity": "sha512-CdoJukjXXxqLNK4y/VOiVzQVjibqoj/xHgInekviUJV73y/BSIcwvJ/4aNHPBPKcPWFnd4/lO9uqRV65jXhcLA==", + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/oauth-methods": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@octokit/oauth-methods/-/oauth-methods-4.1.0.tgz", + "integrity": "sha512-4tuKnCRecJ6CG6gr0XcEXdZtkTDbfbnD5oaHBmLERTjTMZNi2CbfEHZxPU41xXLDG4DfKf+sonu00zvKI9NSbw==", + "dependencies": { + "@octokit/oauth-authorization-url": "^6.0.2", + "@octokit/request": "^8.3.1", + "@octokit/request-error": "^5.1.0", + "@octokit/types": "^13.0.0", + "btoa-lite": "^1.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/oauth-methods/node_modules/@octokit/types": { + "version": "13.6.1", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.1.tgz", + "integrity": "sha512-PHZE9Z+kWXb23Ndik8MKPirBPziOc0D2/3KH1P+6jK5nGWe96kadZuE4jev2/Jq7FvIfTlT2Ltg8Fv2x1v0a5g==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" + } + }, + "node_modules/octokit-auth-probot/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/octokit-auth-probot/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, + "node_modules/octokit-auth-probot/node_modules/universal-github-app-jwt": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.2.0.tgz", + "integrity": "sha512-dncpMpnsKBk0eetwfN8D8OUHGfiDhhJ+mtsbMl+7PfW7mYjiH8LIcqRmYMtzYLgSh47HjfdBtrBwIQ/gizKR3g==", + "dependencies": { + "@types/jsonwebtoken": "^9.0.0", + "jsonwebtoken": "^9.0.2" + } + }, "node_modules/oidc-token-hash": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz", @@ -16604,6 +17073,15 @@ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" }, + "node_modules/stream-wormhole": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stream-wormhole/-/stream-wormhole-1.1.0.tgz", + "integrity": "sha512-gHFfL3px0Kctd6Po0M8TzEvt3De/xu6cnRrjlfYNhwbhLPLwigI2t1nc6jrzNuaYg5C4YF78PPFuQPzRiqn9ew==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -18143,13 +18621,9 @@ } }, "node_modules/universal-github-app-jwt": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.2.tgz", - "integrity": "sha512-t1iB2FmLFE+yyJY9+3wMx0ejB+MQpEVkH0gQv7dR6FZyltyq+ZZO0uDpbopxhrZ3SLEO4dCEkIujOMldEQ2iOA==", - "dependencies": { - "@types/jsonwebtoken": "^9.0.0", - "jsonwebtoken": "^9.0.2" - } + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-2.2.0.tgz", + "integrity": "sha512-G5o6f95b5BggDGuUfKDApKaCgNYy2x7OdHY0zSMF081O0EJobw+1130VONhrA7ezGSV2FNOGyM+KQpQZAr9bIQ==" }, "node_modules/universal-user-agent": { "version": "6.0.1", diff --git a/backend/package.json b/backend/package.json index a51a5b3e4..a713728e9 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,13 +45,19 @@ "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts", "generate:component": "tsx ./scripts/create-backend-file.ts", "generate:schema": "tsx ./scripts/generate-schema-types.ts", + "auditlog-migration:latest": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:latest", + "auditlog-migration:up": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:up", + "auditlog-migration:down": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:down", + "auditlog-migration:list": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:list", + "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status", + "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback", "migration:new": "tsx ./scripts/create-migration.ts", - "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up", - "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down", - "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list", - "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest", - "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status", - "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback", + "migration:up": "npm run auditlog-migration:up && knex --knexfile ./src/db/knexfile.ts --client pg migrate:up", + "migration:down": "npm run auditlog-migration:down && knex --knexfile ./src/db/knexfile.ts --client pg migrate:down", + "migration:list": "npm run auditlog-migration:list && knex --knexfile ./src/db/knexfile.ts --client pg migrate:list", + "migration:latest": "npm run auditlog-migration:latest && knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest", + "migration:status": "npm run auditlog-migration:status && knex --knexfile ./src/db/knexfile.ts --client pg migrate:status", + "migration:rollback": "npm run auditlog-migration:rollback && knex --knexfile ./src/db/knexfile.ts migrate:rollback", "migrate:org": "tsx ./scripts/migrate-organization.ts", "seed:new": "tsx ./scripts/create-seed-file.ts", "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run", @@ -120,12 +126,14 @@ "@fastify/etag": "^5.1.0", "@fastify/formbody": "^7.4.0", "@fastify/helmet": "^11.1.1", + "@fastify/multipart": "8.3.0", "@fastify/passport": "^2.4.0", "@fastify/rate-limit": "^9.0.0", "@fastify/session": "^10.7.0", "@fastify/swagger": "^8.14.0", "@fastify/swagger-ui": "^2.1.0", "@node-saml/passport-saml": "^4.0.4", + "@octokit/auth-app": "^7.1.1", "@octokit/plugin-retry": "^5.0.5", "@octokit/rest": "^20.0.2", "@octokit/webhooks-types": "^7.3.1", diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index 43984ecfa..fc398c2ac 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -90,7 +90,12 @@ const main = async () => { .whereRaw("table_schema = current_schema()") .select<{ tableName: string }[]>("table_name as tableName") .orderBy("table_name") - ).filter((el) => !el.tableName.includes("_migrations")); + ).filter( + (el) => + !el.tableName.includes("_migrations") && + !el.tableName.includes("audit_logs_") && + el.tableName !== "intermediate_audit_logs" + ); for (let i = 0; i < tables.length; i += 1) { const { tableName } = tables[i]; diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 65da73072..21c44a3b5 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -39,6 +39,7 @@ import { TCertificateServiceFactory } from "@app/services/certificate/certificat import { TCertificateAuthorityServiceFactory } from "@app/services/certificate-authority/certificate-authority-service"; import { TCertificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; +import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -185,6 +186,7 @@ declare module "fastify" { workflowIntegration: TWorkflowIntegrationServiceFactory; cmek: TCmekServiceFactory; migration: TExternalMigrationServiceFactory; + externalGroupOrgRoleMapping: TExternalGroupOrgRoleMappingServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 624915276..fb78bce4d 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -336,6 +336,11 @@ import { TWorkflowIntegrationsInsert, TWorkflowIntegrationsUpdate } from "@app/db/schemas"; +import { + TExternalGroupOrgRoleMappings, + TExternalGroupOrgRoleMappingsInsert, + TExternalGroupOrgRoleMappingsUpdate +} from "@app/db/schemas/external-group-org-role-mappings"; import { TSecretV2TagJunction, TSecretV2TagJunctionInsert, @@ -808,5 +813,10 @@ declare module "knex/types/tables" { TWorkflowIntegrationsInsert, TWorkflowIntegrationsUpdate >; + [TableName.ExternalGroupOrgRoleMapping]: KnexOriginal.CompositeTableType< + TExternalGroupOrgRoleMappings, + TExternalGroupOrgRoleMappingsInsert, + TExternalGroupOrgRoleMappingsUpdate + >; } } diff --git a/backend/src/db/auditlog-knexfile.ts b/backend/src/db/auditlog-knexfile.ts new file mode 100644 index 000000000..3ceef65a0 --- /dev/null +++ b/backend/src/db/auditlog-knexfile.ts @@ -0,0 +1,75 @@ +// eslint-disable-next-line +import "ts-node/register"; + +import dotenv from "dotenv"; +import type { Knex } from "knex"; +import path from "path"; + +// Update with your config settings. . +dotenv.config({ + path: path.join(__dirname, "../../../.env.migration") +}); +dotenv.config({ + path: path.join(__dirname, "../../../.env") +}); + +if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) { + console.info("Dedicated audit log database not found. No further migrations necessary"); + process.exit(0); +} + +console.info("Executing migration on audit log database..."); + +export default { + development: { + client: "postgres", + connection: { + connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI, + host: process.env.AUDIT_LOGS_DB_HOST, + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, + pool: { + min: 2, + max: 10 + }, + seeds: { + directory: "./seeds" + }, + migrations: { + tableName: "infisical_migrations" + } + }, + production: { + client: "postgres", + connection: { + connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI, + host: process.env.AUDIT_LOGS_DB_HOST, + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, + pool: { + min: 2, + max: 10 + }, + migrations: { + tableName: "infisical_migrations" + } + } +} as Knex.Config; diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index 75992e2c6..abebdf65a 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -1,2 +1,2 @@ export type { TDbClient } from "./instance"; -export { initDbConnection } from "./instance"; +export { initAuditLogDbConnection, initDbConnection } from "./instance"; diff --git a/backend/src/db/instance.ts b/backend/src/db/instance.ts index f6162ad9c..d4a2a5b2c 100644 --- a/backend/src/db/instance.ts +++ b/backend/src/db/instance.ts @@ -70,3 +70,45 @@ export const initDbConnection = ({ return db; }; + +export const initAuditLogDbConnection = ({ + dbConnectionUri, + dbRootCert +}: { + dbConnectionUri: string; + dbRootCert?: string; +}) => { + // akhilmhdh: the default Knex is knex.Knex. but when assigned with knex({}) the value is knex.Knex + // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[] + // eslint-disable-next-line + const db: Knex = knex({ + client: "pg", + connection: { + connectionString: dbConnectionUri, + host: process.env.AUDIT_LOGS_DB_HOST, + // @ts-expect-error I have no clue why only for the port there is a type error + // eslint-disable-next-line + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: dbRootCert + ? { + rejectUnauthorized: true, + ca: Buffer.from(dbRootCert, "base64").toString("ascii") + } + : false + } + }); + + // we add these overrides so that auditLogDb and the primary DB are interchangeable + db.primaryNode = () => { + return db; + }; + + db.replicaNode = () => { + return db; + }; + + return db; +}; diff --git a/backend/src/db/manual-migrations/partition-audit-logs.ts b/backend/src/db/manual-migrations/partition-audit-logs.ts new file mode 100644 index 000000000..382ef0dbf --- /dev/null +++ b/backend/src/db/manual-migrations/partition-audit-logs.ts @@ -0,0 +1,161 @@ +import kx, { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const INTERMEDIATE_AUDIT_LOG_TABLE = "intermediate_audit_logs"; + +const formatPartitionDate = (date: Date) => { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + + return `${year}-${month}-${day}`; +}; + +const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { + const startDateStr = formatPartitionDate(startDate); + const endDateStr = formatPartitionDate(endDate); + + const partitionName = `${TableName.AuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(/-/g, "")}`; + + await knex.schema.raw( + `CREATE TABLE ${partitionName} PARTITION OF ${TableName.AuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')` + ); +}; + +const up = async (knex: Knex): Promise => { + console.info("Dropping primary key of audit log table..."); + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // remove existing keys + t.dropPrimary(); + }); + + // Get all indices of the audit log table and drop them + const indexNames: { rows: { indexname: string }[] } = await knex.raw( + ` + SELECT indexname + FROM pg_indexes + WHERE tablename = '${TableName.AuditLog}' + ` + ); + + console.log( + "Deleting existing audit log indices:", + indexNames.rows.map((e) => e.indexname) + ); + + for await (const row of indexNames.rows) { + await knex.raw(`DROP INDEX IF EXISTS ${row.indexname}`); + } + + // renaming audit log to intermediate table + console.log("Renaming audit log table to the intermediate name"); + await knex.schema.renameTable(TableName.AuditLog, INTERMEDIATE_AUDIT_LOG_TABLE); + + if (!(await knex.schema.hasTable(TableName.AuditLog))) { + const createTableSql = knex.schema + .createTable(TableName.AuditLog, (t) => { + t.uuid("id").defaultTo(knex.fn.uuid()); + t.string("actor").notNullable(); + t.jsonb("actorMetadata").notNullable(); + t.string("ipAddress"); + t.string("eventType").notNullable(); + t.jsonb("eventMetadata"); + t.string("userAgent"); + t.string("userAgentType"); + t.datetime("expiresAt"); + t.timestamps(true, true, true); + t.uuid("orgId"); + t.string("projectId"); + t.string("projectName"); + t.primary(["id", "createdAt"]); + }) + .toString(); + + console.info("Creating partition table..."); + await knex.schema.raw(` + ${createTableSql} PARTITION BY RANGE ("createdAt"); + `); + + console.log("Adding indices..."); + await knex.schema.alterTable(TableName.AuditLog, (t) => { + t.index(["projectId", "createdAt"]); + t.index(["orgId", "createdAt"]); + t.index("expiresAt"); + t.index("orgId"); + t.index("projectId"); + }); + + console.log("Adding GIN indices..."); + + await knex.raw( + `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` + ); + console.log("GIN index for actorMetadata done"); + + await knex.raw( + `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)` + ); + console.log("GIN index for eventMetadata done"); + + // create default partition + console.log("Creating default partition..."); + await knex.schema.raw(`CREATE TABLE ${TableName.AuditLog}_default PARTITION OF ${TableName.AuditLog} DEFAULT`); + + const nextDate = new Date(); + nextDate.setDate(nextDate.getDate() + 1); + const nextDateStr = formatPartitionDate(nextDate); + + console.log("Attaching existing audit log table as a partition..."); + await knex.schema.raw(` + ALTER TABLE ${INTERMEDIATE_AUDIT_LOG_TABLE} ADD CONSTRAINT audit_log_old + CHECK ( "createdAt" < DATE '${nextDateStr}' ); + + ALTER TABLE ${TableName.AuditLog} ATTACH PARTITION ${INTERMEDIATE_AUDIT_LOG_TABLE} + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); + `); + + // create partition from now until end of month + console.log("Creating audit log partitions ahead of time... next date:", nextDateStr); + await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); + + // create partitions 4 years ahead + const partitionMonths = 4 * 12; + const partitionPromises: Promise[] = []; + for (let x = 1; x <= partitionMonths; x += 1) { + partitionPromises.push( + createAuditLogPartition( + knex, + new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1), + new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1) + ) + ); + } + + await Promise.all(partitionPromises); + console.log("Partition migration complete"); + } +}; + +export const executeMigration = async (url: string) => { + console.log("Executing migration..."); + const knex = kx({ + client: "pg", + connection: url + }); + + await knex.transaction(async (tx) => { + await up(tx); + }); +}; + +const dbUrl = process.env.AUDIT_LOGS_DB_CONNECTION_URI; +if (!dbUrl) { + console.error("Please provide a DB connection URL to the AUDIT_LOGS_DB_CONNECTION_URI env"); + process.exit(1); +} + +void executeMigration(dbUrl).then(() => { + console.log("Migration: partition-audit-logs DONE"); + process.exit(0); +}); diff --git a/backend/src/db/migrations/20241005170802_kms-keys-temp-slug-col.ts b/backend/src/db/migrations/20241005170802_kms-keys-temp-slug-col.ts new file mode 100644 index 000000000..45af3e4b8 --- /dev/null +++ b/backend/src/db/migrations/20241005170802_kms-keys-temp-slug-col.ts @@ -0,0 +1,30 @@ +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.KmsKey)) { + const hasSlug = await knex.schema.hasColumn(TableName.KmsKey, "slug"); + + if (!hasSlug) { + // add slug back temporarily and set value equal to name + await knex.schema + .alterTable(TableName.KmsKey, (table) => { + table.string("slug", 32); + }) + .then(() => knex(TableName.KmsKey).update("slug", knex.ref("name"))); + } + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.KmsKey)) { + const hasSlug = await knex.schema.hasColumn(TableName.KmsKey, "slug"); + + if (hasSlug) { + await knex.schema.alterTable(TableName.KmsKey, (table) => { + table.dropColumn("slug"); + }); + } + } +} diff --git a/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts b/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts new file mode 100644 index 000000000..b6b98b9bc --- /dev/null +++ b/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts @@ -0,0 +1,48 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.AuditLog)) { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + await knex.schema.alterTable(TableName.AuditLog, (t) => { + if (doesOrgIdExist) { + t.dropForeign("orgId"); + } + + if (doesProjectIdExist) { + t.dropForeign("projectId"); + } + + // add normalized field + if (!doesProjectNameExist) { + t.string("projectName"); + } + }); + } +} + +export async function down(knex: Knex): Promise { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + if (await knex.schema.hasTable(TableName.AuditLog)) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized field + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } + }); + } +} diff --git a/backend/src/db/migrations/20241007202149_default-org-membership-roles.ts b/backend/src/db/migrations/20241007202149_default-org-membership-roles.ts new file mode 100644 index 000000000..d80853a8f --- /dev/null +++ b/backend/src/db/migrations/20241007202149_default-org-membership-roles.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; + +export async function up(knex: Knex): Promise { + // org default role + if (await knex.schema.hasTable(TableName.Organization)) { + const hasDefaultRoleCol = await knex.schema.hasColumn(TableName.Organization, "defaultMembershipRole"); + + if (!hasDefaultRoleCol) { + await knex.schema.alterTable(TableName.Organization, (tb) => { + tb.string("defaultMembershipRole").notNullable().defaultTo("member"); + }); + } + } +} + +export async function down(knex: Knex): Promise { + // org default role + if (await knex.schema.hasTable(TableName.Organization)) { + const hasDefaultRoleCol = await knex.schema.hasColumn(TableName.Organization, "defaultMembershipRole"); + + if (hasDefaultRoleCol) { + await knex.schema.alterTable(TableName.Organization, (tb) => { + tb.dropColumn("defaultMembershipRole"); + }); + } + } +} diff --git a/backend/src/db/migrations/20241015084434_increase-identity-metadata-col-length.ts b/backend/src/db/migrations/20241015084434_increase-identity-metadata-col-length.ts new file mode 100644 index 000000000..e7cdf31cf --- /dev/null +++ b/backend/src/db/migrations/20241015084434_increase-identity-metadata-col-length.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) { + await knex.schema.alterTable(TableName.IdentityMetadata, (t) => { + t.string("value", 1020).alter(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) { + await knex.schema.alterTable(TableName.IdentityMetadata, (t) => { + t.string("value", 255).alter(); + }); + } +} diff --git a/backend/src/db/migrations/20241015145450_external-group-org-role-mapping.ts b/backend/src/db/migrations/20241015145450_external-group-org-role-mapping.ts new file mode 100644 index 000000000..728d49c25 --- /dev/null +++ b/backend/src/db/migrations/20241015145450_external-group-org-role-mapping.ts @@ -0,0 +1,32 @@ +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "@app/db/utils"; + +export async function up(knex: Knex): Promise { + // add external group to org role mapping table + if (!(await knex.schema.hasTable(TableName.ExternalGroupOrgRoleMapping))) { + await knex.schema.createTable(TableName.ExternalGroupOrgRoleMapping, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("groupName").notNullable(); + t.index("groupName"); + t.string("role").notNullable(); + t.uuid("roleId"); + t.foreign("roleId").references("id").inTable(TableName.OrgRoles); + t.uuid("orgId").notNullable(); + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + t.timestamps(true, true, true); + t.unique(["orgId", "groupName"]); + }); + + await createOnUpdateTrigger(knex, TableName.ExternalGroupOrgRoleMapping); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.ExternalGroupOrgRoleMapping)) { + await dropOnUpdateTrigger(knex, TableName.ExternalGroupOrgRoleMapping); + + await knex.schema.dropTable(TableName.ExternalGroupOrgRoleMapping); + } +} diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index b8906698b..d1c239724 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), orgId: z.string().uuid().nullable().optional(), - projectId: z.string().nullable().optional() + projectId: z.string().nullable().optional(), + projectName: z.string().nullable().optional() }); export type TAuditLogs = z.infer; diff --git a/backend/src/db/schemas/external-group-org-role-mappings.ts b/backend/src/db/schemas/external-group-org-role-mappings.ts new file mode 100644 index 000000000..f7e6eab25 --- /dev/null +++ b/backend/src/db/schemas/external-group-org-role-mappings.ts @@ -0,0 +1,27 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const ExternalGroupOrgRoleMappingsSchema = z.object({ + id: z.string().uuid(), + groupName: z.string(), + role: z.string(), + roleId: z.string().uuid().nullable().optional(), + orgId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TExternalGroupOrgRoleMappings = z.infer; +export type TExternalGroupOrgRoleMappingsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TExternalGroupOrgRoleMappingsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/kms-keys.ts b/backend/src/db/schemas/kms-keys.ts index b56fab7bf..dffaeec24 100644 --- a/backend/src/db/schemas/kms-keys.ts +++ b/backend/src/db/schemas/kms-keys.ts @@ -16,7 +16,8 @@ export const KmsKeysSchema = z.object({ name: z.string(), createdAt: z.date(), updatedAt: z.date(), - projectId: z.string().nullable().optional() + projectId: z.string().nullable().optional(), + slug: z.string().nullable().optional() }); export type TKmsKeys = z.infer; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 08f3e79ce..7b48bb6fc 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -17,6 +17,7 @@ export enum TableName { Groups = "groups", GroupProjectMembership = "group_project_memberships", GroupProjectMembershipRole = "group_project_membership_roles", + ExternalGroupOrgRoleMapping = "external_group_org_role_mappings", UserGroupMembership = "user_group_membership", UserAliases = "user_aliases", UserEncryptionKey = "user_encryption_keys", diff --git a/backend/src/db/schemas/organizations.ts b/backend/src/db/schemas/organizations.ts index aa63423c9..7bd20d94d 100644 --- a/backend/src/db/schemas/organizations.ts +++ b/backend/src/db/schemas/organizations.ts @@ -19,7 +19,8 @@ export const OrganizationsSchema = z.object({ authEnforced: z.boolean().default(false).nullable().optional(), scimEnabled: z.boolean().default(false).nullable().optional(), kmsDefaultKeyId: z.string().uuid().nullable().optional(), - kmsEncryptedDataKey: zodBuffer.nullable().optional() + kmsEncryptedDataKey: zodBuffer.nullable().optional(), + defaultMembershipRole: z.string().default("member") }); export type TOrganizations = z.infer; diff --git a/backend/src/ee/routes/v1/saml-router.ts b/backend/src/ee/routes/v1/saml-router.ts index d96eb7c1f..aaebd9b6f 100644 --- a/backend/src/ee/routes/v1/saml-router.ts +++ b/backend/src/ee/routes/v1/saml-router.ts @@ -128,7 +128,10 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => { .map((key) => { // for the ones like in format: http://schemas.xmlsoap.org/ws/2005/05/identity/claims/email const formatedKey = key.startsWith("http") ? key.split("/").at(-1) || "" : key; - return { key: formatedKey, value: String((profile.attributes as Record)[key]) }; + return { + key: formatedKey, + value: String((profile.attributes as Record)[key]).substring(0, 1020) + }; }) .filter((el) => el.key && !["email", "firstName", "lastName"].includes(el.key)); diff --git a/backend/src/ee/routes/v1/scim-router.ts b/backend/src/ee/routes/v1/scim-router.ts index 427c77fa7..cd5f2f9f3 100644 --- a/backend/src/ee/routes/v1/scim-router.ts +++ b/backend/src/ee/routes/v1/scim-router.ts @@ -20,7 +20,7 @@ const ScimUserSchema = z.object({ z.object({ primary: z.boolean(), value: z.string().email(), - type: z.string().trim() + type: z.string().trim().default("work") }) ) .optional(), @@ -210,8 +210,7 @@ export const registerScimRouter = async (server: FastifyZodProvider) => { .array( z.object({ primary: z.boolean(), - value: z.string().email(), - type: z.string().trim() + value: z.string().email() }) ) .optional(), @@ -281,8 +280,7 @@ export const registerScimRouter = async (server: FastifyZodProvider) => { .array( z.object({ primary: z.boolean(), - value: z.string().email(), - type: z.string().trim() + value: z.string().email() }) ) .optional(), @@ -301,7 +299,7 @@ export const registerScimRouter = async (server: FastifyZodProvider) => { z.object({ primary: z.boolean(), value: z.string().email(), - type: z.string().trim() + type: z.string().trim().default("work") }) ), displayName: z.string().trim(), diff --git a/backend/src/ee/routes/v1/secret-scanning-router.ts b/backend/src/ee/routes/v1/secret-scanning-router.ts index 2604d7232..89784600a 100644 --- a/backend/src/ee/routes/v1/secret-scanning-router.ts +++ b/backend/src/ee/routes/v1/secret-scanning-router.ts @@ -2,6 +2,8 @@ import { z } from "zod"; import { GitAppOrgSchema, SecretScanningGitRisksSchema } from "@app/db/schemas"; import { SecretScanningRiskStatus } from "@app/ee/services/secret-scanning/secret-scanning-types"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError } from "@app/lib/errors"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -23,6 +25,13 @@ export const registerSecretScanningRouter = async (server: FastifyZodProvider) = }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const appCfg = getConfig(); + if (!appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(req.auth.orgId)) { + throw new BadRequestError({ + message: "Secret scanning is temporarily unavailable." + }); + } + const session = await server.services.secretScanning.createInstallationSession({ actor: req.permission.type, actorId: req.permission.id, @@ -30,6 +39,7 @@ export const registerSecretScanningRouter = async (server: FastifyZodProvider) = actorOrgId: req.permission.orgId, orgId: req.body.organizationId }); + return session; } }); diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 5e5e6872b..b2c80aa0b 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,8 +1,9 @@ -import { Knex } from "knex"; +// weird commonjs-related error in the CI requires us to do the import like this +import knex from "knex"; import { TDbClient } from "@app/db"; -import { AuditLogsSchema, TableName } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { TableName } from "@app/db/schemas"; +import { DatabaseError, GatewayTimeoutError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -46,7 +47,7 @@ export const auditLogDALFactory = (db: TDbClient) => { eventType?: EventType[]; eventMetadata?: Record; }, - tx?: Knex + tx?: knex.Knex ) => { if (!orgId && !projectId) { throw new Error("Either orgId or projectId must be provided"); @@ -55,11 +56,10 @@ export const auditLogDALFactory = (db: TDbClient) => { try { // Find statements const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) - .leftJoin(TableName.Project, `${TableName.AuditLog}.projectId`, `${TableName.Project}.id`) // eslint-disable-next-line func-names .where(function () { if (orgId) { - void this.where(`${TableName.Project}.orgId`, orgId).orWhere(`${TableName.AuditLog}.orgId`, orgId); + void this.where(`${TableName.AuditLog}.orgId`, orgId); } else if (projectId) { void this.where(`${TableName.AuditLog}.projectId`, projectId); } @@ -72,23 +72,19 @@ export const auditLogDALFactory = (db: TDbClient) => { // Select statements void sqlQuery .select(selectAllTableCols(TableName.AuditLog)) - .select( - db.ref("name").withSchema(TableName.Project).as("projectName"), - db.ref("slug").withSchema(TableName.Project).as("projectSlug") - ) .limit(limit) .offset(offset) .orderBy(`${TableName.AuditLog}.createdAt`, "desc"); // Special case: Filter by actor ID if (actorId) { - void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actorId]); + void sqlQuery.whereRaw(`"actorMetadata" @> jsonb_build_object('userId', ?::text)`, [actorId]); } // Special case: Filter by key/value pairs in eventMetadata field if (eventMetadata && Object.keys(eventMetadata).length) { Object.entries(eventMetadata).forEach(([key, value]) => { - void sqlQuery.whereRaw(`"eventMetadata"->>'${key}' = ?`, [value]); + void sqlQuery.whereRaw(`"eventMetadata" @> jsonb_build_object(?::text, ?::text)`, [key, value]); }); } @@ -109,30 +105,25 @@ export const auditLogDALFactory = (db: TDbClient) => { if (endDate) { void sqlQuery.where(`${TableName.AuditLog}.createdAt`, "<=", endDate); } - const docs = await sqlQuery; - return docs.map((doc) => { - // Our type system refuses to acknowledge that the project name and slug are present in the doc, due to the disjointed query structure above. - // This is a quick and dirty way to get around the types. - const projectDoc = doc as unknown as { projectName: string; projectSlug: string }; + // we timeout long running queries to prevent DB resource issues (2 minutes) + const docs = await sqlQuery.timeout(1000 * 120); - return { - ...AuditLogsSchema.parse(doc), - ...(projectDoc?.projectSlug && { - project: { - name: projectDoc.projectName, - slug: projectDoc.projectSlug - } - }) - }; - }); + return docs; } catch (error) { + if (error instanceof knex.KnexTimeoutError) { + throw new GatewayTimeoutError({ + error, + message: "Failed to fetch audit logs due to timeout. Add more search filters." + }); + } + throw new DatabaseError({ error }); } }; // delete all audit log that have expired - const pruneAuditLog = async (tx?: Knex) => { + const pruneAuditLog = async (tx?: knex.Knex) => { const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000; const MAX_RETRY_ON_FAILURE = 3; @@ -148,6 +139,7 @@ export const auditLogDALFactory = (db: TDbClient) => { .where("expiresAt", "<", today) .select("id") .limit(AUDIT_LOG_PRUNE_BATCH_SIZE); + // eslint-disable-next-line no-await-in-loop deletedAuditLogIds = await (tx || db)(TableName.AuditLog) .whereIn("id", findExpiredLogSubQuery) diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index 3fde40c8e..83a2fafa6 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -74,6 +74,7 @@ export const auditLogQueueServiceFactory = ({ actorMetadata: actor.metadata, userAgent, projectId, + projectName: project?.name, ipAddress, orgId, eventType: event.type, diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 9a2875f69..2744b34c2 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -123,6 +123,7 @@ export enum EventType { UPDATE_WEBHOOK_STATUS = "update-webhook-status", DELETE_WEBHOOK = "delete-webhook", GET_SECRET_IMPORTS = "get-secret-imports", + GET_SECRET_IMPORT = "get-secret-import", CREATE_SECRET_IMPORT = "create-secret-import", UPDATE_SECRET_IMPORT = "update-secret-import", DELETE_SECRET_IMPORT = "delete-secret-import", @@ -189,7 +190,9 @@ export enum EventType { DELETE_CMEK = "delete-cmek", GET_CMEKS = "get-cmeks", CMEK_ENCRYPT = "cmek-encrypt", - CMEK_DECRYPT = "cmek-decrypt" + CMEK_DECRYPT = "cmek-decrypt", + UPDATE_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS = "update-external-group-org-role-mapping", + GET_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS = "get-external-group-org-role-mapping" } interface UserActorMetadata { @@ -1011,6 +1014,14 @@ interface GetSecretImportsEvent { }; } +interface GetSecretImportEvent { + type: EventType.GET_SECRET_IMPORT; + metadata: { + secretImportId: string; + folderId: string; + }; +} + interface CreateSecretImportEvent { type: EventType.CREATE_SECRET_IMPORT; metadata: { @@ -1595,6 +1606,18 @@ interface CmekDecryptEvent { }; } +interface GetExternalGroupOrgRoleMappingsEvent { + type: EventType.GET_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS; + metadata?: Record; // not needed, based off orgId +} + +interface UpdateExternalGroupOrgRoleMappingsEvent { + type: EventType.UPDATE_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS; + metadata: { + mappings: { groupName: string; roleSlug: string }[]; + }; +} + export type Event = | GetSecretsEvent | GetSecretEvent @@ -1674,6 +1697,7 @@ export type Event = | UpdateWebhookStatusEvent | DeleteWebhookEvent | GetSecretImportsEvent + | GetSecretImportEvent | CreateSecretImportEvent | UpdateSecretImportEvent | DeleteSecretImportEvent @@ -1740,4 +1764,6 @@ export type Event = | DeleteCmekEvent | GetCmeksEvent | CmekEncryptEvent - | CmekDecryptEvent; + | CmekDecryptEvent + | GetExternalGroupOrgRoleMappingsEvent + | UpdateExternalGroupOrgRoleMappingsEvent; diff --git a/backend/src/ee/services/dynamic-secret/providers/ldap.ts b/backend/src/ee/services/dynamic-secret/providers/ldap.ts index 056bb582a..a4c6408cb 100644 --- a/backend/src/ee/services/dynamic-secret/providers/ldap.ts +++ b/backend/src/ee/services/dynamic-secret/providers/ldap.ts @@ -1,4 +1,4 @@ -import { compile } from "handlebars"; +import handlebars from "handlebars"; import ldapjs from "ldapjs"; import ldif from "ldif"; import { customAlphabet } from "nanoid"; @@ -40,7 +40,7 @@ const generateLDIF = ({ EncodedPassword: encodePassword(password) }; - const renderTemplate = compile(ldifTemplate); + const renderTemplate = handlebars.compile(ldifTemplate); const renderedLdif = renderTemplate(data); return renderedLdif; diff --git a/backend/src/ee/services/ldap-config/ldap-config-service.ts b/backend/src/ee/services/ldap-config/ldap-config-service.ts index ad2da3e23..7caaa5596 100644 --- a/backend/src/ee/services/ldap-config/ldap-config-service.ts +++ b/backend/src/ee/services/ldap-config/ldap-config-service.ts @@ -1,14 +1,7 @@ import { ForbiddenError } from "@casl/ability"; import jwt from "jsonwebtoken"; -import { - OrgMembershipRole, - OrgMembershipStatus, - SecretKeyEncoding, - TableName, - TLdapConfigsUpdate, - TUsers -} from "@app/db/schemas"; +import { OrgMembershipStatus, SecretKeyEncoding, TableName, TLdapConfigsUpdate, TUsers } from "@app/db/schemas"; import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; import { addUsersToGroupByUserIds, removeUsersFromGroupByUserIds } from "@app/ee/services/group/group-fns"; import { TUserGroupMembershipDALFactory } from "@app/ee/services/group/user-group-membership-dal"; @@ -28,6 +21,7 @@ import { TokenType } from "@app/services/auth-token/auth-token-types"; import { TGroupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal"; import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; @@ -444,11 +438,14 @@ export const ldapConfigServiceFactory = ({ { tx } ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgDAL.createMembership( { userId: userAlias.userId, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: OrgMembershipStatus.Accepted, isActive: true }, @@ -529,12 +526,15 @@ export const ldapConfigServiceFactory = ({ ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgMembershipDAL.create( { userId: newUser.id, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, diff --git a/backend/src/ee/services/oidc/oidc-config-service.ts b/backend/src/ee/services/oidc/oidc-config-service.ts index a8589b309..b68d5497f 100644 --- a/backend/src/ee/services/oidc/oidc-config-service.ts +++ b/backend/src/ee/services/oidc/oidc-config-service.ts @@ -3,7 +3,7 @@ import { ForbiddenError } from "@casl/ability"; import jwt from "jsonwebtoken"; import { Issuer, Issuer as OpenIdIssuer, Strategy as OpenIdStrategy, TokenSet } from "openid-client"; -import { OrgMembershipRole, OrgMembershipStatus, SecretKeyEncoding, TableName, TUsers } from "@app/db/schemas"; +import { OrgMembershipStatus, SecretKeyEncoding, TableName, TUsers } from "@app/db/schemas"; import { TOidcConfigsUpdate } from "@app/db/schemas/oidc-configs"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; @@ -23,6 +23,7 @@ import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-se import { TokenType } from "@app/services/auth-token/auth-token-types"; import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal"; import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { getServerCfg } from "@app/services/super-admin/super-admin-service"; @@ -187,12 +188,15 @@ export const oidcConfigServiceFactory = ({ { tx } ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgMembershipDAL.create( { userId: userAlias.userId, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: foundUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, @@ -261,12 +265,15 @@ export const oidcConfigServiceFactory = ({ ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgMembershipDAL.create( { userId: newUser.id, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, diff --git a/backend/src/ee/services/saml-config/saml-config-service.ts b/backend/src/ee/services/saml-config/saml-config-service.ts index 0c979d023..69d3626f0 100644 --- a/backend/src/ee/services/saml-config/saml-config-service.ts +++ b/backend/src/ee/services/saml-config/saml-config-service.ts @@ -2,7 +2,6 @@ import { ForbiddenError } from "@casl/ability"; import jwt from "jsonwebtoken"; import { - OrgMembershipRole, OrgMembershipStatus, SecretKeyEncoding, TableName, @@ -26,6 +25,7 @@ import { TokenType } from "@app/services/auth-token/auth-token-types"; import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal"; import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal"; import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { getServerCfg } from "@app/services/super-admin/super-admin-service"; @@ -369,12 +369,15 @@ export const samlConfigServiceFactory = ({ { tx } ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgMembershipDAL.create( { userId: userAlias.userId, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: foundUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, @@ -472,12 +475,15 @@ export const samlConfigServiceFactory = ({ ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole); + await orgMembershipDAL.create( { userId: newUser.id, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, diff --git a/backend/src/ee/services/scim/scim-service.ts b/backend/src/ee/services/scim/scim-service.ts index 732f37f42..9165408fa 100644 --- a/backend/src/ee/services/scim/scim-service.ts +++ b/backend/src/ee/services/scim/scim-service.ts @@ -3,7 +3,7 @@ import slugify from "@sindresorhus/slugify"; import jwt from "jsonwebtoken"; import { scimPatch } from "scim-patch"; -import { OrgMembershipRole, OrgMembershipStatus, TableName, TOrgMemberships, TUsers } from "@app/db/schemas"; +import { OrgMembershipRole, OrgMembershipStatus, TableName, TGroups, TOrgMemberships, TUsers } from "@app/db/schemas"; import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; import { addUsersToGroupByUserIds, removeUsersFromGroupByUserIds } from "@app/ee/services/group/group-fns"; import { TUserGroupMembershipDALFactory } from "@app/ee/services/group/user-group-membership-dal"; @@ -13,9 +13,11 @@ import { BadRequestError, NotFoundError, ScimRequestError, UnauthorizedError } f import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TOrgPermission } from "@app/lib/types"; import { AuthTokenType } from "@app/services/auth/auth-type"; +import { TExternalGroupOrgRoleMappingDALFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-dal"; import { TGroupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { TOrgDALFactory } from "@app/services/org/org-dal"; import { deleteOrgMembershipFn } from "@app/services/org/org-fns"; +import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; @@ -70,7 +72,10 @@ type TScimServiceFactoryDep = { | "transaction" | "updateMembershipById" >; - orgMembershipDAL: Pick; + orgMembershipDAL: Pick< + TOrgMembershipDALFactory, + "find" | "findOne" | "create" | "updateById" | "findById" | "update" + >; projectDAL: Pick; projectMembershipDAL: Pick; groupDAL: Pick< @@ -101,6 +106,7 @@ type TScimServiceFactoryDep = { permissionService: Pick; smtpService: Pick; projectUserAdditionalPrivilegeDAL: Pick; + externalGroupOrgRoleMappingDAL: TExternalGroupOrgRoleMappingDALFactory; }; export type TScimServiceFactory = ReturnType; @@ -121,7 +127,8 @@ export const scimServiceFactory = ({ projectBotDAL, permissionService, projectUserAdditionalPrivilegeDAL, - smtpService + smtpService, + externalGroupOrgRoleMappingDAL }: TScimServiceFactoryDep) => { const createScimToken = async ({ actor, @@ -318,12 +325,15 @@ export const scimServiceFactory = ({ ); if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(org.defaultMembershipRole); + orgMembership = await orgMembershipDAL.create( { userId: userAlias.userId, inviteEmail: email, orgId, - role: OrgMembershipRole.NoAccess, + role, + roleId, status: user.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, @@ -391,12 +401,15 @@ export const scimServiceFactory = ({ orgMembership = foundOrgMembership; if (!orgMembership) { + const { role, roleId } = await getDefaultOrgMembershipRole(org.defaultMembershipRole); + orgMembership = await orgMembershipDAL.create( { userId: user.id, inviteEmail: email, orgId, - role: OrgMembershipRole.Member, + role, + roleId, status: user.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later isActive: true }, @@ -685,6 +698,43 @@ export const scimServiceFactory = ({ }); }; + const $syncNewMembersRoles = async (group: TGroups, members: TScimGroup["members"]) => { + // this function handles configuring newly provisioned users org membership if an external group mapping exists + + if (!members.length) return; + + const externalGroupMapping = await externalGroupOrgRoleMappingDAL.findOne({ + orgId: group.orgId, + groupName: group.name + }); + + // no mapping, user will have default org membership + if (!externalGroupMapping) return; + + // only get org memberships that are new (invites) + const newOrgMemberships = await orgMembershipDAL.find({ + status: "invited", + $in: { + id: members.map((member) => member.value) + } + }); + + if (!newOrgMemberships.length) return; + + // set new membership roles to group mapping value + await orgMembershipDAL.update( + { + $in: { + id: newOrgMemberships.map((membership) => membership.id) + } + }, + { + role: externalGroupMapping.role, + roleId: externalGroupMapping.roleId + } + ); + }; + const createScimGroup = async ({ displayName, orgId, members }: TCreateScimGroupDTO) => { const plan = await licenseService.getPlan(orgId); if (!plan.groups) @@ -738,6 +788,8 @@ export const scimServiceFactory = ({ tx }); + await $syncNewMembersRoles(group, members); + return { group, newMembers }; } @@ -813,22 +865,41 @@ export const scimServiceFactory = ({ orgId: string, { displayName, members = [] }: { displayName: string; members: { value: string }[] } ) => { - const updatedGroup = await groupDAL.transaction(async (tx) => { - const [group] = await groupDAL.update( - { - id: groupId, - orgId - }, - { - name: displayName - } - ); + let group = await groupDAL.findOne({ + id: groupId, + orgId + }); - if (!group) { - throw new ScimRequestError({ - detail: "Group Not Found", - status: 404 - }); + if (!group) { + throw new ScimRequestError({ + detail: "Group Not Found", + status: 404 + }); + } + + const updatedGroup = await groupDAL.transaction(async (tx) => { + if (group.name !== displayName) { + await externalGroupOrgRoleMappingDAL.update( + { + groupName: group.name, + orgId + }, + { + groupName: displayName + } + ); + + const [modifiedGroup] = await groupDAL.update( + { + id: groupId, + orgId + }, + { + name: displayName + } + ); + + group = modifiedGroup; } const orgMemberships = members.length @@ -885,6 +956,8 @@ export const scimServiceFactory = ({ return group; }); + await $syncNewMembersRoles(group, members); + return updatedGroup; }; diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts index 1b19fd7f5..1907ddd9a 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts @@ -1,6 +1,6 @@ import { ProbotOctokit } from "probot"; -import { OrgMembershipRole } from "@app/db/schemas"; +import { OrgMembershipRole, TableName } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; @@ -61,7 +61,7 @@ export const secretScanningQueueFactory = ({ const getOrgAdminEmails = async (organizationId: string) => { // get emails of admins const adminsOfWork = await orgMemberDAL.findMembership({ - orgId: organizationId, + [`${TableName.Organization}.id` as string]: organizationId, role: OrgMembershipRole.Admin }); return adminsOfWork.filter((userObject) => userObject.email).map((userObject) => userObject.email as string); diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts index 913972cd1..945164094 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-service.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-service.ts @@ -90,7 +90,7 @@ export const secretScanningServiceFactory = ({ const { data: { repositories } } = await octokit.apps.listReposAccessibleToInstallation(); - if (!appCfg.DISABLE_SECRET_SCANNING) { + if (appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(actorOrgId)) { await Promise.all( repositories.map(({ id, full_name }) => secretScanningQueue.startFullRepoScan({ @@ -164,7 +164,7 @@ export const secretScanningServiceFactory = ({ }); if (!installationLink) return; - if (!appCfg.DISABLE_SECRET_SCANNING) { + if (appCfg.SECRET_SCANNING_ORG_WHITELIST?.includes(installationLink.orgId)) { await secretScanningQueue.startPushEventScan({ commits, pusher: { name: pusher.name, email: pusher.email }, diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index a1efb25ed..de285a0f2 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -240,7 +240,8 @@ export const secretSnapshotServiceFactory = ({ }, tx ); - const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( + + const snapshotSecrets = await snapshotSecretV2BridgeDAL.batchInsert( secretVersions.map(({ id }) => ({ secretVersionId: id, envId: folder.environment.envId, @@ -248,7 +249,8 @@ export const secretSnapshotServiceFactory = ({ })), tx ); - const snapshotFolders = await snapshotFolderDAL.insertMany( + + const snapshotFolders = await snapshotFolderDAL.batchInsert( folderVersions.map(({ id }) => ({ folderVersionId: id, envId: folder.environment.envId, diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index d15b4fd86..0d519ab8c 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -533,7 +533,8 @@ export const ENVIRONMENTS = { CREATE: { workspaceId: "The ID of the project to create the environment in.", name: "The name of the environment to create.", - slug: "The slug of the environment to create." + slug: "The slug of the environment to create.", + position: "The position of the environment. The lowest number will be displayed as the first environment." }, UPDATE: { workspaceId: "The ID of the project to update the environment in.", @@ -675,6 +676,9 @@ export const SECRET_IMPORTS = { environment: "The slug of the environment to list secret imports from.", path: "The path to list secret imports from." }, + GET: { + secretImportId: "The ID of the secret import to fetch." + }, CREATE: { environment: "The slug of the environment to import into.", path: "The path to import into.", diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 06b60f27e..60f4b74a9 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -34,6 +34,12 @@ const envSchema = z DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default( `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}` ), + AUDIT_LOGS_DB_CONNECTION_URI: zpStr( + z.string().describe("Postgres database connection string for Audit logs").optional() + ), + AUDIT_LOGS_DB_ROOT_CERT: zpStr( + z.string().describe("Postgres database base64-encoded CA cert for Audit logs").optional() + ), MAX_LEASE_LIMIT: z.coerce.number().default(10000), DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()), DB_HOST: zpStr(z.string().describe("Postgres database host").optional()), @@ -111,9 +117,16 @@ const envSchema = z // gcp secret manager CLIENT_ID_GCP_SECRET_MANAGER: zpStr(z.string().optional()), CLIENT_SECRET_GCP_SECRET_MANAGER: zpStr(z.string().optional()), - // github + // github oauth CLIENT_ID_GITHUB: zpStr(z.string().optional()), CLIENT_SECRET_GITHUB: zpStr(z.string().optional()), + // github app + CLIENT_ID_GITHUB_APP: zpStr(z.string().optional()), + CLIENT_SECRET_GITHUB_APP: zpStr(z.string().optional()), + CLIENT_PRIVATE_KEY_GITHUB_APP: zpStr(z.string().optional()), + CLIENT_APP_ID_GITHUB_APP: z.coerce.number().optional(), + CLIENT_SLUG_GITHUB_APP: zpStr(z.string().optional()), + // azure CLIENT_ID_AZURE: zpStr(z.string().optional()), CLIENT_SECRET_AZURE: zpStr(z.string().optional()), @@ -129,6 +142,7 @@ const envSchema = z SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()), SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()), SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional()), + SECRET_SCANNING_ORG_WHITELIST: zpStr(z.string().optional()), // LICENSE LICENSE_SERVER_URL: zpStr(z.string().optional().default("https://portal.infisical.com")), LICENSE_SERVER_KEY: zpStr(z.string().optional()), @@ -164,7 +178,8 @@ const envSchema = z Boolean(data.SECRET_SCANNING_GIT_APP_ID) && Boolean(data.SECRET_SCANNING_PRIVATE_KEY) && Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET), - samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG + samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG, + SECRET_SCANNING_ORG_WHITELIST: data.SECRET_SCANNING_ORG_WHITELIST?.split(",") })); let envCfg: Readonly>; diff --git a/backend/src/lib/errors/index.ts b/backend/src/lib/errors/index.ts index fd5631788..0818cfe7d 100644 --- a/backend/src/lib/errors/index.ts +++ b/backend/src/lib/errors/index.ts @@ -23,6 +23,18 @@ export class InternalServerError extends Error { } } +export class GatewayTimeoutError extends Error { + name: string; + + error: unknown; + + constructor({ name, error, message }: { message?: string; name?: string; error?: unknown }) { + super(message || "Timeout error"); + this.name = name || "GatewayTimeoutError"; + this.error = error; + } +} + export class UnauthorizedError extends Error { name: string; @@ -59,6 +71,13 @@ export class BadRequestError extends Error { } } +export class RateLimitError extends Error { + constructor({ message }: { message?: string }) { + super(message || "Rate limit exceeded"); + this.name = "RateLimitExceeded"; + } +} + export class NotFoundError extends Error { name: string; diff --git a/backend/src/lib/fn/array.ts b/backend/src/lib/fn/array.ts index 959d01aef..e7db061f3 100644 --- a/backend/src/lib/fn/array.ts +++ b/backend/src/lib/fn/array.ts @@ -70,3 +70,14 @@ export const objectify = ( {} as Record ); }; + +/** + * Chunks an array into smaller arrays of the given size. + */ +export const chunkArray = (array: T[], chunkSize: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; +}; diff --git a/backend/src/lib/knex/scim.ts b/backend/src/lib/knex/scim.ts index 530a7d45a..64f7fc2f6 100644 --- a/backend/src/lib/knex/scim.ts +++ b/backend/src/lib/knex/scim.ts @@ -8,12 +8,14 @@ const appendParentToGroupingOperator = (parentPath: string, filter: Filter) => { return filter; }; -export const generateKnexQueryFromScim = ( +const processDynamicQuery = ( rootQuery: Knex.QueryBuilder, - rootScimFilter: string, - getAttributeField: (attr: string) => string | null + scimRootFilterAst: Filter, + getAttributeField: (attr: string) => string | null, + depth = 0 ) => { - const scimRootFilterAst = parse(rootScimFilter); + if (depth > 20) return; + const stack = [ { scimFilterAst: scimRootFilterAst, @@ -75,42 +77,35 @@ export const generateKnexQueryFromScim = ( break; } case "and": { - void query.andWhere((subQueryBuilder) => { - scimFilterAst.filters.forEach((el) => { - stack.push({ - query: subQueryBuilder, - scimFilterAst: el - }); + scimFilterAst.filters.forEach((el) => { + void query.andWhere((subQueryBuilder) => { + processDynamicQuery(subQueryBuilder, el, getAttributeField, depth + 1); }); }); break; } case "or": { - void query.orWhere((subQueryBuilder) => { - scimFilterAst.filters.forEach((el) => { - stack.push({ - query: subQueryBuilder, - scimFilterAst: el - }); + scimFilterAst.filters.forEach((el) => { + void query.orWhere((subQueryBuilder) => { + processDynamicQuery(subQueryBuilder, el, getAttributeField, depth + 1); }); }); break; } case "not": { void query.whereNot((subQueryBuilder) => { - stack.push({ - query: subQueryBuilder, - scimFilterAst: scimFilterAst.filter - }); + processDynamicQuery(subQueryBuilder, scimFilterAst.filter, getAttributeField, depth + 1); }); break; } case "[]": { - void query.whereNot((subQueryBuilder) => { - stack.push({ - query: subQueryBuilder, - scimFilterAst: appendParentToGroupingOperator(scimFilterAst.attrPath, scimFilterAst.valFilter) - }); + void query.where((subQueryBuilder) => { + processDynamicQuery( + subQueryBuilder, + appendParentToGroupingOperator(scimFilterAst.attrPath, scimFilterAst.valFilter), + getAttributeField, + depth + 1 + ); }); break; } @@ -119,3 +114,12 @@ export const generateKnexQueryFromScim = ( } } }; + +export const generateKnexQueryFromScim = ( + rootQuery: Knex.QueryBuilder, + rootScimFilter: string, + getAttributeField: (attr: string) => string | null +) => { + const scimRootFilterAst = parse(rootScimFilter); + return processDynamicQuery(rootQuery, scimRootFilterAst, getAttributeField); +}; diff --git a/backend/src/main.ts b/backend/src/main.ts index a1c5dfd09..f71a1fe95 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -1,7 +1,7 @@ import dotenv from "dotenv"; import path from "path"; -import { initDbConnection } from "./db"; +import { initAuditLogDbConnection, initDbConnection } from "./db"; import { keyStoreFactory } from "./keystore/keystore"; import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env"; import { isMigrationMode } from "./lib/fn"; @@ -25,6 +25,13 @@ const run = async () => { })) }); + const auditLogDb = appCfg.AUDIT_LOGS_DB_CONNECTION_URI + ? initAuditLogDbConnection({ + dbConnectionUri: appCfg.AUDIT_LOGS_DB_CONNECTION_URI, + dbRootCert: appCfg.AUDIT_LOGS_DB_ROOT_CERT + }) + : undefined; + // Case: App is running in packaged mode (binary), and migration mode is enabled. // Run the migrations and exit the process after completion. if (IS_PACKAGED && isMigrationMode()) { @@ -46,7 +53,7 @@ const run = async () => { const queue = queueServiceFactory(appCfg.REDIS_URL); const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const server = await main({ db, smtp, logger, queue, keyStore }); + const server = await main({ db, auditLogDb, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 0606f9dba..457eebcc1 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,7 +1,7 @@ import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; -import { SecretKeyEncoding } from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, @@ -32,7 +32,8 @@ export enum QueueName { SecretReplication = "secret-replication", SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", - AccessTokenStatusUpdate = "access-token-status-update" + AccessTokenStatusUpdate = "access-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export enum QueueJobs { @@ -56,7 +57,8 @@ export enum QueueJobs { SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", IdentityAccessTokenStatusUpdate = "identity-access-token-status-update", - ServiceTokenStatusUpdate = "service-token-status-update" + ServiceTokenStatusUpdate = "service-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export type TQueueJobTypes = { @@ -166,6 +168,19 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.ImportSecretsFromExternalSource]: { + name: QueueJobs.ImportSecretsFromExternalSource; + payload: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 8456eed8d..b768d0db5 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -30,6 +30,7 @@ import { fastifySwagger } from "./plugins/swagger"; import { registerRoutes } from "./routes"; type TMain = { + auditLogDb?: Knex; db: Knex; smtp: TSmtpService; logger?: Logger; @@ -38,7 +39,7 @@ type TMain = { }; // Run the server! -export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { +export const main = async ({ db, auditLogDb, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); const server = fastify({ logger: appCfg.NODE_ENV === "test" ? false : logger, @@ -94,7 +95,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { await server.register(maintenanceMode); - await server.register(registerRoutes, { smtp, queue, db, keyStore }); + await server.register(registerRoutes, { smtp, queue, db, auditLogDb, keyStore }); if (appCfg.isProductionMode) { await server.register(registerExternalNextjs, { diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index bdbf80371..176d44183 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -2,6 +2,7 @@ import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-lim import { Redis } from "ioredis"; import { getConfig } from "@app/lib/config/env"; +import { RateLimitError } from "@app/lib/errors"; export const globalRateLimiterCfg = (): RateLimitPluginOptions => { const appCfg = getConfig(); @@ -10,6 +11,11 @@ export const globalRateLimiterCfg = (): RateLimitPluginOptions => { : null; return { + errorResponseBuilder: (_, context) => { + throw new RateLimitError({ + message: `Rate limit exceeded. Please try again in ${context.after}` + }); + }, timeWindow: 60 * 1000, max: 600, redis, diff --git a/backend/src/server/plugins/add-errors-to-response-schemas.ts b/backend/src/server/plugins/add-errors-to-response-schemas.ts index 75844040c..8eb358a1b 100644 --- a/backend/src/server/plugins/add-errors-to-response-schemas.ts +++ b/backend/src/server/plugins/add-errors-to-response-schemas.ts @@ -3,9 +3,12 @@ import fp from "fastify-plugin"; import { DefaultResponseErrorsSchema } from "../routes/sanitizedSchemas"; +const isScimRoutes = (pathname: string) => + pathname.startsWith("/api/v1/scim/Users") || pathname.startsWith("/api/v1/scim/Groups"); + export const addErrorsToResponseSchemas = fp(async (server) => { server.addHook("onRoute", (routeOptions) => { - if (routeOptions.schema && routeOptions.schema.response) { + if (routeOptions.schema && routeOptions.schema.response && !isScimRoutes(routeOptions.path)) { routeOptions.schema.response = { ...DefaultResponseErrorsSchema, ...routeOptions.schema.response diff --git a/backend/src/server/plugins/error-handler.ts b/backend/src/server/plugins/error-handler.ts index 8ea8b8223..be8665a84 100644 --- a/backend/src/server/plugins/error-handler.ts +++ b/backend/src/server/plugins/error-handler.ts @@ -7,8 +7,10 @@ import { BadRequestError, DatabaseError, ForbiddenRequestError, + GatewayTimeoutError, InternalServerError, NotFoundError, + RateLimitError, ScimRequestError, UnauthorizedError } from "@app/lib/errors"; @@ -25,7 +27,9 @@ enum HttpStatusCodes { Unauthorized = 401, Forbidden = 403, // eslint-disable-next-line @typescript-eslint/no-shadow - InternalServerError = 500 + InternalServerError = 500, + GatewayTimeout = 504, + TooManyRequests = 429 } export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider) => { @@ -47,6 +51,10 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider void res .status(HttpStatusCodes.InternalServerError) .send({ statusCode: HttpStatusCodes.InternalServerError, message: "Something went wrong", error: error.name }); + } else if (error instanceof GatewayTimeoutError) { + void res + .status(HttpStatusCodes.GatewayTimeout) + .send({ statusCode: HttpStatusCodes.GatewayTimeout, message: error.message, error: error.name }); } else if (error instanceof ZodError) { void res .status(HttpStatusCodes.Unauthorized) @@ -63,6 +71,12 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider message: error.message, error: error.name }); + } else if (error instanceof RateLimitError) { + void res.status(HttpStatusCodes.TooManyRequests).send({ + statusCode: HttpStatusCodes.TooManyRequests, + message: error.message, + error: error.name + }); } else if (error instanceof ScimRequestError) { void res.status(error.status).send({ schemas: error.schemas, @@ -91,7 +105,11 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider message }); } else { - void res.send(error); + void res.status(HttpStatusCodes.InternalServerError).send({ + statusCode: HttpStatusCodes.InternalServerError, + error: "InternalServerError", + message: "Something went wrong" + }); } }); }); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index b8b4303dd..68df7f2d9 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,9 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal"; import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; import { cmekServiceFactory } from "@app/services/cmek/cmek-service"; +import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-dal"; +import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; +import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; @@ -214,16 +217,15 @@ import { registerV3Routes } from "./v3"; export const registerRoutes = async ( server: FastifyZodProvider, { + auditLogDb, db, smtp: smtpService, queue: queueService, keyStore - }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } + }: { auditLogDb?: Knex; db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } ) => { const appCfg = getConfig(); - if (!appCfg.DISABLE_SECRET_SCANNING) { - await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); - } + await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); // db layers const userDAL = userDALFactory(db); @@ -283,7 +285,7 @@ export const registerRoutes = async ( const identityOidcAuthDAL = identityOidcAuthDALFactory(db); const identityAzureAuthDAL = identityAzureAuthDALFactory(db); - const auditLogDAL = auditLogDALFactory(db); + const auditLogDAL = auditLogDALFactory(auditLogDb ?? db); const auditLogStreamDAL = auditLogStreamDALFactory(db); const trustedIpDAL = trustedIpDALFactory(db); const telemetryDAL = telemetryDALFactory(db); @@ -334,6 +336,8 @@ export const registerRoutes = async ( const projectSlackConfigDAL = projectSlackConfigDALFactory(db); const workflowIntegrationDAL = workflowIntegrationDALFactory(db); + const externalGroupOrgRoleMappingDAL = externalGroupOrgRoleMappingDALFactory(db); + const permissionService = permissionServiceFactory({ permissionDAL, orgRoleDAL, @@ -440,7 +444,8 @@ export const registerRoutes = async ( projectKeyDAL, projectBotDAL, permissionService, - smtpService + smtpService, + externalGroupOrgRoleMappingDAL }); const ldapService = ldapConfigServiceFactory({ @@ -491,6 +496,9 @@ export const registerRoutes = async ( authDAL, userDAL }); + + const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL, projectDAL }); + const orgService = orgServiceFactory({ userAliasDAL, identityMetadataDAL, @@ -513,7 +521,8 @@ export const registerRoutes = async ( userDAL, groupDAL, orgBotDAL, - oidcConfigDAL + oidcConfigDAL, + projectBotService }); const signupService = authSignupServiceFactory({ tokenService, @@ -531,7 +540,12 @@ export const registerRoutes = async ( orgService, licenseService }); - const orgRoleService = orgRoleServiceFactory({ permissionService, orgRoleDAL }); + const orgRoleService = orgRoleServiceFactory({ + permissionService, + orgRoleDAL, + orgDAL, + externalGroupOrgRoleMappingDAL + }); const superAdminService = superAdminServiceFactory({ userDAL, authService: loginService, @@ -572,7 +586,6 @@ export const registerRoutes = async ( secretScanningDAL, secretScanningQueue }); - const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL, projectDAL }); const projectMembershipService = projectMembershipServiceFactory({ projectMembershipDAL, @@ -836,7 +849,10 @@ export const registerRoutes = async ( integrationAuthDAL, snapshotDAL, snapshotSecretV2BridgeDAL, - secretApprovalRequestDAL + secretApprovalRequestDAL, + projectKeyDAL, + projectUserMembershipRoleDAL, + orgService }); const secretImportService = secretImportServiceFactory({ licenseService, @@ -1201,12 +1217,33 @@ export const registerRoutes = async ( permissionService }); - const migrationService = externalMigrationServiceFactory({ - projectService, - orgService, + const externalMigrationQueue = externalMigrationQueueFactory({ projectEnvService, + projectDAL, + projectService, + smtpService, + kmsService, + projectEnvDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderDAL, + secretDAL: secretV2BridgeDAL, + queueService, + secretV2BridgeService + }); + + const migrationService = externalMigrationServiceFactory({ + externalMigrationQueue, + userDAL, + permissionService + }); + + const externalGroupOrgRoleMappingService = externalGroupOrgRoleMappingServiceFactory({ permissionService, - secretService + licenseService, + orgRoleDAL, + externalGroupOrgRoleMappingDAL }); await superAdminService.initServerCfg(); @@ -1294,7 +1331,8 @@ export const registerRoutes = async ( orgAdmin: orgAdminService, slack: slackService, workflowIntegration: workflowIntegrationService, - migration: migrationService + migration: migrationService, + externalGroupOrgRoleMapping: externalGroupOrgRoleMappingService }); const cronJobs: CronJob[] = []; diff --git a/backend/src/server/routes/v1/admin-router.ts b/backend/src/server/routes/v1/admin-router.ts index 9a3480288..e6da3ad73 100644 --- a/backend/src/server/routes/v1/admin-router.ts +++ b/backend/src/server/routes/v1/admin-router.ts @@ -109,7 +109,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { firstName: true, lastName: true, email: true, - id: true + id: true, + superAdmin: true }).array() }) } diff --git a/backend/src/server/routes/v1/external-group-org-role-mapping-router.ts b/backend/src/server/routes/v1/external-group-org-role-mapping-router.ts new file mode 100644 index 000000000..032deda7d --- /dev/null +++ b/backend/src/server/routes/v1/external-group-org-role-mapping-router.ts @@ -0,0 +1,83 @@ +import slugify from "@sindresorhus/slugify"; +import { z } from "zod"; + +import { ExternalGroupOrgRoleMappingsSchema } from "@app/db/schemas/external-group-org-role-mappings"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerExternalGroupOrgRoleMappingRouter = async (server: FastifyZodProvider) => { + // get mappings for current org + server.route({ + method: "GET", + url: "/", + config: { + rateLimit: readLimit + }, + schema: { + response: { + 200: ExternalGroupOrgRoleMappingsSchema.array() + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const mappings = server.services.externalGroupOrgRoleMapping.listExternalGroupOrgRoleMappings(req.permission); + + await server.services.auditLog.createAuditLog({ + orgId: req.permission.orgId, + ...req.auditLogInfo, + event: { + type: EventType.GET_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS + } + }); + + return mappings; + } + }); + + // update mappings for current org + server.route({ + method: "PUT", // using put since this endpoint creates, updates and deletes mappings + url: "/", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + mappings: z + .object({ + groupName: z.string().trim().min(1), + roleSlug: z + .string() + .min(1) + .toLowerCase() + .refine((v) => slugify(v) === v, { + message: "Role must be a valid slug" + }) + }) + .array() + }), + response: { + 200: ExternalGroupOrgRoleMappingsSchema.array() + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { body, permission } = req; + + const mappings = server.services.externalGroupOrgRoleMapping.updateExternalGroupOrgRoleMappings(body, permission); + + await server.services.auditLog.createAuditLog({ + orgId: permission.orgId, + ...req.auditLogInfo, + event: { + type: EventType.UPDATE_EXTERNAL_GROUP_ORG_ROLE_MAPPINGS, + metadata: body + } + }); + + return mappings; + } + }); +}; diff --git a/backend/src/server/routes/v1/identity-aws-iam-auth-router.ts b/backend/src/server/routes/v1/identity-aws-iam-auth-router.ts index 9721e46c5..9199c21f1 100644 --- a/backend/src/server/routes/v1/identity-aws-iam-auth-router.ts +++ b/backend/src/server/routes/v1/identity-aws-iam-auth-router.ts @@ -22,7 +22,7 @@ export const registerIdentityAwsAuthRouter = async (server: FastifyZodProvider) schema: { description: "Login with AWS Auth", body: z.object({ - identityId: z.string().describe(AWS_AUTH.LOGIN.identityId), + identityId: z.string().trim().describe(AWS_AUTH.LOGIN.identityId), iamHttpRequestMethod: z.string().default("POST").describe(AWS_AUTH.LOGIN.iamHttpRequestMethod), iamRequestBody: z.string().describe(AWS_AUTH.LOGIN.iamRequestBody), iamRequestHeaders: z.string().describe(AWS_AUTH.LOGIN.iamRequestHeaders) diff --git a/backend/src/server/routes/v1/identity-azure-auth-router.ts b/backend/src/server/routes/v1/identity-azure-auth-router.ts index 7526d097b..6aee4504f 100644 --- a/backend/src/server/routes/v1/identity-azure-auth-router.ts +++ b/backend/src/server/routes/v1/identity-azure-auth-router.ts @@ -21,7 +21,7 @@ export const registerIdentityAzureAuthRouter = async (server: FastifyZodProvider schema: { description: "Login with Azure Auth", body: z.object({ - identityId: z.string().describe(AZURE_AUTH.LOGIN.identityId), + identityId: z.string().trim().describe(AZURE_AUTH.LOGIN.identityId), jwt: z.string() }), response: { diff --git a/backend/src/server/routes/v1/identity-gcp-auth-router.ts b/backend/src/server/routes/v1/identity-gcp-auth-router.ts index 2022aef58..88c5af45f 100644 --- a/backend/src/server/routes/v1/identity-gcp-auth-router.ts +++ b/backend/src/server/routes/v1/identity-gcp-auth-router.ts @@ -19,7 +19,7 @@ export const registerIdentityGcpAuthRouter = async (server: FastifyZodProvider) schema: { description: "Login with GCP Auth", body: z.object({ - identityId: z.string().describe(GCP_AUTH.LOGIN.identityId), + identityId: z.string().trim().describe(GCP_AUTH.LOGIN.identityId).trim(), jwt: z.string() }), response: { diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index 55b323656..f9edfc18c 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -7,6 +7,7 @@ import { registerProjectBotRouter } from "./bot-router"; import { registerCaRouter } from "./certificate-authority-router"; import { registerCertRouter } from "./certificate-router"; import { registerCertificateTemplateRouter } from "./certificate-template-router"; +import { registerExternalGroupOrgRoleMappingRouter } from "./external-group-org-role-mapping-router"; import { registerIdentityAccessTokenRouter } from "./identity-access-token-router"; import { registerIdentityAwsAuthRouter } from "./identity-aws-iam-auth-router"; import { registerIdentityAzureAuthRouter } from "./identity-azure-auth-router"; @@ -106,4 +107,5 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await server.register(registerUserEngagementRouter, { prefix: "/user-engagement" }); await server.register(registerDashboardRouter, { prefix: "/dashboard" }); await server.register(registerCmekRouter, { prefix: "/kms" }); + await server.register(registerExternalGroupOrgRoleMappingRouter, { prefix: "/external-group-mappings" }); }; diff --git a/backend/src/server/routes/v1/integration-auth-router.ts b/backend/src/server/routes/v1/integration-auth-router.ts index 4baa39f76..1d2959f5b 100644 --- a/backend/src/server/routes/v1/integration-auth-router.ts +++ b/backend/src/server/routes/v1/integration-auth-router.ts @@ -189,6 +189,7 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider) workspaceId: z.string().trim(), code: z.string().trim(), integration: z.string().trim(), + installationId: z.string().trim().optional(), url: z.string().trim().url().optional() }), response: { @@ -452,6 +453,40 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider) } }); + server.route({ + method: "POST", + url: "/:integrationAuthId/duplicate", + config: { + rateLimit: writeLimit + }, + onRequest: verifyAuth([AuthMode.JWT]), + schema: { + params: z.object({ + integrationAuthId: z.string().trim() + }), + body: z.object({ + projectId: z.string().trim() + }), + response: { + 200: z.object({ + integrationAuth: integrationAuthPubSchema + }) + } + }, + handler: async (req) => { + const integrationAuth = await server.services.integrationAuth.duplicateIntegrationAuth({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod, + id: req.params.integrationAuthId, + projectId: req.body.projectId + }); + + return { integrationAuth }; + } + }); + server.route({ method: "GET", url: "/:integrationAuthId/github/envs", diff --git a/backend/src/server/routes/v1/integration-router.ts b/backend/src/server/routes/v1/integration-router.ts index 2a09da526..86d321852 100644 --- a/backend/src/server/routes/v1/integration-router.ts +++ b/backend/src/server/routes/v1/integration-router.ts @@ -52,7 +52,13 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - integration: IntegrationsSchema + integration: IntegrationsSchema.extend({ + environment: z.object({ + slug: z.string().trim(), + name: z.string().trim(), + id: z.string().trim() + }) + }) }) } }, @@ -138,7 +144,13 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - integration: IntegrationsSchema + integration: IntegrationsSchema.extend({ + environment: z.object({ + slug: z.string().trim(), + name: z.string().trim(), + id: z.string().trim() + }) + }) }) } }, diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index b113b9f9d..e7e5fb532 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -1,3 +1,4 @@ +import slugify from "@sindresorhus/slugify"; import { z } from "zod"; import { @@ -11,13 +12,13 @@ import { } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; import { AUDIT_LOGS, ORGANIZATIONS } from "@app/lib/api-docs"; -import { getConfig } from "@app/lib/config/env"; -import { BadRequestError } from "@app/lib/errors"; import { getLastMidnightDateISO } from "@app/lib/fn"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; +import { integrationAuthPubSchema } from "../sanitizedSchemas"; + export const registerOrgRouter = async (server: FastifyZodProvider) => { server.route({ method: "GET", @@ -69,6 +70,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + method: "GET", + url: "/:organizationId/integration-authorizations", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + organizationId: z.string().trim() + }), + response: { + 200: z.object({ + authorizations: integrationAuthPubSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const authorizations = await server.services.integrationAuth.listOrgIntegrationAuth({ + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actor: req.permission.type, + actorOrgId: req.permission.orgId + }); + + return { authorizations }; + } + }); + server.route({ method: "GET", url: "/audit-logs", @@ -125,12 +155,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }) .merge( z.object({ - project: z - .object({ - name: z.string(), - slug: z.string() - }) - .optional(), event: z.object({ type: z.string(), metadata: z.any() @@ -145,13 +169,8 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }) } }, - onRequest: verifyAuth([AuthMode.JWT]), + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const appCfg = getConfig(); - if (appCfg.isCloud) { - throw new BadRequestError({ message: "Infisical cloud audit log is in maintenance mode." }); - } - const auditLogs = await server.services.auditLog.listAuditLogs({ filter: { ...req.query, @@ -168,6 +187,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, actor: req.permission.type }); + return { auditLogs }; } }); @@ -191,7 +211,8 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { email: true, firstName: true, lastName: true, - id: true + id: true, + superAdmin: true }).merge(z.object({ publicKey: z.string().nullable() })) }) ) @@ -229,7 +250,15 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { .regex(/^[a-zA-Z0-9-]+$/, "Slug must only contain alphanumeric characters or hyphens") .optional(), authEnforced: z.boolean().optional(), - scimEnabled: z.boolean().optional() + scimEnabled: z.boolean().optional(), + defaultMembershipRoleSlug: z + .string() + .min(1) + .trim() + .refine((v) => slugify(v) === v, { + message: "Membership role must be a valid slug" + }) + .optional() }), response: { 200: z.object({ diff --git a/backend/src/server/routes/v1/password-router.ts b/backend/src/server/routes/v1/password-router.ts index c94e5d4cb..316ddcb53 100644 --- a/backend/src/server/routes/v1/password-router.ts +++ b/backend/src/server/routes/v1/password-router.ts @@ -65,7 +65,7 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => { const appCfg = getConfig(); await server.services.password.changePassword({ ...req.body, userId: req.permission.id }); - void res.cookie("jid", appCfg.COOKIE_SECRET_SIGN_KEY, { + void res.cookie("jid", "", { httpOnly: true, path: "/", sameSite: "strict", diff --git a/backend/src/server/routes/v1/project-env-router.ts b/backend/src/server/routes/v1/project-env-router.ts index fed609196..c5ded83e4 100644 --- a/backend/src/server/routes/v1/project-env-router.ts +++ b/backend/src/server/routes/v1/project-env-router.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { ProjectEnvironmentsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { ENVIRONMENTS } from "@app/lib/api-docs"; -import { writeLimit } from "@app/server/config/rateLimiter"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -23,6 +23,7 @@ export const registerProjectEnvRouter = async (server: FastifyZodProvider) => { } ], params: z.object({ + // NOTE(daniel): workspaceId isn't used, but we need to keep it for backwards compatibility. The endpoint defined below, uses no project ID, and is takes a pure environment ID. workspaceId: z.string().trim().describe(ENVIRONMENTS.GET.workspaceId), envId: z.string().trim().describe(ENVIRONMENTS.GET.id) }), @@ -39,7 +40,53 @@ export const registerProjectEnvRouter = async (server: FastifyZodProvider) => { actor: req.permission.type, actorOrgId: req.permission.orgId, actorAuthMethod: req.permission.authMethod, - projectId: req.params.workspaceId, + id: req.params.envId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: environment.projectId, + event: { + type: EventType.GET_ENVIRONMENT, + metadata: { + id: environment.id + } + } + }); + + return { environment }; + } + }); + + server.route({ + method: "GET", + url: "/environments/:envId", + config: { + rateLimit: readLimit + }, + schema: { + description: "Get Environment by ID", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + envId: z.string().trim().describe(ENVIRONMENTS.GET.id) + }), + response: { + 200: z.object({ + environment: ProjectEnvironmentsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const environment = await server.services.projectEnv.getEnvironmentById({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod, id: req.params.envId }); @@ -76,6 +123,7 @@ export const registerProjectEnvRouter = async (server: FastifyZodProvider) => { }), body: z.object({ name: z.string().trim().describe(ENVIRONMENTS.CREATE.name), + position: z.number().min(1).optional().describe(ENVIRONMENTS.CREATE.position), slug: z .string() .trim() diff --git a/backend/src/server/routes/v1/secret-folder-router.ts b/backend/src/server/routes/v1/secret-folder-router.ts index f67f9ec67..21fc1bd27 100644 --- a/backend/src/server/routes/v1/secret-folder-router.ts +++ b/backend/src/server/routes/v1/secret-folder-router.ts @@ -365,7 +365,15 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) => }), response: { 200: z.object({ - folder: SecretFoldersSchema + folder: SecretFoldersSchema.extend({ + environment: z.object({ + envId: z.string(), + envName: z.string(), + envSlug: z.string() + }), + path: z.string(), + projectId: z.string() + }) }) } }, diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index ec48803f6..aa6efdf36 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -312,6 +312,64 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => } }); + server.route({ + url: "/:secretImportId", + method: "GET", + config: { + rateLimit: readLimit + }, + schema: { + description: "Get single secret import", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + secretImportId: z.string().trim().describe(SECRET_IMPORTS.GET.secretImportId) + }), + response: { + 200: z.object({ + secretImport: SecretImportsSchema.omit({ importEnv: true }).extend({ + environment: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() + }), + projectId: z.string(), + importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }), + secretPath: z.string() + }) + }) + } + }, + + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const secretImport = await server.services.secretImport.getImportById({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + id: req.params.secretImportId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: secretImport.projectId, + event: { + type: EventType.GET_SECRET_IMPORT, + metadata: { + secretImportId: secretImport.id, + folderId: secretImport.folderId + } + } + }); + + return { secretImport }; + } + }); + server.route({ url: "/secrets", method: "GET", diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 71861bf4c..865287157 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -1,30 +1,50 @@ -import { z } from "zod"; +import fastifyMultipart from "@fastify/multipart"; +import { BadRequestError } from "@app/lib/errors"; import { readLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +const MB25_IN_BYTES = 26214400; + export const registerExternalMigrationRouter = async (server: FastifyZodProvider) => { + await server.register(fastifyMultipart); + server.route({ method: "POST", + bodyLimit: MB25_IN_BYTES, url: "/env-key", config: { rateLimit: readLimit }, - schema: { - body: z.object({ - decryptionKey: z.string().trim().min(1), - encryptedJson: z.object({ - nonce: z.string().trim().min(1), - data: z.string().trim().min(1) - }) - }) - }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { + const data = await req.file({ + limits: { + fileSize: MB25_IN_BYTES + } + }); + + if (!data) { + throw new BadRequestError({ message: "No file provided" }); + } + + const fullFile = Buffer.from(await data.toBuffer()).toString("utf8"); + const parsedJsonFile = JSON.parse(fullFile) as { nonce: string; data: string }; + + const decryptionKey = (data.fields.decryptionKey as { value: string }).value; + + if (!parsedJsonFile.nonce || !parsedJsonFile.data) { + throw new BadRequestError({ message: "Invalid file format. Nonce or data missing." }); + } + + if (!decryptionKey) { + throw new BadRequestError({ message: "Decryption key is required" }); + } + await server.services.migration.importEnvKeyData({ - decryptionKey: req.body.decryptionKey, - encryptedJson: req.body.encryptedJson, + decryptionKey, + encryptedJson: parsedJsonFile, actorId: req.permission.id, actor: req.permission.type, actorOrgId: req.permission.orgId, diff --git a/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-dal.ts b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-dal.ts new file mode 100644 index 000000000..6f8f5973c --- /dev/null +++ b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-dal.ts @@ -0,0 +1,46 @@ +import { Tables } from "knex/types/tables"; + +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { TExternalGroupOrgRoleMappings } from "@app/db/schemas/external-group-org-role-mappings"; +import { ormify } from "@app/lib/knex"; + +export type TExternalGroupOrgRoleMappingDALFactory = ReturnType; + +export const externalGroupOrgRoleMappingDALFactory = (db: TDbClient) => { + const externalGroupOrgRoleMappingOrm = ormify(db, TableName.ExternalGroupOrgRoleMapping); + + const updateExternalGroupOrgRoleMappingForOrg = async ( + orgId: string, + newMappings: readonly Tables[TableName.ExternalGroupOrgRoleMapping]["insert"][] + ) => { + const currentMappings = await externalGroupOrgRoleMappingOrm.find({ orgId }); + + const newMap = new Map(newMappings.map((mapping) => [mapping.groupName, mapping])); + const currentMap = new Map(currentMappings.map((mapping) => [mapping.groupName, mapping])); + + const mappingsToDelete = currentMappings.filter((mapping) => !newMap.has(mapping.groupName)); + const mappingsToUpdate = currentMappings + .filter((mapping) => newMap.has(mapping.groupName)) + .map((mapping) => ({ id: mapping.id, ...newMap.get(mapping.groupName) })); + const mappingsToInsert = newMappings.filter((mapping) => !currentMap.has(mapping.groupName)); + + const mappings = await externalGroupOrgRoleMappingOrm.transaction(async (tx) => { + await externalGroupOrgRoleMappingOrm.delete({ $in: { id: mappingsToDelete.map((mapping) => mapping.id) } }, tx); + + const updatedMappings: TExternalGroupOrgRoleMappings[] = []; + for await (const { id, ...mappingData } of mappingsToUpdate) { + const updatedMapping = await externalGroupOrgRoleMappingOrm.update({ id }, mappingData, tx); + updatedMappings.push(updatedMapping[0]); + } + + const insertedMappings = await externalGroupOrgRoleMappingOrm.insertMany(mappingsToInsert, tx); + + return [...updatedMappings, ...insertedMappings]; + }); + + return mappings; + }; + + return { ...externalGroupOrgRoleMappingOrm, updateExternalGroupOrgRoleMappingForOrg }; +}; diff --git a/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-fns.ts b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-fns.ts new file mode 100644 index 000000000..fe6724251 --- /dev/null +++ b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-fns.ts @@ -0,0 +1,67 @@ +import { OrgMembershipRole, TOrgRoles } from "@app/db/schemas"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { TOrgRoleDALFactory } from "@app/services/org/org-role-dal"; +import { isCustomOrgRole } from "@app/services/org/org-role-fns"; + +import { TExternalGroupOrgMembershipRoleMappingDTO } from "./external-group-org-role-mapping-types"; + +export const constructGroupOrgMembershipRoleMappings = async ({ + mappingsDTO, + orgId, + orgRoleDAL, + licenseService +}: { + mappingsDTO: TExternalGroupOrgMembershipRoleMappingDTO[]; + orgRoleDAL: TOrgRoleDALFactory; + licenseService: TLicenseServiceFactory; + orgId: string; +}) => { + const plan = await licenseService.getPlan(orgId); + + // prevent setting custom values if not in plan + if (mappingsDTO.some((map) => isCustomOrgRole(map.roleSlug)) && !plan?.rbac) + throw new BadRequestError({ + message: + "Failed to set group organization role mapping due to plan RBAC restriction. Upgrade plan to set custom role mapping." + }); + + const customRoleSlugs = mappingsDTO + .filter((mapping) => isCustomOrgRole(mapping.roleSlug)) + .map((mapping) => mapping.roleSlug); + + let customRolesMap: Map = new Map(); + if (customRoleSlugs.length > 0) { + const customRoles = await orgRoleDAL.find({ + $in: { + slug: customRoleSlugs + } + }); + + customRolesMap = new Map(customRoles.map((role) => [role.slug, role])); + } + + const mappings = mappingsDTO.map(({ roleSlug, groupName }) => { + if (isCustomOrgRole(roleSlug)) { + const customRole = customRolesMap.get(roleSlug); + + if (!customRole) throw new NotFoundError({ message: `Custom role ${roleSlug} not found.` }); + + return { + groupName, + role: OrgMembershipRole.Custom, + roleId: customRole.id, + orgId + }; + } + + return { + groupName, + role: roleSlug, + roleId: null, // need to set explicitly null for updates + orgId + }; + }); + + return mappings; +}; diff --git a/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-service.ts b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-service.ts new file mode 100644 index 000000000..2d116eb38 --- /dev/null +++ b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-service.ts @@ -0,0 +1,78 @@ +import { ForbiddenError } from "@casl/ability"; +import { FastifyRequest } from "fastify"; + +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { constructGroupOrgMembershipRoleMappings } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-fns"; +import { TSyncExternalGroupOrgMembershipRoleMappingsDTO } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-types"; +import { TOrgRoleDALFactory } from "@app/services/org/org-role-dal"; + +import { TExternalGroupOrgRoleMappingDALFactory } from "./external-group-org-role-mapping-dal"; + +type TExternalGroupOrgRoleMappingServiceFactoryDep = { + externalGroupOrgRoleMappingDAL: TExternalGroupOrgRoleMappingDALFactory; + permissionService: TPermissionServiceFactory; + licenseService: TLicenseServiceFactory; + orgRoleDAL: TOrgRoleDALFactory; +}; + +export type TExternalGroupOrgRoleMappingServiceFactory = ReturnType; + +export const externalGroupOrgRoleMappingServiceFactory = ({ + externalGroupOrgRoleMappingDAL, + licenseService, + permissionService, + orgRoleDAL +}: TExternalGroupOrgRoleMappingServiceFactoryDep) => { + const listExternalGroupOrgRoleMappings = async (actor: FastifyRequest["permission"]) => { + const { permission } = await permissionService.getOrgPermission( + actor.type, + actor.id, + actor.orgId, + actor.authMethod, + actor.orgId + ); + + // TODO: will need to change if we add support for ldap, oidc, etc. + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Scim); + + const mappings = await externalGroupOrgRoleMappingDAL.find({ + orgId: actor.orgId + }); + + return mappings; + }; + + const updateExternalGroupOrgRoleMappings = async ( + dto: TSyncExternalGroupOrgMembershipRoleMappingsDTO, + actor: FastifyRequest["permission"] + ) => { + const { permission } = await permissionService.getOrgPermission( + actor.type, + actor.id, + actor.orgId, + actor.authMethod, + actor.orgId + ); + + // TODO: will need to change if we add support for ldap, oidc, etc. + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Scim); + + const mappings = await constructGroupOrgMembershipRoleMappings({ + mappingsDTO: dto.mappings, + orgRoleDAL, + licenseService, + orgId: actor.orgId + }); + + const data = await externalGroupOrgRoleMappingDAL.updateExternalGroupOrgRoleMappingForOrg(actor.orgId, mappings); + + return data; + }; + + return { + updateExternalGroupOrgRoleMappings, + listExternalGroupOrgRoleMappings + }; +}; diff --git a/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-types.ts b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-types.ts new file mode 100644 index 000000000..2b2ccc10c --- /dev/null +++ b/backend/src/services/external-group-org-role-mapping/external-group-org-role-mapping-types.ts @@ -0,0 +1,8 @@ +export type TExternalGroupOrgMembershipRoleMappingDTO = { + groupName: string; + roleSlug: string; +}; + +export type TSyncExternalGroupOrgMembershipRoleMappingsDTO = { + mappings: TExternalGroupOrgMembershipRoleMappingDTO[]; +}; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 0e3cc576f..6d996022a 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,22 +4,41 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; -import { BadRequestError } from "@app/lib/errors"; +import { SecretType } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { TOrgServiceFactory } from "../org/org-service"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { fnSecretBulkInsert, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; +import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; export type TImportDataIntoInfisicalDTO = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; + projectService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; input: TImportInfisicalDataCreate; }; @@ -46,13 +65,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise(), - environments: new Map(), - secrets: new Map() + projects: [], + environments: [], + secrets: [] }; parsedJson.apps.forEach((app: { name: string; id: string }) => { - infisicalImportData.projects.set(app.id, { name: app.name, id: app.id }); + infisicalImportData.projects.push({ name: app.name, id: app.id }); }); // string to string map for env templates @@ -63,7 +82,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { // Import data to infisical @@ -103,95 +126,145 @@ export const importDataIntoInfisicalFn = async ({ const originalToNewProjectId = new Map(); const originalToNewEnvironmentId = new Map(); + const projectsNotImported: string[] = []; - for await (const [id, project] of data.projects) { - const newProject = await projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false - }) - .catch(() => { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); - }); - - originalToNewProjectId.set(project.id, newProject.id); - } - - // Invite user importing projects - const invites = await orgService.inviteUserToOrganization({ - actorAuthMethod, - actorId, - actorOrgId, - actor, - inviteeEmails: [], - orgId: actorOrgId, - organizationRoleSlug: OrgMembershipRole.NoAccess, - projects: Array.from(originalToNewProjectId.values()).map((project) => ({ - id: project, - projectRoleSlug: [ProjectMembershipRole.Member] - })) - }); - if (!invites) { - throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); - } - - // Import environments - if (data.environments) { - for await (const [id, environment] of data.environments) { - try { - const newEnvironment = await projectEnvService.createEnvironment({ + await projectDAL.transaction(async (tx) => { + for await (const project of data.projects) { + const newProject = await projectService + .createProject({ actor, actorId, actorOrgId, actorAuthMethod, - name: environment.name, - projectId: originalToNewProjectId.get(environment.projectId)!, - slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) + workspaceName: project.name, + createDefaultEnvs: false, + tx + }) + .catch((e) => { + logger.error(e, `Failed to import to project [name:${project.name}]`); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); }); + originalToNewProjectId.set(project.id, newProject.id); + } - if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + // Import environments + if (data.environments) { + for await (const environment of data.environments) { + const projectId = originalToNewProjectId.get(environment.projectId); + const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); + + if (!projectId) { + projectsNotImported.push(environment.projectId); + // eslint-disable-next-line no-continue + continue; + } + + const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); + + if (existingEnv) { throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + message: `Environment with slug '${slug}' already exist`, + name: "CreateEnvironment" }); } - originalToNewEnvironmentId.set(id, newEnvironment.slug); - } catch (error) { - throw new BadRequestError({ - message: `Failed to import environment: ${environment.name}]`, - name: "EnvKeyMigrationImportEnvironment" + + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + originalToNewEnvironmentId.set(environment.id, doc.slug); + } + } + + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + }[] + >(); + + for (const secret of data.secrets) { + if (!originalToNewEnvironmentId.get(secret.environmentId)) { + // eslint-disable-next-line no-continue + continue; + } + + if (!mappedToEnvironmentId.has(secret.environmentId)) { + mappedToEnvironmentId.set(secret.environmentId, []); + } + mappedToEnvironmentId.get(secret.environmentId)!.push({ + secretKey: secret.name, + secretValue: secret.value || "" }); } - } - } - // Import secrets - if (data.secrets) { - for await (const [id, secret] of data.secrets) { - const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; - if (!dataProjectId) { - throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` }); - } - const projectId = originalToNewProjectId.get(dataProjectId); - const newSecret = await secretService.createSecretRaw({ - actorId, - actor, - actorOrgId, - environment: originalToNewEnvironmentId.get(secret.environmentId)!, - actorAuthMethod, - projectId: projectId!, - secretPath: "/", - secretName: secret.name, - type: SecretType.Shared, - secretValue: secret.value - }); - if (!newSecret) { - throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); + // for each of the mappedEnvironmentId + for await (const [envId, secrets] of mappedToEnvironmentId) { + const environment = data.environments.find((env) => env.id === envId); + const projectId = originalToNewProjectId.get(environment?.projectId as string)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( + { + type: KmsDataKey.SecretManager, + projectId + }, + tx + ); + + const envSlug = originalToNewEnvironmentId.get(envId)!; + const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); + if (!folder) + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, + name: "Create secret" + }); + + const secretBatches = chunkArray(secrets, 2500); + for await (const secretBatch of secretBatches) { + const secretsByKeys = await secretDAL.findBySecretKeys( + folder.id, + secretBatch.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })), + tx + ); + if (secretsByKeys.length) { + throw new BadRequestError({ + message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` + }); + } + await fnSecretBulkInsert({ + inputSecrets: secretBatch.map((el) => { + const references = getAllNestedSecretReferences(el.secretValue); + + return { + version: 1, + encryptedValue: el.secretValue + ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + key: el.secretKey, + references, + type: SecretType.Shared + }; + }), + folderId: folder.id, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }); + } } } - } + }); + + return { projectsNotImported }; }; diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts new file mode 100644 index 000000000..76c7ceb1e --- /dev/null +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -0,0 +1,152 @@ +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TKmsServiceFactory } from "../kms/kms-service"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; +import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; +import { importDataIntoInfisicalFn } from "./external-migration-fns"; +import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; + +export type TExternalMigrationQueueFactoryDep = { + smtpService: TSmtpService; + queueService: TQueueServiceFactory; + + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; + projectService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; +}; + +export type TExternalMigrationQueueFactory = ReturnType; + +export const externalMigrationQueueFactory = ({ + queueService, + projectService, + smtpService, + projectDAL, + projectEnvService, + secretV2BridgeService, + kmsService, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL +}: TExternalMigrationQueueFactoryDep) => { + const startImport = async (dto: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }) => { + await queueService.queue( + QueueName.ImportSecretsFromExternalSource, + QueueJobs.ImportSecretsFromExternalSource, + dto, + { + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => { + try { + const { data, actorEmail } = job.data; + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import started", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportStarted + }); + + const decrypted = infisicalSymmetricDecrypt({ + ciphertext: data.ciphertext, + iv: data.iv, + keyEncoding: data.encoding, + tag: data.tag + }); + + const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate; + + const { projectsNotImported } = await importDataIntoInfisicalFn({ + input: decryptedJson, + projectDAL, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + kmsService, + projectService, + projectEnvService, + secretV2BridgeService + }); + + if (projectsNotImported.length) { + logger.info( + { + actorEmail, + actorOrgId: decryptedJson.actorOrgId, + projectsNotImported + }, + "One or more projects were not imported during import from external source" + ); + } + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import successful", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportSuccessful + }); + } catch (err) { + await smtpService.sendMail({ + recipients: [job.data.actorEmail], + subjectLine: "Infisical import failed", + substitutions: { + provider: ExternalPlatforms.EnvKey, + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment + error: (err as any)?.message || "Unknown error" + }, + template: SmtpTemplates.ExternalImportFailed + }); + + logger.error(err, "Failed to import data from external source"); + } + }); + return { + startImport + }; +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index a65a278f5..700819022 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,30 +1,25 @@ import { OrgMembershipRole } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { ForbiddenRequestError } from "@app/lib/errors"; -import { TOrgServiceFactory } from "../org/org-service"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; -import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TUserDALFactory } from "../user/user-dal"; +import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TExternalMigrationQueueFactory } from "./external-migration-queue"; import { TImportEnvKeyDataCreate } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; permissionService: TPermissionServiceFactory; + externalMigrationQueue: TExternalMigrationQueueFactory; + userDAL: Pick; }; export type TExternalMigrationServiceFactory = ReturnType; export const externalMigrationServiceFactory = ({ - projectService, - orgService, - projectEnvService, permissionService, - secretService + externalMigrationQueue, + userDAL }: TExternalMigrationServiceFactoryDep) => { const importEnvKeyData = async ({ decryptionKey, @@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({ actorAuthMethod, actorOrgId ); - if (membership.role !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ message: "Only admins can import data" }); } + const user = await userDAL.findById(actorId); const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson); const envKeyData = await parseEnvKeyDataFn(json); - const response = await importDataIntoInfisicalFn({ - input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }, - projectService, - orgService, - projectEnvService, - secretService + + const stringifiedJson = JSON.stringify({ + data: envKeyData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = infisicalSymmetricEncypt(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: encrypted }); - return response; }; return { diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index cb58ad9a2..53c954bf9 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,26 +1,9 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { - projects: Map; - - environments?: Map< - string, - { - name: string; - id: string; - projectId: string; - } - >; - - secrets?: Map< - string, - { - name: string; - id: string; - environmentId: string; - value: string; - } - >; + projects: Array<{ name: string; id: string }>; + environments: Array<{ name: string; id: string; projectId: string }>; + secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; }; export type TImportEnvKeyDataCreate = { @@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = { } >; }; + +export enum ExternalPlatforms { + EnvKey = "EnvKey" +} diff --git a/backend/src/services/identity/identity-org-dal.ts b/backend/src/services/identity/identity-org-dal.ts index 1bea4c9f4..857f24562 100644 --- a/backend/src/services/identity/identity-org-dal.ts +++ b/backend/src/services/identity/identity-org-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TIdentityOrgMemberships } from "@app/db/schemas"; +import { TableName, TIdentityOrgMemberships, TOrgRoles } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { OrderByDirection } from "@app/lib/types"; @@ -42,30 +42,50 @@ export const identityOrgDALFactory = (db: TDbClient) => { tx?: Knex ) => { try { - const paginatedFetchIdentity = (tx || db.replicaNode())(TableName.Identity) - .as(TableName.Identity) - .orderBy(`${TableName.Identity}.${orderBy}`, orderDirection); + const paginatedIdentity = (tx || db.replicaNode())(TableName.Identity) + .join( + TableName.IdentityOrgMembership, + `${TableName.IdentityOrgMembership}.identityId`, + `${TableName.Identity}.id` + ) + .orderBy(`${TableName.Identity}.${orderBy}`, orderDirection) + .select( + selectAllTableCols(TableName.IdentityOrgMembership), + db.ref("name").withSchema(TableName.Identity).as("identityName"), + db.ref("authMethod").withSchema(TableName.Identity).as("identityAuthMethod") + ) + .where(filter) + .as("paginatedIdentity"); if (search?.length) { - void paginatedFetchIdentity.whereILike(`${TableName.Identity}.name`, `%${search}%`); + void paginatedIdentity.whereILike(`${TableName.Identity}.name`, `%${search}%`); } if (limit) { - void paginatedFetchIdentity.offset(offset).limit(limit); + void paginatedIdentity.offset(offset).limit(limit); } - const query = (tx || db.replicaNode())(TableName.IdentityOrgMembership) - .where(filter) - .join>(paginatedFetchIdentity, (queryBuilder) => { - queryBuilder.on(`${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`); - }) - .leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`) + // akhilmhdh: refer this for pagination with multiple left queries + type TSubquery = Awaited; + const query = (tx || db.replicaNode()) + .from(paginatedIdentity) + .leftJoin(TableName.OrgRoles, `paginatedIdentity.roleId`, `${TableName.OrgRoles}.id`) .leftJoin(TableName.IdentityMetadata, (queryBuilder) => { void queryBuilder - .on(`${TableName.IdentityOrgMembership}.identityId`, `${TableName.IdentityMetadata}.identityId`) - .andOn(`${TableName.IdentityOrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`); + .on(`paginatedIdentity.identityId`, `${TableName.IdentityMetadata}.identityId`) + .andOn(`paginatedIdentity.orgId`, `${TableName.IdentityMetadata}.orgId`); }) - .select(selectAllTableCols(TableName.IdentityOrgMembership)) + .select( + db.ref("id").withSchema("paginatedIdentity"), + db.ref("role").withSchema("paginatedIdentity"), + db.ref("roleId").withSchema("paginatedIdentity"), + db.ref("orgId").withSchema("paginatedIdentity"), + db.ref("createdAt").withSchema("paginatedIdentity"), + db.ref("updatedAt").withSchema("paginatedIdentity"), + db.ref("identityId").withSchema("paginatedIdentity"), + db.ref("identityName").withSchema("paginatedIdentity"), + db.ref("identityAuthMethod").withSchema("paginatedIdentity") + ) // cr stands for custom role .select(db.ref("id").as("crId").withSchema(TableName.OrgRoles)) .select(db.ref("name").as("crName").withSchema(TableName.OrgRoles)) @@ -73,17 +93,14 @@ export const identityOrgDALFactory = (db: TDbClient) => { .select(db.ref("description").as("crDescription").withSchema(TableName.OrgRoles)) .select(db.ref("permissions").as("crPermission").withSchema(TableName.OrgRoles)) .select(db.ref("permissions").as("crPermission").withSchema(TableName.OrgRoles)) - .select(db.ref("id").as("identityId").withSchema(TableName.Identity)) - .select( - db.ref("name").as("identityName").withSchema(TableName.Identity), - db.ref("authMethod").as("identityAuthMethod").withSchema(TableName.Identity) - ) .select( db.ref("id").withSchema(TableName.IdentityMetadata).as("metadataId"), db.ref("key").withSchema(TableName.IdentityMetadata).as("metadataKey"), db.ref("value").withSchema(TableName.IdentityMetadata).as("metadataValue") - ) - .orderBy(`${TableName.Identity}.${orderBy}`, orderDirection); + ); + if (orderBy === OrgIdentityOrderBy.Name) { + void query.orderBy("identityName", orderDirection); + } const docs = await query; const formattedDocs = sqlNestRelationships({ diff --git a/backend/src/services/integration-auth/integration-app-list.ts b/backend/src/services/integration-auth/integration-app-list.ts index dd0cb16dc..44ac825c7 100644 --- a/backend/src/services/integration-auth/integration-app-list.ts +++ b/backend/src/services/integration-auth/integration-app-list.ts @@ -1,9 +1,13 @@ /* eslint-disable no-await-in-loop */ +import { createAppAuth } from "@octokit/auth-app"; import { Octokit } from "@octokit/rest"; +import { TIntegrationAuths } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { NotFoundError } from "@app/lib/errors"; +import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema"; import { Integrations, IntegrationUrls } from "./integration-list"; // akhilmhdh: check this part later. Copied from old base @@ -230,7 +234,13 @@ const getAppsNetlify = async ({ accessToken }: { accessToken: string }) => { /** * Return list of repositories for Github integration */ -const getAppsGithub = async ({ accessToken }: { accessToken: string }) => { +const getAppsGithub = async ({ + accessToken, + authMetadata +}: { + accessToken: string; + authMetadata?: TIntegrationAuthMetadata; +}) => { interface GitHubApp { id: string; name: string; @@ -242,6 +252,29 @@ const getAppsGithub = async ({ accessToken }: { accessToken: string }) => { }; } + if (authMetadata?.installationId) { + const appCfg = getConfig(); + const octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.CLIENT_APP_ID_GITHUB_APP, + privateKey: appCfg.CLIENT_PRIVATE_KEY_GITHUB_APP, + installationId: authMetadata.installationId + } + }); + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + const repos = await octokit.paginate("GET /installation/repositories", { + per_page: 100 + }); + + return repos.map((a) => ({ + appId: String(a.id), + name: a.name, + owner: a.owner.login + })); + } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion const repos = (await new Octokit({ auth: accessToken @@ -455,6 +488,31 @@ const getAppsCircleCI = async ({ accessToken }: { accessToken: string }) => { return apps; }; +/** + * Return list of projects for Databricks integration + */ +const getAppsDatabricks = async ({ url, accessToken }: { url?: string | null; accessToken: string }) => { + const databricksApiUrl = `${url}/api`; + + const res = await request.get<{ scopes: { name: string; backend_type: string }[] }>( + `${databricksApiUrl}/2.0/secrets/scopes/list`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" + } + } + ); + + const scopes = + res.data?.scopes?.map((a) => ({ + name: a.name, // name maps to unique scope name in Databricks + backend_type: a.backend_type + })) ?? []; + + return scopes; +}; + const getAppsTravisCI = async ({ accessToken }: { accessToken: string }) => { const res = ( await request.get<{ id: string; slug: string }[]>(`${IntegrationUrls.TRAVISCI_API_URL}/repos`, { @@ -1031,6 +1089,7 @@ const getAppsAzureDevOps = async ({ accessToken, orgName }: { accessToken: strin export const getApps = async ({ integration, + integrationAuth, accessToken, accessId, teamId, @@ -1041,6 +1100,7 @@ export const getApps = async ({ integration: string; accessToken: string; accessId?: string; + integrationAuth: TIntegrationAuths; teamId?: string | null; azureDevOpsOrgName?: string | null; workspaceSlug?: string; @@ -1074,7 +1134,8 @@ export const getApps = async ({ case Integrations.GITHUB: return getAppsGithub({ - accessToken + accessToken, + authMetadata: IntegrationAuthMetadataSchema.parse(integrationAuth.metadata || {}) }); case Integrations.GITLAB: @@ -1104,6 +1165,12 @@ export const getApps = async ({ accessToken }); + case Integrations.DATABRICKS: + return getAppsDatabricks({ + url, + accessToken + }); + case Integrations.LARAVELFORGE: return getAppsLaravelForge({ accessToken, diff --git a/backend/src/services/integration-auth/integration-auth-dal.ts b/backend/src/services/integration-auth/integration-auth-dal.ts index d32cd1579..7a56afcbb 100644 --- a/backend/src/services/integration-auth/integration-auth-dal.ts +++ b/backend/src/services/integration-auth/integration-auth-dal.ts @@ -3,7 +3,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, TIntegrationAuths, TIntegrationAuthsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; -import { ormify } from "@app/lib/knex"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TIntegrationAuthDALFactory = ReturnType; @@ -28,8 +28,23 @@ export const integrationAuthDALFactory = (db: TDbClient) => { } }; + const getByOrg = async (orgId: string, tx?: Knex) => { + try { + const integrationAuths = await (tx || db)(TableName.IntegrationAuth) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.IntegrationAuth}.projectId`) + .join(TableName.Organization, `${TableName.Organization}.id`, `${TableName.Project}.orgId`) + .where(`${TableName.Organization}.id`, "=", orgId) + .select(selectAllTableCols(TableName.IntegrationAuth)); + + return integrationAuths; + } catch (error) { + throw new DatabaseError({ error, name: "get by org" }); + } + }; + return { ...integrationAuthOrm, - bulkUpdate + bulkUpdate, + getByOrg }; }; diff --git a/backend/src/services/integration-auth/integration-auth-schema.ts b/backend/src/services/integration-auth/integration-auth-schema.ts new file mode 100644 index 000000000..94a68cc72 --- /dev/null +++ b/backend/src/services/integration-auth/integration-auth-schema.ts @@ -0,0 +1,7 @@ +import { z } from "zod"; + +export const IntegrationAuthMetadataSchema = z.object({ + installationId: z.string().optional() +}); + +export type TIntegrationAuthMetadata = z.infer; diff --git a/backend/src/services/integration-auth/integration-auth-service.ts b/backend/src/services/integration-auth/integration-auth-service.ts index c3e0dfe06..728e417cf 100644 --- a/backend/src/services/integration-auth/integration-auth-service.ts +++ b/backend/src/services/integration-auth/integration-auth-service.ts @@ -1,14 +1,16 @@ import { ForbiddenError } from "@casl/ability"; +import { createAppAuth } from "@octokit/auth-app"; import { Octokit } from "@octokit/rest"; import AWS from "aws-sdk"; import { SecretEncryptionAlgo, SecretKeyEncoding, TIntegrationAuths, TIntegrationAuthsInsert } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; -import { TProjectPermission } from "@app/lib/types"; +import { TGenericPermission, TProjectPermission } from "@app/lib/types"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TKmsServiceFactory } from "../kms/kms-service"; @@ -16,11 +18,13 @@ import { KmsDataKey } from "../kms/kms-types"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { getApps } from "./integration-app-list"; import { TIntegrationAuthDALFactory } from "./integration-auth-dal"; +import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema"; import { TBitbucketWorkspace, TChecklyGroups, TDeleteIntegrationAuthByIdDTO, TDeleteIntegrationAuthsDTO, + TDuplicateGithubIntegrationAuthDTO, TGetIntegrationAuthDTO, TGetIntegrationAuthTeamCityBuildConfigDTO, THerokuPipelineCoupling, @@ -86,6 +90,24 @@ export const integrationAuthServiceFactory = ({ return authorizations; }; + const listOrgIntegrationAuth = async ({ actorId, actor, actorOrgId, actorAuthMethod }: TGenericPermission) => { + const authorizations = await integrationAuthDAL.getByOrg(actorOrgId as string); + + return Promise.all( + authorizations.filter(async (auth) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + auth.projectId, + actorAuthMethod, + actorOrgId + ); + + return permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); + }) + ); + }; + const getIntegrationAuth = async ({ actor, id, actorId, actorAuthMethod, actorOrgId }: TGetIntegrationAuthDTO) => { const integrationAuth = await integrationAuthDAL.findById(id); if (!integrationAuth) throw new NotFoundError({ message: "Failed to find integration" }); @@ -109,7 +131,8 @@ export const integrationAuthServiceFactory = ({ actorAuthMethod, integration, url, - code + code, + installationId }: TOauthExchangeDTO) => { if (!Object.values(Integrations).includes(integration as Integrations)) throw new BadRequestError({ message: "Invalid integration" }); @@ -123,7 +146,7 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Integrations); - const tokenExchange = await exchangeCode({ integration, code, url }); + const tokenExchange = await exchangeCode({ integration, code, url, installationId }); const updateDoc: TIntegrationAuthsInsert = { projectId, integration, @@ -141,6 +164,16 @@ export const integrationAuthServiceFactory = ({ updateDoc.metadata = { authMethod: "oauth2" }; + } else if (integration === Integrations.GITHUB && installationId) { + updateDoc.metadata = { + installationId, + installationName: tokenExchange.installationName, + authMethod: "app" + }; + } + + if (installationId && integration === Integrations.GITHUB) { + return integrationAuthDAL.create(updateDoc); } const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId); @@ -176,12 +209,23 @@ export const integrationAuthServiceFactory = ({ updateDoc.accessCiphertext = accessEncToken.ciphertext; } } + return integrationAuthDAL.transaction(async (tx) => { - const doc = await integrationAuthDAL.findOne({ projectId, integration }, tx); - if (!doc) { + const integrationAuths = await integrationAuthDAL.find({ projectId, integration }, { tx }); + let existingIntegrationAuth: TIntegrationAuths | undefined; + + // we need to ensure that the integration auth that we use for Github is actually Oauth + if (integration === Integrations.GITHUB) { + existingIntegrationAuth = integrationAuths.find((integAuth) => !integAuth.metadata); + } else { + [existingIntegrationAuth] = integrationAuths; + } + + if (!existingIntegrationAuth) { return integrationAuthDAL.create(updateDoc, tx); } - return integrationAuthDAL.updateById(doc.id, updateDoc, tx); + + return integrationAuthDAL.updateById(existingIntegrationAuth.id, updateDoc, tx); }); }; @@ -334,6 +378,13 @@ export const integrationAuthServiceFactory = ({ ) { return { accessToken: "", accessId: "" }; } + if ( + integrationAuth.integration === Integrations.GITHUB && + IntegrationAuthMetadataSchema.parse(integrationAuth.metadata || {}).installationId + ) { + return { accessToken: "", accessId: "" }; + } + if (shouldUseSecretV2Bridge) { const { decryptor: secretManagerDecryptor, encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ @@ -460,6 +511,7 @@ export const integrationAuthServiceFactory = ({ const { accessToken, accessId } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const apps = await getApps({ integration: integrationAuth.integration, + integrationAuth, accessToken, accessId, teamId, @@ -575,6 +627,7 @@ export const integrationAuthServiceFactory = ({ }; const getGithubOrgs = async ({ actorId, actor, actorOrgId, actorAuthMethod, id }: TIntegrationAuthGithubOrgsDTO) => { + const appCfg = getConfig(); const integrationAuth = await integrationAuthDAL.findById(id); if (!integrationAuth) throw new NotFoundError({ message: "Failed to find integration" }); @@ -587,9 +640,44 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); - const octokit = new Octokit({ + let octokit: Octokit; + const { installationId } = (integrationAuth.metadata as TIntegrationAuthMetadata) || {}; + if (installationId) { + octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.CLIENT_APP_ID_GITHUB_APP, + privateKey: appCfg.CLIENT_PRIVATE_KEY_GITHUB_APP, + installationId + } + }); + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + const repos = await octokit.paginate("GET /installation/repositories", { + per_page: 100 + }); + + const orgSet: Set = new Set(); + + return repos + .filter((repo) => repo.owner.type === "Organization") + .map((repo) => ({ + name: repo.owner.login, + orgId: String(repo.owner.id) + })) + .filter((org) => { + const isOrgProcessed = orgSet.has(org.orgId); + if (!isOrgProcessed) { + orgSet.add(org.orgId); + } + + return !isOrgProcessed; + }); + } + + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); + octokit = new Octokit({ auth: accessToken }); @@ -598,7 +686,9 @@ export const integrationAuthServiceFactory = ({ "X-GitHub-Api-Version": "2022-11-28" } }); - if (!data) return []; + if (!data) { + return []; + } return data.map(({ login: name, id: orgId }) => ({ name, orgId: String(orgId) })); }; @@ -626,9 +716,24 @@ export const integrationAuthServiceFactory = ({ const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); - const octokit = new Octokit({ - auth: accessToken - }); + let octokit: Octokit; + const appCfg = getConfig(); + + const authMetadata = IntegrationAuthMetadataSchema.parse(integrationAuth.metadata || {}); + if (authMetadata.installationId) { + octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.CLIENT_APP_ID_GITHUB_APP, + privateKey: appCfg.CLIENT_PRIVATE_KEY_GITHUB_APP, + installationId: authMetadata.installationId + } + }); + } else { + octokit = new Octokit({ + auth: accessToken + }); + } const { data: { environments } @@ -1315,8 +1420,58 @@ export const integrationAuthServiceFactory = ({ return delIntegrationAuth; }; + // At the moment, we only use this for Github App integration as it's a special case + const duplicateIntegrationAuth = async ({ + id, + actorId, + actor, + actorAuthMethod, + actorOrgId, + projectId + }: TDuplicateGithubIntegrationAuthDTO) => { + const integrationAuth = await integrationAuthDAL.findById(id); + if (!integrationAuth) { + throw new NotFoundError({ message: "Failed to find integration" }); + } + + const { permission: sourcePermission } = await permissionService.getProjectPermission( + actor, + actorId, + integrationAuth.projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(sourcePermission).throwUnlessCan( + ProjectPermissionActions.Create, + ProjectPermissionSub.Integrations + ); + + const { permission: targetPermission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(targetPermission).throwUnlessCan( + ProjectPermissionActions.Create, + ProjectPermissionSub.Integrations + ); + + const newIntegrationAuth: Omit & { id?: string } = { + ...integrationAuth, + id: undefined, + projectId + }; + + return integrationAuthDAL.create(newIntegrationAuth); + }; + return { listIntegrationAuthByProjectId, + listOrgIntegrationAuth, getIntegrationOptions, getIntegrationAuth, oauthExchange, @@ -1343,6 +1498,7 @@ export const integrationAuthServiceFactory = ({ getNorthFlankSecretGroups, getTeamcityBuildConfigs, getBitbucketWorkspaces, - getIntegrationAccessToken + getIntegrationAccessToken, + duplicateIntegrationAuth }; }; diff --git a/backend/src/services/integration-auth/integration-auth-types.ts b/backend/src/services/integration-auth/integration-auth-types.ts index af390297a..eb8b8044d 100644 --- a/backend/src/services/integration-auth/integration-auth-types.ts +++ b/backend/src/services/integration-auth/integration-auth-types.ts @@ -9,6 +9,7 @@ export type TOauthExchangeDTO = { integration: string; code: string; url?: string; + installationId?: string; } & TProjectPermission; export type TSaveIntegrationAccessTokenDTO = { @@ -107,6 +108,10 @@ export type TDeleteIntegrationAuthByIdDTO = { id: string; } & Omit; +export type TDuplicateGithubIntegrationAuthDTO = { + id: string; +} & TProjectPermission; + export type TGetIntegrationAuthTeamCityBuildConfigDTO = { id: string; appId: string; diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index bd9619ad4..7cf77cb26 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -1,7 +1,10 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import { createAppAuth } from "@octokit/auth-app"; import { retry } from "@octokit/plugin-retry"; import { Octokit } from "@octokit/rest"; import { TIntegrationAuths, TIntegrations } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; @@ -15,6 +18,7 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema"; import { TIntegrationAuthServiceFactory } from "./integration-auth-service"; import { Integrations } from "./integration-list"; @@ -154,10 +158,12 @@ const getIntegrationSecretsV1 = async ( export const deleteGithubSecrets = async ({ integration, + authMetadata, secrets, accessToken }: { integration: Omit; + authMetadata: TIntegrationAuthMetadata; secrets: Record; accessToken: string; }) => { @@ -170,9 +176,23 @@ export const deleteGithubSecrets = async ({ } const OctokitWithRetry = Octokit.plugin(retry); - const octokit = new OctokitWithRetry({ - auth: accessToken - }); + let octokit: Octokit; + const appCfg = getConfig(); + + if (authMetadata.installationId) { + octokit = new OctokitWithRetry({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.CLIENT_APP_ID_GITHUB_APP, + privateKey: appCfg.CLIENT_PRIVATE_KEY_GITHUB_APP, + installationId: authMetadata.installationId + } + }); + } else { + octokit = new OctokitWithRetry({ + auth: accessToken + }); + } enum GithubScope { Repo = "github-repo", @@ -192,6 +212,7 @@ export const deleteGithubSecrets = async ({ break; } case GithubScope.Env: { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment encryptedGithubSecrets = ( await octokit.request("GET /repositories/{repository_id}/environments/{environment_name}/secrets", { repository_id: Number(integration.appId), @@ -346,6 +367,7 @@ export const deleteIntegrationSecrets = async ({ case Integrations.GITHUB: { await deleteGithubSecrets({ integration, + authMetadata: IntegrationAuthMetadataSchema.parse(integrationAuth.metadata || {}), accessToken, secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets }); diff --git a/backend/src/services/integration-auth/integration-list.ts b/backend/src/services/integration-auth/integration-list.ts index b91654474..af9d358ed 100644 --- a/backend/src/services/integration-auth/integration-list.ts +++ b/backend/src/services/integration-auth/integration-list.ts @@ -15,6 +15,7 @@ export enum Integrations { FLYIO = "flyio", LARAVELFORGE = "laravel-forge", CIRCLECI = "circleci", + DATABRICKS = "databricks", TRAVISCI = "travisci", TEAMCITY = "teamcity", SUPABASE = "supabase", @@ -73,6 +74,7 @@ export enum IntegrationUrls { RAILWAY_API_URL = "https://backboard.railway.app/graphql/v2", FLYIO_API_URL = "https://api.fly.io/graphql", CIRCLECI_API_URL = "https://circleci.com/api", + DATABRICKS_API_URL = "https:/xxxx.com/api", TRAVISCI_API_URL = "https://api.travis-ci.com", SUPABASE_API_URL = "https://api.supabase.com", LARAVELFORGE_API_URL = "https://forge.laravel.com", @@ -94,7 +96,9 @@ export enum IntegrationUrls { GCP_SECRET_MANAGER_SERVICE_NAME = "secretmanager.googleapis.com", GCP_SECRET_MANAGER_URL = `https://${GCP_SECRET_MANAGER_SERVICE_NAME}`, GCP_SERVICE_USAGE_URL = "https://serviceusage.googleapis.com", - GCP_CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" + GCP_CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform", + + GITHUB_USER_INSTALLATIONS = "https://api.github.com/user/installations" } export const getIntegrationOptions = async () => { @@ -136,6 +140,7 @@ export const getIntegrationOptions = async () => { isAvailable: true, type: "oauth", clientId: appCfg.CLIENT_ID_GITHUB, + clientSlug: appCfg.CLIENT_SLUG_GITHUB_APP, docsLink: "" }, { @@ -210,6 +215,15 @@ export const getIntegrationOptions = async () => { clientId: "", docsLink: "" }, + { + name: "Databricks", + slug: "databricks", + image: "Databricks.png", + isAvailable: true, + type: "pat", + clientId: "", + docsLink: "" + }, { name: "GitLab", slug: "gitlab", diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 5085d4253..7913b4029 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -9,6 +9,7 @@ import { CreateSecretCommand, + DeleteSecretCommand, DescribeSecretCommand, GetSecretValueCommand, ResourceNotFoundException, @@ -18,6 +19,7 @@ import { UpdateSecretCommand } from "@aws-sdk/client-secrets-manager"; import { AssumeRoleCommand, STSClient } from "@aws-sdk/client-sts"; +import { createAppAuth } from "@octokit/auth-app"; import { Octokit } from "@octokit/rest"; import AWS, { AWSError } from "aws-sdk"; import { AxiosError } from "axios"; @@ -35,6 +37,7 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/ import { TIntegrationDALFactory } from "../integration/integration-dal"; import { IntegrationMetadataSchema } from "../integration/integration-schema"; +import { IntegrationAuthMetadataSchema } from "./integration-auth-schema"; import { TIntegrationsWithEnvironment } from "./integration-auth-types"; import { IntegrationInitialSyncBehavior, @@ -727,7 +730,7 @@ const syncSecretsAWSParameterStore = async ({ awsParameterStoreSecretsObj[key].KeyId !== metadata.kmsKeyId; // we ensure that the KMS key configured in the integration is applied for ALL parameters on AWS - if (shouldUpdateKms || awsParameterStoreSecretsObj[key].Value !== secrets[key].value) { + if (secrets[key].value && (shouldUpdateKms || awsParameterStoreSecretsObj[key].Value !== secrets[key].value)) { await ssm .putParameter({ Name: `${integration.path}${key}`, @@ -788,7 +791,7 @@ const syncSecretsAWSParameterStore = async ({ logger.info( `getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [step=2]` ); - if (!(key in secrets)) { + if (!(key in secrets) || !secrets[key].value) { logger.info( `getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [step=3]` ); @@ -899,12 +902,21 @@ const syncSecretsAWSSecretManager = async ({ } if (!isEqual(secretToCompare, secretValue)) { - await secretsManager.send( - new UpdateSecretCommand({ - SecretId: secretId, - SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue) - }) - ); + if (secretValue) { + await secretsManager.send( + new UpdateSecretCommand({ + SecretId: secretId, + SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue) + }) + ); + // delete it + } else { + await secretsManager.send( + new DeleteSecretCommand({ + SecretId: secretId + }) + ); + } } const secretAWSTag = metadata.secretAWSTag as { key: string; value: string }[] | undefined; @@ -989,16 +1001,21 @@ const syncSecretsAWSSecretManager = async ({ } catch (err) { // case 1: when AWS manager can't find the specified secret if (err instanceof ResourceNotFoundException && secretsManager) { - await secretsManager.send( - new CreateSecretCommand({ - Name: secretId, - SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue), - ...(metadata.kmsKeyId && { KmsKeyId: metadata.kmsKeyId }), - Tags: metadata.secretAWSTag - ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({ Key: tag.key, Value: tag.value })) - : [] - }) - ); + if (secretValue) { + await secretsManager.send( + new CreateSecretCommand({ + Name: secretId, + SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue), + ...(metadata.kmsKeyId && { KmsKeyId: metadata.kmsKeyId }), + Tags: metadata.secretAWSTag + ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({ + Key: tag.key, + Value: tag.value + })) + : [] + }) + ); + } // case 2: something unexpected went wrong, so we'll throw the error to reflect the error in the integration sync status } else { throw err; @@ -1527,11 +1544,13 @@ const syncSecretsNetlify = async ({ */ const syncSecretsGitHub = async ({ integration, + integrationAuth, secrets, accessToken, appendices }: { integration: TIntegrations; + integrationAuth: TIntegrationAuths; secrets: Record; accessToken: string; appendices?: { prefix: string; suffix: string }; @@ -1553,9 +1572,24 @@ const syncSecretsGitHub = async ({ selected_repositories_url?: string | undefined; } - const octokit = new Octokit({ - auth: accessToken - }); + const authMetadata = IntegrationAuthMetadataSchema.parse(integrationAuth.metadata || {}); + let octokit: Octokit; + const appCfg = getConfig(); + + if (authMetadata.installationId) { + octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.CLIENT_APP_ID_GITHUB_APP, + privateKey: appCfg.CLIENT_PRIVATE_KEY_GITHUB_APP, + installationId: authMetadata.installationId + } + }); + } else { + octokit = new Octokit({ + auth: accessToken + }); + } enum GithubScope { Repo = "github-repo", @@ -2085,6 +2119,80 @@ const syncSecretsCircleCI = async ({ ); }; +/** + * Sync/push [secrets] to Databricks project + */ +const syncSecretsDatabricks = async ({ + integration, + integrationAuth, + secrets, + accessToken +}: { + integration: TIntegrations; + integrationAuth: TIntegrationAuths; + secrets: Record; + accessToken: string; +}) => { + const databricksApiUrl = `${integrationAuth.url}/api`; + + // sync secrets to Databricks + await Promise.all( + Object.keys(secrets).map(async (key) => + request.post( + `${databricksApiUrl}/2.0/secrets/put`, + { + scope: integration.app, + key, + string_value: secrets[key].value + }, + { + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" + } + } + ) + ) + ); + + // get secrets from Databricks + const getSecretsRes = ( + await request.get<{ secrets: { key: string; last_updated_timestamp: number }[] }>( + `${databricksApiUrl}/2.0/secrets/list`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + }, + params: { + scope: integration.app + } + } + ) + ).data.secrets; + + // delete secrets from Databricks + await Promise.all( + getSecretsRes.map(async (sec) => { + if (!(sec.key in secrets)) { + return request.post( + `${databricksApiUrl}/2.0/secrets/delete`, + { + scope: integration.app, + key: sec.key + }, + { + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + } + ); + } + }) + ); +}; + /** * Sync/push [secrets] to TravisCI project */ @@ -3980,6 +4088,7 @@ export const syncIntegrationSecrets = async ({ case Integrations.GITHUB: await syncSecretsGitHub({ integration, + integrationAuth, secrets, accessToken, appendices @@ -4021,6 +4130,14 @@ export const syncIntegrationSecrets = async ({ accessToken }); break; + case Integrations.DATABRICKS: + await syncSecretsDatabricks({ + integration, + integrationAuth, + secrets, + accessToken + }); + break; case Integrations.LARAVELFORGE: await syncSecretsLaravelForge({ integration, diff --git a/backend/src/services/integration-auth/integration-token.ts b/backend/src/services/integration-auth/integration-token.ts index ba26a3aaa..9b4e5c20f 100644 --- a/backend/src/services/integration-auth/integration-token.ts +++ b/backend/src/services/integration-auth/integration-token.ts @@ -2,7 +2,7 @@ import jwt from "jsonwebtoken"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { BadRequestError, ForbiddenRequestError, InternalServerError, NotFoundError } from "@app/lib/errors"; import { Integrations, IntegrationUrls } from "./integration-list"; @@ -234,12 +234,73 @@ const exchangeCodeNetlify = async ({ code }: { code: string }) => { }; }; -const exchangeCodeGithub = async ({ code }: { code: string }) => { +const exchangeCodeGithub = async ({ code, installationId }: { code: string; installationId?: string }) => { const appCfg = getConfig(); - if (!appCfg.CLIENT_ID_GITHUB || !appCfg.CLIENT_SECRET_GITHUB) { - throw new BadRequestError({ message: "Missing client id and client secret" }); + + if (!installationId && (!appCfg.CLIENT_ID_GITHUB || !appCfg.CLIENT_SECRET_GITHUB)) { + throw new InternalServerError({ message: "Missing client id and client secret" }); } + if (installationId && (!appCfg.CLIENT_ID_GITHUB_APP || !appCfg.CLIENT_SECRET_GITHUB_APP)) { + throw new InternalServerError({ + message: "Missing Github app client ID and client secret" + }); + } + + if (installationId) { + // handle app installations + const oauthRes = ( + await request.get(IntegrationUrls.GITHUB_TOKEN_URL, { + params: { + client_id: appCfg.CLIENT_ID_GITHUB_APP, + client_secret: appCfg.CLIENT_SECRET_GITHUB_APP, + code, + redirect_uri: `${appCfg.SITE_URL}/integrations/github/oauth2/callback` + }, + headers: { + Accept: "application/json", + "Accept-Encoding": "application/json" + } + }) + ).data; + + // use access token to validate installation ID + const installationsRes = ( + await request.get<{ + installations: { + id: number; + account: { + login: string; + }; + }[]; + }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${oauthRes.access_token}`, + "Accept-Encoding": "application/json" + } + }) + ).data; + + const matchingInstallation = installationsRes.installations.find( + (installation) => installation.id === +installationId + ); + + if (!matchingInstallation) { + throw new ForbiddenRequestError({ + message: "User has no access to the provided installation" + }); + } + + return { + accessToken: "", // for github app integrations, we only need the installationID from the metadata + refreshToken: null, + accessExpiresAt: null, + installationName: matchingInstallation.account.login + }; + } + + // handle oauth github integration const res = ( await request.get(IntegrationUrls.GITHUB_TOKEN_URL, { params: { @@ -346,6 +407,7 @@ type TExchangeReturn = { url?: string; teamId?: string; accountId?: string; + installationName?: string; }; /** @@ -355,11 +417,13 @@ type TExchangeReturn = { export const exchangeCode = async ({ integration, code, - url + url, + installationId }: { integration: string; code: string; url?: string; + installationId?: string; }): Promise => { switch (integration) { case Integrations.GCP_SECRET_MANAGER: @@ -384,7 +448,8 @@ export const exchangeCode = async ({ }); case Integrations.GITHUB: return exchangeCodeGithub({ - code + code, + installationId }); case Integrations.GITLAB: return exchangeCodeGitlab({ diff --git a/backend/src/services/integration/integration-service.ts b/backend/src/services/integration/integration-service.ts index 4f4b26e24..47a92c384 100644 --- a/backend/src/services/integration/integration-service.ts +++ b/backend/src/services/integration/integration-service.ts @@ -120,7 +120,13 @@ export const integrationServiceFactory = ({ secretPath, projectId: integrationAuth.projectId }); - return { integration, integrationAuth }; + return { + integration: { + ...integration, + environment: folder.environment + }, + integrationAuth + }; }; const updateIntegration = async ({ @@ -183,7 +189,10 @@ export const integrationServiceFactory = ({ projectId: folder.projectId }); - return updatedIntegration; + return { + ...updatedIntegration, + environment: folder.environment + }; }; const getIntegration = async ({ id, actor, actorAuthMethod, actorId, actorOrgId }: TGetIntegrationDTO) => { @@ -249,27 +258,7 @@ export const integrationServiceFactory = ({ }); } - const deletedIntegration = await integrationDAL.transaction(async (tx) => { - // delete integration - const deletedIntegrationResult = await integrationDAL.deleteById(id, tx); - - // check if there are other integrations that share the same integration auth - const integrations = await integrationDAL.find( - { - integrationAuthId: integration.integrationAuthId - }, - tx - ); - - if (integrations.length === 0) { - // no other integration shares the same integration auth - // -> delete the integration auth - await integrationAuthDAL.deleteById(integration.integrationAuthId, tx); - } - - return deletedIntegrationResult; - }); - + const deletedIntegration = await integrationDAL.deleteById(id); return { ...integration, ...deletedIntegration }; }; diff --git a/backend/src/services/kms/kms-service.ts b/backend/src/services/kms/kms-service.ts index 50dc0bd08..e1166d8c0 100644 --- a/backend/src/services/kms/kms-service.ts +++ b/backend/src/services/kms/kms-service.ts @@ -160,8 +160,8 @@ export const kmsServiceFactory = ({ * In mean time the rest of the request will wait until creation is finished followed by getting the created on * In real time this would be milliseconds */ - const getOrgKmsKeyId = async (orgId: string) => { - let org = await orgDAL.findById(orgId); + const getOrgKmsKeyId = async (orgId: string, trx?: Knex) => { + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -180,9 +180,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const keyId = await orgDAL.transaction(async (tx) => { + const keyId = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsDefaultKeyId) { return org.kmsDefaultKeyId; @@ -240,11 +240,12 @@ export const kmsServiceFactory = ({ const decryptWithKmsKey = async ({ kmsId, - depth = 0 - }: Omit & { depth?: number }) => { + depth = 0, + tx + }: Omit & { depth?: number; tx?: Knex }) => { if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" }); - const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx); if (!kmsDoc) { throw new NotFoundError({ message: "KMS ID not found" }); } @@ -261,7 +262,8 @@ export const kmsServiceFactory = ({ // we put a limit of depth to avoid too many cycles const orgKmsDecryptor = await decryptWithKmsKey({ kmsId: kmsDoc.orgKms.id, - depth: depth + 1 + depth: depth + 1, + tx }); const orgKmsDataKey = await orgKmsDecryptor({ @@ -375,9 +377,9 @@ export const kmsServiceFactory = ({ }; }; - const $getOrgKmsDataKey = async (orgId: string) => { - const kmsKeyId = await getOrgKmsKeyId(orgId); - let org = await orgDAL.findById(orgId); + const $getOrgKmsDataKey = async (orgId: string, trx?: Knex) => { + const kmsKeyId = await getOrgKmsKeyId(orgId, trx); + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -396,9 +398,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org data key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const orgDataKey = await orgDAL.transaction(async (tx) => { + const orgDataKey = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsEncryptedDataKey) { return; @@ -455,8 +457,8 @@ export const kmsServiceFactory = ({ }); }; - const getProjectSecretManagerKmsKeyId = async (projectId: string) => { - let project = await projectDAL.findById(projectId); + const getProjectSecretManagerKmsKeyId = async (projectId: string, trx?: Knex) => { + let project = await projectDAL.findById(projectId, trx); if (!project) { throw new NotFoundError({ message: "Project not found" }); } @@ -477,7 +479,7 @@ export const kmsServiceFactory = ({ project = await projectDAL.findById(projectId); } else { - const kmsKeyId = await projectDAL.transaction(async (tx) => { + const kmsKeyId = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerKeyId) { return project.kmsSecretManagerKeyId; @@ -520,9 +522,9 @@ export const kmsServiceFactory = ({ return project.kmsSecretManagerKeyId; }; - const $getProjectSecretManagerKmsDataKey = async (projectId: string) => { - const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId); - let project = await projectDAL.findById(projectId); + const $getProjectSecretManagerKmsDataKey = async (projectId: string, trx?: Knex) => { + const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId, trx); + let project = await projectDAL.findById(projectId, trx); if (!project.kmsSecretManagerEncryptedDataKey) { const lock = await keyStore @@ -538,18 +540,21 @@ export const kmsServiceFactory = ({ delay: 500 }); - project = await projectDAL.findById(projectId); + project = await projectDAL.findById(projectId, trx); } else { - const projectDataKey = await projectDAL.transaction(async (tx) => { + const projectDataKey = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerEncryptedDataKey) { return; } const dataKey = randomSecureBytes(); - const kmsEncryptor = await encryptWithKmsKey({ - kmsId: kmsKeyId - }); + const kmsEncryptor = await encryptWithKmsKey( + { + kmsId: kmsKeyId + }, + tx + ); const { cipherTextBlob } = await kmsEncryptor({ plainText: dataKey @@ -585,7 +590,8 @@ export const kmsServiceFactory = ({ } const kmsDecryptor = await decryptWithKmsKey({ - kmsId: kmsKeyId + kmsId: kmsKeyId, + tx: trx }); return kmsDecryptor({ @@ -593,13 +599,13 @@ export const kmsServiceFactory = ({ }); }; - const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => { + const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { switch (dto.type) { case KmsDataKey.SecretManager: { - return $getProjectSecretManagerKmsDataKey(dto.projectId); + return $getProjectSecretManagerKmsDataKey(dto.projectId, trx); } default: { - return $getOrgKmsDataKey(dto.orgId); + return $getOrgKmsDataKey(dto.orgId, trx); } } }; @@ -607,8 +613,9 @@ export const kmsServiceFactory = ({ // by keeping the decrypted data key in inner scope // none of the entities outside can interact directly or expose the data key // NOTICE: If changing here update migrations/utils/kms - const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => { - const dataKey = await $getDataKey(encryptionContext); + const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { + const dataKey = await $getDataKey(encryptionContext, trx); + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); return { diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index 0093c4f69..24f1d55b0 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -106,14 +106,19 @@ export const orgDALFactory = (db: TDbClient) => { db.ref("firstName").withSchema(TableName.Users), db.ref("lastName").withSchema(TableName.Users), db.ref("id").withSchema(TableName.Users).as("userId"), + db.ref("superAdmin").withSchema(TableName.Users), db.ref("publicKey").withSchema(TableName.UserEncryptionKey) ) - .where({ isGhost: false }); // MAKE SURE USER IS NOT A GHOST USER + .where({ isGhost: false }) // MAKE SURE USER IS NOT A GHOST USER + .orderBy("firstName") + .orderBy("lastName"); - return members.map(({ email, isEmailVerified, username, firstName, lastName, userId, publicKey, ...data }) => ({ - ...data, - user: { email, isEmailVerified, username, firstName, lastName, id: userId, publicKey } - })); + return members.map( + ({ email, isEmailVerified, username, firstName, lastName, userId, publicKey, superAdmin, ...data }) => ({ + ...data, + user: { email, isEmailVerified, username, firstName, lastName, id: userId, publicKey, superAdmin } + }) + ); } catch (error) { throw new DatabaseError({ error, name: "Find all org members" }); } @@ -370,6 +375,7 @@ export const orgDALFactory = (db: TDbClient) => { db.ref("firstName").withSchema(TableName.Users), db.ref("lastName").withSchema(TableName.Users), db.ref("scimEnabled").withSchema(TableName.Organization), + db.ref("defaultMembershipRole").withSchema(TableName.Organization), db.ref("externalId").withSchema(TableName.UserAliases) ) .where({ isGhost: false }); diff --git a/backend/src/services/org/org-role-fns.ts b/backend/src/services/org/org-role-fns.ts new file mode 100644 index 000000000..f460e18a4 --- /dev/null +++ b/backend/src/services/org/org-role-fns.ts @@ -0,0 +1,52 @@ +import { OrgMembershipRole } from "@app/db/schemas"; +import { TFeatureSet } from "@app/ee/services/license/license-types"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { TOrgRoleDALFactory } from "@app/services/org/org-role-dal"; + +const RESERVED_ORG_ROLE_SLUGS = Object.values(OrgMembershipRole).filter((role) => role !== "custom"); + +export const isCustomOrgRole = (roleSlug: string) => !RESERVED_ORG_ROLE_SLUGS.includes(roleSlug as OrgMembershipRole); + +// this is only for updating an org +export const getDefaultOrgMembershipRoleForUpdateOrg = async ({ + membershipRoleSlug, + orgRoleDAL, + plan, + orgId +}: { + orgId: string; + membershipRoleSlug: string; + orgRoleDAL: TOrgRoleDALFactory; + plan: TFeatureSet; +}) => { + if (isCustomOrgRole(membershipRoleSlug)) { + if (!plan?.rbac) + throw new BadRequestError({ + message: + "Failed to set custom default role due to plan RBAC restriction. Upgrade plan to set custom default org membership role." + }); + + const customRole = await orgRoleDAL.findOne({ slug: membershipRoleSlug, orgId }); + if (!customRole) throw new NotFoundError({ name: "UpdateOrg", message: "Organization role not found" }); + + // use ID for default role + return customRole.id; + } + + // not custom, use reserved slug + return membershipRoleSlug; +}; + +// this is only for creating an org membership +export const getDefaultOrgMembershipRole = async ( + defaultOrgMembershipRole: string // can either be ID or reserved slug +) => { + if (isCustomOrgRole(defaultOrgMembershipRole)) + return { + roleId: defaultOrgMembershipRole, + role: OrgMembershipRole.Custom + }; + + // will be reserved slug + return { roleId: undefined, role: defaultOrgMembershipRole as OrgMembershipRole }; +}; diff --git a/backend/src/services/org/org-role-service.ts b/backend/src/services/org/org-role-service.ts index 023cbeccf..f11d53aa0 100644 --- a/backend/src/services/org/org-role-service.ts +++ b/backend/src/services/org/org-role-service.ts @@ -11,6 +11,8 @@ import { } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { TExternalGroupOrgRoleMappingDALFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-dal"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; import { ActorAuthMethod } from "../auth/auth-type"; import { TOrgRoleDALFactory } from "./org-role-dal"; @@ -18,11 +20,18 @@ import { TOrgRoleDALFactory } from "./org-role-dal"; type TOrgRoleServiceFactoryDep = { orgRoleDAL: TOrgRoleDALFactory; permissionService: TPermissionServiceFactory; + orgDAL: TOrgDALFactory; + externalGroupOrgRoleMappingDAL: TExternalGroupOrgRoleMappingDALFactory; }; export type TOrgRoleServiceFactory = ReturnType; -export const orgRoleServiceFactory = ({ orgRoleDAL, permissionService }: TOrgRoleServiceFactoryDep) => { +export const orgRoleServiceFactory = ({ + orgRoleDAL, + orgDAL, + permissionService, + externalGroupOrgRoleMappingDAL +}: TOrgRoleServiceFactoryDep) => { const createRole = async ( userId: string, orgId: string, @@ -129,6 +138,30 @@ export const orgRoleServiceFactory = ({ orgRoleDAL, permissionService }: TOrgRol ) => { const { permission } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.Role); + + const org = await orgDAL.findOrgById(orgId); + + if (!org) + throw new NotFoundError({ + message: "Failed to find organization" + }); + + if (org.defaultMembershipRole === roleId) + throw new BadRequestError({ + message: "Cannot delete default org membership role. Please re-assign and try again." + }); + + const externalGroupMapping = await externalGroupOrgRoleMappingDAL.findOne({ + orgId, + roleId + }); + + if (externalGroupMapping) + throw new BadRequestError({ + message: + "Cannot delete role assigned to external group organization role mapping. Please re-assign external mapping and try again." + }); + const [deletedRole] = await orgRoleDAL.delete({ id: roleId, orgId }); if (!deletedRole) throw new NotFoundError({ message: "Organization role not found", name: "Update role" }); diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 4d6681669..b4b8775f0 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -32,6 +32,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro import { groupBy } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { isDisposableEmail } from "@app/lib/validator"; +import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal"; @@ -40,8 +41,9 @@ import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service"; import { TokenType } from "../auth-token/auth-token-types"; import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal"; import { TProjectDALFactory } from "../project/project-dal"; -import { assignWorkspaceKeysToMembers } from "../project/project-fns"; +import { assignWorkspaceKeysToMembers, createProjectKey } from "../project/project-fns"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; @@ -79,7 +81,7 @@ type TOrgServiceFactoryDep = { TProjectMembershipDALFactory, "findProjectMembershipsByUserId" | "delete" | "create" | "find" | "insertMany" | "transaction" >; - projectKeyDAL: Pick; + projectKeyDAL: Pick; orgMembershipDAL: Pick; incidentContactDAL: TIncidentContactsDALFactory; samlConfigDAL: Pick; @@ -93,8 +95,9 @@ type TOrgServiceFactoryDep = { >; projectUserAdditionalPrivilegeDAL: Pick; projectRoleDAL: Pick; - projectBotDAL: Pick; - projectUserMembershipRoleDAL: Pick; + projectBotDAL: Pick; + projectUserMembershipRoleDAL: Pick; + projectBotService: Pick; }; export type TOrgServiceFactory = ReturnType; @@ -121,7 +124,8 @@ export const orgServiceFactory = ({ oidcConfigDAL, projectBotDAL, projectUserMembershipRoleDAL, - identityMetadataDAL + identityMetadataDAL, + projectBotService }: TOrgServiceFactoryDep) => { /* * Get organization details by the organization id @@ -264,7 +268,7 @@ export const orgServiceFactory = ({ actorOrgId, actorAuthMethod, orgId, - data: { name, slug, authEnforced, scimEnabled } + data: { name, slug, authEnforced, scimEnabled, defaultMembershipRoleSlug } }: TUpdateOrgDTO) => { const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); @@ -298,11 +302,22 @@ export const orgServiceFactory = ({ }); } + let defaultMembershipRole: string | undefined; + if (defaultMembershipRoleSlug) { + defaultMembershipRole = await getDefaultOrgMembershipRoleForUpdateOrg({ + membershipRoleSlug: defaultMembershipRoleSlug, + orgId, + orgRoleDAL, + plan + }); + } + const org = await orgDAL.updateById(orgId, { name, slug: slug ? slugify(slug) : undefined, authEnforced, - scimEnabled + scimEnabled, + defaultMembershipRole }); if (!org) throw new NotFoundError({ message: "Organization not found" }); return org; @@ -706,20 +721,67 @@ export const orgServiceFactory = ({ const customRolesGroupBySlug = groupBy(customRoles, ({ slug }) => slug); - const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx); - if (!ghostUser) { - throw new NotFoundError({ - name: "InviteUser", - message: "Failed to find project owner" - }); - } + // this will auto generate bot + const { botKey, bot: autoGeneratedBot } = await projectBotService.getBotKey(projectId, true); - const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.id, projectId, tx); - if (!ghostUserLatestKey) { - throw new NotFoundError({ - name: "InviteUser", - message: "Failed to find project owner's latest key" + const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx); + let ghostUserId = ghostUser?.id; + + // backfill missing ghost user + if (!ghostUserId) { + const newGhostUser = await addGhostUser(project.orgId, tx); + const projectMembership = await projectMembershipDAL.create( + { + userId: newGhostUser.user.id, + projectId: project.id + }, + tx + ); + await projectUserMembershipRoleDAL.create( + { projectMembershipId: projectMembership.id, role: ProjectMembershipRole.Admin }, + tx + ); + + const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({ + publicKey: newGhostUser.keys.publicKey, + privateKey: newGhostUser.keys.plainPrivateKey, + plainProjectKey: botKey }); + + // 4. Save the project key for the ghost user. + await projectKeyDAL.create( + { + projectId: project.id, + receiverId: newGhostUser.user.id, + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv, + senderId: newGhostUser.user.id + }, + tx + ); + + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt( + newGhostUser.keys.plainPrivateKey + ); + if (autoGeneratedBot) { + await projectBotDAL.updateById( + autoGeneratedBot.id, + { + tag, + iv, + encryptedProjectKey, + encryptedProjectKeyNonce: encryptedProjectKeyIv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: newGhostUser.keys.publicKey, + senderId: newGhostUser.user.id, + algorithm, + keyEncoding: encoding + }, + tx + ); + } + ghostUserId = newGhostUser.user.id; } const bot = await projectBotDAL.findOne({ projectId }, tx); @@ -730,6 +792,14 @@ export const orgServiceFactory = ({ }); } + const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUserId, projectId, tx); + if (!ghostUserLatestKey) { + throw new NotFoundError({ + name: "InviteUser", + message: "Failed to find project owner's latest key" + }); + } + const botPrivateKey = infisicalSymmetricDecrypt({ keyEncoding: bot.keyEncoding as SecretKeyEncoding, iv: bot.iv, @@ -773,7 +843,7 @@ export const orgServiceFactory = ({ newWsMembers.map((el) => ({ encryptedKey: el.workspaceEncryptedKey, nonce: el.workspaceEncryptedNonce, - senderId: ghostUser.id, + senderId: ghostUserId, receiverId: el.orgMembershipId, projectId })), diff --git a/backend/src/services/org/org-types.ts b/backend/src/services/org/org-types.ts index 085226be0..d62a2c25b 100644 --- a/backend/src/services/org/org-types.ts +++ b/backend/src/services/org/org-types.ts @@ -26,18 +26,13 @@ export type TDeleteOrgMembershipDTO = { }; export type TInviteUserToOrgDTO = { - actorId: string; - actor: ActorType; - orgId: string; - actorOrgId: string | undefined; - actorAuthMethod: ActorAuthMethod; inviteeEmails: string[]; organizationRoleSlug: string; projects?: { id: string; projectRoleSlug?: string[]; }[]; -}; +} & TOrgPermission; export type TVerifyUserToOrgDTO = { email: string; @@ -63,7 +58,13 @@ export type TFindAllWorkspacesDTO = { }; export type TUpdateOrgDTO = { - data: Partial<{ name: string; slug: string; authEnforced: boolean; scimEnabled: boolean }>; + data: Partial<{ + name: string; + slug: string; + authEnforced: boolean; + scimEnabled: boolean; + defaultMembershipRoleSlug: string; + }>; } & TOrgPermission; export type TGetOrgGroupsDTO = TOrgPermission; diff --git a/backend/src/services/project-bot/project-bot-fns.ts b/backend/src/services/project-bot/project-bot-fns.ts index 252fd03e5..315ef2e0d 100644 --- a/backend/src/services/project-bot/project-bot-fns.ts +++ b/backend/src/services/project-bot/project-bot-fns.ts @@ -24,14 +24,14 @@ export const getBotKeyFnFactory = ( projectBotDAL: TProjectBotDALFactory, projectDAL: Pick ) => { - const getBotKeyFn = async (projectId: string) => { + const getBotKeyFn = async (projectId: string, shouldGetBotKey?: boolean) => { const project = await projectDAL.findById(projectId); if (!project) throw new NotFoundError({ message: "Project not found during bot lookup. Are you sure you are using the correct project ID?" }); - if (project.version === 3) { + if (project.version === 3 && !shouldGetBotKey) { return { project, shouldUseSecretV2Bridge: true }; } @@ -65,8 +65,9 @@ export const getBotKeyFnFactory = ( const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(botKey.privateKey); const encryptedWorkspaceKey = encryptAsymmetric(workspaceKey, botKey.publicKey, userPrivateKey); + let botId; if (!bot) { - await projectBotDAL.create({ + const newBot = await projectBotDAL.create({ name: "Infisical Bot (Ghost)", projectId, isActive: true, @@ -80,8 +81,9 @@ export const getBotKeyFnFactory = ( encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce, senderId: projectV1Keys.userId }); + botId = newBot.id; } else { - await projectBotDAL.updateById(bot.id, { + const updatedBot = await projectBotDAL.updateById(bot.id, { isActive: true, tag, iv, @@ -93,8 +95,10 @@ export const getBotKeyFnFactory = ( encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce, senderId: projectV1Keys.userId }); + botId = updatedBot.id; } - return { botKey: workspaceKey, project, shouldUseSecretV2Bridge: false }; + + return { botKey: workspaceKey, project, shouldUseSecretV2Bridge: false, bot: { id: botId } }; } const botPrivateKey = getBotPrivateKey({ bot }); @@ -104,7 +108,7 @@ export const getBotKeyFnFactory = ( nonce: bot.encryptedProjectKeyNonce, publicKey: bot.sender.publicKey }); - return { botKey, project, shouldUseSecretV2Bridge: false }; + return { botKey, project, shouldUseSecretV2Bridge: false, bot: { id: bot.id } }; }; return getBotKeyFn; diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index cc327df54..6a6178c9f 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -27,8 +27,8 @@ export const projectBotServiceFactory = ({ }: TProjectBotServiceFactoryDep) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); - const getBotKey = async (projectId: string) => { - return getBotKeyFn(projectId); + const getBotKey = async (projectId: string, shouldGetBotKey?: boolean) => { + return getBotKeyFn(projectId, shouldGetBotKey); }; const findBotByProjectId = async ({ diff --git a/backend/src/services/project-env/project-env-dal.ts b/backend/src/services/project-env/project-env-dal.ts index 8d42aab86..15e37bfdc 100644 --- a/backend/src/services/project-env/project-env-dal.ts +++ b/backend/src/services/project-env/project-env-dal.ts @@ -65,10 +65,16 @@ export const projectEnvDALFactory = (db: TDbClient) => { } }; + const shiftPositions = async (projectId: string, pos: number, tx?: Knex) => { + // Shift all positions >= the new position up by 1 + await (tx || db)(TableName.Environment).where({ projectId }).where("position", ">=", pos).increment("position", 1); + }; + return { ...projectEnvOrm, findBySlugs, findLastEnvPosition, - updateAllPosition + updateAllPosition, + shiftPositions }; }; diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index 645f34ade..67bdd867a 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -37,6 +37,7 @@ export const projectEnvServiceFactory = ({ actor, actorOrgId, actorAuthMethod, + position, name, slug }: TCreateEnvDTO) => { @@ -83,9 +84,25 @@ export const projectEnvServiceFactory = ({ } const env = await projectEnvDAL.transaction(async (tx) => { + if (position !== undefined) { + // Check if there's an environment at the specified position + const existingEnvWithPosition = await projectEnvDAL.findOne({ projectId, position }, tx); + + // If there is, then shift positions + if (existingEnvWithPosition) { + await projectEnvDAL.shiftPositions(projectId, position, tx); + } + + const doc = await projectEnvDAL.create({ slug, name, projectId, position }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + return doc; + } + // If no position is specified, add to the end const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx); await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + return doc; }); @@ -150,7 +167,11 @@ export const projectEnvServiceFactory = ({ const env = await projectEnvDAL.transaction(async (tx) => { if (position) { - await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + const existingEnvWithPosition = await projectEnvDAL.findOne({ projectId, position }, tx); + + if (existingEnvWithPosition && existingEnvWithPosition.id !== oldEnv.id) { + await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + } } return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx); }); @@ -199,7 +220,6 @@ export const projectEnvServiceFactory = ({ name: "DeleteEnvironment" }); - await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx); return doc; }); @@ -215,29 +235,26 @@ export const projectEnvServiceFactory = ({ } }; - const getEnvironmentById = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => { + const getEnvironmentById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => { + const environment = await projectEnvDAL.findById(id); + + if (!environment) { + throw new NotFoundError({ + message: "Environment does not exist" + }); + } + const { permission } = await permissionService.getProjectPermission( actor, actorId, - projectId, + environment.projectId, actorAuthMethod, actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Environments); - const [env] = await projectEnvDAL.find({ - id, - projectId - }); - - if (!env) { - throw new NotFoundError({ - message: "Environment does not exist" - }); - } - - return env; + return environment; }; return { diff --git a/backend/src/services/project-env/project-env-types.ts b/backend/src/services/project-env/project-env-types.ts index 27d808a47..a87c76d4d 100644 --- a/backend/src/services/project-env/project-env-types.ts +++ b/backend/src/services/project-env/project-env-types.ts @@ -3,6 +3,7 @@ import { TProjectPermission } from "@app/lib/types"; export type TCreateEnvDTO = { name: string; slug: string; + position?: number; } & TProjectPermission; export type TUpdateEnvDTO = { @@ -23,4 +24,4 @@ export type TReorderEnvDTO = { export type TGetEnvDTO = { id: string; -} & TProjectPermission; +} & Omit; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index ec6c54dcc..d99212744 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -147,6 +147,7 @@ export const projectServiceFactory = ({ workspaceName, slug: projectSlug, kmsKeyId, + tx: trx, createDefaultEnvs = true }: TCreateProjectDTO) => { const organization = await orgDAL.findOne({ id: actorOrgId }); @@ -169,7 +170,7 @@ export const projectServiceFactory = ({ }); } - const results = await projectDAL.transaction(async (tx) => { + const results = await (trx || projectDAL).transaction(async (tx) => { const ghostUser = await orgService.addGhostUser(organization.id, tx); if (kmsKeyId) { diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index 7193f1121..d35fcb24f 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TProjectKeys } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; @@ -30,6 +32,7 @@ export type TCreateProjectDTO = { slug?: string; kmsKeyId?: string; createDefaultEnvs?: boolean; + tx?: Knex; }; export type TDeleteProjectBySlugDTO = { diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 7062a0c4f..b16d90b6b 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -502,12 +502,21 @@ export const secretFolderServiceFactory = ({ const getFolderById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetFolderByIdDTO) => { const folder = await folderDAL.findById(id); - if (!folder) throw new NotFoundError({ message: "folder not found" }); + if (!folder) throw new NotFoundError({ message: "Folder not found" }); // folder list is allowed to be read by anyone // permission to check does user has access await permissionService.getProjectPermission(actor, actorId, folder.projectId, actorAuthMethod, actorOrgId); - return folder; + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]); + + if (!folderWithPath) { + throw new NotFoundError({ message: "Folder path not found" }); + } + + return { + ...folder, + path: folderWithPath.path + }; }; return { diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index 893a6d6b4..da25f4d30 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -97,6 +97,34 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; + const findById = async (id: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())(TableName.SecretImport) + .where({ [`${TableName.SecretImport}.id` as "id"]: id }) + .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) + .select( + db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment), + db.ref("id").withSchema(TableName.Environment).as("envId") + ) + .first(); + + if (!doc) { + return null; + } + + const { envId, slug, name, ...el } = doc; + + return { + ...el, + importEnv: { id: envId, slug, name } + }; + } catch (error) { + throw new DatabaseError({ error, name: "Find secret imports" }); + } + }; + const getProjectImportCount = async ( { search, ...filter }: Partial, tx?: Knex @@ -144,6 +172,7 @@ export const secretImportDALFactory = (db: TDbClient) => { return { ...secretImportOrm, find, + findById, findByFolderIds, findLastImportPosition, updateAllPosition, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 9f40c9702..5551b0180 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -24,6 +24,7 @@ import { fnSecretsFromImports, fnSecretsV2FromImports } from "./secret-import-fn import { TCreateSecretImportDTO, TDeleteSecretImportDTO, + TGetSecretImportByIdDTO, TGetSecretImportsDTO, TGetSecretsFromImportDTO, TResyncSecretImportReplicationDTO, @@ -455,6 +456,64 @@ export const secretImportServiceFactory = ({ return secImports; }; + const getImportById = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + id: importId + }: TGetSecretImportByIdDTO) => { + const importDoc = await secretImportDAL.findById(importId); + + if (!importDoc) { + throw new NotFoundError({ message: "Secret import not found" }); + } + + // the folder to import into + const folder = await folderDAL.findById(importDoc.folderId); + + if (!folder) throw new NotFoundError({ message: "Secret import folder not found" }); + + // the folder to import into, with path + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]); + + if (!folderWithPath) throw new NotFoundError({ message: "Folder path not found" }); + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + folder.projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: folder.environment.envSlug, + secretPath: folderWithPath.path + }) + ); + + const importIntoEnv = await projectEnvDAL.findOne({ + projectId: folder.projectId, + slug: folder.environment.envSlug + }); + + if (!importIntoEnv) throw new NotFoundError({ message: "Environment to import into not found" }); + + return { + ...importDoc, + projectId: folder.projectId, + secretPath: folderWithPath.path, + environment: { + id: importIntoEnv.id, + slug: importIntoEnv.slug, + name: importIntoEnv.name + } + }; + }; + const getSecretsFromImports = async ({ path: secretPath, environment, @@ -565,6 +624,7 @@ export const secretImportServiceFactory = ({ updateImport, deleteImport, getImports, + getImportById, getSecretsFromImports, getRawSecretsFromImports, resyncSecretImportReplication, diff --git a/backend/src/services/secret-import/secret-import-types.ts b/backend/src/services/secret-import/secret-import-types.ts index 0a72c4da2..638e36cb1 100644 --- a/backend/src/services/secret-import/secret-import-types.ts +++ b/backend/src/services/secret-import/secret-import-types.ts @@ -37,6 +37,10 @@ export type TGetSecretImportsDTO = { offset?: number; } & TProjectPermission; +export type TGetSecretImportByIdDTO = { + id: string; +} & Omit; + export type TGetSecretsFromImportDTO = { environment: string; path: string; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 78e21b579..1ae7ce6dc 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { TableName, TSecretFolders, TSecretsV2 } from "@app/db/schemas"; -import { ForbiddenRequestError } from "@app/lib/errors"; +import { ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -82,7 +82,10 @@ export const fnSecretBulkInsert = async ({ }) ); - const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId }))); + const newSecrets = await secretDAL.insertMany( + sanitizedInputSecrets.map((el) => ({ ...el, folderId })), + tx + ); const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key); const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) => secretTags.map((tag) => ({ @@ -339,7 +342,7 @@ export const recursivelyGetSecretPaths = async ({ }); if (!env) { - throw new Error(`'${environment}' environment not found in project with ID ${projectId}`); + throw new NotFoundError({ message: `'${environment}' environment not found in project with ID ${projectId}` }); } // Fetch all folders in env once with a single query diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 6e8d95e1c..0d2797800 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -193,14 +193,16 @@ export const secretV2BridgeServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - secretPath, - actorId, - actor, - projectId, - environmentSlug: folder.environment.slug - }); + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } return reshapeBridgeSecret(projectId, environment, secretPath, { ...secret[0], @@ -349,14 +351,17 @@ export const secretV2BridgeServiceFactory = ({ projectId }); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - actor, - actorId, - secretPath, - projectId, - environmentSlug: folder.environment.slug - }); + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } + return reshapeBridgeSecret(projectId, environment, secretPath, { ...updatedSecret[0], value: inputSecret.secretValue || "", @@ -427,14 +432,16 @@ export const secretV2BridgeServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - actor, - actorId, - secretPath, - projectId, - environmentSlug: folder.environment.slug - }); + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 3afde2f1e..1d0b89b46 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -152,7 +152,7 @@ export const recursivelyGetSecretPaths = ({ }); if (!env) { - throw new Error(`'${environment}' environment not found in project with ID ${projectId}`); + throw new NotFoundError({ message: `'${environment}' environment not found in project with ID ${projectId}` }); } // Fetch all folders in env once with a single query diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 4b3f7dfbf..4076b179f 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -17,6 +17,7 @@ import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/sn import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { getTimeDifferenceInSeconds, groupBy, isSamePath, unique } from "@app/lib/fn"; @@ -37,10 +38,14 @@ import { syncIntegrationSecrets } from "../integration-auth/integration-sync-sec import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TOrgDALFactory } from "../org/org-dal"; +import { TOrgServiceFactory } from "../org/org-service"; import { TProjectDALFactory } from "../project/project-dal"; +import { createProjectKey } from "../project/project-fns"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; +import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; @@ -77,7 +82,8 @@ type TSecretQueueFactoryDep = { projectEnvDAL: Pick; projectDAL: TProjectDALFactory; projectBotDAL: TProjectBotDALFactory; - projectMembershipDAL: Pick; + projectKeyDAL: Pick; + projectMembershipDAL: Pick; smtpService: TSmtpService; orgDAL: Pick; secretVersionDAL: TSecretVersionDALFactory; @@ -85,7 +91,7 @@ type TSecretQueueFactoryDep = { secretTagDAL: TSecretTagDALFactory; userDAL: Pick; secretVersionTagDAL: TSecretVersionTagDALFactory; - kmsService: Pick; + kmsService: TKmsServiceFactory; secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; @@ -95,6 +101,8 @@ type TSecretQueueFactoryDep = { snapshotSecretV2BridgeDAL: Pick; keyStore: Pick; auditLogService: Pick; + orgService: Pick; + projectUserMembershipRoleDAL: Pick; }; export type TGetSecrets = { @@ -111,6 +119,8 @@ type TIntegrationSecret = Record< string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined } >; + +// TODO(akhilmhdh): split this into multiple queue export const secretQueueFactory = ({ queueService, integrationDAL, @@ -141,7 +151,10 @@ export const secretQueueFactory = ({ snapshotSecretV2BridgeDAL, secretApprovalRequestDAL, keyStore, - auditLogService + auditLogService, + orgService, + projectUserMembershipRoleDAL, + projectKeyDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -1028,11 +1041,13 @@ export const secretQueueFactory = ({ const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3, - project + project, + bot } = await projectBotService.getBotKey(projectId); if (isProjectUpgradedToV3 || project.upgradeStatus === ProjectUpgradeStatus.InProgress) { return; } + if (!botKey) throw new NotFoundError({ message: "Project bot not found" }); await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress }); @@ -1044,6 +1059,57 @@ export const secretQueueFactory = ({ const folders = await folderDAL.findByProjectId(projectId); // except secret version and snapshot migrate rest of everything first in a transaction await secretDAL.transaction(async (tx) => { + // if project v1 create the project ghost user + if (project.version === ProjectVersion.V1) { + const ghostUser = await orgService.addGhostUser(project.orgId, tx); + const projectMembership = await projectMembershipDAL.create( + { + userId: ghostUser.user.id, + projectId: project.id + }, + tx + ); + await projectUserMembershipRoleDAL.create( + { projectMembershipId: projectMembership.id, role: ProjectMembershipRole.Admin }, + tx + ); + + const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({ + publicKey: ghostUser.keys.publicKey, + privateKey: ghostUser.keys.plainPrivateKey, + plainProjectKey: botKey + }); + + // 4. Save the project key for the ghost user. + await projectKeyDAL.create( + { + projectId: project.id, + receiverId: ghostUser.user.id, + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv, + senderId: ghostUser.user.id + }, + tx + ); + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); + await projectBotDAL.updateById( + bot.id, + { + tag, + iv, + encryptedProjectKey, + encryptedProjectKeyNonce: encryptedProjectKeyIv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: ghostUser.keys.publicKey, + senderId: ghostUser.user.id, + algorithm, + keyEncoding: encoding + }, + tx + ); + } + for (const folder of folders) { const folderId = folder.id; /* diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 8fcd7ad10..9d3037ec0 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -264,14 +264,16 @@ export const secretServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - secretPath: path, - actorId, - actor, - projectId, - environmentSlug: folder.environment.slug - }); + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath: path, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; @@ -399,14 +401,16 @@ export const secretServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - actor, - actorId, - secretPath: path, - projectId, - environmentSlug: folder.environment.slug - }); + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath: path, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; @@ -474,15 +478,17 @@ export const secretServiceFactory = ({ }) ); - await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ - actor, - actorId, - secretPath: path, - projectId, - environmentSlug: folder.environment.slug - }); - // TODO(akhilmhdh-pg): license check, posthog service and snapshot + if (inputSecret.type === SecretType.Shared) { + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath: path, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + } + return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; }; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 823da4cca..1f38babb3 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -34,7 +34,10 @@ export enum SmtpTemplates { WorkspaceInvite = "workspaceInvitation.handlebars", ScimUserProvisioned = "scimUserProvisioned.handlebars", PkiExpirationAlert = "pkiExpirationAlert.handlebars", - IntegrationSyncFailed = "integrationSyncFailed.handlebars" + IntegrationSyncFailed = "integrationSyncFailed.handlebars", + ExternalImportSuccessful = "externalImportSuccessful.handlebars", + ExternalImportFailed = "externalImportFailed.handlebars", + ExternalImportStarted = "externalImportStarted.handlebars" } export enum SmtpHost { diff --git a/backend/src/services/smtp/templates/externalImportFailed.handlebars b/backend/src/services/smtp/templates/externalImportFailed.handlebars new file mode 100644 index 000000000..c7869af27 --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportFailed.handlebars @@ -0,0 +1,21 @@ + + + + + + Import failed + + + +

An import from {{provider}} to Infisical has failed

+

An import from + {{provider}} + to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you + can contact the Infisical team at team@infisical.com. +

+ +

Error: {{error}}

+ + + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportStarted.handlebars b/backend/src/services/smtp/templates/externalImportStarted.handlebars new file mode 100644 index 000000000..551f972cc --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportStarted.handlebars @@ -0,0 +1,17 @@ + + + + + + Import in progress + + + +

An import from {{provider}} to Infisical is in progress

+

An import from + {{provider}} + to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import + has finished or if it fails.

+ + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportSuccessful.handlebars b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars new file mode 100644 index 000000000..51a1c465e --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars @@ -0,0 +1,14 @@ + + + + + + Import successful + + + +

An import from {{provider}} to Infisical was successful

+

An import from {{provider}} was successful. Your data is now available in Infisical.

+ + + \ No newline at end of file diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index c6ecd8bcd..995385b65 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -415,6 +415,10 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques req.SetQueryParam("recursive", "true") } + if request.ExpandSecretReferences { + req.SetQueryParam("expandSecretReferences", "true") + } + response, err := req.Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL)) if err != nil { diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index f954531e2..4b6c5a761 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -569,12 +569,13 @@ type CreateDynamicSecretLeaseV1Response struct { } type GetRawSecretsV3Request struct { - Environment string `json:"environment"` - WorkspaceId string `json:"workspaceId"` - SecretPath string `json:"secretPath"` - IncludeImport bool `json:"include_imports"` - Recursive bool `json:"recursive"` - TagSlugs string `json:"tagSlugs,omitempty"` + Environment string `json:"environment"` + WorkspaceId string `json:"workspaceId"` + SecretPath string `json:"secretPath"` + IncludeImport bool `json:"include_imports"` + Recursive bool `json:"recursive"` + TagSlugs string `json:"tagSlugs,omitempty"` + ExpandSecretReferences bool `json:"expandSecretReferences,omitempty"` } type GetRawSecretsV3Response struct { @@ -587,6 +588,7 @@ type GetRawSecretsV3Response struct { SecretKey string `json:"secretKey"` SecretValue string `json:"secretValue"` SecretComment string `json:"secretComment"` + SecretPath string `json:"secretPath"` } `json:"secrets"` Imports []ImportedRawSecretV3 `json:"imports"` ETag string @@ -610,6 +612,7 @@ type GetRawSecretV3ByNameResponse struct { SecretKey string `json:"secretKey"` SecretValue string `json:"secretValue"` SecretComment string `json:"secretComment"` + SecretPath string `json:"secretPath"` } `json:"secret"` ETag string } diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 118aca119..10ace5efe 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "os" @@ -311,9 +312,34 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { return config, nil } -func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { - return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { - res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, false, "") +type secretArguments struct { + IsRecursive bool `json:"recursive"` + ShouldExpandSecretReferences *bool `json:"expandSecretReferences,omitempty"` +} + +func (s *secretArguments) SetDefaults() { + if s.ShouldExpandSecretReferences == nil { + var bool = true + s.ShouldExpandSecretReferences = &bool + } +} + +func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string, ...string) ([]models.SingleEnvironmentVariable, error) { + // ...string is because golang doesn't have optional arguments. + // thus we make it slice and pick it only first element + return func(projectID, envSlug, secretPath string, args ...string) ([]models.SingleEnvironmentVariable, error) { + var parsedArguments secretArguments + // to make it optional + if len(args) > 0 { + err := json.Unmarshal([]byte(args[0]), &parsedArguments) + if err != nil { + return nil, err + } + } + + parsedArguments.SetDefaults() + + res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences) if err != nil { return nil, err } @@ -322,9 +348,7 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *currentEtag = res.Etag } - expandedSecrets := util.ExpandSecrets(res.Secrets, models.ExpandSecretsAuthentication{UniversalAuthAccessToken: accessToken}, "") - - return expandedSecrets, nil + return res.Secrets, nil } } @@ -456,7 +480,6 @@ func ProcessLiteralTemplate(templateId int, templateString string, data interfac return &buf, nil } - type AgentManager struct { accessToken string accessTokenTTL time.Duration diff --git a/cli/packages/cmd/export.go b/cli/packages/cmd/export.go index f6b028b7a..6f02408fd 100644 --- a/cli/packages/cmd/export.go +++ b/cli/packages/cmd/export.go @@ -87,11 +87,12 @@ var exportCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - TagSlugs: tagSlugs, - WorkspaceId: projectId, - SecretsPath: secretsPath, - IncludeImport: includeImports, + Environment: environmentName, + TagSlugs: tagSlugs, + WorkspaceId: projectId, + SecretsPath: secretsPath, + IncludeImport: includeImports, + ExpandSecretReferences: shouldExpandSecrets, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -137,18 +138,6 @@ var exportCmd = &cobra.Command{ } var output string - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } secrets = util.FilterSecretsByTag(secrets, tagSlugs) secrets = util.SortSecretsByKeys(secrets) diff --git a/cli/packages/cmd/login.go b/cli/packages/cmd/login.go index f66197517..6cba897f3 100644 --- a/cli/packages/cmd/login.go +++ b/cli/packages/cmd/login.go @@ -216,7 +216,9 @@ var loginCmd = &cobra.Command{ } //override domain domainQuery := true - if config.INFISICAL_URL_MANUAL_OVERRIDE != "" && config.INFISICAL_URL_MANUAL_OVERRIDE != util.INFISICAL_DEFAULT_API_URL { + if config.INFISICAL_URL_MANUAL_OVERRIDE != "" && + config.INFISICAL_URL_MANUAL_OVERRIDE != fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_EU_URL) && + config.INFISICAL_URL_MANUAL_OVERRIDE != fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_US_URL) { overrideDomain, err := DomainOverridePrompt() if err != nil { util.HandleError(err) @@ -526,16 +528,17 @@ func askForDomain() error { // query user to choose between Infisical cloud or self hosting const ( - INFISICAL_CLOUD = "Infisical Cloud" - SELF_HOSTING = "Self Hosting" - ADD_NEW_DOMAIN = "Add a new domain" + INFISICAL_CLOUD_US = "Infisical Cloud (US Region)" + INFISICAL_CLOUD_EU = "Infisical Cloud (EU Region)" + SELF_HOSTING = "Self Hosting" + ADD_NEW_DOMAIN = "Add a new domain" ) - options := []string{INFISICAL_CLOUD, SELF_HOSTING} + options := []string{INFISICAL_CLOUD_US, INFISICAL_CLOUD_EU, SELF_HOSTING} optionsPrompt := promptui.Select{ Label: "Select your hosting option", Items: options, - Size: 2, + Size: 3, } _, selectedHostingOption, err := optionsPrompt.Run() @@ -543,10 +546,15 @@ func askForDomain() error { return err } - if selectedHostingOption == INFISICAL_CLOUD { - //cloud option - config.INFISICAL_URL = fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_URL) - config.INFISICAL_LOGIN_URL = fmt.Sprintf("%s/login", util.INFISICAL_DEFAULT_URL) + if selectedHostingOption == INFISICAL_CLOUD_US { + // US cloud option + config.INFISICAL_URL = fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_US_URL) + config.INFISICAL_LOGIN_URL = fmt.Sprintf("%s/login", util.INFISICAL_DEFAULT_US_URL) + return nil + } else if selectedHostingOption == INFISICAL_CLOUD_EU { + // EU cloud option + config.INFISICAL_URL = fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_EU_URL) + config.INFISICAL_LOGIN_URL = fmt.Sprintf("%s/login", util.INFISICAL_DEFAULT_EU_URL) return nil } diff --git a/cli/packages/cmd/root.go b/cli/packages/cmd/root.go index e40c07022..c533f3415 100644 --- a/cli/packages/cmd/root.go +++ b/cli/packages/cmd/root.go @@ -40,7 +40,7 @@ func init() { cobra.OnInitialize(initLog) rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (trace, debug, info, warn, error, fatal)") rootCmd.PersistentFlags().Bool("telemetry", true, "Infisical collects non-sensitive telemetry data to enhance features and improve user experience. Participation is voluntary") - rootCmd.PersistentFlags().StringVar(&config.INFISICAL_URL, "domain", util.INFISICAL_DEFAULT_API_URL, "Point the CLI to your own backend [can also set via environment variable name: INFISICAL_API_URL]") + rootCmd.PersistentFlags().StringVar(&config.INFISICAL_URL, "domain", fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_US_URL), "Point the CLI to your own backend [can also set via environment variable name: INFISICAL_API_URL]") rootCmd.PersistentFlags().Bool("silent", false, "Disable output of tip/info messages. Useful when running in scripts or CI/CD pipelines.") rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { silent, err := cmd.Flags().GetBool("silent") diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index fa5176d89..a232896f1 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -137,15 +137,16 @@ var runCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpandSecrets, } - injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token) + injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token) if err != nil { util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") } @@ -153,7 +154,7 @@ var runCmd = &cobra.Command{ log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables) if watchMode { - executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token) + executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, secretOverriding, token) } else { if cmd.Flags().Changed("command") { command := cmd.Flag("command").Value.String() @@ -306,7 +307,7 @@ func waitForExitCommand(cmd *exec.Cmd) (int, error) { return waitStatus.ExitStatus(), nil } -func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) { +func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) { var cmd *exec.Cmd var err error @@ -420,7 +421,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt <-recheckSecretsChannel watchMutex.Lock() - newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token) + newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token) if err != nil { log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets") continue @@ -437,7 +438,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt } } -func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) { +func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) { if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { request.InfisicalToken = token.Token @@ -457,19 +458,6 @@ func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, proje secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) - } - secretsByKey := getSecretsByKeys(secrets) environmentVariables := make(map[string]string) diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index e2987cc7d..eff011c5e 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -79,12 +79,13 @@ var secretsCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpandSecrets, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -104,17 +105,6 @@ var secretsCmd = &cobra.Command{ secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpandSecrets { - authParams := models.ExpandSecretsAuthentication{} - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } - // Sort the secrets by key so we can create a consistent output secrets = util.SortSecretsByKeys(secrets) @@ -382,12 +372,13 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpand, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -407,17 +398,6 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpand { - authParams := models.ExpandSecretsAuthentication{} - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } - requestedSecrets := []models.SingleEnvironmentVariable{} secretsMap := getSecretsByKeys(secrets) diff --git a/cli/packages/cmd/user.go b/cli/packages/cmd/user.go index 844213e18..d3e6096a9 100644 --- a/cli/packages/cmd/user.go +++ b/cli/packages/cmd/user.go @@ -2,6 +2,7 @@ package cmd import ( "errors" + "fmt" "net/url" "github.com/Infisical/infisical-merge/packages/config" @@ -119,7 +120,7 @@ var domainCmd = &cobra.Command{ domain := "" domainQuery := true - if config.INFISICAL_URL_MANUAL_OVERRIDE != util.INFISICAL_DEFAULT_API_URL { + if config.INFISICAL_URL_MANUAL_OVERRIDE != fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_EU_URL) && config.INFISICAL_URL_MANUAL_OVERRIDE != fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_US_URL) { override, err := DomainOverridePrompt() if err != nil { diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 62ff07190..8b9fef6f6 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -30,6 +30,7 @@ type SingleEnvironmentVariable struct { Value string `json:"value"` Type string `json:"type"` ID string `json:"_id"` + SecretPath string `json:"secretPath"` Tags []struct { ID string `json:"_id"` Name string `json:"name"` @@ -103,6 +104,7 @@ type GetAllSecretsParameters struct { SecretsPath string IncludeImport bool Recursive bool + ExpandSecretReferences bool } type InjectableEnvironmentResult struct { diff --git a/cli/packages/util/constants.go b/cli/packages/util/constants.go index 1f9ec0c65..8b4c586e6 100644 --- a/cli/packages/util/constants.go +++ b/cli/packages/util/constants.go @@ -3,8 +3,8 @@ package util const ( CONFIG_FILE_NAME = "infisical-config.json" CONFIG_FOLDER_NAME = ".infisical" - INFISICAL_DEFAULT_API_URL = "https://app.infisical.com/api" - INFISICAL_DEFAULT_URL = "https://app.infisical.com" + INFISICAL_DEFAULT_US_URL = "https://app.infisical.com" + INFISICAL_DEFAULT_EU_URL = "https://eu.infisical.com" INFISICAL_WORKSPACE_CONFIG_FILE_NAME = ".infisical.json" INFISICAL_TOKEN_NAME = "INFISICAL_TOKEN" INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN_NAME = "INFISICAL_UNIVERSAL_AUTH_ACCESS_TOKEN" diff --git a/cli/packages/util/init.go b/cli/packages/util/init.go index 33350f3b7..4aecb2ab3 100644 --- a/cli/packages/util/init.go +++ b/cli/packages/util/init.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/config" "github.com/Infisical/infisical-merge/packages/models" ) @@ -11,7 +12,7 @@ func GetOrganizationsNameList(organizationResponse api.GetOrganizationsResponse) organizations := organizationResponse.Organizations if len(organizations) == 0 { - message := fmt.Sprintf("You don't have any organization created in Infisical. You must first create a organization at %s", INFISICAL_DEFAULT_URL) + message := fmt.Sprintf("You don't have any organization created in Infisical. You must first create a organization at %s", config.INFISICAL_URL) PrintErrorMessageAndExit(message) } @@ -37,7 +38,7 @@ func GetWorkspacesInOrganization(workspaceResponse api.GetWorkSpacesResponse, or } if len(filteredWorkspaces) == 0 { - message := fmt.Sprintf("You don't have any projects created in Infisical organization. You must first create a project at %s", INFISICAL_DEFAULT_URL) + message := fmt.Sprintf("You don't have any projects created in Infisical organization. You must first create a project at %s", config.INFISICAL_URL) PrintErrorMessageAndExit(message) } diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index b6ec7f752..5e19ea664 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" "os" - "path" - "regexp" "strings" "unicode" @@ -21,7 +19,7 @@ import ( "github.com/zalando/go-keyring" ) -func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) ([]models.SingleEnvironmentVariable, error) { serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4) if len(serviceTokenParts) < 4 { return nil, fmt.Errorf("invalid service token entered. Please double check your service token and try again") @@ -49,12 +47,13 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{ - WorkspaceId: serviceTokenDetails.Workspace, - Environment: environment, - SecretPath: secretPath, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, + WorkspaceId: serviceTokenDetails.Workspace, + Environment: environment, + SecretPath: secretPath, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, }) if err != nil { @@ -78,17 +77,18 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } -func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string) (models.PlaintextSecretResult, error) { +func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") getSecretsRequest := api.GetRawSecretsV3Request{ - WorkspaceId: workspaceId, - Environment: environmentName, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, + WorkspaceId: workspaceId, + Environment: environmentName, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, } if secretsPath != "" { @@ -104,7 +104,7 @@ func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentNa plainTextSecrets := []models.SingleEnvironmentVariable{} for _, secret := range rawSecrets.Secrets { - plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue, Type: secret.Type, WorkspaceId: secret.Workspace}) + plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue, Type: secret.Type, WorkspaceId: secret.Workspace, SecretPath: secret.SecretPath}) } if includeImports { @@ -145,6 +145,7 @@ func GetSinglePlainTextSecretByNameV3(accessToken string, workspaceId string, en Type: rawSecret.Secret.Type, ID: rawSecret.Secret.ID, Comment: rawSecret.Secret.SecretComment, + SecretPath: rawSecret.Secret.SecretPath, } return formattedSecrets, rawSecret.ETag, nil @@ -283,7 +284,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } res, err := GetPlainTextSecretsV3(loggedInUserDetails.UserCredentials.JTWToken, infisicalDotJson.WorkspaceId, - params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err) if err == nil { @@ -312,7 +313,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") - secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) } else if params.UniversalAuthAccessToken != "" { if params.WorkspaceId == "" { @@ -320,7 +321,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } log.Debug().Msg("Trying to fetch secrets using universal auth") - res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) errorToReturn = err secretsToReturn = res.Secrets @@ -330,44 +331,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo return secretsToReturn, errorToReturn } -var secRefRegex = regexp.MustCompile(`\${([^\}]*)}`) - -func recursivelyExpandSecret(expandedSecs map[string]string, interpolatedSecs map[string]string, crossSecRefFetch func(env string, path []string, key string) string, key string) string { - if v, ok := expandedSecs[key]; ok { - return v - } - - interpolatedVal, ok := interpolatedSecs[key] - if !ok { - HandleError(fmt.Errorf("could not find refered secret - %s", key), "Kindly check whether its provided") - } - - refs := secRefRegex.FindAllStringSubmatch(interpolatedVal, -1) - for _, val := range refs { - // key: "${something}" val: [${something},something] - interpolatedExp, interpolationKey := val[0], val[1] - ref := strings.Split(interpolationKey, ".") - - // ${KEY1} => [key1] - if len(ref) == 1 { - val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey) - interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val) - continue - } - - // cross board reference ${env.folder.key1} => [env folder key1] - if len(ref) > 1 { - secEnv, tmpSecPath, secKey := ref[0], ref[1:len(ref)-1], ref[len(ref)-1] - interpolatedSecs[interpolationKey] = crossSecRefFetch(secEnv, tmpSecPath, secKey) // get the reference value - val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey) - interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val) - } - - } - expandedSecs[key] = interpolatedVal - return interpolatedVal -} - func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]models.SingleEnvironmentVariable { secretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets)) @@ -378,70 +341,6 @@ func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]mod return secretMapByName } -func ExpandSecrets(secrets []models.SingleEnvironmentVariable, auth models.ExpandSecretsAuthentication, projectConfigPathDir string) []models.SingleEnvironmentVariable { - expandedSecs := make(map[string]string) - interpolatedSecs := make(map[string]string) - // map[env.secret-path][keyname]Secret - crossEnvRefSecs := make(map[string]map[string]models.SingleEnvironmentVariable) // a cache to hold all cross board reference secrets - - for _, sec := range secrets { - // get all references in a secret - refs := secRefRegex.FindAllStringSubmatch(sec.Value, -1) - // nil means its a secret without reference - if refs == nil { - expandedSecs[sec.Key] = sec.Value // atomic secrets without any interpolation - } else { - interpolatedSecs[sec.Key] = sec.Value - } - } - - for i, sec := range secrets { - // already present pick that up - if expandedVal, ok := expandedSecs[sec.Key]; ok { - secrets[i].Value = expandedVal - continue - } - - expandedVal := recursivelyExpandSecret(expandedSecs, interpolatedSecs, func(env string, secPaths []string, secKey string) string { - secPaths = append([]string{"/"}, secPaths...) - secPath := path.Join(secPaths...) - - secPathDot := strings.Join(secPaths, ".") - uniqKey := fmt.Sprintf("%s.%s", env, secPathDot) - - if crossRefSec, ok := crossEnvRefSecs[uniqKey]; !ok { - - var refSecs []models.SingleEnvironmentVariable - var err error - - // if not in cross reference cache, fetch it from server - if auth.InfisicalToken != "" { - refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, InfisicalToken: auth.InfisicalToken, SecretsPath: secPath}, projectConfigPathDir) - } else if auth.UniversalAuthAccessToken != "" { - refSecs, err = GetAllEnvironmentVariables((models.GetAllSecretsParameters{Environment: env, UniversalAuthAccessToken: auth.UniversalAuthAccessToken, SecretsPath: secPath, WorkspaceId: sec.WorkspaceId}), projectConfigPathDir) - } else if IsLoggedIn() { - refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, SecretsPath: secPath}, projectConfigPathDir) - } else { - HandleError(errors.New("no authentication provided"), "Please provide authentication to fetch secrets") - } - if err != nil { - HandleError(err, fmt.Sprintf("Could not fetch secrets in environment: %s secret-path: %s", env, secPath), "If you are using a service token to fetch secrets, please ensure it is valid") - } - refSecsByKey := getSecretsByKeys(refSecs) - // save it to avoid calling api again for same environment and folder path - crossEnvRefSecs[uniqKey] = refSecsByKey - return refSecsByKey[secKey].Value - - } else { - return crossRefSec[secKey].Value - } - }, sec.Key) - - secrets[i].Value = expandedVal - } - return secrets -} - func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable { personalSecrets := make(map[string]models.SingleEnvironmentVariable) sharedSecrets := make(map[string]models.SingleEnvironmentVariable) diff --git a/cli/secret-render-template b/cli/secret-render-template index 32ab2331a..41489a074 100644 --- a/cli/secret-render-template +++ b/cli/secret-render-template @@ -1,5 +1,5 @@ -{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }} +{{- with secret "8fac9f01-4a81-44d7-8ff0-3d7be684f56f" "staging" "/" `{"recursive":true, "expandSecretReferences": false}` }} {{- range . }} {{ .Key }}={{ .Value }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/docs/changelog/overview.mdx b/docs/changelog/overview.mdx index b66b07288..60a73df05 100644 --- a/docs/changelog/overview.mdx +++ b/docs/changelog/overview.mdx @@ -4,6 +4,27 @@ title: "Changelog" The changelog below reflects new product developments and updates on a monthly basis. +## September 2024 +- Improved paginations for identities and secrets. +- Significant improvements to the [Infisical Terraform Provider](https://registry.terraform.io/providers/Infisical/infisical/latest/docs). +- Created [Slack Integration](https://infisical.com/docs/documentation/platform/workflow-integrations/slack-integration#slack-integration) for Access Requests and Approval Workflows. +- Added Dynamic Secrets for [Elaticsearch](https://infisical.com/docs/documentation/platform/dynamic-secrets/elastic-search) and [MongoDB](https://infisical.com/docs/documentation/platform/dynamic-secrets/mongo-db). +- More authentication methods are now supported by Infisical SDKs and Agent. +- Integrations now have dedicated audit logs and an overview screen. +- Added support for secret referencing in the Terraform Provider. +- Released support for [older versions of .NET](https://www.nuget.org/packages/Infisical.Sdk#supportedframeworks-body-tab) via SDK. +- Released Infisical PKI Issuer which works alongside `cert-manager` to manage certificates in Kubernetes. + +## August 2024 +- Added [Azure DevOps integration](https://infisical.com/docs/integrations/cloud/azure-devops). +- Released ability to hot-reload variables in CLI ([--watch flag](https://infisical.com/docs/cli/commands/run#infisical-run:watch)). +- Added Dynamic Secrets for [Redis](https://infisical.com/docs/documentation/platform/dynamic-secrets/redis). +- Added [Alerting](https://infisical.com/docs/documentation/platform/pki/alerting) for Certificate Management. +- You can now specify roles and project memberships when adding new users. +- Approval workflows now have email notifications. +- Access requests are now integrated with User Groups. +- Released ability to use IAM Roles for AWS Integrations. + ## July 2024 - Released the official [Ruby SDK](https://infisical.com/docs/sdks/languages/ruby). - Increased the speed and efficiency of secret operations. diff --git a/docs/cli/commands/token.mdx b/docs/cli/commands/token.mdx index 5b0d4ad5c..9f631f07c 100644 --- a/docs/cli/commands/token.mdx +++ b/docs/cli/commands/token.mdx @@ -4,7 +4,7 @@ description: "Manage your Infisical identity access tokens" --- ```bash -infisical service-token renew +infisical token renew ``` ## Description diff --git a/docs/contributing/platform/developing.mdx b/docs/contributing/platform/developing.mdx index a6675b6f6..68add4603 100644 --- a/docs/contributing/platform/developing.mdx +++ b/docs/contributing/platform/developing.mdx @@ -1,6 +1,6 @@ --- -title: 'Local development' -description: 'This guide will help you set up and run the Infisical platform in local development.' +title: "Local development" +description: "This guide will help you set up and run the Infisical platform in local development." --- ## Fork and clone the repo @@ -15,28 +15,28 @@ git checkout -b MY_BRANCH_NAME ## Set up environment variables - Start by creating a .env file at the root of the Infisical directory then copy the contents of the file linked [here](https://github.com/Infisical/infisical/blob/main/.env.example). View all available [environment variables](https://infisical.com/docs/self-hosting/configuration/envars) and guidance for each. ## Starting Infisical for development We use Docker to spin up all required services for Infisical in local development. If you are unfamiliar with Docker, don’t worry, all you have to do is install Docker for your -machine and run the command below to start up the development server. +machine and run the command below to start up the development server. -#### Start local server +#### Start local server ```bash -docker-compose -f docker-compose.dev.yml up --build --force-recreate +docker compose -f docker-compose.dev.yml up --build --force-recreate ``` -#### Access local server + +#### Access local server Once all the services have spun up, browse to http://localhost:8080. -#### Shutdown local server +#### Shutdown local server ```bash # To stop environment use Control+C (on Mac) CTRL+C (on Win) or -docker-compose -f docker-compose.dev.yml down +docker compose -f docker-compose.dev.yml down ``` ## Starting Infisical docs locally @@ -56,9 +56,10 @@ yarn global add mintlify ``` #### Running the docs + Go to `docs` directory and run `mintlify dev`. This will start up the docs on `localhost:3000` ```bash # From the root directory cd docs; mintlify dev; -``` \ No newline at end of file +``` diff --git a/docs/documentation/platform/admin-panel/org-admin-console.mdx b/docs/documentation/platform/admin-panel/org-admin-console.mdx new file mode 100644 index 000000000..39d7819a4 --- /dev/null +++ b/docs/documentation/platform/admin-panel/org-admin-console.mdx @@ -0,0 +1,31 @@ +--- +title: "Organization Admin Console" +description: "View and manage resources across your organization" +--- + + + The Organization Admin Console can only be accessed by organization members with admin status. + + + +## Accessing the Organization Admin Console + +On the sidebar, tap on your initials to access the settings dropdown and press the **Organization Admin Console** option. + +![Access Organization Admin Console](/images/platform/admin-panels/access-org-admin-console.png) + +## Projects Tab + +The Projects tab lists all the projects within your organization, including those which you are not a member of. You can easily filter projects by name or slug using the search bar. + +![Projects Section](/images/platform/admin-panels/org-admin-console-projects.png) + + +### Accessing a Project in Your Organization + +You can access a project that you are not a member of by tapping on the options menu of the project row and pressing the **Access** button. +Doing so will grant you admin permissions for the selected project and add you as a member. + +![Access project](/images/platform/admin-panels/org-admin-console-access.png) + + diff --git a/docs/documentation/platform/admin-panel/overview.mdx b/docs/documentation/platform/admin-panel/overview.mdx new file mode 100644 index 000000000..968728bfc --- /dev/null +++ b/docs/documentation/platform/admin-panel/overview.mdx @@ -0,0 +1,25 @@ +--- +description: "Learn about Infisical's Admin Consoles" +--- + +Infisical offers a server and organization level console for admins to customize their settings and manage various resources across the platform. + + + + Configure and manage server related features. + + + + View and access resources across your organization. + + diff --git a/docs/documentation/platform/admin-panel/server-admin.mdx b/docs/documentation/platform/admin-panel/server-admin.mdx new file mode 100644 index 000000000..355679f82 --- /dev/null +++ b/docs/documentation/platform/admin-panel/server-admin.mdx @@ -0,0 +1,69 @@ +--- +title: "Server Admin Console" +description: "Configure and manage server related features" +--- + +The Server Admin Console provides **server administrators** with the ability to +customize settings and manage users for their entire Infisical instance. + + + The first user to setup an account on your Infisical instance is designated as the server administrator by default. + + +## Accessing the Server Admin Console + + +On the sidebar, tap on your initials to access the settings dropdown and press the **Server Admin Console** option. + +![Access Server Admin Console](/images/platform/admin-panels/access-server-admin-panel.png) + +## General Tab +Configure general settings for your instance. + +![General Settings](/images/platform/admin-panels/admin-panel-general.png) + + +### Allow User Signups + +User signups are enabled by default, allowing **Anyone** with access to your instance to sign up. This can alternatively be **Disabled** to prevent any users from signing up. + +### Restrict Signup Domain + +Signup can be restricted to users matching one or more email domains, such as your organization's domain, to control who has access to your instance. + +### Default Organization + +If you're using SAML/LDAP for only one organization on your instance, you can specify a default organization to use at login to skip requiring users to manually enter the organization slug. + +### Trust Emails + +By default, users signing up through SAML/LDAP/OIDC will still need to verify their email address to prevent email spoofing. This requirement can be skipped by enabling the switch to trust logins through the respective method. + + +## Authentication Tab + +From this tab, you can configure which login methods are enabled for your instance. + +![Authentication Settings](/images/platform/admin-panels/admin-panel-auths.png) + + +## Rate Limit Tab + +This tab allows you to set various rate limits for your Infisical instance. You do not need to redeploy when making changes to rate limits as these will be propagated automatically. + +![Rate Limit Settings](/images/platform/admin-panels/admin-panel-rate-limits.png) + + + + Note that rate limit configuration is a paid feature. Please contact sales@infisical.com to purchase a license for its use. + + +## User Management Tab + +From this tab, you can view all the users who have signed up for your instance. You can search for users using the search bar and remove them from your instance by pressing the **X** button on their respective row. + +![User Management](/images/platform/admin-panels/admin-panel-users.png) + + + Note that rate limit configuration is a paid feature. Please contact sales@infisical.com to purchase a license for its use. + diff --git a/docs/documentation/platform/identities/oidc-auth/circleci.mdx b/docs/documentation/platform/identities/oidc-auth/circleci.mdx new file mode 100644 index 000000000..ddf74e3fa --- /dev/null +++ b/docs/documentation/platform/identities/oidc-auth/circleci.mdx @@ -0,0 +1,174 @@ +--- +title: CircleCI +description: "Learn how to authenticate CircleCI jobs with Infisical using OpenID Connect (OIDC)." +--- + +**OIDC Auth** is a platform-agnostic JWT-based authentication method that can be used to authenticate from any platform or environment using an identity provider with OpenID Connect. + +## Diagram + +The following sequence diagram illustrates the OIDC Auth workflow for authenticating CircleCI jobs with Infisical. + +```mermaid +sequenceDiagram + participant Client as CircleCI Job + participant Idp as CircleCI Identity Provider + participant Infis as Infisical + + Idp->>Client: Step 1: Inject JWT with verifiable claims + + Note over Client,Infis: Step 2: Login Operation + Client->>Infis: Send signed JWT to /api/v1/auth/oidc-auth/login + + Note over Infis,Idp: Step 3: Query verification + Infis->>Idp: Request JWT public key using OIDC Discovery + Idp-->>Infis: Return public key + + Note over Infis: Step 4: JWT validation + Infis->>Client: Return short-lived access token + + Note over Client,Infis: Step 5: Access Infisical API with Token + Client->>Infis: Make authenticated requests using the short-lived access token +``` + +## Concept + +At a high-level, Infisical authenticates a client by verifying the JWT and checking that it meets specific requirements (e.g. it is issued by a trusted identity provider) at the `/api/v1/auth/oidc-auth/login` endpoint. If successful, +then Infisical returns a short-lived access token that can be used to make authenticated requests to the Infisical API. + +To be more specific: + +1. CircleCI provides the running job with a valid OIDC token specific to the execution. +2. The CircleCI OIDC token is sent to Infisical at the `/api/v1/auth/oidc-auth/login` endpoint. +3. Infisical fetches the public key that was used to sign the identity token provided by CircleCI. +4. Infisical validates the JWT using the public key provided by the identity provider and checks that the subject, audience, and claims of the token matches with the set criteria. +5. If all is well, Infisical returns a short-lived access token that CircleCI jobs can use to make authenticated requests to the Infisical API. + +Infisical needs network-level access to the CircleCI servers. + +## Guide + +In the following steps, we explore how to create and use identities to access the Infisical API using the OIDC Auth authentication method. + + + + To create an identity, head to your Organization Settings > Access Control > Machine Identities and press **Create identity**. + + ![identities organization](/images/platform/identities/identities-org.png) + + When creating an identity, you specify an organization level [role](/documentation/platform/role-based-access-controls) for it to assume; you can configure roles in Organization Settings > Access Control > Organization Roles. + + ![identities organization create](/images/platform/identities/identities-org-create.png) + + Now input a few details for your new identity. Here's some guidance for each field: + + - Name (required): A friendly name for the identity. + - Role (required): A role from the **Organization Roles** tab for the identity to assume. The organization role assigned will determine what organization level resources this identity can have access to. + + Once you've created an identity, you'll be redirected to a page where you can manage the identity. + + ![identities page](/images/platform/identities/identities-page.png) + + Since the identity has been configured with Universal Auth by default, you should re-configure it to use OIDC Auth instead. To do this, press to edit the **Authentication** section, + remove the existing Universal Auth configuration, and add a new OIDC Auth configuration onto the identity. + + ![identities page remove default auth](/images/platform/identities/identities-page-remove-default-auth.png) + + ![identities create oidc auth method](/images/platform/identities/identities-org-create-oidc-auth-method.png) + + Restrict access by configuring the Subject, Audiences, and Claims fields + + Here's some more guidance on each field: + - OIDC Discovery URL: The URL used to retrieve the OpenID Connect configuration from the identity provider. This will be used to fetch the public key needed for verifying the provided JWT. This should be set to `https://oidc.circleci.com/org/` where `organization_id` refers to the CircleCI organization where the job is being run. + - Issuer: The unique identifier of the identity provider issuing the JWT. This value is used to verify the iss (issuer) claim in the JWT to ensure the token is issued by a trusted provider. This should be set to `https://oidc.circleci.com/org/` as well. + - CA Certificate: The PEM-encoded CA cert for establishing secure communication with the Identity Provider endpoints. This can be left as blank. + - Subject: The expected principal that is the subject of the JWT. The format of the sub field for CircleCI OIDC tokens is `org//project//user/` where organization_id, project_id, and user_id are UUIDs that identify the CircleCI organization, project, and user, respectively. The user is the CircleCI user that caused this job to run. + - Audiences: A list of intended recipients. This value is checked against the aud (audience) claim in the token. Set this to the CircleCI `organization_id` corresponding to where the job is running. + - Claims: Additional information or attributes that should be present in the JWT for it to be valid. Refer to CircleCI's [documentation](https://circleci.com/docs/openid-connect-tokens) for the complete list of supported claims. + - Access Token TTL (default is `2592000` equivalent to 30 days): The lifetime for an acccess token in seconds. This value will be referenced at renewal time. + - Access Token Max TTL (default is `2592000` equivalent to 30 days): The maximum lifetime for an acccess token in seconds. This value will be referenced at renewal time. + - Access Token Max Number of Uses (default is `0`): The maximum number of times that an access token can be used; a value of `0` implies infinite number of uses. + - Access Token Trusted IPs: The IPs or CIDR ranges that access tokens can be used from. By default, each token is given the `0.0.0.0/0`, allowing usage from any network address. + For more details on the appropriate values for the OIDC fields, refer to CircleCI's [documentation](https://circleci.com/docs/openid-connect-tokens). + The `subject`, `audiences`, and `claims` fields support glob pattern matching; however, we highly recommend using hardcoded values whenever possible. + + + To enable the identity to access project-level resources such as secrets within a specific project, you should add it to that project. + + To do this, head over to the project you want to add the identity to and go to Project Settings > Access Control > Machine Identities and press **Add identity**. + + Next, select the identity you want to add to the project and the project level role you want to allow it to assume. The project role assigned will determine what project level resources this identity can have access to. + + ![identities project](/images/platform/identities/identities-project.png) + + ![identities project create](/images/platform/identities/identities-project-create.png) + + + The following is an example of how to use the `$CIRCLE_OIDC_TOKEN` with the Infisical [terraform provider](https://registry.terraform.io/providers/Infisical/infisical/latest/docs) to manage resources in a CircleCI pipeline. + + ```yml config.yml + version: 2.1 + + jobs: + terraform-apply: + docker: + - image: hashicorp/terraform:latest + + steps: + - checkout + - run: + command: | + export INFISICAL_AUTH_JWT="$CIRCLE_OIDC_TOKEN" + terraform init + terraform apply -auto-approve + + workflows: + version: 2 + build-and-test: + jobs: + - terraform-apply + ``` + The Infisical terraform provider expects the `INFISICAL_AUTH_JWT` environment variable to be set to the CircleCI OIDC token. + ```hcl main.tf + terraform { + required_providers { + infisical = { + source = "infisical/infisical" + } + } + } + + provider "infisical" { + host = "https://app.infisical.com" + auth = { + oidc = { + identity_id = "f2f5ee4c-6223-461a-87c3-406a6b481462" + } + } + } + + resource "infisical_access_approval_policy" "prod-access-approval" { + project_id = "09eda1f8-85a3-47a9-8a6f-e27f133b2a36" + name = "my-approval-policy" + environment_slug = "prod" + secret_path = "/" + approvers = [ + { + type = "user" + username = "sheen+200@infisical.com" + }, + ] + required_approvals = 1 + enforcement_level = "soft" + } + ``` + + Each identity access token has a time-to-live (TLL) which you can infer from the response of the login operation; + the default TTL is `7200` seconds which can be adjusted. + + If an identity access token expires, it can no longer authenticate with the Infisical API. In this case, + a new access token should be obtained by performing another login operation. + + + + diff --git a/docs/documentation/platform/ldap/overview.mdx b/docs/documentation/platform/ldap/overview.mdx index 4502158d0..c1a980b04 100644 --- a/docs/documentation/platform/ldap/overview.mdx +++ b/docs/documentation/platform/ldap/overview.mdx @@ -36,7 +36,7 @@ If the documentation for your required identity provider is not shown in the lis verification step upon their first login. If you're running a self-hosted instance of Infisical and would like it to trust emails from external identity providers, - you can configure this behavior in the admin panel. + you can configure this behavior in the Server Admin Console. diff --git a/docs/documentation/platform/organization.mdx b/docs/documentation/platform/organization.mdx index 5e75f1a3b..f1c62ff37 100644 --- a/docs/documentation/platform/organization.mdx +++ b/docs/documentation/platform/organization.mdx @@ -16,8 +16,10 @@ as well as create a new project. The **Settings** page lets you manage information about your organization including: -- Name: The name of your organization. -- Incident contacts: Emails that should be alerted if anything abnormal is detected within the organization. +- **Name**: The name of your organization. +- **Slug**: The slug of your organization. +- **Default Organization Member Role**: The role assigned to users when joining your organization unless otherwise specified. +- **Incident Contacts**: Emails that should be alerted if anything abnormal is detected within the organization. ![organization settings general](../../images/platform/organization/organization-settings-general.png) diff --git a/docs/documentation/platform/scim/azure.mdx b/docs/documentation/platform/scim/azure.mdx index ff46fe4e7..74a6c2030 100644 --- a/docs/documentation/platform/scim/azure.mdx +++ b/docs/documentation/platform/scim/azure.mdx @@ -28,6 +28,13 @@ Prerequisites: ![SCIM copy token](/images/platform/scim/scim-copy-token.png) + + In Azure, navigate to Enterprise Application > Users and Groups. Add any users and/or groups to your application that you would like + to be provisioned over to Infisical. + + ![SCIM Azure Users and Groups](/images/platform/scim/azure/scim-azure-add-users-and-groups.png) + + In Azure, head to your Enterprise Application > Provisioning > Overview and press **Get started**. @@ -39,7 +46,7 @@ Prerequisites: - Tenant URL: Input **SCIM URL** from Step 1. - Secret Token: Input the **New SCIM Token** from Step 1. - Afterwards, press the **Test Connection** button to check that SCIM is configured properly. + Afterwards, click **Enable SCIM** and press the **Test Connection** button to check that SCIM is configured properly. ![SCIM Azure](/images/platform/scim/azure/scim-azure-config.png) @@ -71,4 +78,4 @@ Prerequisites: For this reason, SCIM-provisioned users are initialized but must finish setting up their account when logging in the first time by creating a master encryption/decryption key. With this implementation, IdPs and SCIM providers cannot and will not have access to the decryption key needed to decrypt your secrets. - \ No newline at end of file + diff --git a/docs/documentation/platform/scim/group-mappings.mdx b/docs/documentation/platform/scim/group-mappings.mdx new file mode 100644 index 000000000..acce52e1e --- /dev/null +++ b/docs/documentation/platform/scim/group-mappings.mdx @@ -0,0 +1,26 @@ +--- +title: "SCIM Group Mappings" +description: "Learn how to enhance your SCIM implementation using group mappings" +--- + + + SCIM provisioning, and by extension group mapping, is a paid feature. + + If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, + then you should contact sales@infisical.com to purchase an enterprise license to use it. + + +## SCIM Group to Organization Role Mapping + +By default, when users are provisioned via SCIM, they will be assigned the default organization role configured in [Organization General Settings](/documentation/platform/organization#settings). + +For more precise control over membership roles, you can set up SCIM Group to Organization Role Mappings. This enables you to assign specific roles based on the group from which a user is provisioned. + +![SCIM Group Mapping](/images/platform/scim/scim-group-mapping.png) + +To configure a mapping, simply enter the SCIM group's name and select the role you would like users to be assigned from this group. Be sure +to tap **Update Mappings** once complete. + + + SCIM Group Mappings only apply when users are first provisioned. Previously provisioned users will not be affected, allowing you to customize user roles after they are added. + diff --git a/docs/documentation/platform/sso/overview.mdx b/docs/documentation/platform/sso/overview.mdx index 4bb45cf48..227a7502f 100644 --- a/docs/documentation/platform/sso/overview.mdx +++ b/docs/documentation/platform/sso/overview.mdx @@ -45,7 +45,7 @@ If your required identity provider is not shown in the list above, please reach verification step upon their first login. If you're running a self-hosted instance of Infisical and would like it to trust emails from external identity providers, - you can configure this behavior in the admin panel. + you can configure this behavior in the Server Admin Console. diff --git a/docs/images/integrations/databricks/integrations-databricks-auth.png b/docs/images/integrations/databricks/integrations-databricks-auth.png new file mode 100644 index 000000000..de17cfd5c Binary files /dev/null and b/docs/images/integrations/databricks/integrations-databricks-auth.png differ diff --git a/docs/images/integrations/databricks/integrations-databricks-create.png b/docs/images/integrations/databricks/integrations-databricks-create.png new file mode 100644 index 000000000..058f08369 Binary files /dev/null and b/docs/images/integrations/databricks/integrations-databricks-create.png differ diff --git a/docs/images/integrations/databricks/integrations-databricks.png b/docs/images/integrations/databricks/integrations-databricks.png new file mode 100644 index 000000000..f48f6b95a Binary files /dev/null and b/docs/images/integrations/databricks/integrations-databricks.png differ diff --git a/docs/images/integrations/databricks/pat-token.png b/docs/images/integrations/databricks/pat-token.png new file mode 100644 index 000000000..02a264710 Binary files /dev/null and b/docs/images/integrations/databricks/pat-token.png differ diff --git a/docs/images/integrations/github/app/github-app-installation.png b/docs/images/integrations/github/app/github-app-installation.png new file mode 100644 index 000000000..60a2ec4fc Binary files /dev/null and b/docs/images/integrations/github/app/github-app-installation.png differ diff --git a/docs/images/integrations/github/app/github-app-method-selection.png b/docs/images/integrations/github/app/github-app-method-selection.png new file mode 100644 index 000000000..3f66a396e Binary files /dev/null and b/docs/images/integrations/github/app/github-app-method-selection.png differ diff --git a/docs/images/integrations/github/app/integration-overview.png b/docs/images/integrations/github/app/integration-overview.png new file mode 100644 index 000000000..1dad2fb64 Binary files /dev/null and b/docs/images/integrations/github/app/integration-overview.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-basic-details.png b/docs/images/integrations/github/app/self-hosted-github-app-basic-details.png new file mode 100644 index 000000000..463adabd8 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-basic-details.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-create-confirm.png b/docs/images/integrations/github/app/self-hosted-github-app-create-confirm.png new file mode 100644 index 000000000..15dc7f9d6 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-create-confirm.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-create.png b/docs/images/integrations/github/app/self-hosted-github-app-create.png new file mode 100644 index 000000000..d55a49b66 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-create.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-credentials.png b/docs/images/integrations/github/app/self-hosted-github-app-credentials.png new file mode 100644 index 000000000..6e4480bc7 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-credentials.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-enable-oauth.png b/docs/images/integrations/github/app/self-hosted-github-app-enable-oauth.png new file mode 100644 index 000000000..45d50c7b2 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-enable-oauth.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-organization.png b/docs/images/integrations/github/app/self-hosted-github-app-organization.png new file mode 100644 index 000000000..60ba84151 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-organization.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-private-key.png b/docs/images/integrations/github/app/self-hosted-github-app-private-key.png new file mode 100644 index 000000000..ce03f740e Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-private-key.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-repository.png b/docs/images/integrations/github/app/self-hosted-github-app-repository.png new file mode 100644 index 000000000..edf1d1087 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-repository.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-secret.png b/docs/images/integrations/github/app/self-hosted-github-app-secret.png new file mode 100644 index 000000000..8c9918404 Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-secret.png differ diff --git a/docs/images/integrations/github/app/self-hosted-github-app-webhook.png b/docs/images/integrations/github/app/self-hosted-github-app-webhook.png new file mode 100644 index 000000000..2b7493fbf Binary files /dev/null and b/docs/images/integrations/github/app/self-hosted-github-app-webhook.png differ diff --git a/docs/images/integrations/github/github-oauth-method-selection.png b/docs/images/integrations/github/github-oauth-method-selection.png new file mode 100644 index 000000000..eb1d00cfc Binary files /dev/null and b/docs/images/integrations/github/github-oauth-method-selection.png differ diff --git a/docs/images/integrations/github/integration-overview.png b/docs/images/integrations/github/integration-overview.png new file mode 100644 index 000000000..1dad2fb64 Binary files /dev/null and b/docs/images/integrations/github/integration-overview.png differ diff --git a/docs/images/platform/admin-panels/access-org-admin-console.png b/docs/images/platform/admin-panels/access-org-admin-console.png new file mode 100644 index 000000000..057c82944 Binary files /dev/null and b/docs/images/platform/admin-panels/access-org-admin-console.png differ diff --git a/docs/images/platform/admin-panels/access-server-admin-panel.png b/docs/images/platform/admin-panels/access-server-admin-panel.png new file mode 100644 index 000000000..a27735de0 Binary files /dev/null and b/docs/images/platform/admin-panels/access-server-admin-panel.png differ diff --git a/docs/images/platform/admin-panels/admin-panel-auths.png b/docs/images/platform/admin-panels/admin-panel-auths.png new file mode 100644 index 000000000..a0abd5d9a Binary files /dev/null and b/docs/images/platform/admin-panels/admin-panel-auths.png differ diff --git a/docs/images/platform/admin-panels/admin-panel-general.png b/docs/images/platform/admin-panels/admin-panel-general.png new file mode 100644 index 000000000..bce175cf0 Binary files /dev/null and b/docs/images/platform/admin-panels/admin-panel-general.png differ diff --git a/docs/images/platform/admin-panels/admin-panel-integration.png b/docs/images/platform/admin-panels/admin-panel-integration.png new file mode 100644 index 000000000..43bedd17e Binary files /dev/null and b/docs/images/platform/admin-panels/admin-panel-integration.png differ diff --git a/docs/images/platform/admin-panels/admin-panel-rate-limits.png b/docs/images/platform/admin-panels/admin-panel-rate-limits.png new file mode 100644 index 000000000..d8f689f1e Binary files /dev/null and b/docs/images/platform/admin-panels/admin-panel-rate-limits.png differ diff --git a/docs/images/platform/admin-panels/admin-panel-users.png b/docs/images/platform/admin-panels/admin-panel-users.png new file mode 100644 index 000000000..94add6d85 Binary files /dev/null and b/docs/images/platform/admin-panels/admin-panel-users.png differ diff --git a/docs/images/platform/admin-panels/org-admin-console-access.png b/docs/images/platform/admin-panels/org-admin-console-access.png new file mode 100644 index 000000000..6aba5b21a Binary files /dev/null and b/docs/images/platform/admin-panels/org-admin-console-access.png differ diff --git a/docs/images/platform/admin-panels/org-admin-console-projects.png b/docs/images/platform/admin-panels/org-admin-console-projects.png new file mode 100644 index 000000000..13b8bcfce Binary files /dev/null and b/docs/images/platform/admin-panels/org-admin-console-projects.png differ diff --git a/docs/images/platform/organization/organization-settings-general.png b/docs/images/platform/organization/organization-settings-general.png index 2c60090fe..affcf32ff 100644 Binary files a/docs/images/platform/organization/organization-settings-general.png and b/docs/images/platform/organization/organization-settings-general.png differ diff --git a/docs/images/platform/scim/azure/scim-azure-add-users-and-groups.png b/docs/images/platform/scim/azure/scim-azure-add-users-and-groups.png new file mode 100644 index 000000000..ec8b4428a Binary files /dev/null and b/docs/images/platform/scim/azure/scim-azure-add-users-and-groups.png differ diff --git a/docs/images/platform/scim/scim-group-mapping.png b/docs/images/platform/scim/scim-group-mapping.png new file mode 100644 index 000000000..76baa8d8d Binary files /dev/null and b/docs/images/platform/scim/scim-group-mapping.png differ diff --git a/docs/integrations/cicd/circleci.mdx b/docs/integrations/cicd/circleci.mdx index ed14b891e..0753f40f7 100644 --- a/docs/integrations/cicd/circleci.mdx +++ b/docs/integrations/cicd/circleci.mdx @@ -21,12 +21,6 @@ Prerequisites: ![integrations circleci authorization](../../images/integrations/circleci/integrations-circleci-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which CircleCI project and press create integration to start syncing secrets to CircleCI. diff --git a/docs/integrations/cicd/codefresh.mdx b/docs/integrations/cicd/codefresh.mdx index e41e00f50..cf69ae04d 100644 --- a/docs/integrations/cicd/codefresh.mdx +++ b/docs/integrations/cicd/codefresh.mdx @@ -22,12 +22,6 @@ Prerequisites: ![integrations codefresh authorization](../../images/integrations/codefresh/integrations-codefresh-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Codefresh service and press create integration to start syncing secrets to Codefresh. diff --git a/docs/integrations/cicd/githubactions.mdx b/docs/integrations/cicd/githubactions.mdx index 93b1347f7..8f009716c 100644 --- a/docs/integrations/cicd/githubactions.mdx +++ b/docs/integrations/cicd/githubactions.mdx @@ -4,33 +4,32 @@ description: "How to sync secrets from Infisical to GitHub Actions" --- - Alternatively, you can use Infisical's official Github Action + Alternatively, you can use Infisical's official GitHub Action [here](https://github.com/Infisical/secrets-action). Infisical lets you sync secrets to GitHub at the organization-level, repository-level, and repository environment-level. -Prerequisites: - -- Set up and add envars to [Infisical Cloud](https://app.infisical.com) -- Ensure that you have admin privileges to the repository you want to sync secrets to. +## Connecting with GitHub App (Recommended) - - Navigate to your project's integrations tab in Infisical. + + Navigate to your project's integrations tab in Infisical and press on the GitHub tile. - ![integrations](../../images/integrations.png) + ![integrations](../../images/integrations/github/app/integration-overview.png) - Press on the GitHub tile and grant Infisical access to your GitHub account (repo privileges only). + Select GitHub App as the authentication method and click **Connect to GitHub**. - ![integrations github authorization](../../images/integrations/github/integrations-github-auth.png) + ![integrations github app auth selection](../../images/integrations/github/app/github-app-method-selection.png) + + You will then be redirected to the GitHub app installation page. + + ![integrations github app installation](../../images/integrations/github/app/github-app-installation.png) + + Install and authorize the GitHub application. This will redirect you back to the Infisical integration page. - - If this is your project's first cloud integration, then you'll have to grant Infisical access to your project's environment variables. - Although this step breaks E2EE, it's necessary for Infisical to sync the environment variables to the cloud platform. - Select which Infisical environment secrets you want to sync to which GitHub organization, repository, or repository environment. @@ -42,7 +41,115 @@ Prerequisites: ![integrations github](../../images/integrations/github/integrations-github-scope-org.png) - When using the organization scope, your secrets will be saved in the top-level of your Github Organization. + When using the organization scope, your secrets will be saved in the top-level of your GitHub Organization. + + You can choose the visibility, which defines which repositories can access the secrets. The options are: + - **All public repositories**: All public repositories in the organization can access the secrets. + - **All private repositories**: All private repositories in the organization can access the secrets. + - **Selected repositories**: Only the selected repositories can access the secrets. This gives a more fine-grained control over which repositories can access the secrets. You can select _both_ private and public repositories with this option. + + + ![integrations github](../../images/integrations/github/integrations-github-scope-env.png) + + + + Finally, press create integration to start syncing secrets to GitHub. + + ![integrations github](../../images/integrations/github/integrations-github.png) + + + + + + Using the GitHub integration with app authentication on a self-hosted instance of Infisical requires configuring an application on GitHub + and registering your instance with it. + + + Navigate to the GitHub app settings [here](https://github.com/settings/apps). Click **New GitHub App**. + + ![integrations github app create](../../images/integrations/github/app/self-hosted-github-app-create.png) + + Give the application a name, a homepage URL (your self-hosted domain i.e. `https://your-domain.com`), and a callback URL (i.e. `https://your-domain.com/integrations/github/oauth2/callback`). + + ![integrations github app basic details](../../images/integrations/github/app/self-hosted-github-app-basic-details.png) + + Enable request user authorization during app installation. + ![integrations github app enable auth](../../images/integrations/github/app/self-hosted-github-app-enable-oauth.png) + + Disable webhook by unchecking the Active checkbox. + ![integrations github app webhook](../../images/integrations/github/app/self-hosted-github-app-webhook.png) + + Set the repository permissions as follows: Metadata: Read-only, Secrets: Read and write, Environments: Read and write. + ![integrations github app repository](../../images/integrations/github/app/self-hosted-github-app-repository.png) + + Similarly, set the organization permissions as follows: Secrets: Read and write. + ![integrations github app organization](../../images/integrations/github/app/self-hosted-github-app-organization.png) + + Create the Github application. + ![integrations github app create confirm](../../images/integrations/github/app/self-hosted-github-app-create-confirm.png) + + + If you have a GitHub organization, you can create an application under it + in your organization Settings > Developer settings > GitHub Apps > New GitHub App. + + + + Generate a new **Client Secret** for your GitHub application. + ![integrations github app create secret](../../images/integrations/github/app/self-hosted-github-app-secret.png) + + Generate a new **Private Key** for your Github application. + ![integrations github app create private key](../../images/integrations/github/app/self-hosted-github-app-private-key.png) + + Obtain the necessary Github application credentials. This would be the application slug, client ID, app ID, client secret, and private key. + ![integrations github app credentials](../../images/integrations/github/app/self-hosted-github-app-credentials.png) + + Back in your Infisical instance, add the five new environment variables for the credentials of your GitHub application: + + - `CLIENT_ID_GITHUB_APP`: The **Client ID** of your GitHub application. + - `CLIENT_SECRET_GITHUB_APP`: The **Client Secret** of your GitHub application. + - `CLIENT_SLUG_GITHUB_APP`: The **Slug** of your GitHub application. This is the one found in the URL. + - `CLIENT_APP_ID_GITHUB_APP`: The **App ID** of your GitHub application. + - `CLIENT_PRIVATE_KEY_GITHUB_APP`: The **Private Key** of your GitHub application. + + Once added, restart your Infisical instance and use the GitHub integration via app authentication. + + + + + + +## Connecting with GitHub OAuth + +Prerequisites: + +- Set up and add envars to [Infisical Cloud](https://app.infisical.com) +- Ensure that you have admin privileges to the repository you want to sync secrets to. + + + + + + Navigate to your project's integrations tab in Infisical and press on the GitHub tile. + ![integrations](../../images/integrations/github/integration-overview.png) + + Select OAuth as the authentication method and click **Connect to GitHub**. + ![integrations github oauth auth selection](../../images/integrations/github/github-oauth-method-selection.png) + + Grant Infisical access to your GitHub account (organization and repo privileges). + ![integrations github authorization](../../images/integrations/github/integrations-github-auth.png) + + + + Select which Infisical environment secrets you want to sync to which GitHub organization, repository, or repository environment. + + + + ![integrations github](../../images/integrations/github/integrations-github-scope-repo.png) + + + ![integrations github](../../images/integrations/github/integrations-github-scope-org.png) + + When using the organization scope, your secrets will be saved in the top-level of your GitHub Organization. You can choose the visibility, which defines which repositories can access the secrets. The options are: - **All public repositories**: All public repositories in the organization can access the secrets. diff --git a/docs/integrations/cicd/gitlab.mdx b/docs/integrations/cicd/gitlab.mdx index c6d72a11e..2da61ef77 100644 --- a/docs/integrations/cicd/gitlab.mdx +++ b/docs/integrations/cicd/gitlab.mdx @@ -20,12 +20,6 @@ description: "How to sync secrets from Infisical to GitLab" ![integrations gitlab authorization](../../images/integrations/gitlab/integrations-gitlab-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which GitLab repository and press create integration to start syncing secrets to GitLab. diff --git a/docs/integrations/cicd/rundeck.mdx b/docs/integrations/cicd/rundeck.mdx index a0743fd01..bda7d8162 100644 --- a/docs/integrations/cicd/rundeck.mdx +++ b/docs/integrations/cicd/rundeck.mdx @@ -21,13 +21,6 @@ Prerequisites: ![integrations rundeck authorization](../../images/integrations/rundeck/integrations-rundeck-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - - Select which Infisical environment secrets you want to sync to a Rundeck Key Storage Path and press create integration to start syncing secrets to Rundeck. diff --git a/docs/integrations/cicd/travisci.mdx b/docs/integrations/cicd/travisci.mdx index 4e70c7696..873c371b6 100644 --- a/docs/integrations/cicd/travisci.mdx +++ b/docs/integrations/cicd/travisci.mdx @@ -21,12 +21,6 @@ Prerequisites: ![integrations travis ci authorization](../../images/integrations/travis-ci/integrations-travisci-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Travis CI repository and press create integration to start syncing secrets to Travis CI. diff --git a/docs/integrations/cloud/aws-parameter-store.mdx b/docs/integrations/cloud/aws-parameter-store.mdx index 467789647..9ce84bd22 100644 --- a/docs/integrations/cloud/aws-parameter-store.mdx +++ b/docs/integrations/cloud/aws-parameter-store.mdx @@ -60,13 +60,6 @@ Prerequisites: ![integration auth](../../images/integrations/aws/integrations-aws-parameter-store-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - - Select which Infisical environment secrets you want to sync to which AWS Parameter Store region and indicate the path for your secrets. Then, press create integration to start syncing secrets to AWS Parameter Store. diff --git a/docs/integrations/cloud/checkly.mdx b/docs/integrations/cloud/checkly.mdx index 618082a3b..00ec38d2f 100644 --- a/docs/integrations/cloud/checkly.mdx +++ b/docs/integrations/cloud/checkly.mdx @@ -22,12 +22,6 @@ Prerequisites: ![integrations checkly authorization](../../images/integrations/checkly/integrations-checkly-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to Checkly and press create integration to start syncing secrets. diff --git a/docs/integrations/cloud/cloud-66.mdx b/docs/integrations/cloud/cloud-66.mdx index ab362137c..c087f6564 100644 --- a/docs/integrations/cloud/cloud-66.mdx +++ b/docs/integrations/cloud/cloud-66.mdx @@ -31,13 +31,6 @@ Copy and save your token. Click on the Cloud 66 tile and enter your API token to grant Infisical access to your Cloud 66 account. ![integrations cloud 66 tile in infisical dashboard](../../images/integrations/cloud-66/integrations-cloud-66-infisical-dashboard.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - - Enter your Cloud 66 Personal Access Token here. Then click "Connect to Cloud 66". ![integrations cloud 66 tile in infisical dashboard](../../images/integrations/cloud-66/integrations-cloud-66-paste-pat.png) diff --git a/docs/integrations/cloud/cloudflare-pages.mdx b/docs/integrations/cloud/cloudflare-pages.mdx index 4d28cc574..addba4fcd 100644 --- a/docs/integrations/cloud/cloudflare-pages.mdx +++ b/docs/integrations/cloud/cloudflare-pages.mdx @@ -29,12 +29,6 @@ Prerequisites: ![integrations cloudflare authorization](../../images/integrations/cloudflare/integrations-cloudflare-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to Cloudflare and press create integration to start syncing secrets. diff --git a/docs/integrations/cloud/cloudflare-workers.mdx b/docs/integrations/cloud/cloudflare-workers.mdx index 9a126a0e3..10a579701 100644 --- a/docs/integrations/cloud/cloudflare-workers.mdx +++ b/docs/integrations/cloud/cloudflare-workers.mdx @@ -29,13 +29,6 @@ Prerequisites: ![integrations cloudflare authorization](../../images/integrations/cloudflare/integration-cloudflare-workers-connect.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - - Select which Infisical environment secrets you want to sync to Cloudflare Workers and press create integration to start syncing secrets. diff --git a/docs/integrations/cloud/databricks.mdx b/docs/integrations/cloud/databricks.mdx new file mode 100644 index 000000000..7fee3acd3 --- /dev/null +++ b/docs/integrations/cloud/databricks.mdx @@ -0,0 +1,31 @@ +--- +title: "Databricks" +description: "Learn how to sync secrets from Infisical to Databricks." +--- + +Prerequisites: + +- Set up and add secrets to [Infisical Cloud](https://app.infisical.com) + + + + Obtain a Personal Access Token in **User Settings** > **Developer** > **Access Tokens**. + + ![integrations databricks token](../../images/integrations/databricks/pat-token.png) + + Navigate to your project's integrations tab in Infisical. + + ![integrations](../../images/integrations.png) + + Press on the Databricks tile and enter your Databricks instance URL in the following format: `https://xxx.cloud.databricks.com`. Then, input your Databricks Access Token to grant Infisical the necessary permissions in your Databricks account. + + ![integrations databricks authorization](../../images/integrations/databricks/integrations-databricks-auth.png) + + + + Select which Infisical environment and secret path you want to sync to which Databricks scope. Then, press create integration to start syncing secrets to Databricks. + + ![create integration Databricks](../../images/integrations/databricks/integrations-databricks-create.png) + ![integrations Databricks](../../images/integrations/databricks/integrations-databricks.png) + + \ No newline at end of file diff --git a/docs/integrations/cloud/digital-ocean-app-platform.mdx b/docs/integrations/cloud/digital-ocean-app-platform.mdx index 1ed255a48..a0ed545cc 100644 --- a/docs/integrations/cloud/digital-ocean-app-platform.mdx +++ b/docs/integrations/cloud/digital-ocean-app-platform.mdx @@ -20,13 +20,6 @@ Name it **infisical**, choose **No expiry**, and make sure to check **Write (opt Click on the **Digital Ocean App Platform** tile and enter your API token to grant Infisical access to your Digital Ocean account. ![integrations](../../images/integrations.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - - Then enter your Digital Ocean Personal Access Token here. Then click "Connect to Digital Ocean App Platform". ![integrations infisical dashboard digital ocean integration](../../images/integrations/digital-ocean/integrations-do-enter-token.png) diff --git a/docs/integrations/cloud/flyio.mdx b/docs/integrations/cloud/flyio.mdx index 71b24e2a9..2aa14a919 100644 --- a/docs/integrations/cloud/flyio.mdx +++ b/docs/integrations/cloud/flyio.mdx @@ -22,12 +22,6 @@ Prerequisites: ![integrations fly authorization](../../images/integrations/flyio/integrations-flyio-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Fly.io app and press create integration to start syncing secrets to Fly.io. diff --git a/docs/integrations/cloud/gcp-secret-manager.mdx b/docs/integrations/cloud/gcp-secret-manager.mdx index 99edcd115..e57a976f0 100644 --- a/docs/integrations/cloud/gcp-secret-manager.mdx +++ b/docs/integrations/cloud/gcp-secret-manager.mdx @@ -24,12 +24,6 @@ description: "How to sync secrets from Infisical to GCP Secret Manager" ![integrations GCP authorization](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - In the **Connection** tab, select which Infisical environment secrets you want to sync to which GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. @@ -85,12 +79,6 @@ description: "How to sync secrets from Infisical to GCP Secret Manager" ![integrations GCP authorization options](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-auth-options.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - In the **Connection** tab, select which Infisical environment secrets you want to sync to the GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. diff --git a/docs/integrations/cloud/hasura-cloud.mdx b/docs/integrations/cloud/hasura-cloud.mdx index 48d9d301b..f88c1eb50 100644 --- a/docs/integrations/cloud/hasura-cloud.mdx +++ b/docs/integrations/cloud/hasura-cloud.mdx @@ -21,12 +21,6 @@ Prerequisites: ![integrations hasura cloud authorization](../../images/integrations/hasura-cloud/integrations-hasura-cloud-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Hasura Cloud project and press create integration to start syncing secrets to Hasura Cloud. diff --git a/docs/integrations/cloud/heroku.mdx b/docs/integrations/cloud/heroku.mdx index 903ab8270..a63c3f381 100644 --- a/docs/integrations/cloud/heroku.mdx +++ b/docs/integrations/cloud/heroku.mdx @@ -19,12 +19,6 @@ description: "How to sync secrets from Infisical to Heroku" ![integrations heroku authorization](../../images/integrations/heroku/integrations-heroku-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Heroku app and press create integration to start syncing secrets to Heroku. diff --git a/docs/integrations/cloud/laravel-forge.mdx b/docs/integrations/cloud/laravel-forge.mdx index 5797692ea..c58c4a7be 100644 --- a/docs/integrations/cloud/laravel-forge.mdx +++ b/docs/integrations/cloud/laravel-forge.mdx @@ -27,12 +27,6 @@ Prerequisites: ![integrations laravel forge authorization](../../images/integrations/laravel-forge/integrations-laravelforge-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Laravel Forge site and press create integration to start syncing secrets to Laravel Forge. diff --git a/docs/integrations/cloud/netlify.mdx b/docs/integrations/cloud/netlify.mdx index f9b20abda..f793aae10 100644 --- a/docs/integrations/cloud/netlify.mdx +++ b/docs/integrations/cloud/netlify.mdx @@ -25,12 +25,6 @@ description: "How to sync secrets from Infisical to Netlify" ![integrations netlify authorization](../../images/integrations/netlify/integrations-netlify-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Netlify app and context. Lastly, press create integration to start syncing secrets to Netlify. diff --git a/docs/integrations/cloud/northflank.mdx b/docs/integrations/cloud/northflank.mdx index 117ac73a8..10dcb288e 100644 --- a/docs/integrations/cloud/northflank.mdx +++ b/docs/integrations/cloud/northflank.mdx @@ -23,12 +23,6 @@ Prerequisites: ![integrations northflank authorization](../../images/integrations/northflank/integrations-northflank-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Northflank project and secret group. Finally, press create integration to start syncing secrets to Northflank. diff --git a/docs/integrations/cloud/qovery.mdx b/docs/integrations/cloud/qovery.mdx index 6eae9b954..13aa6af46 100644 --- a/docs/integrations/cloud/qovery.mdx +++ b/docs/integrations/cloud/qovery.mdx @@ -21,12 +21,6 @@ Prerequisites: ![integrations qovery authorization](../../images/integrations/qovery/integrations-qovery-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it is necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to Qovery and press create integration to start syncing secrets. diff --git a/docs/integrations/cloud/railway.mdx b/docs/integrations/cloud/railway.mdx index 3f08d75a0..77b315517 100644 --- a/docs/integrations/cloud/railway.mdx +++ b/docs/integrations/cloud/railway.mdx @@ -30,12 +30,6 @@ Prerequisites: ![integrations railway authorization](../../images/integrations/railway/integrations-railway-authorization.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Railway project and environment (and optionally service). Lastly, press create integration to start syncing secrets to Railway. diff --git a/docs/integrations/cloud/render.mdx b/docs/integrations/cloud/render.mdx index 970789a31..366316171 100644 --- a/docs/integrations/cloud/render.mdx +++ b/docs/integrations/cloud/render.mdx @@ -22,12 +22,6 @@ Prerequisites: ![integrations render authorization](../../images/integrations/render/integrations-render-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Render service and press create integration to start syncing secrets to Render. diff --git a/docs/integrations/cloud/supabase.mdx b/docs/integrations/cloud/supabase.mdx index 3e94c0f51..b5179c45f 100644 --- a/docs/integrations/cloud/supabase.mdx +++ b/docs/integrations/cloud/supabase.mdx @@ -28,12 +28,6 @@ Prerequisites: ![integrations supabase authorization](../../images/integrations/supabase/integrations-supabase-authorization.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Supabase project. Lastly, press create integration to start syncing secrets to Supabase. diff --git a/docs/integrations/cloud/teamcity.mdx b/docs/integrations/cloud/teamcity.mdx index 20a9dfbea..3e713cc6a 100644 --- a/docs/integrations/cloud/teamcity.mdx +++ b/docs/integrations/cloud/teamcity.mdx @@ -28,12 +28,6 @@ Prerequisites: ![integrations teamcity authorization](../../images/integrations/teamcity/integrations-teamcity-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which TeamCity project (and optionally build configuration) and press create integration to start syncing secrets to TeamCity. diff --git a/docs/integrations/cloud/windmill.mdx b/docs/integrations/cloud/windmill.mdx index 0fc6fddeb..d0b2b9643 100644 --- a/docs/integrations/cloud/windmill.mdx +++ b/docs/integrations/cloud/windmill.mdx @@ -22,12 +22,6 @@ Prerequisites: ![integrations windmill authorization](../../images/integrations/windmill/integrations-windmill-auth.png) - - If this is your project's first cloud integration, then you'll have to grant - Infisical access to your project's environment variables. Although this step - breaks E2EE, it's necessary for Infisical to sync the environment variables to - the cloud platform. - Select which Infisical environment secrets you want to sync to which Windmill workspace and press create integration to start syncing secrets to Windmill. diff --git a/docs/integrations/platforms/docker-compose.mdx b/docs/integrations/platforms/docker-compose.mdx index fea6b79a1..09bd82ff7 100644 --- a/docs/integrations/platforms/docker-compose.mdx +++ b/docs/integrations/platforms/docker-compose.mdx @@ -17,13 +17,7 @@ Follow this [guide](./docker) to configure the Infisical CLI for each service th Generate a machine identity for each service you want to inject secrets into. You can do this by following the steps in the [Machine Identity](/documentation/platform/identities/machine-identities) guide. ### Set the machine identity client ID and client secret as environment variables - For each service you want to inject secrets into, set two environment variable called `INFISICAL_MACHINE_IDENTITY_CLIENT_ID`, and `INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET` equal to the client ID and client secret of the machine identity(s) you created in the previous step. - - In the example below, we set two sets of client ID and client secret for the services. - - For the web service we set `INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_WEB` and `INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_WEB` as the client ID and client secret respectively. - - For the API service we set `INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_API` and `INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_API` as the client ID and client secret respectively. + For each service you want to inject secrets into, generate the required `INFISICAL_TOKEN_SERVICE_A` and `INFISICAL_TOKEN_SERVICE_B`. ```yaml # Example Docker Compose file @@ -32,31 +26,25 @@ Follow this [guide](./docker) to configure the Infisical CLI for each service th build: . image: example-service-1 environment: - - INFISICAL_MACHINE_IDENTITY_CLIENT_ID=${INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_WEB} - - INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET=${INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_WEB} + - INFISICAL_TOKEN=${INFISICAL_TOKEN_SERVICE_A} api: build: . image: example-service-2 environment: - - INFISICAL_MACHINE_IDENTITY_CLIENT_ID=${INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_API} - - INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET=${INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_API} + - INFISICAL_TOKEN=${INFISICAL_TOKEN_SERVICE_B} ``` ### Export shell variables - Next, set the shell variables you defined in your compose file. This can be done manually or via your CI/CD environment. Once done, it will be used to populate the corresponding `INFISICAL_MACHINE_IDENTITY_CLIENT_ID` and `INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET` in your Docker Compose file. + Next, set the shell variables you defined in your compose file. This can be done manually or via your CI/CD environment. Once done, it will be used to populate the corresponding `INFISICAL_TOKEN_SERVICE_A` and `INFISICAL_TOKEN_SERVICE_B` in your Docker Compose file. ```bash #Example # Token refers to the token we generated in step 2 for this service - export INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_WEB= - export INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_WEB= - - # Token refers to the token we generated in step 2 for this service - export INFISICAL_MACHINE_IDENTITY_CLIENT_ID_FOR_API= - export INFISICAL_MACHINE_IDENTITY_CLIENT_SECRET_FOR_API= + export INFISICAL_TOKEN_SERVICE_A=$(infisical login --method=universal-auth --client-id= --client-secret= --silent --plain) + export INFISICAL_TOKEN_SERVICE_B=$(infisical login --method=universal-auth --client-id= --client-secret= --silent --plain) # Then run your compose file in the same terminal. docker-compose ... diff --git a/docs/integrations/platforms/docker.mdx b/docs/integrations/platforms/docker.mdx index 6c059f8dd..8e429461a 100644 --- a/docs/integrations/platforms/docker.mdx +++ b/docs/integrations/platforms/docker.mdx @@ -81,6 +81,44 @@ CMD ["infisical", "run", "--projectId", "", "--command", "npm r +### Using a Starting Script + +The drawback of the previous method is that you would have to generate the `INFISICAL_TOKEN` manually. To automate this process, you can use a shell script as your starting command. + + + + Create a machine identity for your project by following the steps in the [Machine Identity](/documentation/platform/identities/machine-identities) guide. This identity will enable authentication and secret retrieval from Infisical. + + + + Create a shell script to obtain an access token for the machine identity: + + ```bash script.sh + #!/bin/sh + export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id=$INFISICAL_MACHINE_CLIENT_ID --client-secret=$INFISICAL_MACHINE_CLIENT_SECRET --plain --silent) + exec infisical run --token $INFISICAL_TOKEN --projectId $PROJECT_ID --env $INFISICAL_SECRET_ENV --domain $INFISICAL_API_URL -- + ``` + + > **Note:** The access token has a limited lifespan. Use the [infisical token renew](/cli/commands/token) CLI command to renew it when necessary. + + Caution: Implementing this directly in your Dockerfile presents two key issues: + + 1. Lack of persistence: Variables set in one build step are not automatically carried over to subsequent steps, complicating the process. + 2. Security risk: It exposes sensitive credentials inside your container, potentially allowing anyone with container access to retrieve them. + + + + + Grant the Infisical CLI access to the access token, inside your Docker container. This allows the CLI to fetch and inject secrets into your application. + + Add the following line to your Dockerfile: + + ```dockerfile + CMD ["./script.sh"] + ``` + + + ```dockerfile diff --git a/docs/integrations/platforms/infisical-agent.mdx b/docs/integrations/platforms/infisical-agent.mdx index 61e6f41ca..b25411c74 100644 --- a/docs/integrations/platforms/infisical-agent.mdx +++ b/docs/integrations/platforms/infisical-agent.mdx @@ -312,21 +312,37 @@ infisical agent --config example-agent-config-file.yaml ```bash - listSecrets "" "environment-slug" "" + listSecrets "" "environment-slug" "" "" ``` - ```bash example-template-usage - {{- with listSecrets "6553ccb2b7da580d7f6e7260" "dev" "/" }} + ```bash example-template-usage-1 + {{- with listSecrets "6553ccb2b7da580d7f6e7260" "dev" "/" `{"recursive": false, "expandSecretReferences": true}` }} {{- range . }} {{ .Key }}={{ .Value }} {{- end }} {{- end }} ``` + ```bash example-template-usage-2 +{{- with secret "da8056c8-01e2-4d24-b39f-cb4e004b8d44" "staging" "/" `{"recursive": true, "expandSecretReferences": true}` }} +{{- range . }} +{{- if eq .SecretPath "/"}} +{{ .Key }}={{ .Value }} +{{- else}} +{{ .SecretPath }}/{{ .Key }}={{ .Value }} +{{- end}} +{{- end }} +{{- end }} + ``` + + **Function name**: listSecrets -**Description**: This function can be used to render the full list of secrets within a given project, environment and secret path. +**Description**: This function can be used to render the full list of secrets within a given project, environment and secret path. -**Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment` +An optional JSON argument is also available. It includes the properties `recursive`, which defaults to false, and `expandSecretReferences`, which defaults to true and expands the returned secrets. + + +**Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, SecretPath, Type, ID, and Comment` diff --git a/docs/mint.json b/docs/mint.json index 65f03f3c7..662d008ff 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -187,6 +187,14 @@ "documentation/platform/workflow-integrations/slack-integration" ] }, + { + "group": "Admin Consoles", + "pages": [ + "documentation/platform/admin-panel/overview", + "documentation/platform/admin-panel/server-admin", + "documentation/platform/admin-panel/org-admin-console" + ] + }, "documentation/platform/secret-sharing" ] }, @@ -205,7 +213,8 @@ "group": "OIDC Auth", "pages": [ "documentation/platform/identities/oidc-auth/general", - "documentation/platform/identities/oidc-auth/github" + "documentation/platform/identities/oidc-auth/github", + "documentation/platform/identities/oidc-auth/circleci" ] }, "documentation/platform/mfa", @@ -240,7 +249,8 @@ "documentation/platform/scim/overview", "documentation/platform/scim/okta", "documentation/platform/scim/azure", - "documentation/platform/scim/jumpcloud" + "documentation/platform/scim/jumpcloud", + "documentation/platform/scim/group-mappings" ] } ] @@ -356,20 +366,21 @@ "integrations/cloud/cloudflare-workers" ] }, - "integrations/cloud/heroku", - "integrations/cloud/render", + "integrations/cloud/terraform-cloud", + "integrations/cloud/databricks", { "group": "View more", "pages": [ "integrations/cloud/digital-ocean-app-platform", + "integrations/cloud/heroku", "integrations/cloud/netlify", "integrations/cloud/railway", "integrations/cloud/flyio", + "integrations/cloud/render", "integrations/cloud/laravel-forge", "integrations/cloud/supabase", "integrations/cloud/northflank", "integrations/cloud/hasura-cloud", - "integrations/cloud/terraform-cloud", "integrations/cloud/qovery", "integrations/cloud/hashicorp-vault", "integrations/cloud/cloud-66", @@ -856,5 +867,166 @@ "koala": { "publicApiKey": "pk_b50d7184e0e39ddd5cdb43cf6abeadd9b97d" } + }, + "footer": { + "socials": { + "x": "https://www.twitter.com/infisical/", + "linkedin": "https://www.linkedin.com/company/infisical/", + "github": "https://github.com/Infisical/infisical-cli", + "slack": "https://infisical.com/slack" + }, + "links": [ + { + "title": "PRODUCT", + "links": [ + { "label": "Secret Management", "url": "https://infisical.com/" }, + { "label": "Secret Scanning", "url": "https://infisical.com/radar" }, + { + "label": "Share Secrets", + "url": "https://app.infisical.com/share-secret" + }, + { "label": "Pricing", "url": "https://infisical.com/pricing" }, + { + "label": "Security", + "url": "https://infisical.com/docs/internals/security" + }, + { + "label": "Blog", + "url": "https://infisical.com/blog" + }, + { + "label": "Infisical vs Vault", + "url": "https://infisical.com/infisical-vs-hashicorp-vault" + }, + { + "label": "Forum", + "url": "https://questions.infisical.com/" + } + ] + }, + { + "title": "USE CASES", + "links": [ + { + "label": "Infisical Agent", + "url": "https://infisical.com/docs/documentation/getting-started/introduction" + }, + { + "label": "Kubernetes", + "url": "https://infisical.com/docs/integrations/platforms/kubernetes" + }, + { + "label": "Dynamic Secrets", + "url": "https://infisical.com/docs/documentation/platform/dynamic-secrets/overview" + }, + { + "label": "Terraform", + "url": "https://infisical.com/docs/integrations/frameworks/terraform" + }, + { + "label": "Ansible", + "url": "https://infisical.com/docs/integrations/platforms/ansible" + }, + { + "label": "Jenkins", + "url": "https://infisical.com/docs/integrations/cicd/jenkins" + }, + { + "label": "Docker", + "url": "https://infisical.com/docs/integrations/platforms/docker-intro" + }, + { + "label": "AWS ECS", + "url": "https://infisical.com/docs/integrations/platforms/ecs-with-agent" + }, + { + "label": "GitLab", + "url": "https://infisical.com/docs/integrations/cicd/gitlab" + }, + { + "label": "GitHub", + "url": "https://infisical.com/docs/integrations/cicd/githubactions" + }, + { + "label": "SDK", + "url": "https://infisical.com/docs/sdks/overview" + } + ] + }, + { + "title": "DEVELOPERS", + "links": [ + { + "label": "Changelog", + "url": "https://www.infisical.com/docs/changelog" + }, + { + "label": "Status", + "url": "https://status.infisical.com/" + }, + { + "label": "Feedback & Requests", + "url": "https://github.com/Infisical/infisical/issues" + }, + { + "label": "Trust of Center", + "url": "https://app.vanta.com/infisical.com/trust/hoop8cr78cuarxo9sztvs" + }, + { + "label": "Open Source Friends", + "url": "https://infisical.com/infisical-friends" + }, + { + "label": "How to contribute", + "url": "https://www.infisical.com/infisical-heroes" + } + ] + }, + { + "title": "OTHERS", + "links": [ + { + "label": "Customers", + "url": "https://infisical.com/customers/traba" + }, + { + "label": "Company Handbook", + "url": "https://infisical.com/wiki/handbook/overview" + }, + { + "label": "Careers", + "url": "https://infisical.com/careers" + }, + { + "label": "Terms of Service", + "url": "https://infisical.com/terms" + }, + { + "label": "Privacy Policy", + "url": "https://infisical.com/privacy" + }, + { + "label": "Subprocessors", + "url": "https://infisical.com/subprocessors" + }, + { + "label": "SLA", + "url": "https://infisical.com/sla" + }, + { + "label": "Team Email", + "url": "mailto:team@infisical.com" + }, + { + "label": "Sales", + "url": "mailto:sales@infisical.com" + }, + { + "label": "Support", + "url": "https://infisical.com/slack" + } + ] + } + ] } } diff --git a/docs/self-hosting/configuration/requirements.mdx b/docs/self-hosting/configuration/requirements.mdx index c0e9cab01..e0e992b7f 100644 --- a/docs/self-hosting/configuration/requirements.mdx +++ b/docs/self-hosting/configuration/requirements.mdx @@ -59,6 +59,7 @@ Redis requirements: - Use Redis versions 6.x or 7.x. We advise upgrading to at least Redis 6.2. - Redis Cluster mode is currently not supported; use Redis Standalone, with or without High Availability (HA). - Redis storage needs are minimal: a setup with 2 vCPU, 4 GB RAM, and 30GB SSD will be sufficient for small deployments. +- Set cache eviction policy to `noeviction`. ## Supported Web Browsers diff --git a/frontend/next.config.js b/frontend/next.config.js index 9b9db1346..e07695ed6 100644 --- a/frontend/next.config.js +++ b/frontend/next.config.js @@ -2,6 +2,7 @@ const path = require("path"); const ContentSecurityPolicy = ` default-src 'self'; + connect-src 'self' https://*.posthog.com; script-src 'self' https://*.posthog.com https://js.stripe.com https://api.stripe.com https://widget.intercom.io https://js.intercomcdn.com https://hcaptcha.com https://*.hcaptcha.com 'unsafe-inline' 'unsafe-eval'; style-src 'self' https://rsms.me 'unsafe-inline' https://hcaptcha.com https://*.hcaptcha.com; child-src https://api.stripe.com; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 5227088b6..0111e54a3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,5 +1,5 @@ { - "name": "relock-npm-lock-v2-SvMQeF", + "name": "frontend", "lockfileVersion": 3, "requires": true, "packages": { @@ -40,7 +40,7 @@ "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-tooltip": "^1.0.7", "@reduxjs/toolkit": "^1.8.3", - "@sindresorhus/slugify": "^2.2.1", + "@sindresorhus/slugify": "1.1.0", "@stripe/react-stripe-js": "^1.16.3", "@stripe/stripe-js": "^1.46.0", "@tanstack/react-query": "^4.23.0", @@ -88,6 +88,7 @@ "react-mailchimp-subscribe": "^2.1.3", "react-markdown": "^8.0.3", "react-redux": "^8.0.2", + "react-select": "^5.8.1", "react-table": "^7.8.0", "react-toastify": "^9.1.3", "sanitize-html": "^2.12.1", @@ -2505,15 +2506,16 @@ } }, "node_modules/@emotion/babel-plugin": { - "version": "11.11.0", - "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", - "integrity": "sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==", + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.12.0.tgz", + "integrity": "sha512-y2WQb+oP8Jqvvclh8Q55gLUyb7UFvgv7eJfsj7td5TToBrIUtPay2kMrZi4xjq9qw2vD0ZR5fSho0yqoFgX7Rw==", + "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.16.7", "@babel/runtime": "^7.18.3", - "@emotion/hash": "^0.9.1", - "@emotion/memoize": "^0.8.1", - "@emotion/serialize": "^1.1.2", + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/serialize": "^1.2.0", "babel-plugin-macros": "^3.1.0", "convert-source-map": "^1.5.0", "escape-string-regexp": "^4.0.0", @@ -2522,18 +2524,31 @@ "stylis": "4.2.0" } }, + "node_modules/@emotion/babel-plugin/node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, "node_modules/@emotion/cache": { - "version": "11.11.0", - "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz", - "integrity": "sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==", + "version": "11.13.1", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.13.1.tgz", + "integrity": "sha512-iqouYkuEblRcXmylXIwwOodiEK5Ifl7JcX7o6V4jI3iW4mLXX3dmt5xwBtIkJiQEXFAI+pC8X0i67yiPkH9Ucw==", + "license": "MIT", "dependencies": { - "@emotion/memoize": "^0.8.1", - "@emotion/sheet": "^1.2.2", - "@emotion/utils": "^1.2.1", - "@emotion/weak-memoize": "^0.3.1", + "@emotion/memoize": "^0.9.0", + "@emotion/sheet": "^1.4.0", + "@emotion/utils": "^1.4.0", + "@emotion/weak-memoize": "^0.4.0", "stylis": "4.2.0" } }, + "node_modules/@emotion/cache/node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, "node_modules/@emotion/css": { "version": "11.11.2", "resolved": "https://registry.npmjs.org/@emotion/css/-/css-11.11.2.tgz", @@ -2547,9 +2562,10 @@ } }, "node_modules/@emotion/hash": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz", - "integrity": "sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ==" + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", + "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==", + "license": "MIT" }, "node_modules/@emotion/is-prop-valid": { "version": "0.8.8", @@ -2571,18 +2587,49 @@ "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz", "integrity": "sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA==" }, - "node_modules/@emotion/serialize": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.3.tgz", - "integrity": "sha512-iD4D6QVZFDhcbH0RAG1uVu1CwVLMWUkCvAqqlewO/rxf8+87yIBAlt4+AxMiiKPLs5hFc0owNk/sLLAOROw3cA==", + "node_modules/@emotion/react": { + "version": "11.13.3", + "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.13.3.tgz", + "integrity": "sha512-lIsdU6JNrmYfJ5EbUCf4xW1ovy5wKQ2CkPRM4xogziOxH1nXxBSjpC9YqbFAP7circxMfYp+6x676BqWcEiixg==", + "license": "MIT", "dependencies": { - "@emotion/hash": "^0.9.1", - "@emotion/memoize": "^0.8.1", - "@emotion/unitless": "^0.8.1", - "@emotion/utils": "^1.2.1", + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.12.0", + "@emotion/cache": "^11.13.0", + "@emotion/serialize": "^1.3.1", + "@emotion/use-insertion-effect-with-fallbacks": "^1.1.0", + "@emotion/utils": "^1.4.0", + "@emotion/weak-memoize": "^0.4.0", + "hoist-non-react-statics": "^3.3.1" + }, + "peerDependencies": { + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/serialize": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.2.tgz", + "integrity": "sha512-grVnMvVPK9yUVE6rkKfAJlYZgo0cu3l9iMC77V7DW6E1DUIrU68pSEXRmFZFOFB1QFo57TncmOcvcbMDWsL4yA==", + "license": "MIT", + "dependencies": { + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/unitless": "^0.10.0", + "@emotion/utils": "^1.4.1", "csstype": "^3.0.2" } }, + "node_modules/@emotion/serialize/node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, "node_modules/@emotion/server": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/server/-/server-11.11.0.tgz", @@ -2603,9 +2650,10 @@ } }, "node_modules/@emotion/sheet": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz", - "integrity": "sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA==" + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz", + "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==", + "license": "MIT" }, "node_modules/@emotion/stylis": { "version": "0.8.5", @@ -2613,28 +2661,31 @@ "integrity": "sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ==" }, "node_modules/@emotion/unitless": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz", - "integrity": "sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ==" + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz", + "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==", + "license": "MIT" }, "node_modules/@emotion/use-insertion-effect-with-fallbacks": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz", - "integrity": "sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==", - "dev": true, + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.1.0.tgz", + "integrity": "sha512-+wBOcIV5snwGgI2ya3u99D7/FJquOIniQT1IKyDsBmEgwvpxMNeS65Oib7OnE2d2aY+3BU4OiH+0Wchf8yk3Hw==", + "license": "MIT", "peerDependencies": { "react": ">=16.8.0" } }, "node_modules/@emotion/utils": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz", - "integrity": "sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg==" + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.1.tgz", + "integrity": "sha512-BymCXzCG3r72VKJxaYVwOXATqXIZ85cuvg0YOUDxMGNrKc1DJRZk8MgV5wyXRyEayIMd4FuXJIUgTBXvDNW5cA==", + "license": "MIT" }, "node_modules/@emotion/weak-memoize": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz", - "integrity": "sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww==" + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", + "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==", + "license": "MIT" }, "node_modules/@esbuild/android-arm": { "version": "0.18.20", @@ -5943,54 +5994,44 @@ "dev": true }, "node_modules/@sindresorhus/slugify": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@sindresorhus/slugify/-/slugify-2.2.1.tgz", - "integrity": "sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/slugify/-/slugify-1.1.0.tgz", + "integrity": "sha512-ujZRbmmizX26yS/HnB3P9QNlNa4+UvHh+rIse3RbOXLp8yl6n1TxB4t7NHggtVgS8QmmOtzXo48kCxZGACpkPw==", + "license": "MIT", "dependencies": { - "@sindresorhus/transliterate": "^1.0.0", - "escape-string-regexp": "^5.0.0" + "@sindresorhus/transliterate": "^0.1.1", + "escape-string-regexp": "^4.0.0" }, "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@sindresorhus/slugify/node_modules/escape-string-regexp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "engines": { - "node": ">=12" + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@sindresorhus/transliterate": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/transliterate/-/transliterate-1.6.0.tgz", - "integrity": "sha512-doH1gimEu3A46VX6aVxpHTeHrytJAG6HgdxntYnCFiIFHEM/ZGpG8KiZGBChchjQmG0XFIBL552kBTjVcMZXwQ==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@sindresorhus/transliterate/-/transliterate-0.1.2.tgz", + "integrity": "sha512-5/kmIOY9FF32nicXH+5yLNTX4NJ4atl7jRgqAJuIn/iyDFXBktOKDxCvyGE/EzmF4ngSUvjXxQUQlQiZ5lfw+w==", + "license": "MIT", "dependencies": { - "escape-string-regexp": "^5.0.0" + "escape-string-regexp": "^2.0.0", + "lodash.deburr": "^4.1.0" }, "engines": { - "node": ">=12" + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@sindresorhus/transliterate/node_modules/escape-string-regexp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "license": "MIT", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, "node_modules/@storybook/addon-actions": { @@ -8854,6 +8895,15 @@ "redux": "^4.0.0" } }, + "node_modules/@types/react-transition-group": { + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.11.tgz", + "integrity": "sha512-RM05tAniPZ5DZPzzNFP+DmrcOdD0efDUxMy3145oljWSl3x9ZV5vhme98gTxFrj2lhXvmGNnUiuDyJgY9IKkNA==", + "license": "MIT", + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/resolve": { "version": "1.20.6", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.6.tgz", @@ -12678,6 +12728,16 @@ "utila": "~0.4" } }, + "node_modules/dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, "node_modules/dom-serializer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", @@ -17278,6 +17338,12 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.deburr": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", + "integrity": "sha512-m/M1U1f3ddMCs6Hq2tAsYThTBDaAKFDX3dwDo97GEYzamXi9SqUpjWi/Rrj/gf3X2n8ktwgZrlP1z6E3v/IExQ==", + "license": "MIT" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -21117,6 +21183,33 @@ "react": ">= 16.3" } }, + "node_modules/react-select": { + "version": "5.8.1", + "resolved": "https://registry.npmjs.org/react-select/-/react-select-5.8.1.tgz", + "integrity": "sha512-RT1CJmuc+ejqm5MPgzyZujqDskdvB9a9ZqrdnVLsvAHjJ3Tj0hELnLeVPQlmYdVKCdCpxanepl6z7R5KhXhWzg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.0", + "@emotion/cache": "^11.4.0", + "@emotion/react": "^11.8.1", + "@floating-ui/dom": "^1.0.1", + "@types/react-transition-group": "^4.4.0", + "memoize-one": "^6.0.0", + "prop-types": "^15.6.0", + "react-transition-group": "^4.3.0", + "use-isomorphic-layout-effect": "^1.1.2" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-select/node_modules/memoize-one": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz", + "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==", + "license": "MIT" + }, "node_modules/react-style-singleton": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz", @@ -21171,6 +21264,22 @@ "node": ">=6" } }, + "node_modules/react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -24246,6 +24355,20 @@ } } }, + "node_modules/use-isomorphic-layout-effect": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz", + "integrity": "sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/use-memo-one": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/use-memo-one/-/use-memo-one-1.1.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index 42467ced4..d4a820f8b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,11 @@ "storybook": "storybook dev -p 6006 -s ./public", "build-storybook": "storybook build" }, + "overrides": { + "@storybook/nextjs": { + "sharp": "npm:dry-uninstall" + } + }, "dependencies": { "@casl/ability": "^6.5.0", "@casl/react": "^3.1.0", @@ -48,7 +53,7 @@ "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-tooltip": "^1.0.7", "@reduxjs/toolkit": "^1.8.3", - "@sindresorhus/slugify": "^2.2.1", + "@sindresorhus/slugify": "1.1.0", "@stripe/react-stripe-js": "^1.16.3", "@stripe/stripe-js": "^1.46.0", "@tanstack/react-query": "^4.23.0", @@ -96,6 +101,7 @@ "react-mailchimp-subscribe": "^2.1.3", "react-markdown": "^8.0.3", "react-redux": "^8.0.2", + "react-select": "^5.8.1", "react-table": "^7.8.0", "react-toastify": "^9.1.3", "sanitize-html": "^2.12.1", diff --git a/frontend/public/data/frequentConstants.ts b/frontend/public/data/frequentConstants.ts index 1fd7e7789..80cb51028 100644 --- a/frontend/public/data/frequentConstants.ts +++ b/frontend/public/data/frequentConstants.ts @@ -16,6 +16,7 @@ const integrationSlugNameMapping: Mapping = { railway: "Railway", flyio: "Fly.io", circleci: "CircleCI", + databricks: "Databricks", travisci: "TravisCI", supabase: "Supabase", checkly: "Checkly", diff --git a/frontend/public/images/integrations/Databricks.png b/frontend/public/images/integrations/Databricks.png new file mode 100644 index 000000000..ec0ddbc02 Binary files /dev/null and b/frontend/public/images/integrations/Databricks.png differ diff --git a/frontend/src/components/tags/CreateTagModal/CreateTagModal.tsx b/frontend/src/components/tags/CreateTagModal/CreateTagModal.tsx index acbcdcb2d..a2fde465d 100644 --- a/frontend/src/components/tags/CreateTagModal/CreateTagModal.tsx +++ b/frontend/src/components/tags/CreateTagModal/CreateTagModal.tsx @@ -115,7 +115,10 @@ export const CreateTagModal = ({ isOpen, onToggle }: Props): JSX.Element => { formState: { isSubmitting }, handleSubmit } = useForm({ - resolver: zodResolver(createTagSchema) + resolver: zodResolver(createTagSchema), + defaultValues: { + color: secretTagsColors[0].hex + } }); const { currentWorkspace } = useWorkspace(); diff --git a/frontend/src/components/v2/Checkbox/Checkbox.tsx b/frontend/src/components/v2/Checkbox/Checkbox.tsx index 8c7472b26..751a026c3 100644 --- a/frontend/src/components/v2/Checkbox/Checkbox.tsx +++ b/frontend/src/components/v2/Checkbox/Checkbox.tsx @@ -1,5 +1,5 @@ import { ReactNode } from "react"; -import { faCheck } from "@fortawesome/free-solid-svg-icons"; +import { faCheck, faMinus } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import * as CheckboxPrimitive from "@radix-ui/react-checkbox"; import { twMerge } from "tailwind-merge"; @@ -15,6 +15,7 @@ export type CheckboxProps = Omit< isRequired?: boolean; checkIndicatorBg?: string | undefined; isError?: boolean; + isIndeterminate?: boolean; }; export const Checkbox = ({ @@ -26,6 +27,7 @@ export const Checkbox = ({ isRequired, checkIndicatorBg, isError, + isIndeterminate, ...props }: CheckboxProps): JSX.Element => { return ( @@ -45,7 +47,11 @@ export const Checkbox = ({ id={id} > - + {isIndeterminate ? ( + + ) : ( + + )}