diff --git a/.github/workflows/build-staging-and-deploy-aws.yml b/.github/workflows/build-staging-and-deploy-aws.yml index 4341d19f5..a9b2046ae 100644 --- a/.github/workflows/build-staging-and-deploy-aws.yml +++ b/.github/workflows/build-staging-and-deploy-aws.yml @@ -122,13 +122,13 @@ jobs: uses: pr-mpt/actions-commit-hash@v2 - name: Download task definition run: | - aws ecs describe-task-definition --task-definition infisical-prod-platform --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition infisical-core-platform --query taskDefinition > task-definition.json - name: Render Amazon ECS task definition id: render-web-container uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: infisical-prod-platform + container-name: infisical-core-platform image: infisical/staging_infisical:${{ steps.commit.outputs.short }} environment-variables: "LOG_LEVEL=info" - name: Deploy to Amazon ECS service diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 3453352fa..291197b0b 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -234,6 +234,7 @@ import { TWebhooksInsert, TWebhooksUpdate } from "@app/db/schemas"; +import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references"; declare module "knex/types/tables" { interface Tables { @@ -307,6 +308,11 @@ declare module "knex/types/tables" { >; [TableName.ProjectKeys]: Knex.CompositeTableType; [TableName.Secret]: Knex.CompositeTableType; + [TableName.SecretReference]: Knex.CompositeTableType< + TSecretReferences, + TSecretReferencesInsert, + TSecretReferencesUpdate + >; [TableName.SecretBlindIndex]: Knex.CompositeTableType< TSecretBlindIndexes, TSecretBlindIndexesInsert, diff --git a/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts b/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts new file mode 100644 index 000000000..fa6fb4fea --- /dev/null +++ b/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts @@ -0,0 +1,24 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.SecretReference))) { + await knex.schema.createTable(TableName.SecretReference, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("environment").notNullable(); + t.string("secretPath").notNullable(); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + + await createOnUpdateTrigger(knex, TableName.SecretReference); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SecretReference); + await dropOnUpdateTrigger(knex, TableName.SecretReference); +} diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 863e0a7e6..28a6973b7 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -28,6 +28,7 @@ export enum TableName { ProjectUserMembershipRole = "project_user_membership_roles", ProjectKeys = "project_keys", Secret = "secrets", + SecretReference = "secret_references", SecretBlindIndex = "secret_blind_indexes", SecretVersion = "secret_versions", SecretFolder = "secret_folders", diff --git a/backend/src/db/schemas/secret-references.ts b/backend/src/db/schemas/secret-references.ts new file mode 100644 index 000000000..b3e6a8629 --- /dev/null +++ b/backend/src/db/schemas/secret-references.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretReferencesSchema = z.object({ + id: z.string().uuid(), + environment: z.string(), + secretPath: z.string(), + secretId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretReferences = z.infer; +export type TSecretReferencesInsert = Omit, TImmutableDBKeys>; +export type TSecretReferencesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts b/backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts index 0fecc9d2e..9a1a91672 100644 --- a/backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts +++ b/backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts @@ -8,7 +8,7 @@ import { IDENTITY_ADDITIONAL_PRIVILEGE } from "@app/lib/api-docs"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; -import { PermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas"; +import { ProjectPermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas"; import { AuthMode } from "@app/services/auth/auth-type"; export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: FastifyZodProvider) => { @@ -39,7 +39,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F }) .optional() .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug), - permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions) + permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions) }), response: { 200: z.object({ @@ -90,7 +90,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F }) .optional() .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug), - permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions), + permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions), temporaryMode: z .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode) .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.temporaryMode), @@ -155,7 +155,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F message: "Slug must be a valid slug" }) .describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.newSlug), - permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions), + permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions), isTemporary: z.boolean().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.isTemporary), temporaryMode: z .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 2e66ab2ce..690d308d2 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -7,12 +7,15 @@ import { SecretType, TSecretApprovalRequestsSecretsInsert } from "@app/db/schemas"; +import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, UnauthorizedError } from "@app/lib/errors"; import { groupBy, pick, unique } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; +import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; @@ -53,6 +56,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretVersionDAL: Pick; secretVersionTagDAL: Pick; projectDAL: Pick; + projectBotService: Pick; secretService: Pick< TSecretServiceFactory, | "fnSecretBulkInsert" @@ -80,7 +84,8 @@ export const secretApprovalRequestServiceFactory = ({ snapshotService, secretService, secretVersionDAL, - secretQueueService + secretQueueService, + projectBotService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -352,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({ } const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete); - + const botKey = await projectBotService.getBotKey(projectId).catch(() => null); const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { const newSecrets = secretCreationCommits.length ? await secretService.fnSecretBulkInsert({ @@ -379,7 +384,17 @@ export const secretApprovalRequestServiceFactory = ({ ]), tags: el?.tags.map(({ id }) => id), version: 1, - type: SecretType.Shared + type: SecretType.Shared, + references: botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }) + ) + : undefined })), secretDAL, secretVersionDAL, @@ -414,7 +429,17 @@ export const secretApprovalRequestServiceFactory = ({ "secretReminderNote", "secretReminderRepeatDays", "secretBlindIndex" - ]) + ]), + references: botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }) + ) + : undefined } })), secretDAL, diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 483f8e54c..f9a5ef312 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -89,6 +89,9 @@ export const UNIVERSAL_AUTH = { }, RENEW_ACCESS_TOKEN: { accessToken: "The access token to renew." + }, + REVOKE_ACCESS_TOKEN: { + accessToken: "The access token to revoke." } } as const; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e1149120d..bc8ac88ff 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -65,7 +65,13 @@ export type TQueueJobTypes = { }; [QueueName.IntegrationSync]: { name: QueueJobs.IntegrationSync; - payload: { projectId: string; environment: string; secretPath: string; depth?: number }; + payload: { + projectId: string; + environment: string; + secretPath: string; + depth?: number; + deDupeQueue?: Record; + }; }; [QueueName.SecretFullRepoScan]: { name: QueueJobs.SecretScan; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 013ace491..75c43a9aa 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -610,6 +610,7 @@ export const registerRoutes = async ( }); const sarService = secretApprovalRequestServiceFactory({ permissionService, + projectBotService, folderDAL, secretDAL, secretTagDAL, diff --git a/backend/src/server/routes/sanitizedSchemas.ts b/backend/src/server/routes/sanitizedSchemas.ts index 14155ecf9..cf9f23851 100644 --- a/backend/src/server/routes/sanitizedSchemas.ts +++ b/backend/src/server/routes/sanitizedSchemas.ts @@ -8,6 +8,7 @@ import { UsersSchema } from "@app/db/schemas"; import { UnpackedPermissionSchema } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service"; +import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; // sometimes the return data must be santizied to avoid leaking important values // always prefer pick over omit in zod @@ -64,14 +65,12 @@ export const secretRawSchema = z.object({ secretComment: z.string().optional() }); -export const PermissionSchema = z.object({ +export const ProjectPermissionSchema = z.object({ action: z - .string() - .min(1) + .nativeEnum(ProjectPermissionActions) .describe("Describe what action an entity can take. Possible actions: create, edit, delete, and read"), subject: z - .string() - .min(1) + .nativeEnum(ProjectPermissionSub) .describe("The entity this permission pertains to. Possible options: secrets, environments"), conditions: z .object({ diff --git a/backend/src/server/routes/v1/identity-access-token-router.ts b/backend/src/server/routes/v1/identity-access-token-router.ts index 387c54c13..7ed62e679 100644 --- a/backend/src/server/routes/v1/identity-access-token-router.ts +++ b/backend/src/server/routes/v1/identity-access-token-router.ts @@ -36,4 +36,29 @@ export const registerIdentityAccessTokenRouter = async (server: FastifyZodProvid }; } }); + + server.route({ + url: "/token/revoke", + method: "POST", + config: { + rateLimit: writeLimit + }, + schema: { + description: "Revoke access token", + body: z.object({ + accessToken: z.string().trim().describe(UNIVERSAL_AUTH.REVOKE_ACCESS_TOKEN.accessToken) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + handler: async (req) => { + await server.services.identityAccessToken.revokeAccessToken(req.body.accessToken); + return { + message: "Successfully revoked access token" + }; + } + }); }; diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index e2d8ddb12..6fa574a69 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1926,4 +1926,41 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { return { secrets }; } }); + + server.route({ + method: "POST", + url: "/backfill-secret-references", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Backfill secret references", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + projectId: z.string().trim().min(1) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { projectId } = req.body; + const message = await server.services.secret.backfillSecretReferences({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId + }); + + return message; + } + }); }; diff --git a/backend/src/services/identity-access-token/identity-access-token-dal.ts b/backend/src/services/identity-access-token/identity-access-token-dal.ts index 42fb5bba5..de8eb7ebc 100644 --- a/backend/src/services/identity-access-token/identity-access-token-dal.ts +++ b/backend/src/services/identity-access-token/identity-access-token-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TIdentityAccessTokens } from "@app/db/schemas"; +import { IdentityAuthMethod, TableName, TIdentityAccessTokens } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; @@ -15,23 +15,46 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => { const doc = await (tx || db)(TableName.IdentityAccessToken) .where(filter) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`) - .leftJoin( - TableName.IdentityUaClientSecret, - `${TableName.IdentityAccessToken}.identityUAClientSecretId`, - `${TableName.IdentityUaClientSecret}.id` - ) - .leftJoin( - TableName.IdentityUniversalAuth, - `${TableName.IdentityUaClientSecret}.identityUAId`, - `${TableName.IdentityUniversalAuth}.id` - ) + .leftJoin(TableName.IdentityUaClientSecret, (qb) => { + qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn( + `${TableName.IdentityAccessToken}.identityUAClientSecretId`, + `${TableName.IdentityUaClientSecret}.id` + ); + }) + .leftJoin(TableName.IdentityUniversalAuth, (qb) => { + qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn( + `${TableName.IdentityUaClientSecret}.identityUAId`, + `${TableName.IdentityUniversalAuth}.id` + ); + }) + .leftJoin(TableName.IdentityGcpAuth, (qb) => { + qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.GCP_AUTH])).andOn( + `${TableName.Identity}.id`, + `${TableName.IdentityGcpAuth}.identityId` + ); + }) + .leftJoin(TableName.IdentityAwsAuth, (qb) => { + qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.AWS_AUTH])).andOn( + `${TableName.Identity}.id`, + `${TableName.IdentityAwsAuth}.identityId` + ); + }) .select(selectAllTableCols(TableName.IdentityAccessToken)) .select( - db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth), + db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth).as("accessTokenTrustedIpsUa"), + db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityGcpAuth).as("accessTokenTrustedIpsGcp"), + db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityAwsAuth).as("accessTokenTrustedIpsAws"), db.ref("name").withSchema(TableName.Identity) ) .first(); - return doc; + + if (!doc) return; + + return { + ...doc, + accessTokenTrustedIps: + doc.accessTokenTrustedIpsUa || doc.accessTokenTrustedIpsGcp || doc.accessTokenTrustedIpsAws + }; } catch (error) { throw new DatabaseError({ error, name: "IdAccessTokenFindOne" }); } diff --git a/backend/src/services/identity-access-token/identity-access-token-service.ts b/backend/src/services/identity-access-token/identity-access-token-service.ts index 4b53c8174..898d0bc62 100644 --- a/backend/src/services/identity-access-token/identity-access-token-service.ts +++ b/backend/src/services/identity-access-token/identity-access-token-service.ts @@ -106,6 +106,24 @@ export const identityAccessTokenServiceFactory = ({ return { accessToken, identityAccessToken: updatedIdentityAccessToken }; }; + const revokeAccessToken = async (accessToken: string) => { + const appCfg = getConfig(); + + const decodedToken = jwt.verify(accessToken, appCfg.AUTH_SECRET) as JwtPayload & { + identityAccessTokenId: string; + }; + if (decodedToken.authTokenType !== AuthTokenType.IDENTITY_ACCESS_TOKEN) throw new UnauthorizedError(); + + const identityAccessToken = await identityAccessTokenDAL.findOne({ + [`${TableName.IdentityAccessToken}.id` as "id"]: decodedToken.identityAccessTokenId, + isAccessTokenRevoked: false + }); + if (!identityAccessToken) throw new UnauthorizedError(); + + const revokedToken = await identityAccessTokenDAL.deleteById(identityAccessToken.id); + return { revokedToken }; + }; + const fnValidateIdentityAccessToken = async (token: TIdentityAccessTokenJwtPayload, ipAddress?: string) => { const identityAccessToken = await identityAccessTokenDAL.findOne({ [`${TableName.IdentityAccessToken}.id` as "id"]: token.identityAccessTokenId, @@ -132,5 +150,5 @@ export const identityAccessTokenServiceFactory = ({ return { ...identityAccessToken, orgId: identityOrgMembership.orgId }; }; - return { renewAccessToken, fnValidateIdentityAccessToken }; + return { renewAccessToken, revokeAccessToken, fnValidateIdentityAccessToken }; }; diff --git a/backend/src/services/identity-project/identity-project-service.ts b/backend/src/services/identity-project/identity-project-service.ts index 18a1803ac..13ecf8bbc 100644 --- a/backend/src/services/identity-project/identity-project-service.ts +++ b/backend/src/services/identity-project/identity-project-service.ts @@ -82,6 +82,7 @@ export const identityProjectServiceFactory = ({ role, project.id ); + const hasPriviledge = isAtLeastAsPrivileged(permission, rolePermission); if (!hasPriviledge) throw new ForbiddenRequestError({ @@ -135,16 +136,18 @@ export const identityProjectServiceFactory = ({ message: `Identity with id ${identityId} doesn't exists in project with id ${projectId}` }); - const { permission: identityRolePermission } = await permissionService.getProjectPermission( - ActorType.IDENTITY, - projectIdentity.identityId, - projectIdentity.projectId, - actorAuthMethod, - actorOrgId - ); - const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, identityRolePermission); - if (!hasRequiredPriviledges) - throw new ForbiddenRequestError({ message: "Failed to delete more privileged identity" }); + for await (const { role: requestedRoleChange } of roles) { + const { permission: rolePermission } = await permissionService.getProjectPermissionByRole( + requestedRoleChange, + projectId + ); + + const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, rolePermission); + + if (!hasRequiredPriviledges) { + throw new ForbiddenRequestError({ message: "Failed to change to a more privileged role" }); + } + } // validate custom roles input const customInputRoles = roles.filter( diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 963db1e1e..5f47580e4 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -462,27 +462,39 @@ const syncSecretsAWSParameterStore = async ({ ssm.config.update(config); const metadata = z.record(z.any()).parse(integration.metadata || {}); + const awsParameterStoreSecretsObj: Record = {}; - const params = { - Path: integration.path as string, - Recursive: false, - WithDecryption: true - }; + // now fetch all aws parameter store secrets + let hasNext = true; + let nextToken: string | undefined; + while (hasNext) { + const parameters = await ssm + .getParametersByPath({ + Path: integration.path as string, + Recursive: false, + WithDecryption: true, + MaxResults: 10, + NextToken: nextToken + }) + .promise(); - const parameterList = (await ssm.getParametersByPath(params).promise()).Parameters; + if (parameters.Parameters) { + parameters.Parameters.forEach((parameter) => { + if (parameter.Name) { + const secKey = parameter.Name.substring((integration.path as string).length); + awsParameterStoreSecretsObj[secKey] = parameter; + } + }); + } + hasNext = Boolean(parameters.NextToken); + nextToken = parameters.NextToken; + } - const awsParameterStoreSecretsObj = (parameterList || []) - .filter(({ Name }) => Boolean(Name)) - .reduce( - (obj, secret) => ({ - ...obj, - [(secret.Name as string).substring((integration.path as string).length)]: secret - }), - {} as Record - ); // Identify secrets to create - await Promise.all( - Object.keys(secrets).map(async (key) => { + // don't use Promise.all() and promise map here + // it will cause rate limit + for (const key in secrets) { + if (Object.hasOwn(secrets, key)) { if (!(key in awsParameterStoreSecretsObj)) { // case: secret does not exist in AWS parameter store // -> create secret @@ -517,13 +529,16 @@ const syncSecretsAWSParameterStore = async ({ }) .promise(); } - }) - ); + + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + } + } if (!metadata.shouldDisableDelete) { - // Identify secrets to delete - await Promise.all( - Object.keys(awsParameterStoreSecretsObj).map(async (key) => { + for (const key in awsParameterStoreSecretsObj) { + if (Object.hasOwn(awsParameterStoreSecretsObj, key)) { if (!(key in secrets)) { // case: // -> delete secret @@ -533,8 +548,11 @@ const syncSecretsAWSParameterStore = async ({ }) .promise(); } - }) - ); + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + } + } } }; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 8a5970b83..1a2e414dd 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const upsertSecretReferences = async ( + data: { + secretId: string; + references: Array<{ environment: string; secretPath: string }>; + }[] = [], + tx?: Knex + ) => { + try { + if (!data.length) return; + + await (tx || db)(TableName.SecretReference) + .whereIn( + "secretId", + data.map(({ secretId }) => secretId) + ) + .delete(); + const newSecretReferences = data + .filter(({ references }) => references.length) + .flatMap(({ secretId, references }) => + references.map(({ environment, secretPath }) => ({ + secretPath, + secretId, + environment + })) + ); + if (!newSecretReferences.length) return; + const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences); + return secretReferences; + } catch (error) { + throw new DatabaseError({ error, name: "UpsertSecretReference" }); + } + }; + + const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretReference) + .where({ + secretPath, + environment: envSlug + }) + .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`) + .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + .select(selectAllTableCols(TableName.SecretReference)) + .select("folderId"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindReferencedSecretReferences" }); + } + }; + + // special query to backfill secret value + const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.Secret) + .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + // not empty + .whereNotNull("secretValueCiphertext") + .select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindAllProjectSecretValues" }); + } + }; + return { ...secretOrm, update, @@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => { getSecretTags, findByFolderId, findByFolderIds, - findByBlindIndexes + findByBlindIndexes, + upsertSecretReferences, + findReferencedSecretReferences, + findAllProjectSecretValues }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index fb2b90ba9..6b2b50920 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -194,6 +194,7 @@ type TInterpolateSecretArg = { folderDAL: Pick; }; +const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => { const fetchSecretsCrossEnv = () => { const fetchCache: Record> = {}; @@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD }; }; - const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; const recursivelyExpandSecret = async ( expandedSec: Record, interpolatedSec: Record, @@ -353,7 +353,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD }; export const decryptSecretRaw = ( - secret: TSecrets & { workspace: string; environment: string; secretPath?: string }, + secret: TSecrets & { workspace: string; environment: string; secretPath: string }, key: string ) => { const secretKey = decryptSymmetric128BitHexKeyUTF8({ @@ -396,6 +396,37 @@ export const decryptSecretRaw = ( }; }; +/** + * Grabs and processes nested secret references from a string + * + * This function looks for patterns that match the interpolation syntax in the input string. + * It filters out references that include nested paths, splits them into environment and + * secret path parts, and then returns an array of objects with the environment and the + * joined secret path. + * + * @param {string} maybeSecretReference - The string that has the potential secret references. + * @returns {Array<{ environment: string, secretPath: string }>} - An array of objects + * with the environment and joined secret path. + * + * @example + * const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}"; + * const result = getAllNestedSecretReferences(value); + * // result will be: + * // [ + * // { environment: 'dev', secretPath: '/someFolder/OtherFolder' }, + * // { environment: 'prod', secretPath: '/anotherFolder' } + * // ] + */ +export const getAllNestedSecretReferences = (maybeSecretReference: string) => { + const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]); + return references + .filter((el) => el.includes(".")) + .map((el) => { + const [environment, ...secretPathList] = el.split("."); + return { environment, secretPath: path.join("/", ...secretPathList.slice(0, -1)) }; + }); +}; + /** * Checks and handles secrets using a blind index method. * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes. @@ -467,7 +498,7 @@ export const fnSecretBulkInsert = async ({ tx }: TFnSecretBulkInsert) => { const newSecrets = await secretDAL.insertMany( - inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })), + inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })), tx ); const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string); @@ -478,13 +509,20 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, ...el }) => ({ + inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id })), tx ); + await secretDAL.upsertSecretReferences( + inputSecrets.map(({ references = [], secretBlindIndex }) => ({ + secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id, + references + })), + tx + ); if (newSecretTags.length) { const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); @@ -509,7 +547,7 @@ export const fnSecretBulkUpdate = async ({ secretVersionTagDAL }: TFnSecretBulkUpdate) => { const newSecrets = await secretDAL.bulkUpdate( - inputSecrets.map(({ filter, data: { tags, ...data } }) => ({ + inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({ filter: { ...filter, folderId }, data })), @@ -522,6 +560,15 @@ export const fnSecretBulkUpdate = async ({ })), tx ); + await secretDAL.upsertSecretReferences( + inputSecrets + .filter(({ data: { references } }) => Boolean(references)) + .map(({ data: { references = [] } }, i) => ({ + secretId: newSecrets[i].id, + references + })), + tx + ); const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] ); @@ -591,50 +638,39 @@ export const createManySecretsRawFnFactory = ({ folderId, isNew: true, blindIndexCfg, + userId, secretDAL }); - const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + const inputSecrets = secrets.map((secret) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretReferences = getAllNestedSecretReferences(secret.secretValue || ""); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - if (secret.type === SecretType.Personal) { - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: secretReferences + }; + }); - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to create personal secret override for no corresponding shared secret" - }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return { - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }; - }) - ); + // get all tags + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ @@ -703,56 +739,35 @@ export const updateManySecretsRawFnFactory = ({ userId }); - const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - if (secret.newSecretName === "") { - throw new BadRequestError({ message: "New secret name cannot be empty" }); - } + const inputSecrets = secrets.map((secret) => { + if (secret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretReferences = getAllNestedSecretReferences(secret.secretValue || ""); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - if (secret.type === SecretType.Personal) { - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); - - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to update personal secret override for no corresponding shared secret" - }); - - if (secret.newSecretName) - throw new BadRequestError({ message: "Personal secret cannot change the key name" }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return { - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - newSecretName: secret.newSecretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }; - }) - ); + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + newSecretName: secret.newSecretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: secretReferences + }; + }); const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 32b68fe29..e1c8d61af 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -59,6 +59,7 @@ export type TGetSecrets = { }; const MAX_SYNC_SECRET_DEPTH = 5; +const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`; export const secretQueueFactory = ({ queueService, @@ -102,28 +103,35 @@ export const secretQueueFactory = ({ folderDAL }); - const syncIntegrations = async (dto: TGetSecrets) => { + const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { - attempts: 5, + attempts: 3, delay: 1000, backoff: { type: "exponential", delay: 3000 }, removeOnComplete: true, - removeOnFail: { - count: 5 // keep the most recent jobs - } + removeOnFail: true }); }; - const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => { + const syncSecrets = async ({ + deDupeQueue = {}, + ...dto + }: TGetSecrets & { depth?: number; deDupeQueue?: Record }) => { + const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath); + if (deDupeQueue?.[deDuplicationKey]) { + return; + } + // eslint-disable-next-line + deDupeQueue[deDuplicationKey] = true; logger.info( `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]` ); await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, - removeOnFail: { count: 5 }, + removeOnFail: true, removeOnComplete: true, delay: 1000, attempts: 5, @@ -132,7 +140,7 @@ export const secretQueueFactory = ({ delay: 3000 } }); - await syncIntegrations(dto); + await syncIntegrations({ ...dto, deDupeQueue }); }; const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { @@ -326,7 +334,7 @@ export const secretQueueFactory = ({ }; queueService.start(QueueName.IntegrationSync, async (job) => { - const { environment, projectId, secretPath, depth = 1 } = job.data; + const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { @@ -349,21 +357,68 @@ export const secretQueueFactory = ({ const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id); + logger.info( + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); await Promise.all( imports .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) - .map(({ folderId }) => { - const syncDto = { + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueIntegrationKey( + foldersGroupedById[folderId][0].environmentSlug, + foldersGroupedById[folderId][0].path + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ depth: depth + 1, projectId, secretPath: foldersGroupedById[folderId][0].path, - environment: foldersGroupedById[folderId][0].environmentSlug - }; - logger.info( - `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - return syncSecrets(syncDto); - }) + environment: foldersGroupedById[folderId][0].environmentSlug, + deDupeQueue + }) + ) + ); + } + + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + if (secretReferences.length) { + const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); + const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id); + logger.info( + `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + secretReferences + .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueIntegrationKey( + referencedFoldersGroupedById[folderId][0].environmentSlug, + referencedFoldersGroupedById[folderId][0].path + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + depth: depth + 1, + projectId, + secretPath: referencedFoldersGroupedById[folderId][0].path, + environment: referencedFoldersGroupedById[folderId][0].environmentSlug, + deDupeQueue + }) + ) ); } } else { diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index feb8c01d5..39e47a28e 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -2,12 +2,22 @@ /* eslint-disable no-await-in-loop */ import { ForbiddenError, subject } from "@casl/ability"; -import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas"; +import { + ProjectMembershipRole, + SecretEncryptionAlgo, + SecretKeyEncoding, + SecretsSchema, + SecretType +} from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; -import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { + buildSecretBlindIndexFromName, + decryptSymmetric128BitHexKeyUTF8, + encryptSymmetric128BitHexKeyUTF8 +} from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, pick } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -27,12 +37,14 @@ import { fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate, + getAllNestedSecretReferences, interpolateSecrets, recursivelyGetSecretPaths } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, + TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, TCreateManySecretRawDTO, TCreateSecretDTO, @@ -91,6 +103,22 @@ export const secretServiceFactory = ({ secretImportDAL, secretVersionTagDAL }: TSecretServiceFactoryDep) => { + const getSecretReference = async (projectId: string) => { + // if bot key missing means e2e still exist + const botKey = await projectBotService.getBotKey(projectId).catch(() => null); + return (el: { ciphertext?: string; iv: string; tag: string }) => + botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.ciphertext || "", + iv: el.iv, + tag: el.tag, + key: botKey + }) + ) + : undefined; + }; + // utility function to get secret blind index data const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => { const appCfg = getConfig(); @@ -225,6 +253,7 @@ export const secretServiceFactory = ({ if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); const { secretName, type, ...el } = inputSecret; + const references = await getSecretReference(projectId); const secret = await secretDAL.transaction((tx) => fnSecretBulkInsert({ folderId, @@ -237,7 +266,12 @@ export const secretServiceFactory = ({ userId: inputSecret.type === SecretType.Personal ? actorId : null, algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8, - tags: inputSecret.tags + tags: inputSecret.tags, + references: references({ + ciphertext: inputSecret.secretValueCiphertext, + iv: inputSecret.secretValueIV, + tag: inputSecret.secretValueTag + }) } ], secretDAL, @@ -251,7 +285,7 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return { ...secret[0], environment, workspace: projectId, tags }; + return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; const updateSecret = async ({ @@ -335,6 +369,7 @@ export const secretServiceFactory = ({ const { secretName, ...el } = inputSecret; + const references = await getSecretReference(projectId); const updatedSecret = await secretDAL.transaction(async (tx) => fnSecretBulkUpdate({ folderId, @@ -360,7 +395,12 @@ export const secretServiceFactory = ({ "secretReminderRepeatDays", "tags" ]), - secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName] + secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName], + references: references({ + ciphertext: inputSecret.secretValueCiphertext, + iv: inputSecret.secretValueIV, + tag: inputSecret.secretValueTag + }) } } ], @@ -375,7 +415,7 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return { ...updatedSecret[0], workspace: projectId, environment }; + return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; const deleteSecret = async ({ @@ -444,7 +484,7 @@ export const secretServiceFactory = ({ await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment }; + return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; }; const getSecrets = async ({ @@ -641,7 +681,8 @@ export const secretServiceFactory = ({ return { ...importedSecrets[i].secrets[j], workspace: projectId, - environment: importedSecrets[i].environment + environment: importedSecrets[i].environment, + secretPath: importedSecrets[i].secretPath }; } } @@ -649,7 +690,7 @@ export const secretServiceFactory = ({ } if (!secret) throw new BadRequestError({ message: "Secret not found" }); - return { ...secret, workspace: projectId, environment }; + return { ...secret, workspace: projectId, environment, secretPath: path }; }; const createManySecret = async ({ @@ -700,6 +741,7 @@ export const secretServiceFactory = ({ const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); + const references = await getSecretReference(projectId); const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ @@ -708,7 +750,12 @@ export const secretServiceFactory = ({ secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared, algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 + keyEncoding: SecretKeyEncoding.UTF8, + references: references({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag + }) })), folderId, secretDAL, @@ -783,6 +830,8 @@ export const secretServiceFactory = ({ const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const references = await getSecretReference(projectId); const secrets = await secretDAL.transaction(async (tx) => fnSecretBulkUpdate({ folderId, @@ -799,7 +848,15 @@ export const secretServiceFactory = ({ ? newKeyName2BlindIndex[newSecretName] : keyName2BlindIndex[secretName], algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 + keyEncoding: SecretKeyEncoding.UTF8, + references: + el.secretValueIV && el.secretValueTag + ? references({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag + }) + : undefined } })), secretDAL, @@ -924,34 +981,40 @@ export const secretServiceFactory = ({ }); const batchSecretsExpand = async ( - secretBatch: { - secretKey: string; - secretValue: string; - secretComment?: string; - }[] + secretBatch: { secretKey: string; secretValue: string; secretComment?: string; secretPath: string }[] ) => { - const secretRecord: Record< - string, - { - value: string; - comment?: string; - skipMultilineEncoding?: boolean; + // Group secrets by secretPath + const secretsByPath: Record = {}; + + secretBatch.forEach((secret) => { + if (!secretsByPath[secret.secretPath]) { + secretsByPath[secret.secretPath] = []; } - > = {}; - - secretBatch.forEach((decryptedSecret) => { - secretRecord[decryptedSecret.secretKey] = { - value: decryptedSecret.secretValue, - comment: decryptedSecret.secretComment - }; + secretsByPath[secret.secretPath].push(secret); }); - await expandSecrets(secretRecord); + // Expand secrets for each group + for (const secPath in secretsByPath) { + if (!Object.hasOwn(secretsByPath, path)) { + // eslint-disable-next-line no-continue + continue; + } - secretBatch.forEach((decryptedSecret, index) => { - // eslint-disable-next-line no-param-reassign - secretBatch[index].secretValue = secretRecord[decryptedSecret.secretKey].value; - }); + const secretRecord: Record = {}; + secretsByPath[secPath].forEach((decryptedSecret) => { + secretRecord[decryptedSecret.secretKey] = { + value: decryptedSecret.secretValue, + comment: decryptedSecret.secretComment + }; + }); + + await expandSecrets(secretRecord); + + secretsByPath[secPath].forEach((decryptedSecret) => { + // eslint-disable-next-line no-param-reassign + decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value; + }); + } }; // expand secrets @@ -999,6 +1062,7 @@ export const secretServiceFactory = ({ includeImports, version }); + return decryptSecretRaw(secret, botKey); }; @@ -1171,7 +1235,9 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(secrets[0].folderId); await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const updateManySecretsRaw = async ({ @@ -1223,7 +1289,9 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(secrets[0].folderId); await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const deleteManySecretsRaw = async ({ @@ -1257,7 +1325,9 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(secrets[0].folderId); await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const getSecretVersions = async ({ @@ -1488,6 +1558,52 @@ export const secretServiceFactory = ({ }; }; + // this is a backfilling API for secret references + // what it does is it will go through all the secret values and parse all references + // populate the secret reference to do sync integrations + const backfillSecretReferences = async ({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TBackFillSecretReferencesDTO) => { + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) + throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" }); + + await secretDAL.transaction(async (tx) => { + const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx); + await secretDAL.upsertSecretReferences( + secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({ + secretId: id, + references: getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secretValueCiphertext, + iv: secretValueIV, + tag: secretValueTag, + key: botKey + }) + ) + })), + tx + ); + }); + + return { message: "Successfully backfilled secret references" }; + }; + return { attachTags, detachTags, @@ -1508,6 +1624,7 @@ export const secretServiceFactory = ({ updateManySecretsRaw, deleteManySecretsRaw, getSecretVersions, + backfillSecretReferences, // external services function fnSecretBulkDelete, fnSecretBulkUpdate, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 9f2addc89..7e713a80f 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit & { secretId: string; }; +export type TSecretReference = { environment: string; secretPath: string }; + export type TFnSecretBulkInsert = { folderId: string; tx?: Knex; - inputSecrets: Array & { tags?: string[] }>; - secretDAL: Pick; + inputSecrets: Array & { tags?: string[]; references?: TSecretReference[] }>; + secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; @@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = { export type TFnSecretBulkUpdate = { folderId: string; projectId: string; - inputSecrets: { filter: Partial; data: TSecretsUpdate & { tags?: string[] } }[]; - secretDAL: Pick; + inputSecrets: { + filter: Partial; + data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] }; + }[]; + secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; @@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = { repeatDays: number; }; +export type TBackFillSecretReferencesDTO = TProjectPermission; + // --- export type TCreateManySecretsRawFnFactory = { diff --git a/docs/api-reference/endpoints/universal-auth/revoke-access-token.mdx b/docs/api-reference/endpoints/universal-auth/revoke-access-token.mdx new file mode 100644 index 000000000..082a76544 --- /dev/null +++ b/docs/api-reference/endpoints/universal-auth/revoke-access-token.mdx @@ -0,0 +1,4 @@ +--- +title: "Revoke Access Token" +openapi: "POST /api/v1/auth/token/revoke" +--- diff --git a/docs/cli/commands/export.mdx b/docs/cli/commands/export.mdx index 8f6667a5d..16c226084 100644 --- a/docs/cli/commands/export.mdx +++ b/docs/cli/commands/export.mdx @@ -128,6 +128,12 @@ infisical export --template= + + By default imported secrets are available, you can disable it by setting this option to false. + + Default value: `true` + + Format of the output file. Accepted values: `dotenv`, `dotenv-export`, `csv`, `json` and `yaml` diff --git a/docs/cli/commands/run.mdx b/docs/cli/commands/run.mdx index 9cd8f0a80..74aa84947 100644 --- a/docs/cli/commands/run.mdx +++ b/docs/cli/commands/run.mdx @@ -126,6 +126,12 @@ $ infisical run -- npm run dev + + By default imported secrets are available, you can disable it by setting this option to false. + + Default value: `true` + + {" "} diff --git a/docs/cli/faq.mdx b/docs/cli/faq.mdx index cf95457c9..47e89a48f 100644 --- a/docs/cli/faq.mdx +++ b/docs/cli/faq.mdx @@ -13,6 +13,7 @@ If none of the available stores work for you, you can try using the `file` store If you are still experiencing trouble, please seek support. [Learn more about vault command](./commands/vault) + diff --git a/docs/documentation/platform/identities/gcp-auth.mdx b/docs/documentation/platform/identities/gcp-auth.mdx index 92c3d59e2..c836a946d 100644 --- a/docs/documentation/platform/identities/gcp-auth.mdx +++ b/docs/documentation/platform/identities/gcp-auth.mdx @@ -123,7 +123,7 @@ access the Infisical API using the GCP ID Token authentication method. ```bash curl curl -H "Metadata-Flavor: Google" \ - 'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=' + 'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=&format=full' ``` diff --git a/docs/documentation/platform/secret-versioning.mdx b/docs/documentation/platform/secret-versioning.mdx index 11afbceef..6a0efb8b8 100644 --- a/docs/documentation/platform/secret-versioning.mdx +++ b/docs/documentation/platform/secret-versioning.mdx @@ -3,7 +3,7 @@ title: "Secret Versioning" description: "Learn how secret versioning works in Infisical." --- -Every time a secret change is persformed, a new version of the same secret is created. +Every time a secret change is performed, a new version of the same secret is created. Such versions can be accessed visually by opening up the [secret sidebar](/documentation/platform/project#drawer) (as seen below) or [retrieved via API](/api-reference/endpoints/secrets/read) by specifying the `version` query parameter. diff --git a/docs/mint.json b/docs/mint.json index f74efc401..ed22393a6 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -417,7 +417,8 @@ "api-reference/endpoints/universal-auth/create-client-secret", "api-reference/endpoints/universal-auth/list-client-secrets", "api-reference/endpoints/universal-auth/revoke-client-secret", - "api-reference/endpoints/universal-auth/renew-access-token" + "api-reference/endpoints/universal-auth/renew-access-token", + "api-reference/endpoints/universal-auth/revoke-access-token" ] }, { diff --git a/frontend/src/hooks/api/secrets/index.ts b/frontend/src/hooks/api/secrets/index.ts index b24b5ed19..b58e8779a 100644 --- a/frontend/src/hooks/api/secrets/index.ts +++ b/frontend/src/hooks/api/secrets/index.ts @@ -1,4 +1,5 @@ export { + useBackfillSecretReference, useCreateSecretBatch, useCreateSecretV3, useDeleteSecretBatch, diff --git a/frontend/src/hooks/api/secrets/mutations.tsx b/frontend/src/hooks/api/secrets/mutations.tsx index 448daee3b..e397c7b55 100644 --- a/frontend/src/hooks/api/secrets/mutations.tsx +++ b/frontend/src/hooks/api/secrets/mutations.tsx @@ -87,11 +87,11 @@ export const useCreateSecretV3 = ({ const randomBytes = latestFileKey ? decryptAssymmetric({ - ciphertext: latestFileKey.encryptedKey, - nonce: latestFileKey.nonce, - publicKey: latestFileKey.sender.publicKey, - privateKey: PRIVATE_KEY - }) + ciphertext: latestFileKey.encryptedKey, + nonce: latestFileKey.nonce, + publicKey: latestFileKey.sender.publicKey, + privateKey: PRIVATE_KEY + }) : crypto.randomBytes(16).toString("hex"); const reqBody = { @@ -148,11 +148,11 @@ export const useUpdateSecretV3 = ({ const randomBytes = latestFileKey ? decryptAssymmetric({ - ciphertext: latestFileKey.encryptedKey, - nonce: latestFileKey.nonce, - publicKey: latestFileKey.sender.publicKey, - privateKey: PRIVATE_KEY - }) + ciphertext: latestFileKey.encryptedKey, + nonce: latestFileKey.nonce, + publicKey: latestFileKey.sender.publicKey, + privateKey: PRIVATE_KEY + }) : crypto.randomBytes(16).toString("hex"); const reqBody = { @@ -244,11 +244,11 @@ export const useCreateSecretBatch = ({ const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string; const randomBytes = latestFileKey ? decryptAssymmetric({ - ciphertext: latestFileKey.encryptedKey, - nonce: latestFileKey.nonce, - publicKey: latestFileKey.sender.publicKey, - privateKey: PRIVATE_KEY - }) + ciphertext: latestFileKey.encryptedKey, + nonce: latestFileKey.nonce, + publicKey: latestFileKey.sender.publicKey, + privateKey: PRIVATE_KEY + }) : crypto.randomBytes(16).toString("hex"); const reqBody = { @@ -297,11 +297,11 @@ export const useUpdateSecretBatch = ({ const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string; const randomBytes = latestFileKey ? decryptAssymmetric({ - ciphertext: latestFileKey.encryptedKey, - nonce: latestFileKey.nonce, - publicKey: latestFileKey.sender.publicKey, - privateKey: PRIVATE_KEY - }) + ciphertext: latestFileKey.encryptedKey, + nonce: latestFileKey.nonce, + publicKey: latestFileKey.sender.publicKey, + privateKey: PRIVATE_KEY + }) : crypto.randomBytes(16).toString("hex"); const reqBody = { @@ -379,3 +379,13 @@ export const createSecret = async (dto: CreateSecretDTO) => { const { data } = await apiRequest.post(`/api/v3/secrets/${dto.secretKey}`, dto); return data; }; + +export const useBackfillSecretReference = () => + useMutation<{ message: string }, {}, { projectId: string }>({ + mutationFn: async ({ projectId }) => { + const { data } = await apiRequest.post("/api/v3/secrets/backfill-secret-references", { + projectId + }); + return data.message; + } + }); diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx new file mode 100644 index 000000000..91f71f5eb --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx @@ -0,0 +1,43 @@ +import { createNotification } from "@app/components/notifications"; +import { Button } from "@app/components/v2"; +import { useProjectPermission, useWorkspace } from "@app/context"; +import { useBackfillSecretReference } from "@app/hooks/api"; +import { ProjectMembershipRole } from "@app/hooks/api/roles/types"; + +export const BackfillSecretReferenceSecretion = () => { + const { currentWorkspace } = useWorkspace(); + const { membership } = useProjectPermission(); + const backfillSecretReferences = useBackfillSecretReference(); + + if (!currentWorkspace) return null; + + const handleBackfill = async () => { + if (backfillSecretReferences.isLoading) return; + try { + await backfillSecretReferences.mutateAsync({ projectId: currentWorkspace.id || "" }); + createNotification({ text: "Successfully re-indexed secret references", type: "success" }); + } catch { + createNotification({ text: "Failed to re-index secret references", type: "error" }); + } + }; + + const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin); + return ( +
+
+

Index Secret References

+
+

+ This will index all secret references, enabling integrations to be triggered when their values change going forward. +

+ +
+ ); +}; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx new file mode 100644 index 000000000..3e1651461 --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx @@ -0,0 +1 @@ +export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection"; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx index bbd8e78d4..7d7c30fb0 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx @@ -1,4 +1,5 @@ import { AutoCapitalizationSection } from "../AutoCapitalizationSection"; +import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSection"; import { DeleteProjectSection } from "../DeleteProjectSection"; import { E2EESection } from "../E2EESection"; import { EnvironmentSection } from "../EnvironmentSection"; @@ -13,6 +14,7 @@ export const ProjectGeneralTab = () => { + ); diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx index 72ac106a3..9d97c8bb8 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx @@ -1,4 +1,5 @@ export { AutoCapitalizationSection } from "./AutoCapitalizationSection"; +export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection"; export { DeleteProjectSection } from "./DeleteProjectSection"; export { E2EESection } from "./E2EESection"; export { EnvironmentSection } from "./EnvironmentSection";