feat: added bridge logic in secret replication, snapshot and approval for raw endpoint

This commit is contained in:
=
2024-07-30 23:11:04 +05:30
parent 9e314d7a09
commit ef2816b2ee
14 changed files with 2205 additions and 283 deletions
@@ -356,5 +356,144 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
}
};
return { ...secretApprovalRequestOrm, findById, findProjectRequestCount, findByProjectId };
const findByProjectIdBridgeSecretV2 = async (
{ status, limit = 20, offset = 0, projectId, committer, environment, userId }: TFindQueryFilter,
tx?: Knex
) => {
try {
// akhilmhdh: If ever u wanted a 1 to so many relationship connected with pagination
// this is the place u wanna look at.
const query = (tx || db.replicaNode())(TableName.SecretApprovalRequest)
.join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(
TableName.SecretApprovalPolicy,
`${TableName.SecretApprovalRequest}.policyId`,
`${TableName.SecretApprovalPolicy}.id`
)
.join(
TableName.SecretApprovalPolicyApprover,
`${TableName.SecretApprovalPolicy}.id`,
`${TableName.SecretApprovalPolicyApprover}.policyId`
)
.join<TUsers>(
db(TableName.Users).as("committerUser"),
`${TableName.SecretApprovalRequest}.committerUserId`,
`committerUser.id`
)
.leftJoin(
TableName.SecretApprovalRequestReviewer,
`${TableName.SecretApprovalRequest}.id`,
`${TableName.SecretApprovalRequestReviewer}.requestId`
)
.leftJoin<TSecretApprovalRequestsSecrets>(
TableName.SecretApprovalRequestSecretV2,
`${TableName.SecretApprovalRequestSecretV2}.requestId`,
`${TableName.SecretApprovalRequest}.id`
)
.where(
stripUndefinedInWhere({
projectId,
[`${TableName.Environment}.slug` as "slug"]: environment,
[`${TableName.SecretApprovalRequest}.status`]: status,
committerUserId: committer
})
)
.andWhere(
(bd) =>
void bd
.where(`${TableName.SecretApprovalPolicyApprover}.approverUserId`, userId)
.orWhere(`${TableName.SecretApprovalRequest}.committerUserId`, userId)
)
.select(selectAllTableCols(TableName.SecretApprovalRequest))
.select(
db.ref("projectId").withSchema(TableName.Environment),
db.ref("slug").withSchema(TableName.Environment).as("environment"),
db.ref("id").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerId"),
db.ref("reviewerUserId").withSchema(TableName.SecretApprovalRequestReviewer),
db.ref("status").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerStatus"),
db.ref("id").withSchema(TableName.SecretApprovalPolicy).as("policyId"),
db.ref("name").withSchema(TableName.SecretApprovalPolicy).as("policyName"),
db.ref("op").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitOp"),
db.ref("secretId").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitSecretId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitId"),
db.raw(
`DENSE_RANK() OVER (partition by ${TableName.Environment}."projectId" ORDER BY ${TableName.SecretApprovalRequest}."id" DESC) as rank`
),
db.ref("secretPath").withSchema(TableName.SecretApprovalPolicy).as("policySecretPath"),
db.ref("approvals").withSchema(TableName.SecretApprovalPolicy).as("policyApprovals"),
db.ref("approverUserId").withSchema(TableName.SecretApprovalPolicyApprover),
db.ref("email").withSchema("committerUser").as("committerUserEmail"),
db.ref("username").withSchema("committerUser").as("committerUserUsername"),
db.ref("firstName").withSchema("committerUser").as("committerUserFirstName"),
db.ref("lastName").withSchema("committerUser").as("committerUserLastName")
)
.orderBy("createdAt", "desc");
const docs = await (tx || db)
.with("w", query)
.select("*")
.from<Awaited<typeof query>[number]>("w")
.where("w.rank", ">=", offset)
.andWhere("w.rank", "<", offset + limit);
const formatedDoc = sqlNestRelationships({
data: docs,
key: "id",
parentMapper: (el) => ({
...SecretApprovalRequestsSchema.parse(el),
environment: el.environment,
projectId: el.projectId,
policy: {
id: el.policyId,
name: el.policyName,
approvals: el.policyApprovals,
secretPath: el.policySecretPath
},
committerUser: {
userId: el.committerUserId,
email: el.committerUserEmail,
firstName: el.committerUserFirstName,
lastName: el.committerUserLastName,
username: el.committerUserUsername
}
}),
childrenMapper: [
{
key: "reviewerId",
label: "reviewers" as const,
mapper: ({ reviewerUserId, reviewerStatus: s }) =>
reviewerUserId ? { userId: reviewerUserId, status: s } : undefined
},
{
key: "approverUserId",
label: "approvers" as const,
mapper: ({ approverUserId }) => approverUserId
},
{
key: "commitId",
label: "commits" as const,
mapper: ({ commitSecretId: secretId, commitId: id, commitOp: op }) => ({
op,
id,
secretId
})
}
]
});
return formatedDoc.map((el) => ({
...el,
policy: { ...el.policy, approvers: el.approvers }
}));
} catch (error) {
throw new DatabaseError({ error, name: "FindSAR" });
}
};
return {
...secretApprovalRequestOrm,
findById,
findProjectRequestCount,
findByProjectId,
findByProjectIdBridgeSecretV2
};
};
@@ -3,6 +3,7 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretApprovalRequestsSecretsSchema,
SecretApprovalRequestsSecretsV2Schema,
TableName,
TSecretApprovalRequestsSecrets,
TSecretTags
@@ -15,6 +16,7 @@ export type TSecretApprovalRequestSecretDALFactory = ReturnType<typeof secretApp
export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret);
const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag);
const secretApprovalRequestSecretV2Orm = ormify(db, TableName.SecretApprovalRequestSecretV2);
const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => {
try {
@@ -221,9 +223,141 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
throw new DatabaseError({ error, name: "FindByRequestId" });
}
};
const findByRequestIdBridgeSecretV2 = async (requestId: string, tx?: Knex) => {
try {
const doc = await (tx || db.replicaNode())({
secVerTag: TableName.SecretTag
})
.from(TableName.SecretApprovalRequestSecretV2)
.where({ requestId })
.leftJoin(
TableName.SecretApprovalRequestSecretTagV2,
`${TableName.SecretApprovalRequestSecretV2}.id`,
`${TableName.SecretApprovalRequestSecretTagV2}.secretId`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretApprovalRequestSecretTagV2}.tagId`,
`${TableName.SecretTag}.id`
)
.leftJoin(TableName.SecretV2, `${TableName.SecretApprovalRequestSecretV2}.secretId`, `${TableName.Secret}.id`)
.leftJoin(
TableName.SecretVersionV2,
`${TableName.SecretVersionV2}.id`,
`${TableName.SecretApprovalRequestSecretV2}.secretVersion`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin<TSecretTags>(
db.ref(TableName.SecretTag).as("secVerTag"),
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
db.ref("id").withSchema("secVerTag")
)
.select(selectAllTableCols(TableName.SecretApprovalRequestSecretV2))
.select({
secVerTagId: "secVerTag.id",
secVerTagColor: "secVerTag.color",
secVerTagSlug: "secVerTag.slug",
secVerTagName: "secVerTag.name"
})
.select(
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTagV2).as("tagJnId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
)
.select(
db.ref("version").withSchema(TableName.SecretV2).as("orgSecVersion"),
db.ref("key").withSchema(TableName.SecretV2).as("orgSecKey"),
db.ref("encryptedValue").withSchema(TableName.SecretV2).as("orgSecValue"),
db.ref("encryptedComment").withSchema(TableName.SecretV2).as("orgSecComment")
)
.select(
db.ref("version").withSchema(TableName.SecretVersionV2).as("secVerVersion"),
db.ref("key").withSchema(TableName.SecretVersionV2).as("secVerKey"),
db.ref("encryptedValue").withSchema(TableName.SecretVersionV2).as("secVerValue"),
db.ref("encryptedComment").withSchema(TableName.SecretVersionV2).as("secVerComment")
);
const formatedDoc = sqlNestRelationships({
data: doc,
key: "id",
parentMapper: (data) => SecretApprovalRequestsSecretsV2Schema.omit({ secretVersion: true }).parse(data),
childrenMapper: [
{
key: "tagJnId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color }) => ({
id,
name,
slug,
color
})
},
{
key: "secretId",
label: "secret" as const,
mapper: ({ orgSecVersion, orgSecKey, orgSecValue, orgSecComment, secretId }) =>
secretId
? {
id: secretId,
version: orgSecVersion,
key: orgSecKey,
encryptedValue: orgSecValue,
encryptedComment: orgSecComment
}
: undefined
},
{
key: "secretVersion",
label: "secretVersion" as const,
mapper: ({ secretVersion, secVerVersion, secVerKey, secVerValue, secVerComment }) =>
secretVersion
? {
version: secVerVersion,
id: secretVersion,
key: secVerKey,
encryptedValue: secVerValue,
encryptedComment: secVerComment
}
: undefined,
childrenMapper: [
{
key: "secVerTagId",
label: "tags" as const,
mapper: ({ secVerTagId: id, secVerTagName: name, secVerTagSlug: slug, secVerTagColor: color }) => ({
// eslint-disable-next-line
id,
// eslint-disable-next-line
name,
// eslint-disable-next-line
slug,
// eslint-disable-next-line
color
})
}
]
}
]
});
return formatedDoc?.map(({ secret, secretVersion, ...el }) => ({
...el,
secret: secret?.[0],
secretVersion: secretVersion?.[0]
}));
} catch (error) {
throw new DatabaseError({ error, name: "FindByRequestId" });
}
};
return {
...secretApprovalRequestSecretOrm,
insertV2Bridge: secretApprovalRequestSecretV2Orm.insertMany,
findByRequestId,
findByRequestIdBridgeSecretV2,
bulkUpdateNoVersionIncrement,
insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany
};
@@ -5,7 +5,8 @@ import {
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretType,
TSecretApprovalRequestsSecretsInsert
TSecretApprovalRequestsSecretsInsert,
TSecretApprovalRequestsSecretsV2Insert
} from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
@@ -19,6 +20,7 @@ import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import {
decryptSecretWithBot,
fnSecretBlindIndexCheck,
fnSecretBlindIndexCheckV2,
fnSecretBulkDelete,
@@ -26,6 +28,13 @@ import {
fnSecretBulkUpdate,
getAllNestedSecretReferences
} from "@app/services/secret/secret-fns";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
fnSecretBulkDelete as fnSecretV2BridgeBulkDelete,
getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge,
secretEncryptionHelper
} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { SecretOperations } from "@app/services/secret/secret-types";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -47,12 +56,17 @@ import {
RequestState,
TApprovalRequestCountDTO,
TGenerateSecretApprovalRequestDTO,
TGenerateSecretApprovalRequestV2BridgeDTO,
TListApprovalsDTO,
TMergeSecretApprovalRequestDTO,
TReviewRequestDTO,
TSecretApprovalDetailsDTO,
TStatusChangeDTO
} from "./secret-approval-request-types";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
type TSecretApprovalRequestServiceFactoryDep = {
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
@@ -62,16 +76,29 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
secretDAL: TSecretDALFactory;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretTagDAL: Pick<
TSecretTagDALFactory,
"findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "saveTagsToSecretV2" | "deleteTagsToSecretV2"
>;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findProjectById">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
smtpService: Pick<TSmtpService, "sendMail">;
userDAL: Pick<TUserDALFactory, "find" | "findOne">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findById" | "findProjectById">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
kmsService: Pick<
TKmsServiceFactory,
"getProjectSecretManagerKmsDataKey" | "encryptWithInputKey" | "decryptWithInputKey"
>;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
};
export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -93,7 +120,11 @@ export const secretApprovalRequestServiceFactory = ({
projectBotService,
smtpService,
userDAL,
projectEnvDAL
projectEnvDAL,
kmsService,
secretV2BridgeDAL,
secretVersionV2BridgeDAL,
secretVersionTagV2BridgeDAL
}: TSecretApprovalRequestServiceFactoryDep) => {
const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => {
if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
@@ -125,6 +156,19 @@ export const secretApprovalRequestServiceFactory = ({
if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (shouldUseSecretV2Bridge) {
return secretApprovalRequestDAL.findByProjectIdBridgeSecretV2({
projectId,
committer,
environment,
status,
userId: actorId,
limit,
offset
});
}
const approvals = await secretApprovalRequestDAL.findByProjectId({
projectId,
committer,
@@ -149,11 +193,14 @@ export const secretApprovalRequestServiceFactory = ({
const secretApprovalRequest = await secretApprovalRequestDAL.findById(id);
if (!secretApprovalRequest) throw new BadRequestError({ message: "Secret approval request not found" });
const { projectId } = secretApprovalRequest;
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const { policy } = secretApprovalRequest;
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
secretApprovalRequest.projectId,
projectId,
actorAuthMethod,
actorOrgId
);
@@ -165,7 +212,55 @@ export const secretApprovalRequestServiceFactory = ({
throw new UnauthorizedError({ message: "User has no access" });
}
const secrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
let secrets;
if (shouldUseSecretV2Bridge) {
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2(
secretApprovalRequest.id
);
secrets = encrypedSecrets.map((el) => ({
...el,
secret: {
secretKey: el.secret.key,
id: el.secret.id,
version: el.secret.version,
secretValue: el.secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString()
: undefined,
secretComment: el.secret.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString()
: undefined
},
secretVersion: {
secretKey: el.secretVersion.key,
id: el.secretVersion.id,
version: el.secretVersion.version,
secretValue: el.secretVersion.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString()
: undefined,
secretComment: el.secretVersion.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString()
: undefined
}
}));
} else {
if (!botKey) throw new BadRequestError({ message: "Bot key not found" });
const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
secrets = encrypedSecrets.map((el) => ({
...el,
secret: {
id: el.secret.id,
version: el.secret.version,
...decryptSecretWithBot(el.secret, botKey)
},
secretVersion: {
id: el.secretVersion.id,
version: el.secretVersion.version,
...decryptSecretWithBot(el.secretVersion, botKey)
}
}));
}
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
@@ -300,48 +395,165 @@ export const secretApprovalRequestServiceFactory = ({
secretApprovalRequest.policy.approvers.filter(
({ userId: approverId }) => reviewers[approverId.toString()] === ApprovalStatus.APPROVED
).length;
const isSoftEnforcement = secretApprovalRequest.policy.enforcementLevel === EnforcementLevel.Soft;
const isSoftEnforcement = secretApprovalRequest.policy.enforcementLevel === EnforcementLevel.Soft;
if (!hasMinApproval && !isSoftEnforcement)
throw new BadRequestError({ message: "Doesn't have minimum approvals needed" });
if (!hasMinApproval && !isSoftEnforcement)
throw new BadRequestError({ message: "Doesn't have minimum approvals needed" });
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex };
})
});
secretCreationCommits
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
let mergeStatus;
if (shouldUseSecretV2Bridge) {
// this cycle if for bridged secrets
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2(
secretApprovalRequest.id
);
}
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
userId: "",
inputSecrets: secretUpdationCommits
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
.map(({ secretBlindIndex }) => {
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretCreationCommits.map((el) => ({
key: el.key,
type: SecretType.Shared
}))
);
const creationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key);
secretCreationCommits
.filter(({ key }) => creationConflictSecretsGroupByKey[key])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(({ key }) => !creationConflictSecretsGroupByKey[key]);
}
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretCreationCommits.map((el) => ({
key: el.key,
type: SecretType.Shared
}))
);
const updationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key);
secretUpdationCommits
.filter(({ key, secretId }) => updationConflictSecretsGroupByKey[key] || !secretId)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
secretUpdationCommits = secretUpdationCommits.filter(
({ key, secretId }) => Boolean(secretId) && !updationConflictSecretsGroupByKey[key]
);
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretV2BridgeBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
tagIds: el?.tags.map(({ id }) => id),
version: 1,
encryptedComment: el.encryptedComment,
encryptedValue: el.encryptedValue,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.key,
references: el.encryptedValue
? getAllNestedSecretReferencesV2Bridge(
secretManagerDecryptor({
cipherTextBlob: el.encryptedValue
}).toString()
)
: [],
type: SecretType.Shared
})),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL
})
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretV2BridgeBulkUpdate({
folderId,
tx,
inputSecrets: secretUpdationCommits.map((el) => {
const encryptedValue =
typeof el.encryptedValue !== "undefined"
? {
encryptedValue: el.encryptedValue as Buffer,
references: el.encryptedValue
? getAllNestedSecretReferencesV2Bridge(
secretManagerDecryptor({
cipherTextBlob: el.encryptedValue
}).toString()
)
: []
}
: {};
return {
filter: { id: el.secretId as string, type: SecretType.Shared },
data: {
reminderRepeatDays: el.reminderRepeatDays,
encryptedComment: el.encryptedComment,
reminderNote: el.reminderNote,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.key,
tagIds: el?.tags.map(({ id }) => id),
...encryptedValue
}
};
}),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretV2BridgeBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL: secretV2BridgeDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared }))
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
} else {
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
@@ -349,80 +561,56 @@ export const secretApprovalRequestServiceFactory = ({
}
return { secretBlindIndex };
})
});
secretUpdationCommits
.filter(
({ secretBlindIndex, secretId }) =>
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
secretCreationCommits
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
);
}
secretUpdationCommits = secretUpdationCommits.filter(
({ secretBlindIndex, secretId }) =>
Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true)
);
}
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
userId: "",
inputSecrets: secretUpdationCommits
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex };
})
});
secretUpdationCommits
.filter(
({ secretBlindIndex, secretId }) =>
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
"secretCommentIV",
"secretValueIV",
"secretValueTag",
"secretValueCiphertext",
"secretKeyCiphertext",
"secretKeyTag",
"secretKeyIV",
"metadata",
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"algorithm",
"keyEncoding",
"secretBlindIndex"
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared,
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: secretUpdationCommits.map((el) => ({
filter: {
id: el.secretId as string, // this null check is already checked at top on conflict strategy
type: SecretType.Shared
},
data: {
tags: el?.tags.map(({ id }) => id),
secretUpdationCommits = secretUpdationCommits.filter(
({ secretBlindIndex, secretId }) =>
Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true)
);
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
@@ -437,8 +625,13 @@ export const secretApprovalRequestServiceFactory = ({
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"algorithm",
"keyEncoding",
"secretBlindIndex"
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared,
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
@@ -449,48 +642,94 @@ export const secretApprovalRequestServiceFactory = ({
})
)
: undefined
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex, type: SecretType.Shared };
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId,
bypassReason
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: secretUpdationCommits.map((el) => ({
filter: {
id: el.secretId as string, // this null check is already checked at top on conflict strategy
type: SecretType.Shared
},
data: {
tags: el?.tags.map(({ id }) => id),
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
"secretCommentIV",
"secretValueIV",
"secretValueTag",
"secretValueCiphertext",
"secretKeyCiphertext",
"secretKeyTag",
"secretKeyIV",
"metadata",
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"secretBlindIndex"
]),
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex, type: SecretType.Shared };
})
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
}
await snapshotService.performSnapshot(folderId);
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
@@ -779,8 +1018,248 @@ export const secretApprovalRequestServiceFactory = ({
});
return secretApprovalRequest;
};
const generateSecretApprovalRequestV2Bridge = async ({
data,
actorId,
actor,
actorOrgId,
actorAuthMethod,
policy,
projectId,
secretPath,
environment
}: TGenerateSecretApprovalRequestV2BridgeDTO) => {
if (actor === ActorType.SERVICE || actor === ActorType.Machine)
throw new BadRequestError({ message: "Cannot use service token or machine token over protected branches" });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder)
throw new BadRequestError({
message: "Folder not found for the given environment slug & secret path",
name: "GenSecretApproval"
});
const folderId = folder.id;
const commits: Omit<TSecretApprovalRequestsSecretsV2Insert, "requestId">[] = [];
const commitTagIds: Record<string, string[]> = {};
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerEncryptor = await kmsService.encryptWithInputKey({ key: secretManagerDataKey });
// for created secret approval change
const createdSecrets = data[SecretOperations.Create];
if (createdSecrets && createdSecrets?.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
createdSecrets.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secrets.length)
throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
commits.push(
...createdSecrets.map((createdSecret) => ({
op: SecretOperations.Create,
version: 1,
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, createdSecret.secretComment),
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, createdSecret.secretValue),
skipMultilineEncoding: createdSecret.skipMultilineEncoding,
key: createdSecret.secretKey,
type: SecretType.Shared
}))
);
createdSecrets.forEach(({ tagIds, secretKey }) => {
if (tagIds?.length) commitTagIds[secretKey] = tagIds;
});
}
// not secret approval for update operations
const secretsToUpdate = data[SecretOperations.Update];
if (secretsToUpdate && secretsToUpdate?.length) {
const secretsToUpdateStoredInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretsToUpdate.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secretsToUpdateStoredInDB.length !== secretsToUpdate.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}`
});
// now find any secret that needs to update its name
// same process as above
const secretsWithNewName = secretsToUpdate.filter(({ newSecretName }) => Boolean(newSecretName));
if (secretsWithNewName.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretsWithNewName.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secrets.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}`
});
}
const updatingSecretsGroupByKey = groupBy(secretsToUpdateStoredInDB, (el) => el.key);
const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(
folderId,
secretsToUpdateStoredInDB.map(({ id }) => id)
);
commits.push(
...secretsToUpdate.map(
({
newSecretName,
secretKey,
tagIds,
secretValue,
reminderRepeatDays,
reminderNote,
secretComment,
metadata,
skipMultilineEncoding
}) => {
const secretId = updatingSecretsGroupByKey[secretKey][0].id;
if (tagIds?.length) commitTagIds[secretKey] = tagIds;
return {
...latestSecretVersions[secretId],
key: newSecretName || secretKey,
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretValue) as Buffer,
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretComment) as Buffer,
reminderRepeatDays,
reminderNote,
metadata,
skipMultilineEncoding,
op: SecretOperations.Update as const,
secret: secretId,
secretVersion: latestSecretVersions[secretId].id,
version: updatingSecretsGroupByKey[secretKey][0].version || 1
};
}
)
);
}
// deleted secrets
const deletedSecrets = data[SecretOperations.Delete];
if (deletedSecrets && deletedSecrets.length) {
const secretsToDeleteInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
deletedSecrets.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secretsToDeleteInDB.length !== deletedSecrets.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToDeleteInDB.map((el) => el.key).join(",")}`
});
const secretsGroupedByKey = groupBy(secretsToDeleteInDB, (i) => i.key);
const deletedSecretIds = deletedSecrets.map((el) => secretsGroupedByKey[el.secretKey][0].id);
const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, deletedSecretIds);
commits.push(
...deletedSecrets.map(({ secretKey }) => {
const secretId = secretsGroupedByKey[secretKey][0].id;
return {
op: SecretOperations.Delete as const,
...latestSecretVersions[secretId],
key: secretKey,
secret: secretId,
secretVersion: latestSecretVersions[secretId].id
};
})
);
}
if (!commits.length) throw new BadRequestError({ message: "Empty commits" });
const tagIds = unique(Object.values(commitTagIds).flat());
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const secretApprovalRequest = await secretApprovalRequestDAL.transaction(async (tx) => {
const doc = await secretApprovalRequestDAL.create(
{
folderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerUserId: actorId
},
tx
);
const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge(
commits.map(
({
version,
op,
key,
encryptedComment,
skipMultilineEncoding,
metadata,
reminderNote,
reminderRepeatDays,
encryptedValue,
secretId,
secretVersion
}) => ({
version,
requestId: doc.id,
op,
secretId,
metadata,
secretVersion,
skipMultilineEncoding,
encryptedValue,
reminderRepeatDays,
reminderNote,
encryptedComment,
key
})
),
tx
);
const commitsGroupByKey = groupBy(approvalCommits, (i) => i.key);
if (tagIds.length) {
await secretApprovalRequestSecretDAL.insertApprovalSecretTags(
Object.keys(commitTagIds).flatMap((blindIndex) =>
commitTagIds[blindIndex]
? commitTagIds[blindIndex].map((tagId) => ({
secretId: commitsGroupByKey[blindIndex][0].id,
tagId
}))
: []
),
tx
);
}
return { ...doc, commits: approvalCommits };
});
return secretApprovalRequest;
};
return {
generateSecretApprovalRequest,
generateSecretApprovalRequestV2Bridge,
mergeSecretApprovalRequest,
reviewApproval,
updateApprovalStatus,
@@ -26,6 +26,23 @@ export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
tagIds?: string[];
};
export type TApprovalCreateSecretV2Bridge = {
secretKey: string;
secretValue?: string;
secretComment?: string;
reminderNote?: string | null;
reminderRepeatDays?: number | null;
skipMultilineEncoding?: boolean;
metadata?: Record<string, string>;
tagIds?: string[];
};
export type TApprovalUpdateSecretV2Bridge = Partial<TApprovalCreateSecretV2Bridge> & {
secretKey: string;
newSecretName?: string;
tagIds?: string[];
};
export type TGenerateSecretApprovalRequestDTO = {
environment: string;
secretPath: string;
@@ -37,6 +54,17 @@ export type TGenerateSecretApprovalRequestDTO = {
};
} & TProjectPermission;
export type TGenerateSecretApprovalRequestV2BridgeDTO = {
environment: string;
secretPath: string;
policy: TSecretApprovalPolicies;
data: {
[SecretOperations.Create]?: TApprovalCreateSecretV2Bridge[];
[SecretOperations.Update]?: TApprovalUpdateSecretV2Bridge[];
[SecretOperations.Delete]?: { secretKey: string }[];
};
} & TProjectPermission;
export type TMergeSecretApprovalRequestDTO = {
approvalId: string;
bypassReason?: string;