awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 23:52:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added secret approval policy in replication

Browse Source
This commit is contained in:
Akhil Mohan
2024-05-31 23:10:01 +05:30
committed by =
parent 36b7911bcc
commit 0ae286a80e
16 changed files with 530 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/secret-import/secret-import-service.ts
+4 -2
View File
@@ -111,7 +111,8 @@ export const secretImportServiceFactory = ({
await secretQueueService.syncSecrets({
secretPath: secImport.importPath,
projectId,
environment: importEnv.slug
environmentSlug: importEnv.slug,
excludeReplication: true
});
return { ...secImport, importEnv };
@@ -223,7 +224,8 @@ export const secretImportServiceFactory = ({
await secretQueueService.syncSecrets({
secretPath: path,
projectId,
environment
environmentSlug: environment,
excludeReplication: true
});
return secImport;
backend/src/services/secret-replication/secret-replication-service.ts
+160 -106
View File
@@ -1,28 +1,40 @@
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretReplicationDALFactory } from "./secret-replication-dal";
import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
type TSecretReplicationServiceFactoryDep = {
secretReplicationDAL: TSecretReplicationDALFactory;
secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
secretApprovalRequestSecretDAL: Pick<
TSecretApprovalRequestSecretDALFactory,
"insertMany" | "insertApprovalSecretTags"
>;
};
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
@@ -44,18 +56,21 @@ export const secretReplicationServiceFactory = ({
keyStore,
secretVersionTagDAL,
secretTagDAL,
folderDAL
folderDAL,
secretApprovalPolicyService,
secretApprovalRequestSecretDAL,
secretApprovalRequestDAL
}: TSecretReplicationServiceFactoryDep) => {
queueService.start(QueueName.SecretReplication, async (job) => {
logger.info(job.data, "Replication started");
const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data;
const secretImports = await secretImportDAL.find({
importPath: secretPath,
importEnv: environmentId,
isReplication: true
});
console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
if (!secretImports.length) return;
if (!secretImports.length || !secrets.length) return;
// unfiltered secrets to be replicated
console.log(secrets.length);
@@ -75,7 +90,8 @@ export const secretReplicationServiceFactory = ({
try {
/* eslint-disable no-await-in-loop */
for (const secretImport of secretImports) {
const importFolderId = secretImport.folderId;
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
const importFolderId = importedFolder.id;
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
@@ -86,14 +102,14 @@ export const secretReplicationServiceFactory = ({
const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
return (
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
});
const locallyUpdatedSecrets = secrets.filter(
({ operation, id }) =>
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
@@ -102,35 +118,48 @@ export const secretReplicationServiceFactory = ({
console.log("update", locallyUpdatedSecrets);
console.log("local board", localSecrets);
const locallyDeletedSecrets = secrets
.filter(
({ operation, id }) =>
operation === SecretReplicationOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
)
.map(
({ id }) =>
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
);
const locallyDeletedSecrets = secrets.filter(
({ operation, id }) =>
operation === SecretOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
projectId,
importedFolder.environmentSlug,
importedFolder.path
);
// this means it should be a approval request rather than direct replication
if (policy) {
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
importFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: importFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerId: membershipId
},
tx
);
const commits = locallyCreatedSecrets
.concat(locallyUpdatedSecrets)
.concat(locallyDeletedSecrets)
.map(({ id, operation }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
return {
op: operation,
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
folderId,
type: doc.type,
requestId: approvalRequestDoc.id,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
@@ -143,31 +172,31 @@ export const secretReplicationServiceFactory = ({
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
skipMultilineEncoding: doc.skipMultilineEncoding,
// except create operation other two needs the secret id and version id
...(operation !== SecretOperations.Create
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
: {})
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
});
const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
return { ...approvalRequestDoc, commits: approvalCommits };
});
} else {
let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
@@ -184,39 +213,80 @@ export const secretReplicationServiceFactory = ({
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
},
isReplicated: true,
folderId: importFolderId
},
isReplicated: true,
folderId: importFolderId
},
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
);
}
});
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
folderId: folder.id,
projectId,
secrets: nestedImportSecrets,
secretPath: folder.path,
environmentId: folder.envId
});
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
);
}
});
console.log("Environment ID -> slug", importedFolder.envId, importedFolder.environmentSlug);
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
folderId: importedFolder.id,
projectId,
secrets: nestedImportSecrets,
secretPath: importedFolder.path,
environmentId: importedFolder.envId,
membershipId
});
}
}
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
/* eslint-enable no-await-in-loop */
@@ -228,20 +298,4 @@ export const secretReplicationServiceFactory = ({
queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
logger.error(err, "Failed to replicate secret", job?.data);
});
const replicate = async (data: TSyncSecretReplicationDTO) => {
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
},
removeOnComplete: true,
removeOnFail: true
});
};
return {
replicate
};
};
backend/src/services/secret-replication/secret-replication-types.ts
+1 -15
View File
@@ -1,17 +1,3 @@
export enum SecretReplicationOperations {
Create = "create",
Update = "update",
Delete = "delete"
}
export type TSyncSecretReplicationDTO = {
secretPath: string;
projectId: string;
environmentId: string;
folderId: string;
secrets: {
operation: SecretReplicationOperations;
id: string;
version: number;
}[];
id: string;
};
backend/src/services/secret/secret-fns.ts
+55
View File
@@ -32,6 +32,8 @@ import {
TCreateManySecretsRawFn,
TCreateManySecretsRawFnFactory,
TFnSecretBlindIndexCheck,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TFnSecretBulkInsert,
TFnSecretBulkUpdate,
TUpdateManySecretsRawFn,
@@ -396,6 +398,30 @@ export const decryptSecretRaw = (
};
};
// this is used when secret blind index already exist
// mainly for secret approval
export const fnSecretBlindIndexCheckV2 = async ({
inputSecrets,
folderId,
userId,
secretDAL
}: TFnSecretBlindIndexCheckV2) => {
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretBlindIndex, type }) => ({
blindIndex: secretBlindIndex,
type: type || SecretType.Shared
})),
userId
);
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
return { secsGroupedByBlindIndex, secrets };
};
/**
* Grabs and processes nested secret references from a string
*
@@ -598,6 +624,35 @@ export const fnSecretBulkUpdate = async ({
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const fnSecretBulkDelete = async ({
folderId,
inputSecrets,
tx,
actorId,
secretDAL,
secretQueueService
}: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
blindIndex: secretBlindIndex,
type
})),
folderId,
actorId,
tx
);
await Promise.allSettled(
deletedSecrets
.filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
.map(({ id, secretReminderRepeatDays }) =>
secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
)
);
return deletedSecrets;
};
export const createManySecretsRawFnFactory = ({
projectDAL,
projectBotDAL,
backend/src/services/secret/secret-queue.ts
+67 -19
View File
@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
import { fnTriggerWebhook } from "../webhook/webhook-fns";
import { TSecretDALFactory } from "./secret-dal";
import { interpolateSecrets } from "./secret-fns";
import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
import {
TCreateSecretReminderDTO,
THandleReminderDTO,
TRemoveSecretReminderDTO,
TSyncSecretsDTO
} from "./secret-types";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
@@ -116,21 +121,17 @@ export const secretQueueFactory = ({
});
};
const syncSecrets = async ({
deDupeQueue = {},
...dto
}: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
const syncSecrets = async <T extends boolean = false>({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO<T>) => {
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
);
const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath);
if (deDupeQueue?.[deDuplicationKey]) {
return;
}
// eslint-disable-next-line
deDupeQueue[deDuplicationKey] = true;
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]`
);
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, {
removeOnFail: true,
removeOnComplete: true,
delay: 1000,
@@ -140,7 +141,6 @@ export const secretQueueFactory = ({
delay: 3000
}
});
await syncIntegrations({ ...dto, deDupeQueue });
};
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -333,6 +333,52 @@ export const secretQueueFactory = ({
return content;
};
queueService.start(QueueName.SecretSync, async (job) => {
const {
secretPath,
environmentId,
projectId,
environmentSlug: environment,
secrets,
folderId,
excludeReplication,
membershipId
} = job.data;
await queueService.queue(
QueueName.SecretWebhook,
QueueJobs.SecWebhook,
{ environment, projectId, secretPath },
{
jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`,
removeOnFail: { count: 5 },
removeOnComplete: true,
delay: 1000,
attempts: 5,
backoff: {
type: "exponential",
delay: 3000
}
}
);
await syncIntegrations({ secretPath, projectId, environment });
if (!excludeReplication) {
await queueService.queue(
QueueName.SecretReplication,
QueueJobs.SecretReplication,
{ environmentId, projectId, secretPath, folderId, secrets, membershipId },
{
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
},
removeOnComplete: true,
removeOnFail: true
}
);
}
});
queueService.start(QueueName.IntegrationSync, async (job) => {
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
@@ -375,11 +421,12 @@ export const secretQueueFactory = ({
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
_depth: depth + 1,
projectId,
secretPath: foldersGroupedById[folderId][0].path,
environment: foldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
environmentSlug: foldersGroupedById[folderId][0].environmentSlug,
_deDupeQueue: deDupeQueue,
excludeReplication: true
})
)
);
@@ -412,11 +459,12 @@ export const secretQueueFactory = ({
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
_depth: depth + 1,
projectId,
secretPath: referencedFoldersGroupedById[folderId][0].path,
environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
environmentSlug: referencedFoldersGroupedById[folderId][0].environmentSlug,
_deDupeQueue: deDupeQueue,
excludeReplication: true
})
)
);
@@ -546,7 +594,7 @@ export const secretQueueFactory = ({
return {
// depth is internal only field thus no need to make it available outside
syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
syncSecrets,
syncIntegrations,
addSecretReminder,
removeSecretReminder,
backend/src/services/secret/secret-service.ts
+82 -133
View File
@@ -30,13 +30,12 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
decryptSecretRaw,
fnSecretBlindIndexCheck,
fnSecretBulkDelete,
fnSecretBulkInsert,
fnSecretBulkUpdate,
getAllNestedSecretReferences,
@@ -45,6 +44,7 @@ import {
} from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
SecretOperations,
TAttachSecretTagsDTO,
TBackFillSecretReferencesDTO,
TCreateBulkSecretDTO,
@@ -55,8 +55,6 @@ import {
TDeleteManySecretRawDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TGetASecretDTO,
TGetASecretRawDTO,
TGetSecretsDTO,
@@ -87,7 +85,6 @@ type TSecretServiceFactoryDep = {
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
};
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -104,8 +101,7 @@ export const secretServiceFactory = ({
projectDAL,
projectBotService,
secretImportDAL,
secretVersionTagDAL,
secretReplicationService
secretVersionTagDAL
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
@@ -143,53 +139,6 @@ export const secretServiceFactory = ({
return secretBlindIndex;
};
const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
blindIndex: secretBlindIndex,
type
})),
folderId,
actorId,
tx
);
for (const s of deletedSecrets) {
if (s.secretReminderRepeatDays) {
// eslint-disable-next-line no-await-in-loop
await secretQueueService
.removeSecretReminder({
secretId: s.id,
repeatDays: s.secretReminderRepeatDays
})
.catch((err) => {
logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
});
}
}
return deletedSecrets;
};
// this is used when secret blind index already exist
// mainly for secret approval
const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretBlindIndex, type }) => ({
blindIndex: secretBlindIndex,
type: type || SecretType.Shared
})),
userId
);
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
return { secsGroupedByBlindIndex, secrets };
};
const createSecret = async ({
path,
actor,
@@ -200,7 +149,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TCreateSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -287,16 +236,17 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
secretPath: path,
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Create,
operation: SecretOperations.Create,
id: secret[0].id,
version: 1
}
@@ -315,7 +265,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TUpdateSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -430,15 +380,18 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Update,
operation: SecretOperations.Update,
id: updatedSecret[0].id,
version: updatedSecret[0].version
}
@@ -457,7 +410,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TDeleteSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -499,6 +452,8 @@ export const secretServiceFactory = ({
projectId,
folderId,
actorId,
secretDAL,
secretQueueService,
inputSecrets: [
{
type: inputSecret.type as SecretType,
@@ -510,15 +465,18 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Delete,
operation: SecretOperations.Delete,
id: deletedSecret[0].id,
version: deletedSecret[0].version
}
@@ -744,7 +702,7 @@ export const secretServiceFactory = ({
projectId,
secrets: inputSecrets
}: TCreateBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -808,13 +766,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create }))
});
return newSecrets;
@@ -830,7 +791,7 @@ export const secretServiceFactory = ({
projectId,
secrets: inputSecrets
}: TUpdateBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -915,13 +876,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update }))
});
return secrets;
@@ -937,7 +901,7 @@ export const secretServiceFactory = ({
actorAuthMethod,
actorOrgId
}: TDeleteBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -972,6 +936,8 @@ export const secretServiceFactory = ({
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
secretDAL,
secretQueueService,
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
secretBlindIndex: keyName2BlindIndex[secretName],
type
@@ -984,13 +950,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete }))
});
return secretsDeleted;
@@ -1171,9 +1140,6 @@ export const secretServiceFactory = ({
skipMultilineEncoding
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1212,8 +1178,6 @@ export const secretServiceFactory = ({
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1243,9 +1207,6 @@ export const secretServiceFactory = ({
actorAuthMethod
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1294,12 +1255,7 @@ export const secretServiceFactory = ({
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const updateManySecretsRaw = async ({
@@ -1348,12 +1304,7 @@ export const secretServiceFactory = ({
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const deleteManySecretsRaw = async ({
@@ -1384,12 +1335,7 @@ export const secretServiceFactory = ({
secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const getSecretVersions = async ({
@@ -1510,7 +1456,12 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
await secretQueueService.syncSecrets({
secretPath,
projectId: project.id,
environmentSlug: environment,
excludeReplication: true
});
return {
...updatedSecret[0],
@@ -1612,7 +1563,12 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
await secretQueueService.syncSecrets({
secretPath,
projectId: project.id,
environmentSlug: environment,
excludeReplication: true
});
return {
...updatedSecret[0],
@@ -1685,13 +1641,6 @@ export const secretServiceFactory = ({
createManySecretsRaw,
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
backfillSecretReferences,
// external services function
fnSecretBulkDelete,
fnSecretBulkUpdate,
fnSecretBlindIndexCheck,
fnSecretBulkInsert,
fnSecretBlindIndexCheckV2
getSecretVersions
};
};
backend/src/services/secret/secret-types.ts
+32
View File
@@ -264,6 +264,10 @@ export type TFnSecretBulkDelete = {
inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
actorId: string;
tx?: Knex;
secretDAL: Pick<TSecretDALFactory, "deleteMany">;
secretQueueService: {
removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
};
};
export type TFnSecretBlindIndexCheck = {
@@ -277,6 +281,7 @@ export type TFnSecretBlindIndexCheck = {
// when blind index is already present
export type TFnSecretBlindIndexCheckV2 = {
secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
folderId: string;
userId?: string;
inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
@@ -363,3 +368,30 @@ export type TUpdateManySecretsRawFn = {
}[];
userId?: string;
};
export enum SecretOperations {
Create = "create",
Update = "update",
Delete = "delete"
}
export type TSyncSecretsDTO<T extends boolean = false> = {
_depth?: number;
_deDupeQueue?: Record<string, boolean>;
secretPath: string;
projectId: string;
environmentSlug: string;
// cases for just doing sync integration and webhook
excludeReplication?: T;
} & (T extends true
? object
: {
environmentId: string;
folderId: string;
membershipId: string;
secrets: {
operation: SecretOperations;
id: string;
version: number;
}[];
});
backend/src/services/secret/secret-version-dal.ts
+1
View File
@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
try {
if (!secretIds.length) return {};
const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersion}.secretId`, secretIds)