mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
feat: added secret approval policy in replication
This commit is contained in:
@@ -25,6 +25,9 @@ export const mockKeyStore = (): TKeyStoreFactory => {
|
||||
},
|
||||
incrementBy: async () => {
|
||||
return 1;
|
||||
},
|
||||
acquireLock: () => {
|
||||
throw new Error("Not implemented");
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
@@ -20,6 +20,12 @@ export async function up(knex: Knex): Promise<void> {
|
||||
t.boolean("isReplicated");
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
|
||||
await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
|
||||
t.boolean("isReplicated");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(knex: Knex): Promise<void> {
|
||||
@@ -36,7 +42,13 @@ export async function down(knex: Knex): Promise<void> {
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.SecretVersion)) {
|
||||
await knex.schema.alterTable(TableName.Secret, (t) => {
|
||||
await knex.schema.alterTable(TableName.SecretVersion, (t) => {
|
||||
t.dropColumns("isReplicated");
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
|
||||
await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
|
||||
t.dropColumns("isReplicated");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ export const SecretApprovalRequestsSecretsSchema = z.object({
|
||||
requestId: z.string().uuid(),
|
||||
op: z.string(),
|
||||
secretId: z.string().uuid().nullable().optional(),
|
||||
secretVersion: z.string().uuid().nullable().optional()
|
||||
secretVersion: z.string().uuid().nullable().optional(),
|
||||
isReplicated: z.boolean().nullable().optional()
|
||||
});
|
||||
|
||||
export type TSecretApprovalRequestsSecrets = z.infer<typeof SecretApprovalRequestsSecretsSchema>;
|
||||
|
||||
@@ -16,8 +16,15 @@ import { TProjectDALFactory } from "@app/services/project/project-dal";
|
||||
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
|
||||
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
|
||||
import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
|
||||
import {
|
||||
fnSecretBlindIndexCheck,
|
||||
fnSecretBlindIndexCheckV2,
|
||||
fnSecretBulkDelete,
|
||||
fnSecretBulkInsert,
|
||||
fnSecretBulkUpdate
|
||||
} from "@app/services/secret/secret-fns";
|
||||
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
|
||||
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
|
||||
import { SecretOperations } from "@app/services/secret/secret-types";
|
||||
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
|
||||
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
|
||||
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
|
||||
@@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ
|
||||
import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal";
|
||||
import {
|
||||
ApprovalStatus,
|
||||
CommitType,
|
||||
RequestState,
|
||||
TApprovalRequestCountDTO,
|
||||
TGenerateSecretApprovalRequestDTO,
|
||||
@@ -48,7 +54,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
|
||||
secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
|
||||
secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
|
||||
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
|
||||
secretDAL: TSecretDALFactory;
|
||||
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
|
||||
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
|
||||
@@ -56,16 +62,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
|
||||
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
|
||||
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
|
||||
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
|
||||
secretService: Pick<
|
||||
TSecretServiceFactory,
|
||||
| "fnSecretBulkInsert"
|
||||
| "fnSecretBulkUpdate"
|
||||
| "fnSecretBlindIndexCheck"
|
||||
| "fnSecretBulkDelete"
|
||||
| "fnSecretBlindIndexCheckV2"
|
||||
>;
|
||||
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
|
||||
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
|
||||
};
|
||||
|
||||
export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
|
||||
@@ -82,7 +79,6 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
projectDAL,
|
||||
permissionService,
|
||||
snapshotService,
|
||||
secretService,
|
||||
secretVersionDAL,
|
||||
secretQueueService,
|
||||
projectBotService
|
||||
@@ -302,11 +298,12 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
|
||||
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
|
||||
|
||||
const conflicts: Array<{ secretId: string; op: CommitType }> = [];
|
||||
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create);
|
||||
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
|
||||
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
|
||||
if (secretCreationCommits.length) {
|
||||
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
|
||||
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
|
||||
folderId,
|
||||
secretDAL,
|
||||
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
|
||||
if (!secretBlindIndex) {
|
||||
throw new BadRequestError({
|
||||
@@ -319,17 +316,19 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
secretCreationCommits
|
||||
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
|
||||
.forEach((el) => {
|
||||
conflicts.push({ op: CommitType.Create, secretId: el.id });
|
||||
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
|
||||
});
|
||||
secretCreationCommits = secretCreationCommits.filter(
|
||||
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
|
||||
);
|
||||
}
|
||||
|
||||
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update);
|
||||
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
|
||||
if (secretUpdationCommits.length) {
|
||||
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
|
||||
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
|
||||
folderId,
|
||||
secretDAL,
|
||||
userId: "",
|
||||
inputSecrets: secretUpdationCommits
|
||||
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
|
||||
.map(({ secretBlindIndex }) => {
|
||||
@@ -347,7 +346,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
|
||||
)
|
||||
.forEach((el) => {
|
||||
conflicts.push({ op: CommitType.Update, secretId: el.id });
|
||||
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
|
||||
});
|
||||
|
||||
secretUpdationCommits = secretUpdationCommits.filter(
|
||||
@@ -356,11 +355,11 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
);
|
||||
}
|
||||
|
||||
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
|
||||
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
|
||||
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
|
||||
|
||||
const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
|
||||
const newSecrets = secretCreationCommits.length
|
||||
? await secretService.fnSecretBulkInsert({
|
||||
? await fnSecretBulkInsert({
|
||||
tx,
|
||||
folderId,
|
||||
inputSecrets: secretCreationCommits.map((el) => ({
|
||||
@@ -380,7 +379,8 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
"secretReminderRepeatDays",
|
||||
"algorithm",
|
||||
"keyEncoding",
|
||||
"secretBlindIndex"
|
||||
"secretBlindIndex",
|
||||
"isReplicated"
|
||||
]),
|
||||
tags: el?.tags.map(({ id }) => id),
|
||||
version: 1,
|
||||
@@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
})
|
||||
: [];
|
||||
const updatedSecrets = secretUpdationCommits.length
|
||||
? await secretService.fnSecretBulkUpdate({
|
||||
? await fnSecretBulkUpdate({
|
||||
folderId,
|
||||
projectId,
|
||||
tx,
|
||||
@@ -425,6 +425,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
"secretKeyTag",
|
||||
"secretKeyIV",
|
||||
"metadata",
|
||||
"isReplicated",
|
||||
"skipMultilineEncoding",
|
||||
"secretReminderNote",
|
||||
"secretReminderRepeatDays",
|
||||
@@ -449,11 +450,13 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
})
|
||||
: [];
|
||||
const deletedSecret = secretDeletionCommits.length
|
||||
? await secretService.fnSecretBulkDelete({
|
||||
? await fnSecretBulkDelete({
|
||||
projectId,
|
||||
folderId,
|
||||
tx,
|
||||
actorId: "",
|
||||
secretDAL,
|
||||
secretQueueService,
|
||||
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
|
||||
if (!secretBlindIndex) {
|
||||
throw new BadRequestError({
|
||||
@@ -480,12 +483,33 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
};
|
||||
});
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
const folder = await folderDAL.findById(folderId);
|
||||
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
|
||||
// TODO(akhilmhdh-pg): change query to do secret path from folder
|
||||
await secretQueueService.syncSecrets({
|
||||
projectId,
|
||||
secretPath: "/",
|
||||
environment: folder?.environment.envSlug as string
|
||||
secretPath: folder.path,
|
||||
environmentSlug: folder.environmentSlug,
|
||||
folderId: folder.id,
|
||||
membershipId: membership.id,
|
||||
environmentId: folder.envId,
|
||||
secrets: mergeStatus.secrets.created
|
||||
.map(({ id, version }) => ({
|
||||
operation: SecretOperations.Create,
|
||||
version,
|
||||
id
|
||||
}))
|
||||
.concat(
|
||||
mergeStatus.secrets.updated.map(({ id, version }) => ({
|
||||
operation: SecretOperations.Update,
|
||||
version,
|
||||
id
|
||||
})),
|
||||
mergeStatus.secrets.deleted.map(({ id, version }) => ({
|
||||
operation: SecretOperations.Delete,
|
||||
version,
|
||||
id
|
||||
}))
|
||||
)
|
||||
});
|
||||
return mergeStatus;
|
||||
};
|
||||
@@ -533,9 +557,9 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
const commits: Omit<TSecretApprovalRequestsSecretsInsert, "requestId">[] = [];
|
||||
const commitTagIds: Record<string, string[]> = {};
|
||||
// for created secret approval change
|
||||
const createdSecrets = data[CommitType.Create];
|
||||
const createdSecrets = data[SecretOperations.Create];
|
||||
if (createdSecrets && createdSecrets?.length) {
|
||||
const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
|
||||
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
|
||||
inputSecrets: createdSecrets,
|
||||
folderId,
|
||||
isNew: true,
|
||||
@@ -546,7 +570,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
commits.push(
|
||||
...createdSecrets.map(({ secretName, ...el }) => ({
|
||||
...el,
|
||||
op: CommitType.Create as const,
|
||||
op: SecretOperations.Create as const,
|
||||
version: 1,
|
||||
secretBlindIndex: keyName2BlindIndex[secretName],
|
||||
algorithm: SecretEncryptionAlgo.AES_256_GCM,
|
||||
@@ -558,12 +582,12 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
});
|
||||
}
|
||||
// not secret approval for update operations
|
||||
const updatedSecrets = data[CommitType.Update];
|
||||
const updatedSecrets = data[SecretOperations.Update];
|
||||
if (updatedSecrets && updatedSecrets?.length) {
|
||||
// get all blind index
|
||||
// Find all those secrets
|
||||
// if not throw not found
|
||||
const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({
|
||||
const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({
|
||||
inputSecrets: updatedSecrets,
|
||||
folderId,
|
||||
isNew: false,
|
||||
@@ -574,7 +598,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
// now find any secret that needs to update its name
|
||||
// same process as above
|
||||
const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
|
||||
const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
|
||||
const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
|
||||
inputSecrets: nameUpdatedSecrets,
|
||||
folderId,
|
||||
isNew: true,
|
||||
@@ -599,7 +623,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
return {
|
||||
...latestSecretVersions[secretId],
|
||||
...el,
|
||||
op: CommitType.Update as const,
|
||||
op: SecretOperations.Update as const,
|
||||
secret: secretId,
|
||||
secretVersion: latestSecretVersions[secretId].id,
|
||||
secretBlindIndex,
|
||||
@@ -609,12 +633,12 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
);
|
||||
}
|
||||
// deleted secrets
|
||||
const deletedSecrets = data[CommitType.Delete];
|
||||
const deletedSecrets = data[SecretOperations.Delete];
|
||||
if (deletedSecrets && deletedSecrets.length) {
|
||||
// get all blind index
|
||||
// Find all those secrets
|
||||
// if not throw not found
|
||||
const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({
|
||||
const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({
|
||||
inputSecrets: deletedSecrets,
|
||||
folderId,
|
||||
isNew: false,
|
||||
@@ -635,7 +659,7 @@ export const secretApprovalRequestServiceFactory = ({
|
||||
if (!latestSecretVersions[secretId].secretBlindIndex)
|
||||
throw new BadRequestError({ message: "Failed to find secret blind index" });
|
||||
return {
|
||||
op: CommitType.Delete as const,
|
||||
op: SecretOperations.Delete as const,
|
||||
...latestSecretVersions[secretId],
|
||||
secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string,
|
||||
secret: secretId,
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas";
|
||||
import { TProjectPermission } from "@app/lib/types";
|
||||
|
||||
export enum CommitType {
|
||||
Create = "create",
|
||||
Update = "update",
|
||||
Delete = "delete"
|
||||
}
|
||||
import { SecretOperations } from "@app/services/secret/secret-types";
|
||||
|
||||
export enum RequestState {
|
||||
Open = "open",
|
||||
@@ -18,14 +13,14 @@ export enum ApprovalStatus {
|
||||
REJECTED = "rejected"
|
||||
}
|
||||
|
||||
type TApprovalCreateSecret = Omit<
|
||||
export type TApprovalCreateSecret = Omit<
|
||||
TSecretApprovalRequestsSecrets,
|
||||
TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex"
|
||||
> & {
|
||||
secretName: string;
|
||||
tagIds?: string[];
|
||||
};
|
||||
type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
|
||||
export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
|
||||
secretName: string;
|
||||
newSecretName?: string;
|
||||
tagIds?: string[];
|
||||
@@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = {
|
||||
secretPath: string;
|
||||
policy: TSecretApprovalPolicies;
|
||||
data: {
|
||||
[CommitType.Create]?: TApprovalCreateSecret[];
|
||||
[CommitType.Update]?: TApprovalUpdateSecret[];
|
||||
[CommitType.Delete]?: { secretName: string }[];
|
||||
[SecretOperations.Create]?: TApprovalCreateSecret[];
|
||||
[SecretOperations.Update]?: TApprovalUpdateSecret[];
|
||||
[SecretOperations.Delete]?: { secretName: string }[];
|
||||
};
|
||||
} & TProjectPermission;
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
TScanFullRepoEventPayload,
|
||||
TScanPushEventPayload
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types";
|
||||
import { TSyncSecretsDTO } from "@app/services/secret/secret-types";
|
||||
|
||||
export enum QueueName {
|
||||
SecretRotation = "secret-rotation",
|
||||
@@ -23,7 +23,8 @@ export enum QueueName {
|
||||
SecretPushEventScan = "secret-push-event-scan",
|
||||
UpgradeProjectToGhost = "upgrade-project-to-ghost",
|
||||
DynamicSecretRevocation = "dynamic-secret-revocation",
|
||||
SecretReplication = "secret-replication"
|
||||
SecretReplication = "secret-replication",
|
||||
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
|
||||
}
|
||||
|
||||
export enum QueueJobs {
|
||||
@@ -40,7 +41,8 @@ export enum QueueJobs {
|
||||
UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
|
||||
DynamicSecretRevocation = "dynamic-secret-revocation",
|
||||
DynamicSecretPruning = "dynamic-secret-pruning",
|
||||
SecretReplication = "secret-replication"
|
||||
SecretReplication = "secret-replication",
|
||||
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
|
||||
}
|
||||
|
||||
export type TQueueJobTypes = {
|
||||
@@ -121,7 +123,11 @@ export type TQueueJobTypes = {
|
||||
};
|
||||
[QueueName.SecretReplication]: {
|
||||
name: QueueJobs.SecretReplication;
|
||||
payload: TSyncSecretReplicationDTO;
|
||||
payload: Omit<TSyncSecretsDTO, "environmentSlug">;
|
||||
};
|
||||
[QueueName.SecretSync]: {
|
||||
name: QueueJobs.SecretSync;
|
||||
payload: TSyncSecretsDTO;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -243,8 +243,8 @@ export const registerRoutes = async (
|
||||
const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db);
|
||||
const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db);
|
||||
const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db);
|
||||
const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
|
||||
const sarSecretDAL = secretApprovalRequestSecretDALFactory(db);
|
||||
const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
|
||||
const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db);
|
||||
|
||||
const secretRotationDAL = secretRotationDALFactory(db);
|
||||
const snapshotDAL = snapshotDALFactory(db);
|
||||
@@ -492,7 +492,7 @@ export const registerRoutes = async (
|
||||
projectBotDAL,
|
||||
projectMembershipDAL,
|
||||
secretApprovalRequestDAL,
|
||||
secretApprovalSecretDAL: sarSecretDAL,
|
||||
secretApprovalSecretDAL: secretApprovalRequestSecretDAL,
|
||||
projectUserMembershipRoleDAL
|
||||
});
|
||||
|
||||
@@ -603,17 +603,6 @@ export const registerRoutes = async (
|
||||
secretDAL,
|
||||
secretBlindIndexDAL
|
||||
});
|
||||
const secretReplicationService = secretReplicationServiceFactory({
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretImportDAL,
|
||||
keyStore,
|
||||
queueService,
|
||||
secretReplicationDAL,
|
||||
folderDAL
|
||||
});
|
||||
const secretService = secretServiceFactory({
|
||||
folderDAL,
|
||||
secretVersionDAL,
|
||||
@@ -625,7 +614,6 @@ export const registerRoutes = async (
|
||||
secretTagDAL,
|
||||
snapshotService,
|
||||
secretQueueService,
|
||||
secretReplicationService,
|
||||
secretImportDAL,
|
||||
projectEnvDAL,
|
||||
projectBotService
|
||||
@@ -636,19 +624,18 @@ export const registerRoutes = async (
|
||||
secretSharingDAL
|
||||
});
|
||||
|
||||
const sarService = secretApprovalRequestServiceFactory({
|
||||
const secretApprovalRequestService = secretApprovalRequestServiceFactory({
|
||||
permissionService,
|
||||
projectBotService,
|
||||
folderDAL,
|
||||
secretDAL,
|
||||
secretTagDAL,
|
||||
secretApprovalRequestSecretDAL: sarSecretDAL,
|
||||
secretApprovalRequestReviewerDAL: sarReviewerDAL,
|
||||
secretApprovalRequestSecretDAL,
|
||||
secretApprovalRequestReviewerDAL,
|
||||
projectDAL,
|
||||
secretVersionDAL,
|
||||
secretBlindIndexDAL,
|
||||
secretApprovalRequestDAL,
|
||||
secretService,
|
||||
snapshotService,
|
||||
secretVersionTagDAL,
|
||||
secretQueueService
|
||||
@@ -677,6 +664,21 @@ export const registerRoutes = async (
|
||||
accessApprovalPolicyApproverDAL
|
||||
});
|
||||
|
||||
const secretReplicationService = secretReplicationServiceFactory({
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretImportDAL,
|
||||
keyStore,
|
||||
queueService,
|
||||
secretReplicationDAL,
|
||||
folderDAL,
|
||||
secretApprovalPolicyService,
|
||||
secretBlindIndexDAL,
|
||||
secretApprovalRequestDAL,
|
||||
secretApprovalRequestSecretDAL
|
||||
});
|
||||
const secretRotationQueue = secretRotationQueueFactory({
|
||||
telemetryService,
|
||||
secretRotationDAL,
|
||||
@@ -858,10 +860,10 @@ export const registerRoutes = async (
|
||||
identityGcpAuth: identityGcpAuthService,
|
||||
identityAwsAuth: identityAwsAuthService,
|
||||
identityAzureAuth: identityAzureAuthService,
|
||||
secretApprovalPolicy: sapService,
|
||||
accessApprovalPolicy: accessApprovalPolicyService,
|
||||
accessApprovalRequest: accessApprovalRequestService,
|
||||
secretApprovalRequest: sarService,
|
||||
secretApprovalPolicy: secretApprovalPolicyService,
|
||||
secretApprovalRequest: secretApprovalRequestService,
|
||||
secretRotation: secretRotationService,
|
||||
dynamicSecret: dynamicSecretService,
|
||||
dynamicSecretLease: dynamicSecretLeaseService,
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
ServiceTokenScopes
|
||||
} from "@app/db/schemas";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
|
||||
import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs";
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { removeTrailingSlash } from "@app/lib/fn";
|
||||
@@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log";
|
||||
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
|
||||
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
|
||||
import { ProjectFilterType } from "@app/services/project/project-types";
|
||||
import { SecretOperations } from "@app/services/secret/secret-types";
|
||||
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
|
||||
|
||||
import { secretRawSchema } from "../sanitizedSchemas";
|
||||
@@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Create]: [
|
||||
[SecretOperations.Create]: [
|
||||
{
|
||||
secretName: req.params.secretName,
|
||||
secretValueCiphertext,
|
||||
@@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Update]: [
|
||||
[SecretOperations.Update]: [
|
||||
{
|
||||
secretName: req.params.secretName,
|
||||
newSecretName,
|
||||
@@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Delete]: [
|
||||
[SecretOperations.Delete]: [
|
||||
{
|
||||
secretName: req.params.secretName
|
||||
}
|
||||
@@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Create]: inputSecrets
|
||||
[SecretOperations.Create]: inputSecrets
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared")
|
||||
[SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared")
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
|
||||
projectId,
|
||||
policy,
|
||||
data: {
|
||||
[CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared")
|
||||
[SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared")
|
||||
}
|
||||
});
|
||||
await server.services.auditLog.createAuditLog({
|
||||
|
||||
@@ -111,7 +111,8 @@ export const secretImportServiceFactory = ({
|
||||
await secretQueueService.syncSecrets({
|
||||
secretPath: secImport.importPath,
|
||||
projectId,
|
||||
environment: importEnv.slug
|
||||
environmentSlug: importEnv.slug,
|
||||
excludeReplication: true
|
||||
});
|
||||
|
||||
return { ...secImport, importEnv };
|
||||
@@ -223,7 +224,8 @@ export const secretImportServiceFactory = ({
|
||||
await secretQueueService.syncSecrets({
|
||||
secretPath: path,
|
||||
projectId,
|
||||
environment
|
||||
environmentSlug: environment,
|
||||
excludeReplication: true
|
||||
});
|
||||
|
||||
return secImport;
|
||||
|
||||
@@ -1,28 +1,40 @@
|
||||
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
|
||||
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
|
||||
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
|
||||
import { TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { groupBy } from "@app/lib/fn";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
|
||||
import { TSecretDALFactory } from "../secret/secret-dal";
|
||||
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
|
||||
import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types";
|
||||
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
|
||||
import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
|
||||
import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
|
||||
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
|
||||
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
|
||||
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
|
||||
import { TSecretReplicationDALFactory } from "./secret-replication-dal";
|
||||
import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
|
||||
|
||||
type TSecretReplicationServiceFactoryDep = {
|
||||
secretReplicationDAL: TSecretReplicationDALFactory;
|
||||
secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
|
||||
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
|
||||
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
|
||||
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
|
||||
secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
|
||||
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
|
||||
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
|
||||
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
|
||||
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
|
||||
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
|
||||
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
|
||||
secretApprovalRequestSecretDAL: Pick<
|
||||
TSecretApprovalRequestSecretDALFactory,
|
||||
"insertMany" | "insertApprovalSecretTags"
|
||||
>;
|
||||
};
|
||||
|
||||
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
|
||||
@@ -44,18 +56,21 @@ export const secretReplicationServiceFactory = ({
|
||||
keyStore,
|
||||
secretVersionTagDAL,
|
||||
secretTagDAL,
|
||||
folderDAL
|
||||
folderDAL,
|
||||
secretApprovalPolicyService,
|
||||
secretApprovalRequestSecretDAL,
|
||||
secretApprovalRequestDAL
|
||||
}: TSecretReplicationServiceFactoryDep) => {
|
||||
queueService.start(QueueName.SecretReplication, async (job) => {
|
||||
logger.info(job.data, "Replication started");
|
||||
const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
|
||||
const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data;
|
||||
const secretImports = await secretImportDAL.find({
|
||||
importPath: secretPath,
|
||||
importEnv: environmentId,
|
||||
isReplication: true
|
||||
});
|
||||
console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
|
||||
if (!secretImports.length) return;
|
||||
if (!secretImports.length || !secrets.length) return;
|
||||
|
||||
// unfiltered secrets to be replicated
|
||||
console.log(secrets.length);
|
||||
@@ -75,7 +90,8 @@ export const secretReplicationServiceFactory = ({
|
||||
try {
|
||||
/* eslint-disable no-await-in-loop */
|
||||
for (const secretImport of secretImports) {
|
||||
const importFolderId = secretImport.folderId;
|
||||
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
|
||||
const importFolderId = importedFolder.id;
|
||||
|
||||
const localSecrets = await secretDAL.find({
|
||||
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
|
||||
@@ -86,14 +102,14 @@ export const secretReplicationServiceFactory = ({
|
||||
|
||||
const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
|
||||
return (
|
||||
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
|
||||
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
|
||||
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
);
|
||||
});
|
||||
|
||||
const locallyUpdatedSecrets = secrets.filter(
|
||||
({ operation, id }) =>
|
||||
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
|
||||
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
);
|
||||
|
||||
@@ -102,35 +118,48 @@ export const secretReplicationServiceFactory = ({
|
||||
console.log("update", locallyUpdatedSecrets);
|
||||
console.log("local board", localSecrets);
|
||||
|
||||
const locallyDeletedSecrets = secrets
|
||||
.filter(
|
||||
({ operation, id }) =>
|
||||
operation === SecretReplicationOperations.Delete &&
|
||||
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
)
|
||||
.map(
|
||||
({ id }) =>
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
|
||||
);
|
||||
const locallyDeletedSecrets = secrets.filter(
|
||||
({ operation, id }) =>
|
||||
operation === SecretOperations.Delete &&
|
||||
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
);
|
||||
|
||||
let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
|
||||
await secretReplicationDAL.transaction(async (tx) => {
|
||||
if (locallyCreatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkInsert({
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
|
||||
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
|
||||
projectId,
|
||||
importedFolder.environmentSlug,
|
||||
importedFolder.path
|
||||
);
|
||||
// this means it should be a approval request rather than direct replication
|
||||
if (policy) {
|
||||
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
|
||||
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
|
||||
importFolderId,
|
||||
localSecretsLatestVersions
|
||||
);
|
||||
await secretApprovalRequestDAL.transaction(async (tx) => {
|
||||
const approvalRequestDoc = await secretApprovalRequestDAL.create(
|
||||
{
|
||||
folderId: importFolderId,
|
||||
slug: alphaNumericNanoId(),
|
||||
policyId: policy.id,
|
||||
status: "open",
|
||||
hasMerged: false,
|
||||
committerId: membershipId
|
||||
},
|
||||
tx
|
||||
);
|
||||
const commits = locallyCreatedSecrets
|
||||
.concat(locallyUpdatedSecrets)
|
||||
.concat(locallyDeletedSecrets)
|
||||
.map(({ id, operation }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
|
||||
return {
|
||||
op: operation,
|
||||
keyEncoding: doc.keyEncoding,
|
||||
algorithm: doc.algorithm,
|
||||
folderId,
|
||||
type: doc.type,
|
||||
requestId: approvalRequestDoc.id,
|
||||
metadata: doc.metadata,
|
||||
secretKeyIV: doc.secretKeyIV,
|
||||
secretKeyTag: doc.secretKeyTag,
|
||||
@@ -143,31 +172,31 @@ export const secretReplicationServiceFactory = ({
|
||||
secretCommentTag: doc.secretCommentTag,
|
||||
secretCommentCiphertext: doc.secretCommentCiphertext,
|
||||
isReplicated: true,
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding,
|
||||
// except create operation other two needs the secret id and version id
|
||||
...(operation !== SecretOperations.Create
|
||||
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
|
||||
: {})
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyUpdatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkUpdate({
|
||||
projectId,
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
return {
|
||||
filter: {
|
||||
folderId: importFolderId,
|
||||
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
|
||||
},
|
||||
data: {
|
||||
});
|
||||
const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
|
||||
|
||||
return { ...approvalRequestDoc, commits: approvalCommits };
|
||||
});
|
||||
} else {
|
||||
let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
|
||||
await secretReplicationDAL.transaction(async (tx) => {
|
||||
if (locallyCreatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkInsert({
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
return {
|
||||
keyEncoding: doc.keyEncoding,
|
||||
algorithm: doc.algorithm,
|
||||
type: doc.type,
|
||||
@@ -184,39 +213,80 @@ export const secretReplicationServiceFactory = ({
|
||||
secretCommentCiphertext: doc.secretCommentCiphertext,
|
||||
isReplicated: true,
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding
|
||||
}
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyDeletedSecrets.length) {
|
||||
const newSecrets = await secretDAL.delete(
|
||||
{
|
||||
$in: {
|
||||
id: locallyDeletedSecrets.map(({ id }) => id)
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyUpdatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkUpdate({
|
||||
projectId,
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
return {
|
||||
filter: {
|
||||
folderId: importFolderId,
|
||||
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
|
||||
},
|
||||
data: {
|
||||
keyEncoding: doc.keyEncoding,
|
||||
algorithm: doc.algorithm,
|
||||
type: doc.type,
|
||||
metadata: doc.metadata,
|
||||
secretKeyIV: doc.secretKeyIV,
|
||||
secretKeyTag: doc.secretKeyTag,
|
||||
secretKeyCiphertext: doc.secretKeyCiphertext,
|
||||
secretValueIV: doc.secretValueIV,
|
||||
secretValueTag: doc.secretValueTag,
|
||||
secretValueCiphertext: doc.secretValueCiphertext,
|
||||
secretBlindIndex: doc.secretBlindIndex,
|
||||
secretCommentIV: doc.secretCommentIV,
|
||||
secretCommentTag: doc.secretCommentTag,
|
||||
secretCommentCiphertext: doc.secretCommentCiphertext,
|
||||
isReplicated: true,
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding
|
||||
}
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyDeletedSecrets.length) {
|
||||
const newSecrets = await secretDAL.delete(
|
||||
{
|
||||
$in: {
|
||||
id: locallyDeletedSecrets.map(({ id }) => id)
|
||||
},
|
||||
isReplicated: true,
|
||||
folderId: importFolderId
|
||||
},
|
||||
isReplicated: true,
|
||||
folderId: importFolderId
|
||||
},
|
||||
tx
|
||||
);
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
|
||||
);
|
||||
}
|
||||
});
|
||||
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
|
||||
console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
|
||||
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
secrets: nestedImportSecrets,
|
||||
secretPath: folder.path,
|
||||
environmentId: folder.envId
|
||||
});
|
||||
tx
|
||||
);
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
|
||||
);
|
||||
}
|
||||
});
|
||||
console.log("Environment ID -> slug", importedFolder.envId, importedFolder.environmentSlug);
|
||||
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
|
||||
folderId: importedFolder.id,
|
||||
projectId,
|
||||
secrets: nestedImportSecrets,
|
||||
secretPath: importedFolder.path,
|
||||
environmentId: importedFolder.envId,
|
||||
membershipId
|
||||
});
|
||||
}
|
||||
}
|
||||
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
|
||||
/* eslint-enable no-await-in-loop */
|
||||
@@ -228,20 +298,4 @@ export const secretReplicationServiceFactory = ({
|
||||
queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
|
||||
logger.error(err, "Failed to replicate secret", job?.data);
|
||||
});
|
||||
|
||||
const replicate = async (data: TSyncSecretReplicationDTO) => {
|
||||
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 1000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
replicate
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,17 +1,3 @@
|
||||
export enum SecretReplicationOperations {
|
||||
Create = "create",
|
||||
Update = "update",
|
||||
Delete = "delete"
|
||||
}
|
||||
|
||||
export type TSyncSecretReplicationDTO = {
|
||||
secretPath: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
folderId: string;
|
||||
secrets: {
|
||||
operation: SecretReplicationOperations;
|
||||
id: string;
|
||||
version: number;
|
||||
}[];
|
||||
id: string;
|
||||
};
|
||||
|
||||
@@ -32,6 +32,8 @@ import {
|
||||
TCreateManySecretsRawFn,
|
||||
TCreateManySecretsRawFnFactory,
|
||||
TFnSecretBlindIndexCheck,
|
||||
TFnSecretBlindIndexCheckV2,
|
||||
TFnSecretBulkDelete,
|
||||
TFnSecretBulkInsert,
|
||||
TFnSecretBulkUpdate,
|
||||
TUpdateManySecretsRawFn,
|
||||
@@ -396,6 +398,30 @@ export const decryptSecretRaw = (
|
||||
};
|
||||
};
|
||||
|
||||
// this is used when secret blind index already exist
|
||||
// mainly for secret approval
|
||||
export const fnSecretBlindIndexCheckV2 = async ({
|
||||
inputSecrets,
|
||||
folderId,
|
||||
userId,
|
||||
secretDAL
|
||||
}: TFnSecretBlindIndexCheckV2) => {
|
||||
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
|
||||
throw new BadRequestError({ message: "Missing user id for personal secret" });
|
||||
}
|
||||
const secrets = await secretDAL.findByBlindIndexes(
|
||||
folderId,
|
||||
inputSecrets.map(({ secretBlindIndex, type }) => ({
|
||||
blindIndex: secretBlindIndex,
|
||||
type: type || SecretType.Shared
|
||||
})),
|
||||
userId
|
||||
);
|
||||
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
|
||||
|
||||
return { secsGroupedByBlindIndex, secrets };
|
||||
};
|
||||
|
||||
/**
|
||||
* Grabs and processes nested secret references from a string
|
||||
*
|
||||
@@ -598,6 +624,35 @@ export const fnSecretBulkUpdate = async ({
|
||||
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
|
||||
};
|
||||
|
||||
export const fnSecretBulkDelete = async ({
|
||||
folderId,
|
||||
inputSecrets,
|
||||
tx,
|
||||
actorId,
|
||||
secretDAL,
|
||||
secretQueueService
|
||||
}: TFnSecretBulkDelete) => {
|
||||
const deletedSecrets = await secretDAL.deleteMany(
|
||||
inputSecrets.map(({ type, secretBlindIndex }) => ({
|
||||
blindIndex: secretBlindIndex,
|
||||
type
|
||||
})),
|
||||
folderId,
|
||||
actorId,
|
||||
tx
|
||||
);
|
||||
|
||||
await Promise.allSettled(
|
||||
deletedSecrets
|
||||
.filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
|
||||
.map(({ id, secretReminderRepeatDays }) =>
|
||||
secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
|
||||
)
|
||||
);
|
||||
|
||||
return deletedSecrets;
|
||||
};
|
||||
|
||||
export const createManySecretsRawFnFactory = ({
|
||||
projectDAL,
|
||||
projectBotDAL,
|
||||
|
||||
@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
|
||||
import { fnTriggerWebhook } from "../webhook/webhook-fns";
|
||||
import { TSecretDALFactory } from "./secret-dal";
|
||||
import { interpolateSecrets } from "./secret-fns";
|
||||
import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
|
||||
import {
|
||||
TCreateSecretReminderDTO,
|
||||
THandleReminderDTO,
|
||||
TRemoveSecretReminderDTO,
|
||||
TSyncSecretsDTO
|
||||
} from "./secret-types";
|
||||
|
||||
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
|
||||
type TSecretQueueFactoryDep = {
|
||||
@@ -116,21 +121,17 @@ export const secretQueueFactory = ({
|
||||
});
|
||||
};
|
||||
|
||||
const syncSecrets = async ({
|
||||
deDupeQueue = {},
|
||||
...dto
|
||||
}: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
|
||||
const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
|
||||
const syncSecrets = async <T extends boolean = false>({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO<T>) => {
|
||||
logger.info(
|
||||
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
|
||||
);
|
||||
const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath);
|
||||
if (deDupeQueue?.[deDuplicationKey]) {
|
||||
return;
|
||||
}
|
||||
// eslint-disable-next-line
|
||||
deDupeQueue[deDuplicationKey] = true;
|
||||
logger.info(
|
||||
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]`
|
||||
);
|
||||
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
|
||||
jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
|
||||
await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, {
|
||||
removeOnFail: true,
|
||||
removeOnComplete: true,
|
||||
delay: 1000,
|
||||
@@ -140,7 +141,6 @@ export const secretQueueFactory = ({
|
||||
delay: 3000
|
||||
}
|
||||
});
|
||||
await syncIntegrations({ ...dto, deDupeQueue });
|
||||
};
|
||||
|
||||
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
|
||||
@@ -333,6 +333,52 @@ export const secretQueueFactory = ({
|
||||
return content;
|
||||
};
|
||||
|
||||
queueService.start(QueueName.SecretSync, async (job) => {
|
||||
const {
|
||||
secretPath,
|
||||
environmentId,
|
||||
projectId,
|
||||
environmentSlug: environment,
|
||||
secrets,
|
||||
folderId,
|
||||
excludeReplication,
|
||||
membershipId
|
||||
} = job.data;
|
||||
await queueService.queue(
|
||||
QueueName.SecretWebhook,
|
||||
QueueJobs.SecWebhook,
|
||||
{ environment, projectId, secretPath },
|
||||
{
|
||||
jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`,
|
||||
removeOnFail: { count: 5 },
|
||||
removeOnComplete: true,
|
||||
delay: 1000,
|
||||
attempts: 5,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
}
|
||||
}
|
||||
);
|
||||
await syncIntegrations({ secretPath, projectId, environment });
|
||||
if (!excludeReplication) {
|
||||
await queueService.queue(
|
||||
QueueName.SecretReplication,
|
||||
QueueJobs.SecretReplication,
|
||||
{ environmentId, projectId, secretPath, folderId, secrets, membershipId },
|
||||
{
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 1000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
queueService.start(QueueName.IntegrationSync, async (job) => {
|
||||
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
|
||||
|
||||
@@ -375,11 +421,12 @@ export const secretQueueFactory = ({
|
||||
)
|
||||
.map(({ folderId }) =>
|
||||
syncSecrets({
|
||||
depth: depth + 1,
|
||||
_depth: depth + 1,
|
||||
projectId,
|
||||
secretPath: foldersGroupedById[folderId][0].path,
|
||||
environment: foldersGroupedById[folderId][0].environmentSlug,
|
||||
deDupeQueue
|
||||
environmentSlug: foldersGroupedById[folderId][0].environmentSlug,
|
||||
_deDupeQueue: deDupeQueue,
|
||||
excludeReplication: true
|
||||
})
|
||||
)
|
||||
);
|
||||
@@ -412,11 +459,12 @@ export const secretQueueFactory = ({
|
||||
)
|
||||
.map(({ folderId }) =>
|
||||
syncSecrets({
|
||||
depth: depth + 1,
|
||||
_depth: depth + 1,
|
||||
projectId,
|
||||
secretPath: referencedFoldersGroupedById[folderId][0].path,
|
||||
environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
|
||||
deDupeQueue
|
||||
environmentSlug: referencedFoldersGroupedById[folderId][0].environmentSlug,
|
||||
_deDupeQueue: deDupeQueue,
|
||||
excludeReplication: true
|
||||
})
|
||||
)
|
||||
);
|
||||
@@ -546,7 +594,7 @@ export const secretQueueFactory = ({
|
||||
|
||||
return {
|
||||
// depth is internal only field thus no need to make it available outside
|
||||
syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
|
||||
syncSecrets,
|
||||
syncIntegrations,
|
||||
addSecretReminder,
|
||||
removeSecretReminder,
|
||||
|
||||
@@ -30,13 +30,12 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
|
||||
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
|
||||
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
|
||||
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
|
||||
import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
|
||||
import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
|
||||
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
|
||||
import { TSecretDALFactory } from "./secret-dal";
|
||||
import {
|
||||
decryptSecretRaw,
|
||||
fnSecretBlindIndexCheck,
|
||||
fnSecretBulkDelete,
|
||||
fnSecretBulkInsert,
|
||||
fnSecretBulkUpdate,
|
||||
getAllNestedSecretReferences,
|
||||
@@ -45,6 +44,7 @@ import {
|
||||
} from "./secret-fns";
|
||||
import { TSecretQueueFactory } from "./secret-queue";
|
||||
import {
|
||||
SecretOperations,
|
||||
TAttachSecretTagsDTO,
|
||||
TBackFillSecretReferencesDTO,
|
||||
TCreateBulkSecretDTO,
|
||||
@@ -55,8 +55,6 @@ import {
|
||||
TDeleteManySecretRawDTO,
|
||||
TDeleteSecretDTO,
|
||||
TDeleteSecretRawDTO,
|
||||
TFnSecretBlindIndexCheckV2,
|
||||
TFnSecretBulkDelete,
|
||||
TGetASecretDTO,
|
||||
TGetASecretRawDTO,
|
||||
TGetSecretsDTO,
|
||||
@@ -87,7 +85,6 @@ type TSecretServiceFactoryDep = {
|
||||
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
|
||||
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
|
||||
secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
|
||||
};
|
||||
|
||||
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
|
||||
@@ -104,8 +101,7 @@ export const secretServiceFactory = ({
|
||||
projectDAL,
|
||||
projectBotService,
|
||||
secretImportDAL,
|
||||
secretVersionTagDAL,
|
||||
secretReplicationService
|
||||
secretVersionTagDAL
|
||||
}: TSecretServiceFactoryDep) => {
|
||||
const getSecretReference = async (projectId: string) => {
|
||||
// if bot key missing means e2e still exist
|
||||
@@ -143,53 +139,6 @@ export const secretServiceFactory = ({
|
||||
return secretBlindIndex;
|
||||
};
|
||||
|
||||
const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
|
||||
const deletedSecrets = await secretDAL.deleteMany(
|
||||
inputSecrets.map(({ type, secretBlindIndex }) => ({
|
||||
blindIndex: secretBlindIndex,
|
||||
type
|
||||
})),
|
||||
folderId,
|
||||
actorId,
|
||||
tx
|
||||
);
|
||||
|
||||
for (const s of deletedSecrets) {
|
||||
if (s.secretReminderRepeatDays) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await secretQueueService
|
||||
.removeSecretReminder({
|
||||
secretId: s.id,
|
||||
repeatDays: s.secretReminderRepeatDays
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return deletedSecrets;
|
||||
};
|
||||
|
||||
// this is used when secret blind index already exist
|
||||
// mainly for secret approval
|
||||
const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
|
||||
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
|
||||
throw new BadRequestError({ message: "Missing user id for personal secret" });
|
||||
}
|
||||
const secrets = await secretDAL.findByBlindIndexes(
|
||||
folderId,
|
||||
inputSecrets.map(({ secretBlindIndex, type }) => ({
|
||||
blindIndex: secretBlindIndex,
|
||||
type: type || SecretType.Shared
|
||||
})),
|
||||
userId
|
||||
);
|
||||
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
|
||||
|
||||
return { secsGroupedByBlindIndex, secrets };
|
||||
};
|
||||
|
||||
const createSecret = async ({
|
||||
path,
|
||||
actor,
|
||||
@@ -200,7 +149,7 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
...inputSecret
|
||||
}: TCreateSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -287,16 +236,17 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
secretPath: path,
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Create,
|
||||
operation: SecretOperations.Create,
|
||||
id: secret[0].id,
|
||||
version: 1
|
||||
}
|
||||
@@ -315,7 +265,7 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
...inputSecret
|
||||
}: TUpdateSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -430,15 +380,18 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
secretPath: path,
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Update,
|
||||
operation: SecretOperations.Update,
|
||||
id: updatedSecret[0].id,
|
||||
version: updatedSecret[0].version
|
||||
}
|
||||
@@ -457,7 +410,7 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
...inputSecret
|
||||
}: TDeleteSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -499,6 +452,8 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
folderId,
|
||||
actorId,
|
||||
secretDAL,
|
||||
secretQueueService,
|
||||
inputSecrets: [
|
||||
{
|
||||
type: inputSecret.type as SecretType,
|
||||
@@ -510,15 +465,18 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
secretPath: path,
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Delete,
|
||||
operation: SecretOperations.Delete,
|
||||
id: deletedSecret[0].id,
|
||||
version: deletedSecret[0].version
|
||||
}
|
||||
@@ -744,7 +702,7 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
secrets: inputSecrets
|
||||
}: TCreateBulkSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -808,13 +766,16 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
secretPath: path,
|
||||
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create }))
|
||||
});
|
||||
|
||||
return newSecrets;
|
||||
@@ -830,7 +791,7 @@ export const secretServiceFactory = ({
|
||||
projectId,
|
||||
secrets: inputSecrets
|
||||
}: TUpdateBulkSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -915,13 +876,16 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
secretPath: path,
|
||||
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update }))
|
||||
});
|
||||
|
||||
return secrets;
|
||||
@@ -937,7 +901,7 @@ export const secretServiceFactory = ({
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
}: TDeleteBulkSecretDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
const { permission, membership } = await permissionService.getProjectPermission(
|
||||
actor,
|
||||
actorId,
|
||||
projectId,
|
||||
@@ -972,6 +936,8 @@ export const secretServiceFactory = ({
|
||||
|
||||
const secretsDeleted = await secretDAL.transaction(async (tx) =>
|
||||
fnSecretBulkDelete({
|
||||
secretDAL,
|
||||
secretQueueService,
|
||||
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
|
||||
secretBlindIndex: keyName2BlindIndex[secretName],
|
||||
type
|
||||
@@ -984,13 +950,16 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
await secretQueueService.syncSecrets({
|
||||
// if secret service reached means there was no secret policy
|
||||
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
|
||||
membershipId: membership?.id as string,
|
||||
secretPath: path,
|
||||
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
environmentSlug: folder.environment.slug,
|
||||
environmentId: folder.envId,
|
||||
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete }))
|
||||
});
|
||||
|
||||
return secretsDeleted;
|
||||
@@ -1171,9 +1140,6 @@ export const secretServiceFactory = ({
|
||||
skipMultilineEncoding
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secret.folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return decryptSecretRaw(secret, botKey);
|
||||
};
|
||||
|
||||
@@ -1212,8 +1178,6 @@ export const secretServiceFactory = ({
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secret.folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return decryptSecretRaw(secret, botKey);
|
||||
};
|
||||
|
||||
@@ -1243,9 +1207,6 @@ export const secretServiceFactory = ({
|
||||
actorAuthMethod
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secret.folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return decryptSecretRaw(secret, botKey);
|
||||
};
|
||||
|
||||
@@ -1294,12 +1255,7 @@ export const secretServiceFactory = ({
|
||||
})
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secrets[0].folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return secrets.map((secret) =>
|
||||
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
|
||||
);
|
||||
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
|
||||
};
|
||||
|
||||
const updateManySecretsRaw = async ({
|
||||
@@ -1348,12 +1304,7 @@ export const secretServiceFactory = ({
|
||||
})
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secrets[0].folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return secrets.map((secret) =>
|
||||
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
|
||||
);
|
||||
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
|
||||
};
|
||||
|
||||
const deleteManySecretsRaw = async ({
|
||||
@@ -1384,12 +1335,7 @@ export const secretServiceFactory = ({
|
||||
secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
|
||||
});
|
||||
|
||||
await snapshotService.performSnapshot(secrets[0].folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
|
||||
|
||||
return secrets.map((secret) =>
|
||||
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
|
||||
);
|
||||
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
|
||||
};
|
||||
|
||||
const getSecretVersions = async ({
|
||||
@@ -1510,7 +1456,12 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folder.id);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
|
||||
await secretQueueService.syncSecrets({
|
||||
secretPath,
|
||||
projectId: project.id,
|
||||
environmentSlug: environment,
|
||||
excludeReplication: true
|
||||
});
|
||||
|
||||
return {
|
||||
...updatedSecret[0],
|
||||
@@ -1612,7 +1563,12 @@ export const secretServiceFactory = ({
|
||||
);
|
||||
|
||||
await snapshotService.performSnapshot(folder.id);
|
||||
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
|
||||
await secretQueueService.syncSecrets({
|
||||
secretPath,
|
||||
projectId: project.id,
|
||||
environmentSlug: environment,
|
||||
excludeReplication: true
|
||||
});
|
||||
|
||||
return {
|
||||
...updatedSecret[0],
|
||||
@@ -1685,13 +1641,6 @@ export const secretServiceFactory = ({
|
||||
createManySecretsRaw,
|
||||
updateManySecretsRaw,
|
||||
deleteManySecretsRaw,
|
||||
getSecretVersions,
|
||||
backfillSecretReferences,
|
||||
// external services function
|
||||
fnSecretBulkDelete,
|
||||
fnSecretBulkUpdate,
|
||||
fnSecretBlindIndexCheck,
|
||||
fnSecretBulkInsert,
|
||||
fnSecretBlindIndexCheckV2
|
||||
getSecretVersions
|
||||
};
|
||||
};
|
||||
|
||||
@@ -264,6 +264,10 @@ export type TFnSecretBulkDelete = {
|
||||
inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
|
||||
actorId: string;
|
||||
tx?: Knex;
|
||||
secretDAL: Pick<TSecretDALFactory, "deleteMany">;
|
||||
secretQueueService: {
|
||||
removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
|
||||
};
|
||||
};
|
||||
|
||||
export type TFnSecretBlindIndexCheck = {
|
||||
@@ -277,6 +281,7 @@ export type TFnSecretBlindIndexCheck = {
|
||||
|
||||
// when blind index is already present
|
||||
export type TFnSecretBlindIndexCheckV2 = {
|
||||
secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
|
||||
folderId: string;
|
||||
userId?: string;
|
||||
inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
|
||||
@@ -363,3 +368,30 @@ export type TUpdateManySecretsRawFn = {
|
||||
}[];
|
||||
userId?: string;
|
||||
};
|
||||
|
||||
export enum SecretOperations {
|
||||
Create = "create",
|
||||
Update = "update",
|
||||
Delete = "delete"
|
||||
}
|
||||
|
||||
export type TSyncSecretsDTO<T extends boolean = false> = {
|
||||
_depth?: number;
|
||||
_deDupeQueue?: Record<string, boolean>;
|
||||
secretPath: string;
|
||||
projectId: string;
|
||||
environmentSlug: string;
|
||||
// cases for just doing sync integration and webhook
|
||||
excludeReplication?: T;
|
||||
} & (T extends true
|
||||
? object
|
||||
: {
|
||||
environmentId: string;
|
||||
folderId: string;
|
||||
membershipId: string;
|
||||
secrets: {
|
||||
operation: SecretOperations;
|
||||
id: string;
|
||||
version: number;
|
||||
}[];
|
||||
});
|
||||
|
||||
@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
|
||||
|
||||
const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
|
||||
try {
|
||||
if (!secretIds.length) return {};
|
||||
const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
|
||||
.where("folderId", folderId)
|
||||
.whereIn(`${TableName.SecretVersion}.secretId`, secretIds)
|
||||
|
||||
Reference in New Issue
Block a user