awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-05 11:26:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improved migration wizard to info user prerequisite check list

Browse Source
This commit is contained in:
=
2024-07-30 23:19:32 +05:30
parent e2caa98c74
commit 549d388f59
9 changed files with 254 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts
+16 -1
View File
@@ -489,11 +489,26 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
}
};
const deleteByProjectId = async (projectId: string, tx?: Knex) => {
try {
const query = await (tx || db.replicaNode())(TableName.SecretApprovalRequest)
.join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where({ projectId })
.delete();
return query;
} catch (error) {
throw new DatabaseError({ error, name: "DeleteByProjectId" });
}
};
return {
...secretApprovalRequestOrm,
findById,
findProjectRequestCount,
findByProjectId,
findByProjectIdBridgeSecretV2
findByProjectIdBridgeSecretV2,
deleteByProjectId
};
};
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
+1 -1
View File
@@ -1274,7 +1274,7 @@ export const secretApprovalRequestServiceFactory = ({
const commitsGroupByKey = groupBy(approvalCommits, (i) => i.key);
if (tagIds.length) {
await secretApprovalRequestSecretDAL.insertApprovalSecretTags(
await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(
Object.keys(commitTagIds).flatMap((blindIndex) =>
commitTagIds[blindIndex]
? commitTagIds[blindIndex].map((tagId) => ({
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+16 -2
View File
@@ -743,7 +743,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("envId").withSchema(TableName.SnapshotSecret).as("snapshotEnvId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("secretVersionTagId"),
db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretId"),
db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"),
db.ref("secret_tagsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"),
db.raw(
`DENSE_RANK() OVER (partition by ${TableName.Snapshot}."id" ORDER BY ${TableName.SecretVersion}."createdAt") as rank`
)
@@ -789,6 +789,19 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
const deleteSnapshotsAboveLimit = async (folderId: string, n = 15, tx?: Knex) => {
try {
const query = await (tx || db.replicaNode())(TableName.Snapshot)
.orderBy(`${TableName.Snapshot}.createdAt`, "desc")
.where(`${TableName.Snapshot}.folderId`, folderId)
.offset(n)
.delete();
return query;
} catch (error) {
throw new DatabaseError({ error, name: "DeleteSnapshotsAboveLimit" });
}
};
return {
...secretSnapshotOrm,
findById,
@@ -799,6 +812,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
findSecretSnapshotDataById,
findSecretSnapshotV2DataById,
pruneExcessSnapshots,
findNSecretV1SnapshotByFolderId
findNSecretV1SnapshotByFolderId,
deleteSnapshotsAboveLimit
};
};
backend/src/server/routes/index.ts
+2 -2
View File
@@ -723,8 +723,8 @@ export const registerRoutes = async (
secretRotationDAL,
integrationAuthDAL,
snapshotDAL,
secretApprovalRequestSecretDAL,
snapshotSecretV2BridgeDAL
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL
});
const secretImportService = secretImportServiceFactory({
licenseService,
backend/src/services/secret/secret-queue.ts
+78 -76
View File
@@ -2,7 +2,7 @@
import { AxiosError } from "axios";
import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal";
@@ -76,11 +76,8 @@ type TSecretQueueFactoryDep = {
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
secretApprovalRequestSecretDAL: Pick<
TSecretApprovalRequestSecretDALFactory,
"findByProjectId" | "insertV2Bridge" | "insertApprovalSecretV2Tags"
>;
snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId">;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "deleteByProjectId">;
snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">;
snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany">;
};
@@ -123,9 +120,9 @@ export const secretQueueFactory = ({
kmsService,
secretVersionTagV2BridgeDAL,
secretRotationDAL,
secretApprovalRequestSecretDAL,
snapshotDAL,
snapshotSecretV2BridgeDAL
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL
}: TSecretQueueFactoryDep) => {
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
const appCfg = getConfig();
@@ -792,11 +789,6 @@ export const secretQueueFactory = ({
QueueJobs.ProjectV3Migration,
{ projectId },
{
attempts: 2,
backoff: {
type: "exponential",
delay: 3000
},
removeOnComplete: true,
removeOnFail: true
}
@@ -805,12 +797,17 @@ export const secretQueueFactory = ({
queueService.start(QueueName.ProjectV3Migration, async (job) => {
const { projectId } = job.data;
const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
if (isProjectUpgradedToV3) {
const {
botKey,
shouldUseSecretV2Bridge: isProjectUpgradedToV3,
project
} = await projectBotService.getBotKey(projectId);
if (isProjectUpgradedToV3 || project.upgradeStatus === ProjectUpgradeStatus.InProgress) {
return;
}
if (!botKey) throw new BadRequestError({ message: "Bot not found" });
await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress });
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
projectId,
type: KmsDataKey.SecretManager
@@ -884,7 +881,8 @@ export const secretQueueFactory = ({
await secretV2BridgeDAL.upsertSecretReferences(secretReferences, tx);
}
const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, 10, tx);
const SNAPSHOT_BATCH_SIZE = 15;
const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, SNAPSHOT_BATCH_SIZE, tx);
const projectV3SecretVersionsGroupById: Record<string, TSecretVersionsV2> = {};
const projectV3SecretVersionTags: { secret_versions_v2Id: string; secret_tagsId: string }[] = [];
const projectV3SnapshotSecrets: Omit<TSecretSnapshotSecretsV2, "id">[] = [];
@@ -959,6 +957,7 @@ export const secretQueueFactory = ({
if (projectV3SnapshotSecrets.length) {
await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx);
}
await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx);
}
/*
* Secret Tag Migration
@@ -1056,67 +1055,70 @@ export const secretQueueFactory = ({
);
/*
* approvals
* approvals: we will delete all approvals this is because some secret versions may not be added yet
* Thus doesn't make sense for rest to be there
* */
const projectV1ApprovalSecrets = await secretApprovalRequestSecretDAL.findByProjectId(projectId);
if (projectV1ApprovalSecrets.length) {
await secretApprovalRequestSecretDAL.insertV2Bridge(
projectV1ApprovalSecrets.map((el) => {
const key = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretKeyCiphertext,
iv: el.secretKeyIV,
tag: el.secretKeyTag,
key: botKey
});
const value = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
});
const comment =
el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretCommentCiphertext,
iv: el.secretCommentIV,
tag: el.secretCommentTag,
key: botKey
})
: "";
const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
const encryptedComment = comment
? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
: null;
return {
id: el.id,
createdAt: el.createdAt,
updatedAt: el.updatedAt,
skipMultilineEncoding: el.skipMultilineEncoding,
encryptedComment,
encryptedValue,
key,
version: el.version,
metadata: el.metadata,
reminderNote: el.secretReminderNote,
reminderRepeatDays: el.secretReminderRepeatDays,
requestId: el.requestId,
op: el.op,
secretId: el.secretId,
secretVersion: el.secretVersion
};
}),
tx
);
}
const projectV1SecretApprovalSecretTags = projectV1ApprovalSecrets.flatMap((el) =>
el.tags.map((tag) => ({
secretId: tag.secretApprovalTagSecretId,
tagId: tag.secretApprovalTagId
}))
);
if (projectV1SecretApprovalSecretTags.length) {
await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(projectV1SecretApprovalSecretTags, tx);
}
await secretApprovalRequestDAL.deleteByProjectId(projectId, tx);
// const projectV1ApprovalSecrets = await secretApprovalRequestSecretDAL.findByProjectId(projectId);
// if (projectV1ApprovalSecrets.length) {
// await secretApprovalRequestSecretDAL.insertV2Bridge(
// projectV1ApprovalSecrets.map((el) => {
// const key = decryptSymmetric128BitHexKeyUTF8({
// ciphertext: el.secretKeyCiphertext,
// iv: el.secretKeyIV,
// tag: el.secretKeyTag,
// key: botKey
// });
// const value = decryptSymmetric128BitHexKeyUTF8({
// ciphertext: el.secretValueCiphertext,
// iv: el.secretValueIV,
// tag: el.secretValueTag,
// key: botKey
// });
// const comment =
// el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV
// ? decryptSymmetric128BitHexKeyUTF8({
// ciphertext: el.secretCommentCiphertext,
// iv: el.secretCommentIV,
// tag: el.secretCommentTag,
// key: botKey
// })
// : "";
// const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
// const encryptedComment = comment
// ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
// : null;
// return {
// id: el.id,
// createdAt: el.createdAt,
// updatedAt: el.updatedAt,
// skipMultilineEncoding: el.skipMultilineEncoding,
// encryptedComment,
// encryptedValue,
// key,
// version: el.version,
// metadata: el.metadata,
// reminderNote: el.secretReminderNote,
// reminderRepeatDays: el.secretReminderRepeatDays,
// requestId: el.requestId,
// op: el.op,
// secretId: el.secretId,
// secretVersion: el.secretVersion
// };
// }),
// tx
// );
// }
// const projectV1SecretApprovalSecretTags = projectV1ApprovalSecrets.flatMap((el) =>
// el.tags.map((tag) => ({
// secretId: tag.secretApprovalTagSecretId,
// tagId: tag.secretApprovalTagId
// }))
// );
// if (projectV1SecretApprovalSecretTags.length) {
// await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(projectV1SecretApprovalSecretTags, tx);
// }
await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
});
});
backend/src/services/secret/secret-service.ts
+5 -1
View File
@@ -4,6 +4,7 @@ import { ForbiddenError, subject } from "@casl/ability";
import {
ProjectMembershipRole,
ProjectUpgradeStatus,
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretsSchema,
@@ -2666,8 +2667,11 @@ export const secretServiceFactory = ({
if (!hasRole(ProjectMembershipRole.Admin))
throw new BadRequestError({ message: "Only admins are allowed to take this action" });
const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId);
const { shouldUseSecretV2Bridge: isProjectV3, project } = await projectBotService.getBotKey(projectId);
if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" });
if (project.upgradeStatus === ProjectUpgradeStatus.InProgress)
throw new BadRequestError({ message: "project is upgrading" });
await secretQueueService.startSecretV2Migration(projectId);
return { message: "Migrating project to new KMS architecture" };
};