awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added first version of migrator to secret v2

Browse Source
This commit is contained in:
=
2024-07-24 23:54:18 +05:30
parent ab88e6c414
commit a53d0b2334
10 changed files with 170 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/ee/services/secret-rotation/secret-rotation-dal.ts
View File

@@ -144,6 +144,8 @@ export const secretRotationDALFactory = (db: TDbClient) => {
const findRotationOutputsV2ByRotationId = async (rotationId: string) =>
secretRotationOutputV2Orm.find({ rotationId });
// special query
return {
...secretRotationOrm,
find,

13
backend/src/lib/knex/index.ts
View File

@@ -104,6 +104,19 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
throw new DatabaseError({ error, name: "Create" });
}
},
upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
try {
if (!data.length) return [];
const res = await (tx || db)(tableName)
.insert(data as never)
.onConflict(onConflictField as never)
.merge()
.returning("*");
return res;
} catch (error) {
throw new DatabaseError({ error, name: "Create" });
}
},
updateById: async (
id: string,
{

4
backend/src/server/routes/index.ts
View File

@@ -719,7 +719,9 @@ export const registerRoutes = async (
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL
secretVersionTagV2BridgeDAL,
secretRotationDAL,
integrationAuthDAL
});
const secretImportService = secretImportServiceFactory({
licenseService,

15
backend/src/services/secret-tag/secret-tag-dal.ts
View File

@@ -3,7 +3,7 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify } from "@app/lib/knex";
import { ormify, selectAllTableCols } from "@app/lib/knex";
export type TSecretTagDALFactory = ReturnType<typeof secretTagDALFactory>;
@@ -35,12 +35,25 @@ export const secretTagDALFactory = (db: TDbClient) => {
}
};
// special query for migration
const findSecretTagsByProjectId = async (projectId: string, tx?: Knex) => {
try {
const tags = await (tx || db.replicaNode())(TableName.JnSecretTag)
.join(TableName.SecretTag, `${TableName.JnSecretTag}.secret_tagsId`, `${TableName.SecretTag}.id`)
.where({ projectId })
.select(selectAllTableCols(TableName.JnSecretTag));
return tags;
} catch (error) {
throw new DatabaseError({ error, name: "Find all by ids" });
}
};
return {
...secretTagOrm,
saveTagsToSecret: secretJnTagOrm.insertMany,
deleteTagsToSecret: secretJnTagOrm.delete,
saveTagsToSecretV2: secretV2JnTagOrm.insertMany,
deleteTagsToSecretV2: secretV2JnTagOrm.delete,
findSecretTagsByProjectId,
deleteTagsManySecret,
findManyTagsById
};

129
backend/src/services/secret/secret-queue.ts
View File

@@ -1,6 +1,8 @@
/* eslint-disable no-await-in-loop */
import { AxiosError } from "axios";
import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
@@ -16,6 +18,7 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret";
import { TKmsServiceFactory } from "../kms/kms-service";
@@ -28,6 +31,7 @@ import { TProjectMembershipDALFactory } from "../project-membership/project-memb
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
@@ -41,12 +45,12 @@ import {
TRemoveSecretReminderDTO,
TSyncSecretsDTO
} from "./secret-types";
import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
queueService: TQueueServiceFactory;
integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
integrationAuthDAL: Pick<TIntegrationAuthDALFactory, "upsert" | "find">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
folderDAL: TSecretFolderDALFactory;
@@ -67,6 +71,7 @@ type TSecretQueueFactoryDep = {
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
};
export type TGetSecrets = {
@@ -86,6 +91,7 @@ type TIntegrationSecret = Record<
export const secretQueueFactory = ({
queueService,
integrationDAL,
integrationAuthDAL,
projectBotService,
integrationAuthService,
secretDAL,
@@ -105,7 +111,8 @@ export const secretQueueFactory = ({
secretV2BridgeDAL,
secretVersionV2BridgeDAL,
kmsService,
secretVersionTagV2BridgeDAL
secretVersionTagV2BridgeDAL,
secretRotationDAL
}: TSecretQueueFactoryDep) => {
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
const appCfg = getConfig();
@@ -779,7 +786,6 @@ export const secretQueueFactory = ({
);
};
const MIGRATION_BATCH_SIZE = 10000;
queueService.start(QueueName.ProjectV3Migration, async (job) => {
const { projectId } = job.data;
const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
@@ -798,10 +804,16 @@ export const secretQueueFactory = ({
await secretDAL.transaction(async (tx) => {
for (const folder of folders) {
const folderId = folder.id;
let projectV1Secrets;
do {
// eslint-disable-next-line no-await-in-loop
projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx });
/*
* Secrets Migration
* */
// eslint-disable-next-line no-await-in-loop
const projectV1Secrets = await secretDAL.find({ folderId }, { tx });
if (projectV1Secrets.length) {
const secretReferences: {
secretId: string;
references: { environment: string; secretPath: string; secretKey: string }[];
}[] = [];
await secretV2BridgeDAL.insertMany(
projectV1Secrets.map((el) => {
const key = decryptSymmetric128BitHexKeyUTF8({
@@ -826,6 +838,10 @@ export const secretQueueFactory = ({
})
: "";
const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
// create references
const references = getAllNestedSecretReferences(value);
secretReferences.push({ secretId: el.id, references });
const encryptedComment = comment
? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
: null;
@@ -848,9 +864,104 @@ export const secretQueueFactory = ({
}),
tx
);
projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx);
} while (projectV1Secrets.length > 0);
await secretV2BridgeDAL.upsertSecretReferences(secretReferences);
}
}
/*
* Secret Tag Migration
* */
// eslint-disable-next-line no-await-in-loop
const projectV1SecretTags = await secretTagDAL.findSecretTagsByProjectId(projectId, tx);
if (projectV1SecretTags.length) {
await secretTagDAL.saveTagsToSecretV2(
projectV1SecretTags.map((el) => ({
secrets_v2Id: el.secretsId,
secret_tagsId: el.secret_tagsId
})),
tx
);
}
/*
* Integration Auth Migration
* Saving the new encrypted colum
* */
// eslint-disable-next-line no-await-in-loop
const projectV1IntegrationAuths = await integrationAuthDAL.find({ projectId }, { tx });
await integrationAuthDAL.upsert(
projectV1IntegrationAuths.map((el) => {
const accessToken =
el.accessIV && el.accessTag && el.accessCiphertext
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.accessCiphertext,
iv: el.accessIV,
tag: el.accessTag,
key: botKey
})
: undefined;
const accessId =
el.accessIdIV && el.accessIdTag && el.accessIdCiphertext
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.accessIdCiphertext,
iv: el.accessIdIV,
tag: el.accessIdTag,
key: botKey
})
: undefined;
const refreshToken =
el.refreshIV && el.refreshTag && el.refreshCiphertext
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.refreshCiphertext,
iv: el.refreshIV,
tag: el.refreshTag,
key: botKey
})
: undefined;
const awsAssumeRoleArn =
el.awsAssumeIamRoleArnCipherText && el.awsAssumeIamRoleArnIV && el.awsAssumeIamRoleArnTag
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.awsAssumeIamRoleArnCipherText,
iv: el.awsAssumeIamRoleArnIV,
tag: el.awsAssumeIamRoleArnTag,
key: botKey
})
: undefined;
const encryptedAccess = accessToken
? secretManagerEncryptor({ plainText: Buffer.from(accessToken) }).cipherTextBlob
: null;
const encryptedAccessId = accessId
? secretManagerEncryptor({ plainText: Buffer.from(accessId) }).cipherTextBlob
: null;
const encryptedRefresh = refreshToken
? secretManagerEncryptor({ plainText: Buffer.from(refreshToken) }).cipherTextBlob
: null;
const encryptedAwsAssumeIamRoleArn = awsAssumeRoleArn
? secretManagerEncryptor({ plainText: Buffer.from(awsAssumeRoleArn) }).cipherTextBlob
: null;
return {
...el,
encryptedAccess,
encryptedRefresh,
encryptedAccessId,
encryptedAwsAssumeIamRoleArn
};
}),
"id",
tx
);
/*
* Secret Rotation Secret Migration
* Saving the new encrypted colum
* */
const projectV1SecretRotations = await secretRotationDAL.find({ projectId }, tx);
await secretRotationDAL.secretOutputV2InsertMany(
projectV1SecretRotations.flatMap((el) =>
el.outputs.map((output) => ({ rotationId: el.id, key: output.key, secretId: output.secret.id }))
),
tx
);
await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
});
});

1
frontend/src/hooks/api/users/types.ts
View File

@@ -66,7 +66,6 @@ export type OrgUser = {
export type TProjectMembership = {
id: string;
role: string;
createdAt: string;
updatedAt: string;
projectId: string;

5
frontend/src/hooks/api/workspace/index.tsx
View File

@@ -2,9 +2,8 @@ export {
useAddGroupToWorkspace,
useDeleteGroupFromWorkspace,
useLeaveProject,
useUpdateGroupWorkspaceRole,
useMigrateProjectToV3
} from "./mutations";
useMigrateProjectToV3,
useUpdateGroupWorkspaceRole} from "./mutations";
export {
useAddIdentityToWorkspace,
useCreateWorkspace,

2
frontend/src/hooks/api/workspace/mutations.tsx
View File

@@ -79,7 +79,7 @@ export const useMigrateProjectToV3 = () => {
const queryClient = useQueryClient();
return useMutation<{}, {}, { workspaceId: string }>({
mutationFn: ({ workspaceId }) => {
return apiRequest.delete(`/api/v1/workspace/${workspaceId}/migrate-v3`);
return apiRequest.post(`/api/v1/workspace/${workspaceId}/migrate-v3`);
},
onSuccess: () => {
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);

2
frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx
View File

@@ -69,10 +69,10 @@ import { ProjectVersion } from "@app/hooks/api/workspace/types";
import { FolderForm } from "../SecretMainPage/components/ActionBar/FolderForm";
import { CreateSecretForm } from "./components/CreateSecretForm";
import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs";
import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection";
import { SecretOverviewDynamicSecretRow } from "./components/SecretOverviewDynamicSecretRow";
import { SecretOverviewFolderRow } from "./components/SecretOverviewFolderRow";
import { SecretOverviewTableRow } from "./components/SecretOverviewTableRow";
import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection";
import { SelectionPanel } from "./components/SelectionPanel/SelectionPanel";
export enum EntryType {

42
frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx
View File

@@ -1,13 +1,6 @@
import { createNotification } from "@app/components/notifications";
import { ProjectPermissionCan } from "@app/components/permissions";
import { Button, Spinner } from "@app/components/v2";
import {
ProjectPermissionActions,
ProjectPermissionSub,
useProjectPermission,
useWorkspace
} from "@app/context";
import { useProjectPermission, useWorkspace } from "@app/context";
import { useGetWorkspaceById, useMigrateProjectToV3 } from "@app/hooks/api";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
import { ProjectVersion } from "@app/hooks/api/workspace/types";
@@ -50,8 +43,8 @@ export const SecretV2MigrationSection = () => {
});
}
};
// for non admin this would throw an error
// so no need to render
const isAdmin = membership?.roles.includes(ProjectMembershipRole.Admin);
return (
<div className="mt-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
{isUpgrading && (
@@ -66,25 +59,18 @@ export const SecretV2MigrationSection = () => {
<p className="mb-2 text-lg font-semibold">Action Required</p>
<p className="mb-4 leading-7 text-gray-400">
There is a new update for your project. Introducing Infisical KMS.
<b>
{membership.role !== ProjectMembershipRole.Admin && "This is an admin only operation."}
</b>
<b>{!isAdmin && "This is an admin only operation."}</b>
</p>
<ProjectPermissionCan I={ProjectPermissionActions.Edit} a={ProjectPermissionSub.Settings}>
{(isAllowed) => (
<Button
onClick={handleMigrationSecretV2}
isDisabled={
!isAllowed || membership.role !== ProjectMembershipRole.Admin || isUpgrading
}
color="mineshaft"
type="submit"
isLoading={migrateProjectToV3.isLoading}
>
Start Migration
</Button>
)}
</ProjectPermissionCan>
<Button
onClick={handleMigrationSecretV2}
isDisabled={!isAdmin || isUpgrading}
color="mineshaft"
type="submit"
isLoading={migrateProjectToV3.isLoading}
>
Start Migration
</Button>
</div>
);
};