checkpoint

This commit is contained in:
=
2024-07-24 20:56:27 +05:30
parent 49eb6d6474
commit ab88e6c414
16 changed files with 315 additions and 126 deletions

View File

@@ -349,4 +349,35 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
return backup;
}
});
server.route({
method: "POST",
url: "/:workspaceId/migrate-v3",
config: {
rateLimit: writeLimit
},
schema: {
params: z.object({
workspaceId: z.string().trim()
}),
response: {
200: z.object({
message: z.string()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const migration = await server.services.secret.startSecretV2Migration({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId: req.params.workspaceId
});
return migration;
}
});
};

View File

@@ -25,7 +25,8 @@ export enum QueueName {
DynamicSecretRevocation = "dynamic-secret-revocation",
CaCrlRotation = "ca-crl-rotation",
SecretReplication = "secret-replication",
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
ProjectV3Migration = "project-v3-migration"
}
export enum QueueJobs {
@@ -44,7 +45,8 @@ export enum QueueJobs {
DynamicSecretPruning = "dynamic-secret-pruning",
CaCrlRotation = "ca-crl-rotation-job",
SecretReplication = "secret-replication",
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
ProjectV3Migration = "project-v3-migration"
}
export type TQueueJobTypes = {
@@ -136,6 +138,10 @@ export type TQueueJobTypes = {
name: QueueJobs.SecretSync;
payload: TSyncSecretsDTO;
};
[QueueName.ProjectV3Migration]: {
name: QueueJobs.ProjectV3Migration;
payload: { projectId: string };
};
};
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

View File

@@ -331,6 +331,27 @@ export const secretFolderDALFactory = (db: TDbClient) => {
}
};
// special query for project migration
const findByProjectId = async (projectId: string, tx?: Knex) => {
try {
const folders = await (tx || db.replicaNode())(TableName.SecretFolder)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.select(selectAllTableCols(TableName.SecretFolder))
.where({ projectId })
.select(
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("version").withSchema(TableName.Project).as("projectVersion")
);
return folders;
} catch (error) {
throw new DatabaseError({ error, name: "Find by id" });
}
};
return {
...secretFolderOrm,
update,
@@ -338,6 +359,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
findById,
findByManySecretPath,
findSecretPathByFolderIds,
findClosestFolder
findClosestFolder,
findByProjectId
};
};

View File

@@ -41,6 +41,7 @@ import {
TRemoveSecretReminderDTO,
TSyncSecretsDTO
} from "./secret-types";
import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
@@ -52,7 +53,7 @@ type TSecretQueueFactoryDep = {
secretDAL: TSecretDALFactory;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "find">;
projectDAL: TProjectDALFactory;
projectBotDAL: TProjectBotDALFactory;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
@@ -761,6 +762,108 @@ export const secretQueueFactory = ({
});
});
const startSecretV2Migration = async (projectId: string) => {
await queueService.queue(
QueueName.ProjectV3Migration,
QueueJobs.ProjectV3Migration,
{ projectId },
{
attempts: 2,
backoff: {
type: "exponential",
delay: 3000
},
removeOnComplete: true,
removeOnFail: true
}
);
};
const MIGRATION_BATCH_SIZE = 10000;
queueService.start(QueueName.ProjectV3Migration, async (job) => {
const { projectId } = job.data;
const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
if (isProjectUpgradedToV3) {
return;
}
if (!botKey) throw new BadRequestError({ message: "Bot not found" });
await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress });
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
projectId,
type: KmsDataKey.SecretManager
});
const folders = await folderDAL.findByProjectId(projectId);
// except secret version and snapshot migrate rest of everything first in a transaction
await secretDAL.transaction(async (tx) => {
for (const folder of folders) {
const folderId = folder.id;
let projectV1Secrets;
do {
// eslint-disable-next-line no-await-in-loop
projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx });
await secretV2BridgeDAL.insertMany(
projectV1Secrets.map((el) => {
const key = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretKeyCiphertext,
iv: el.secretKeyIV,
tag: el.secretKeyTag,
key: botKey
});
const value = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
});
const comment =
el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV
? decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretCommentCiphertext,
iv: el.secretCommentIV,
tag: el.secretCommentTag,
key: botKey
})
: "";
const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
const encryptedComment = comment
? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
: null;
return {
id: el.id,
createdAt: el.createdAt,
updatedAt: el.updatedAt,
skipMultilineEncoding: el.skipMultilineEncoding,
encryptedComment,
encryptedValue,
key,
version: el.version,
type: el.type,
userId: el.userId,
folderId: el.folderId,
metadata: el.metadata,
reminderNote: el.secretReminderNote,
reminderRepeatDays: el.secretReminderRepeatDays
};
}),
tx
);
projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx);
} while (projectV1Secrets.length > 0);
}
await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
});
});
// eslint-disable-next-line
queueService.listen(QueueName.ProjectV3Migration, "failed", async (job, err) => {
if (job?.data) {
const { projectId } = job.data;
await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.Failed });
logger.error(err, `Failed to migrate project to v3: ${projectId}`);
}
});
queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => {
logger.error(err, "Failed to sync integration %s", job?.id);
});
@@ -772,6 +875,7 @@ export const secretQueueFactory = ({
return {
// depth is internal only field thus no need to make it available outside
syncSecrets,
startSecretV2Migration,
syncIntegrations,
addSecretReminder,
removeSecretReminder,

View File

@@ -68,6 +68,7 @@ import {
TGetSecretsRawDTO,
TGetSecretVersionsDTO,
TMoveSecretsDTO,
TStartSecretsV2MigrationDTO,
TUpdateBulkSecretDTO,
TUpdateManySecretRawDTO,
TUpdateSecretDTO,
@@ -90,7 +91,10 @@ type TSecretServiceFactoryDep = {
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "handleSecretReminder" | "removeSecretReminder">;
secretQueueService: Pick<
TSecretQueueFactory,
"syncSecrets" | "handleSecretReminder" | "removeSecretReminder" | "startSecretV2Migration"
>;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -2638,6 +2642,30 @@ export const secretServiceFactory = ({
};
};
const startSecretV2Migration = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: TStartSecretsV2MigrationDTO) => {
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
if (!hasRole(ProjectMembershipRole.Admin))
throw new BadRequestError({ message: "Only admins are allowed to take this action" });
const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId);
if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" });
await secretQueueService.startSecretV2Migration(projectId);
return { message: "Migrating project to new KMS architecture" };
};
return {
attachTags,
detachTags,
@@ -2659,6 +2687,7 @@ export const secretServiceFactory = ({
deleteManySecretsRaw,
getSecretVersions,
backfillSecretReferences,
moveSecrets
moveSecrets,
startSecretV2Migration
};
};

View File

@@ -449,3 +449,5 @@ export enum SecretProtectionType {
Approval = "approval",
Direct = "direct"
}
export type TStartSecretsV2MigrationDTO = TProjectPermission;

View File

@@ -2,7 +2,8 @@ export {
useAddGroupToWorkspace,
useDeleteGroupFromWorkspace,
useLeaveProject,
useUpdateGroupWorkspaceRole
useUpdateGroupWorkspaceRole,
useMigrateProjectToV3
} from "./mutations";
export {
useAddIdentityToWorkspace,

View File

@@ -74,3 +74,15 @@ export const useLeaveProject = () => {
}
});
};
export const useMigrateProjectToV3 = () => {
const queryClient = useQueryClient();
return useMutation<{}, {}, { workspaceId: string }>({
mutationFn: ({ workspaceId }) => {
return apiRequest.delete(`/api/v1/workspace/${workspaceId}/migrate-v3`);
},
onSuccess: () => {
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);
}
});
};

View File

@@ -150,11 +150,15 @@ export const useGetWorkspaceSecrets = (workspaceId: string) => {
});
};
export const useGetWorkspaceById = (workspaceId: string) => {
export const useGetWorkspaceById = (
workspaceId: string,
dto?: { refetchInterval?: number | false }
) => {
return useQuery({
queryKey: workspaceKeys.getWorkspaceById(workspaceId),
queryFn: () => fetchWorkspaceById(workspaceId),
enabled: Boolean(workspaceId)
enabled: Boolean(workspaceId),
refetchInterval: dto?.refetchInterval
});
};

View File

@@ -1,6 +1,7 @@
export enum ProjectVersion {
V1 = 1,
V2 = 2
V2 = 2,
V3 = 3
}
export enum ProjectUserMembershipTemporaryMode {

View File

@@ -55,7 +55,6 @@ import {
SelectItem,
UpgradePlanModal
} from "@app/components/v2";
import { UpgradeOverlay } from "@app/components/v2/UpgradeOverlay";
import {
OrgPermissionActions,
OrgPermissionSubjects,
@@ -335,7 +334,6 @@ export const AppLayout = ({ children }: LayoutProps) => {
<aside className="dark w-full border-r border-mineshaft-600 bg-gradient-to-tr from-mineshaft-700 via-mineshaft-800 to-mineshaft-900 md:w-60">
<nav className="items-between flex h-full flex-col justify-between overflow-y-auto dark:[color-scheme:dark]">
<div>
<UpgradeOverlay />
{!router.asPath.includes("personal") && (
<div className="flex h-12 cursor-default items-center px-3 pt-6">
{(router.asPath.includes("project") ||

View File

@@ -59,7 +59,6 @@ import {
useGetFoldersByEnv,
useGetImportedSecretsAllEnvs,
useGetProjectSecretsAllEnv,
useGetUserWsKey,
useUpdateSecretV3
} from "@app/hooks/api";
import { useUpdateFolderBatch } from "@app/hooks/api/secretFolders/queries";
@@ -70,7 +69,7 @@ import { ProjectVersion } from "@app/hooks/api/workspace/types";
import { FolderForm } from "../SecretMainPage/components/ActionBar/FolderForm";
import { CreateSecretForm } from "./components/CreateSecretForm";
import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs";
import { ProjectIndexSecretsSection } from "./components/ProjectIndexSecretsSection";
import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection";
import { SecretOverviewDynamicSecretRow } from "./components/SecretOverviewDynamicSecretRow";
import { SecretOverviewFolderRow } from "./components/SecretOverviewFolderRow";
import { SecretOverviewTableRow } from "./components/SecretOverviewTableRow";
@@ -103,7 +102,6 @@ export const SecretOverviewPage = () => {
const { currentOrg } = useOrganization();
const workspaceId = currentWorkspace?.id as string;
const projectSlug = currentWorkspace?.slug as string;
const { data: latestFileKey } = useGetUserWsKey(workspaceId);
const [searchFilter, setSearchFilter] = useState("");
const secretPath = (router.query?.secretPath as string) || "/";
@@ -475,7 +473,7 @@ export const SecretOverviewPage = () => {
return (
<>
<div className="container mx-auto px-6 text-mineshaft-50 dark:[color-scheme:dark]">
<ProjectIndexSecretsSection decryptFileKey={latestFileKey!} />
<SecretV2MigrationSection />
<div className="relative right-5 ml-4">
<NavHeader pageName={t("dashboard.title")} isProjectRelated />
</div>

View File

@@ -1,109 +0,0 @@
import { ProjectPermissionCan } from "@app/components/permissions";
import {
decryptAssymmetric,
decryptSymmetric
} from "@app/components/utilities/cryptography/crypto";
import { Button, Spinner } from "@app/components/v2";
import {
ProjectPermissionActions,
ProjectPermissionSub,
useProjectPermission,
useWorkspace
} from "@app/context";
import { useToggle } from "@app/hooks";
import { useGetWorkspaceIndexStatus, useNameWorkspaceSecrets } from "@app/hooks/api";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
import { UserWsKeyPair } from "@app/hooks/api/types";
import { fetchWorkspaceSecrets } from "@app/hooks/api/workspace/queries";
// TODO: add check so that this only shows up if user is
// an admin in the workspace
type Props = {
decryptFileKey: UserWsKeyPair;
};
export const ProjectIndexSecretsSection = ({ decryptFileKey }: Props) => {
const { currentWorkspace } = useWorkspace();
const { membership } = useProjectPermission();
const { data: isBlindIndexed, isLoading: isBlindIndexedLoading } = useGetWorkspaceIndexStatus(
currentWorkspace?.id ?? ""
);
const [isIndexing, setIsIndexing] = useToggle();
const nameWorkspaceSecrets = useNameWorkspaceSecrets();
const onEnableBlindIndices = async () => {
if (!currentWorkspace?.id) return;
setIsIndexing.on();
try {
const encryptedSecrets = await fetchWorkspaceSecrets(currentWorkspace.id);
const key = decryptAssymmetric({
ciphertext: decryptFileKey.encryptedKey,
nonce: decryptFileKey.nonce,
publicKey: decryptFileKey.sender.publicKey,
privateKey: localStorage.getItem("PRIVATE_KEY") as string
});
const secretsToUpdate = encryptedSecrets.map((encryptedSecret) => {
const secretName = decryptSymmetric({
ciphertext: encryptedSecret.secretKeyCiphertext,
iv: encryptedSecret.secretKeyIV,
tag: encryptedSecret.secretKeyTag,
key
});
return {
secretName,
secretId: encryptedSecret.id
};
});
await nameWorkspaceSecrets.mutateAsync({
workspaceId: currentWorkspace.id,
secretsToUpdate
});
} catch (err) {
console.log(err);
} finally {
setIsIndexing.off();
}
};
// for non admin this would throw an error
// so no need to render
return !isBlindIndexedLoading && typeof isBlindIndexed === "boolean" && !isBlindIndexed ? (
<div className="mt-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
{isIndexing && (
<div className="absolute top-0 left-0 z-50 flex h-screen w-screen items-center justify-center bg-bunker-500 bg-opacity-80">
<Spinner size="lg" className="text-primary" />
<div className="ml-4 flex flex-col space-y-1">
<div className="text-3xl font-medium">Please wait</div>
<span className="inline-block">Re-indexing your secrets...</span>
</div>
</div>
)}
<p className="mb-2 text-lg font-semibold">Action Required</p>
<p className="mb-4 leading-7 text-gray-400">
Your project was created before the introduction of blind indexing. To continue accessing
secrets by name through the SDK, public API and web dashboard, please enable blind indexing.{" "}
<b>
{membership.role !== ProjectMembershipRole.Admin && "This is an admin only operation."}
</b>
</p>
<ProjectPermissionCan I={ProjectPermissionActions.Edit} a={ProjectPermissionSub.Settings}>
{(isAllowed) => (
<Button
onClick={onEnableBlindIndices}
isDisabled={!isAllowed || membership.role !== ProjectMembershipRole.Admin}
color="mineshaft"
type="submit"
isLoading={isIndexing}
>
Enable Blind Indexing
</Button>
)}
</ProjectPermissionCan>
</div>
) : (
<div />
);
};

View File

@@ -1 +0,0 @@
export { ProjectIndexSecretsSection } from "./ProjectIndexSecretsSection";

View File

@@ -0,0 +1,90 @@
import { createNotification } from "@app/components/notifications";
import { ProjectPermissionCan } from "@app/components/permissions";
import { Button, Spinner } from "@app/components/v2";
import {
ProjectPermissionActions,
ProjectPermissionSub,
useProjectPermission,
useWorkspace
} from "@app/context";
import { useGetWorkspaceById, useMigrateProjectToV3 } from "@app/hooks/api";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
import { ProjectVersion } from "@app/hooks/api/workspace/types";
enum ProjectUpgradeStatus {
InProgress = "IN_PROGRESS",
// Completed -> Will be null if completed. So a completed status is not needed
Failed = "FAILED"
}
export const SecretV2MigrationSection = () => {
const { currentWorkspace } = useWorkspace();
const { data: workspaceDetails } = useGetWorkspaceById(
// if v3 no need to fetch
currentWorkspace?.version === ProjectVersion.V3 ? "" : currentWorkspace?.id || "",
{
refetchInterval:
currentWorkspace?.upgradeStatus === ProjectUpgradeStatus.InProgress ? 3000 : false
}
);
const { membership } = useProjectPermission();
const migrateProjectToV3 = useMigrateProjectToV3();
const isProjectUpgraded = workspaceDetails?.version === ProjectVersion.V3;
if (isProjectUpgraded || currentWorkspace?.version === ProjectVersion.V3) return null;
const isUpgrading = workspaceDetails?.upgradeStatus === ProjectUpgradeStatus.InProgress;
const handleMigrationSecretV2 = async () => {
try {
await migrateProjectToV3.mutateAsync({ workspaceId: currentWorkspace?.id || "" });
createNotification({
text: "Migrated project to new KMS",
type: "success"
});
} catch {
createNotification({
text: "Failed to upgrade project",
type: "error"
});
}
};
// for non admin this would throw an error
// so no need to render
return (
<div className="mt-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
{isUpgrading && (
<div className="absolute top-0 left-0 z-50 flex h-screen w-screen items-center justify-center bg-bunker-500 bg-opacity-80">
<Spinner size="lg" className="text-primary" />
<div className="ml-4 flex flex-col space-y-1">
<div className="text-3xl font-medium">Please wait</div>
<span className="inline-block">Upgrading your project...</span>
</div>
</div>
)}
<p className="mb-2 text-lg font-semibold">Action Required</p>
<p className="mb-4 leading-7 text-gray-400">
There is a new update for your project. Introducing Infisical KMS.
<b>
{membership.role !== ProjectMembershipRole.Admin && "This is an admin only operation."}
</b>
</p>
<ProjectPermissionCan I={ProjectPermissionActions.Edit} a={ProjectPermissionSub.Settings}>
{(isAllowed) => (
<Button
onClick={handleMigrationSecretV2}
isDisabled={
!isAllowed || membership.role !== ProjectMembershipRole.Admin || isUpgrading
}
color="mineshaft"
type="submit"
isLoading={migrateProjectToV3.isLoading}
>
Start Migration
</Button>
)}
</ProjectPermissionCan>
</div>
);
};

View File

@@ -0,0 +1 @@
export { SecretV2MigrationSection } from "./SecretV2MigrationSection";