feat: added first version of migrator to secret v2

backend/src/ee/services/secret-rotation/secret-rotation-dal.ts

@@ -144,6 +144,8 @@ export const secretRotationDALFactory = (db: TDbClient) => {
   const findRotationOutputsV2ByRotationId = async (rotationId: string) =>
     secretRotationOutputV2Orm.find({ rotationId });
 
+  // special query
+
   return {
     ...secretRotationOrm,
     find,

backend/src/lib/knex/index.ts

@@ -104,6 +104,19 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
       throw new DatabaseError({ error, name: "Create" });
     }
   },
+  upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
+    try {
+      if (!data.length) return [];
+      const res = await (tx || db)(tableName)
+        .insert(data as never)
+        .onConflict(onConflictField as never)
+        .merge()
+        .returning("*");
+      return res;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Create" });
+    }
+  },
   updateById: async (
     id: string,
     {

backend/src/server/routes/index.ts

@@ -719,7 +719,9 @@ export const registerRoutes = async (
     kmsService,
     secretVersionV2BridgeDAL,
     secretV2BridgeDAL,
-    secretVersionTagV2BridgeDAL
+    secretVersionTagV2BridgeDAL,
+    secretRotationDAL,
+    integrationAuthDAL
   });
   const secretImportService = secretImportServiceFactory({
     licenseService,

backend/src/services/secret-tag/secret-tag-dal.ts

@@ -3,7 +3,7 @@ import { Knex } from "knex";
 import { TDbClient } from "@app/db";
 import { TableName } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
-import { ormify } from "@app/lib/knex";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
 
 export type TSecretTagDALFactory = ReturnType<typeof secretTagDALFactory>;
 
@@ -35,12 +35,25 @@ export const secretTagDALFactory = (db: TDbClient) => {
     }
   };
 
+  // special query for migration
+  const findSecretTagsByProjectId = async (projectId: string, tx?: Knex) => {
+    try {
+      const tags = await (tx || db.replicaNode())(TableName.JnSecretTag)
+        .join(TableName.SecretTag, `${TableName.JnSecretTag}.secret_tagsId`, `${TableName.SecretTag}.id`)
+        .where({ projectId })
+        .select(selectAllTableCols(TableName.JnSecretTag));
+      return tags;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find all by ids" });
+    }
+  };
   return {
     ...secretTagOrm,
     saveTagsToSecret: secretJnTagOrm.insertMany,
     deleteTagsToSecret: secretJnTagOrm.delete,
     saveTagsToSecretV2: secretV2JnTagOrm.insertMany,
     deleteTagsToSecretV2: secretV2JnTagOrm.delete,
+    findSecretTagsByProjectId,
     deleteTagsManySecret,
     findManyTagsById
   };

backend/src/services/secret/secret-queue.ts

@@ -1,6 +1,8 @@
 /* eslint-disable no-await-in-loop */
 import { AxiosError } from "axios";
 
+import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
+import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
 import { getConfig } from "@app/lib/config/env";
 import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
@@ -16,6 +18,7 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se
 import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
+import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
 import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
 import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret";
 import { TKmsServiceFactory } from "../kms/kms-service";
@@ -28,6 +31,7 @@ import { TProjectMembershipDALFactory } from "../project-membership/project-memb
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
 import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
 import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
 import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
@@ -41,12 +45,12 @@ import {
   TRemoveSecretReminderDTO,
   TSyncSecretsDTO
 } from "./secret-types";
-import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
   queueService: TQueueServiceFactory;
   integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
+  integrationAuthDAL: Pick<TIntegrationAuthDALFactory, "upsert" | "find">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
   folderDAL: TSecretFolderDALFactory;
@@ -67,6 +71,7 @@ type TSecretQueueFactoryDep = {
   secretV2BridgeDAL: TSecretV2BridgeDALFactory;
   secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
   secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
+  secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
 };
 
 export type TGetSecrets = {
@@ -86,6 +91,7 @@ type TIntegrationSecret = Record<
 export const secretQueueFactory = ({
   queueService,
   integrationDAL,
+  integrationAuthDAL,
   projectBotService,
   integrationAuthService,
   secretDAL,
@@ -105,7 +111,8 @@ export const secretQueueFactory = ({
   secretV2BridgeDAL,
   secretVersionV2BridgeDAL,
   kmsService,
-  secretVersionTagV2BridgeDAL
+  secretVersionTagV2BridgeDAL,
+  secretRotationDAL
 }: TSecretQueueFactoryDep) => {
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
     const appCfg = getConfig();
@@ -779,7 +786,6 @@ export const secretQueueFactory = ({
     );
   };
 
-  const MIGRATION_BATCH_SIZE = 10000;
   queueService.start(QueueName.ProjectV3Migration, async (job) => {
     const { projectId } = job.data;
     const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
@@ -798,10 +804,16 @@ export const secretQueueFactory = ({
     await secretDAL.transaction(async (tx) => {
       for (const folder of folders) {
         const folderId = folder.id;
-        let projectV1Secrets;
+        /*
-        do {
+         * Secrets Migration
-          // eslint-disable-next-line no-await-in-loop
+         * */
-          projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx });
+        // eslint-disable-next-line no-await-in-loop
+        const projectV1Secrets = await secretDAL.find({ folderId }, { tx });
+        if (projectV1Secrets.length) {
+          const secretReferences: {
+            secretId: string;
+            references: { environment: string; secretPath: string; secretKey: string }[];
+          }[] = [];
           await secretV2BridgeDAL.insertMany(
             projectV1Secrets.map((el) => {
               const key = decryptSymmetric128BitHexKeyUTF8({
@@ -826,6 +838,10 @@ export const secretQueueFactory = ({
                     })
                   : "";
               const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
+              // create references
+              const references = getAllNestedSecretReferences(value);
+              secretReferences.push({ secretId: el.id, references });
+
               const encryptedComment = comment
                 ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
                 : null;
@@ -848,9 +864,104 @@ export const secretQueueFactory = ({
             }),
             tx
           );
-          projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx);
+          await secretV2BridgeDAL.upsertSecretReferences(secretReferences);
-        } while (projectV1Secrets.length > 0);
+        }
       }
+      /*
+       * Secret Tag Migration
+       * */
+      // eslint-disable-next-line no-await-in-loop
+      const projectV1SecretTags = await secretTagDAL.findSecretTagsByProjectId(projectId, tx);
+      if (projectV1SecretTags.length) {
+        await secretTagDAL.saveTagsToSecretV2(
+          projectV1SecretTags.map((el) => ({
+            secrets_v2Id: el.secretsId,
+            secret_tagsId: el.secret_tagsId
+          })),
+          tx
+        );
+      }
+
+      /*
+       * Integration Auth Migration
+       * Saving the new encrypted colum
+       * */
+      // eslint-disable-next-line no-await-in-loop
+      const projectV1IntegrationAuths = await integrationAuthDAL.find({ projectId }, { tx });
+      await integrationAuthDAL.upsert(
+        projectV1IntegrationAuths.map((el) => {
+          const accessToken =
+            el.accessIV && el.accessTag && el.accessCiphertext
+              ? decryptSymmetric128BitHexKeyUTF8({
+                  ciphertext: el.accessCiphertext,
+                  iv: el.accessIV,
+                  tag: el.accessTag,
+                  key: botKey
+                })
+              : undefined;
+          const accessId =
+            el.accessIdIV && el.accessIdTag && el.accessIdCiphertext
+              ? decryptSymmetric128BitHexKeyUTF8({
+                  ciphertext: el.accessIdCiphertext,
+                  iv: el.accessIdIV,
+                  tag: el.accessIdTag,
+                  key: botKey
+                })
+              : undefined;
+          const refreshToken =
+            el.refreshIV && el.refreshTag && el.refreshCiphertext
+              ? decryptSymmetric128BitHexKeyUTF8({
+                  ciphertext: el.refreshCiphertext,
+                  iv: el.refreshIV,
+                  tag: el.refreshTag,
+                  key: botKey
+                })
+              : undefined;
+          const awsAssumeRoleArn =
+            el.awsAssumeIamRoleArnCipherText && el.awsAssumeIamRoleArnIV && el.awsAssumeIamRoleArnTag
+              ? decryptSymmetric128BitHexKeyUTF8({
+                  ciphertext: el.awsAssumeIamRoleArnCipherText,
+                  iv: el.awsAssumeIamRoleArnIV,
+                  tag: el.awsAssumeIamRoleArnTag,
+                  key: botKey
+                })
+              : undefined;
+
+          const encryptedAccess = accessToken
+            ? secretManagerEncryptor({ plainText: Buffer.from(accessToken) }).cipherTextBlob
+            : null;
+          const encryptedAccessId = accessId
+            ? secretManagerEncryptor({ plainText: Buffer.from(accessId) }).cipherTextBlob
+            : null;
+          const encryptedRefresh = refreshToken
+            ? secretManagerEncryptor({ plainText: Buffer.from(refreshToken) }).cipherTextBlob
+            : null;
+          const encryptedAwsAssumeIamRoleArn = awsAssumeRoleArn
+            ? secretManagerEncryptor({ plainText: Buffer.from(awsAssumeRoleArn) }).cipherTextBlob
+            : null;
+          return {
+            ...el,
+            encryptedAccess,
+            encryptedRefresh,
+            encryptedAccessId,
+            encryptedAwsAssumeIamRoleArn
+          };
+        }),
+        "id",
+        tx
+      );
+      /*
+       * Secret Rotation Secret Migration
+       * Saving the new encrypted colum
+       * */
+
+      const projectV1SecretRotations = await secretRotationDAL.find({ projectId }, tx);
+      await secretRotationDAL.secretOutputV2InsertMany(
+        projectV1SecretRotations.flatMap((el) =>
+          el.outputs.map((output) => ({ rotationId: el.id, key: output.key, secretId: output.secret.id }))
+        ),
+        tx
+      );
       await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
     });
   });

frontend/src/hooks/api/users/types.ts

@@ -66,7 +66,6 @@ export type OrgUser = {
 
 export type TProjectMembership = {
   id: string;
-  role: string;
   createdAt: string;
   updatedAt: string;
   projectId: string;

frontend/src/hooks/api/workspace/index.tsx

@@ -2,9 +2,8 @@ export {
   useAddGroupToWorkspace,
   useDeleteGroupFromWorkspace,
   useLeaveProject,
-  useUpdateGroupWorkspaceRole,
+  useMigrateProjectToV3,
-  useMigrateProjectToV3
+  useUpdateGroupWorkspaceRole} from "./mutations";
-} from "./mutations";
 export {
   useAddIdentityToWorkspace,
   useCreateWorkspace,

frontend/src/hooks/api/workspace/mutations.tsx

@@ -79,7 +79,7 @@ export const useMigrateProjectToV3 = () => {
   const queryClient = useQueryClient();
   return useMutation<{}, {}, { workspaceId: string }>({
     mutationFn: ({ workspaceId }) => {
-      return apiRequest.delete(`/api/v1/workspace/${workspaceId}/migrate-v3`);
+      return apiRequest.post(`/api/v1/workspace/${workspaceId}/migrate-v3`);
     },
     onSuccess: () => {
       queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);

frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx

@@ -69,10 +69,10 @@ import { ProjectVersion } from "@app/hooks/api/workspace/types";
 import { FolderForm } from "../SecretMainPage/components/ActionBar/FolderForm";
 import { CreateSecretForm } from "./components/CreateSecretForm";
 import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs";
-import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection";
 import { SecretOverviewDynamicSecretRow } from "./components/SecretOverviewDynamicSecretRow";
 import { SecretOverviewFolderRow } from "./components/SecretOverviewFolderRow";
 import { SecretOverviewTableRow } from "./components/SecretOverviewTableRow";
+import { SecretV2MigrationSection } from "./components/SecretV2MigrationSection";
 import { SelectionPanel } from "./components/SelectionPanel/SelectionPanel";
 
 export enum EntryType {

frontend/src/views/SecretOverviewPage/components/SecretV2MigrationSection/SecretV2MigrationSection.tsx

@@ -1,13 +1,6 @@
 import { createNotification } from "@app/components/notifications";
-import { ProjectPermissionCan } from "@app/components/permissions";
-
 import { Button, Spinner } from "@app/components/v2";
-import {
+import { useProjectPermission, useWorkspace } from "@app/context";
-  ProjectPermissionActions,
-  ProjectPermissionSub,
-  useProjectPermission,
-  useWorkspace
-} from "@app/context";
 import { useGetWorkspaceById, useMigrateProjectToV3 } from "@app/hooks/api";
 import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
 import { ProjectVersion } from "@app/hooks/api/workspace/types";
@@ -50,8 +43,8 @@ export const SecretV2MigrationSection = () => {
       });
     }
   };
-  // for non admin this would throw an error
+
-  // so no need to render
+  const isAdmin = membership?.roles.includes(ProjectMembershipRole.Admin);
   return (
     <div className="mt-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
       {isUpgrading && (
@@ -66,25 +59,18 @@ export const SecretV2MigrationSection = () => {
       <p className="mb-2 text-lg font-semibold">Action Required</p>
       <p className="mb-4 leading-7 text-gray-400">
         There is a new update for your project. Introducing Infisical KMS.
-        <b>
+        <b>{!isAdmin && "This is an admin only operation."}</b>
-          {membership.role !== ProjectMembershipRole.Admin && "This is an admin only operation."}
-        </b>
       </p>
-      <ProjectPermissionCan I={ProjectPermissionActions.Edit} a={ProjectPermissionSub.Settings}>
+
-        {(isAllowed) => (
+      <Button
-          <Button
+        onClick={handleMigrationSecretV2}
-            onClick={handleMigrationSecretV2}
+        isDisabled={!isAdmin || isUpgrading}
-            isDisabled={
+        color="mineshaft"
-              !isAllowed || membership.role !== ProjectMembershipRole.Admin || isUpgrading
+        type="submit"
-            }
+        isLoading={migrateProjectToV3.isLoading}
-            color="mineshaft"
+      >
-            type="submit"
+        Start Migration
-            isLoading={migrateProjectToV3.isLoading}
+      </Button>
-          >
-            Start Migration
-          </Button>
-        )}
-      </ProjectPermissionCan>
     </div>
   );
 };