From 51c1487ed1f9d73929e2953f2c1507dd7ed8baf5 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Thu, 2 May 2024 19:47:33 +0530 Subject: [PATCH] feat: updated ui to reflect secret import and approval --- backend/src/server/routes/index.ts | 4 +- .../secret-import/secret-import-fns.ts | 2 +- .../secret-import/secret-import-service.ts | 34 ++++++++++---- .../secret-replication-service.ts | 7 ++- backend/src/services/secret/secret-queue.ts | 45 +++++++++++-------- backend/src/services/secret/secret-service.ts | 5 ++- backend/src/services/secret/secret-types.ts | 2 + .../api/secretApprovalRequest/queries.tsx | 1 + .../hooks/api/secretApprovalRequest/types.ts | 1 + frontend/src/hooks/api/secretImports/types.ts | 1 + frontend/src/hooks/api/secrets/queries.tsx | 1 + frontend/src/hooks/api/secrets/types.ts | 2 + .../SecretApprovalRequest.tsx | 8 ++-- .../SecretApprovalRequestChanges.tsx | 1 + .../SecretImportListView/SecretImportItem.tsx | 21 +++++++-- .../SecretImportListView.tsx | 10 ++--- .../components/SecretListView/SecretItem.tsx | 5 +-- 17 files changed, 103 insertions(+), 47 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 605678011..c018cfc26 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -677,7 +677,9 @@ export const registerRoutes = async ( secretApprovalPolicyService, secretBlindIndexDAL, secretApprovalRequestDAL, - secretApprovalRequestSecretDAL + secretApprovalRequestSecretDAL, + secretQueueService, + snapshotService }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index fffc22a99..06ffbc903 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -79,7 +79,7 @@ export const fnSecretsFromImports = async ({ let secretsFromDeeperImports: TSecretImportSecrets[] = []; if (deeperImports.length) { secretsFromDeeperImports = await fnSecretsFromImports({ - allowedImports: deeperImports, + allowedImports: deeperImports.filter(({ isReplication }) => !isReplication), secretImportDAL, folderDAL, secretDAL, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 5503470e4..f002aefbd 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -8,6 +8,7 @@ import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretQueueFactory } from "../secret/secret-queue"; +import { SecretOperations } from "../secret/secret-types"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "./secret-import-dal"; import { fnSecretsFromImports } from "./secret-import-fns"; @@ -26,7 +27,7 @@ type TSecretImportServiceFactoryDep = { projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; - secretQueueService: Pick; + secretQueueService: Pick; }; const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" }); @@ -53,7 +54,7 @@ export const secretImportServiceFactory = ({ isReplication, path }: TCreateSecretImportDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -108,12 +109,27 @@ export const secretImportServiceFactory = ({ ); }); - await secretQueueService.syncSecrets({ - secretPath: secImport.importPath, - projectId, - environmentSlug: importEnv.slug, - excludeReplication: true - }); + if (secImport.isReplication) { + const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); + await secretQueueService.replicateSecrets({ + secretPath: secImport.importPath, + projectId, + environmentSlug: importEnv.slug, + pickOnlyImportIds: [secImport.id], + folderId: sourceFolder?.id as string, + secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), + // TODO(akhilmhdh): approval based replication this will fail for identity + membershipId: membership?.id as string, + environmentId: importEnv.id + }); + } else { + await secretQueueService.syncSecrets({ + secretPath: secImport.importPath, + projectId, + environmentSlug: importEnv.slug, + excludeReplication: true + }); + } return { ...secImport, importEnv }; }; @@ -283,7 +299,7 @@ export const secretImportServiceFactory = ({ if (!folder) return []; // this will already order by position // so anything based on this order will also be in right position - const secretImports = await secretImportDAL.find({ folderId: folder.id }); + const secretImports = await secretImportDAL.find({ folderId: folder.id, isReplication: false }); const allowedImports = secretImports.filter(({ importEnv, importPath }) => permission.can( diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index acb290254..0d126db2f 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -64,12 +64,15 @@ export const secretReplicationServiceFactory = ({ }: TSecretReplicationServiceFactoryDep) => { queueService.start(QueueName.SecretReplication, async (job) => { logger.info(job.data, "Replication started"); - const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data; - const secretImports = await secretImportDAL.find({ + const { secrets, folderId, secretPath, environmentId, projectId, membershipId, pickOnlyImportIds } = job.data; + let secretImports = await secretImportDAL.find({ importPath: secretPath, importEnv: environmentId, isReplication: true }); + secretImports = pickOnlyImportIds + ? secretImports.filter(({ id }) => pickOnlyImportIds?.includes(id)) + : secretImports; if (!secretImports.length || !secrets.length) return; // unfiltered secrets to be replicated diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index e5924fdd3..a4533ebc3 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -143,6 +143,18 @@ export const secretQueueFactory = ({ }); }; + const replicateSecrets = async (dto: TSyncSecretsDTO) => { + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, { + attempts: 3, + backoff: { + type: "exponential", + delay: 2000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); await queueService.stopRepeatableJob( @@ -361,22 +373,17 @@ export const secretQueueFactory = ({ } ); await syncIntegrations({ secretPath, projectId, environment }); - if (!excludeReplication) { - await queueService.queue( - QueueName.SecretReplication, - QueueJobs.SecretReplication, - { environmentId, projectId, secretPath, folderId, secrets, membershipId }, - { - attempts: 3, - backoff: { - type: "exponential", - delay: 2000 - }, - removeOnComplete: true, - removeOnFail: true - } - ); - } + if (!excludeReplication) + await replicateSecrets({ + environmentId, + projectId, + secretPath, + folderId, + secrets, + membershipId, + excludeReplication, + environmentSlug: environment + }); }); queueService.start(QueueName.IntegrationSync, async (job) => { @@ -394,7 +401,8 @@ export const secretQueueFactory = ({ const linkSourceDto = { projectId, importEnv: folder.environment.id, - importPath: secretPath + importPath: secretPath, + isReplication: false }; const imports = await secretImportDAL.find(linkSourceDto); @@ -598,6 +606,7 @@ export const secretQueueFactory = ({ syncIntegrations, addSecretReminder, removeSecretReminder, - handleSecretReminder + handleSecretReminder, + replicateSecrets }; }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index be7333abe..207a3871d 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -550,7 +550,8 @@ export const secretServiceFactory = ({ if (includeImports) { const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); - const allowedImports = secretImports.filter(({ importEnv, importPath }) => + const allowedImports = secretImports.filter(({ importEnv, importPath, isReplication }) => + !isReplication && // if its service token allow full access over imported one actor === ActorType.SERVICE ? true @@ -655,7 +656,7 @@ export const secretServiceFactory = ({ // then search for imported secrets // here we consider the import order also thus starting from bottom if (!secret && includeImports) { - const secretImports = await secretImportDAL.find({ folderId }); + const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); const allowedImports = secretImports.filter(({ importEnv, importPath }) => // if its service token allow full access over imported one actor === ActorType.SERVICE diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index ac1eb6cb8..723aedaf4 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -389,6 +389,8 @@ export type TSyncSecretsDTO = { environmentId: string; folderId: string; membershipId: string; + // used for import creation to trigger replication + pickOnlyImportIds?: string[]; secrets: { operation: SecretOperations; id: string; diff --git a/frontend/src/hooks/api/secretApprovalRequest/queries.tsx b/frontend/src/hooks/api/secretApprovalRequest/queries.tsx index 23017c15d..aaf84941a 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/queries.tsx +++ b/frontend/src/hooks/api/secretApprovalRequest/queries.tsx @@ -220,6 +220,7 @@ export const useGetSecretApprovalRequestCount = ({ }) => useQuery({ queryKey: secretApprovalRequestKeys.count({ workspaceId }), + refetchInterval: 5000, queryFn: () => fetchSecretApprovalRequestCount({ workspaceId }), enabled: Boolean(workspaceId) && (options?.enabled ?? true) }); diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index 32fe31c6b..b7f8784a6 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -44,6 +44,7 @@ export type TSecretApprovalSecChange = { export type TSecretApprovalRequest = { id: string; + isReplication?: boolean; slug: string; createdAt: string; committerId: string; diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 58d38047b..8aafb0e88 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -10,6 +10,7 @@ export type TSecretImport = { position: string; createdAt: string; updatedAt: string; + isReplication?: boolean; }; export type TGetImportedFoldersByEnvDTO = { diff --git a/frontend/src/hooks/api/secrets/queries.tsx b/frontend/src/hooks/api/secrets/queries.tsx index 28999389e..485f9638e 100644 --- a/frontend/src/hooks/api/secrets/queries.tsx +++ b/frontend/src/hooks/api/secrets/queries.tsx @@ -67,6 +67,7 @@ export const decryptSecrets = ( env: encSecret.environment, key: secretKey, value: secretValue, + isReplicated: encSecret.isReplicated, tags: encSecret.tags, comment: secretComment, reminderRepeatDays: encSecret.secretReminderRepeatDays, diff --git a/frontend/src/hooks/api/secrets/types.ts b/frontend/src/hooks/api/secrets/types.ts index f36872e43..d818afe6c 100644 --- a/frontend/src/hooks/api/secrets/types.ts +++ b/frontend/src/hooks/api/secrets/types.ts @@ -14,6 +14,7 @@ export type EncryptedSecret = { secretValueIV: string; secretValueTag: string; __v: number; + isReplicated?: boolean; createdAt: string; updatedAt: string; skipMultilineEncoding?: boolean; @@ -31,6 +32,7 @@ export type DecryptedSecret = { key: string; value: string; comment: string; + isReplicated?: boolean; reminderRepeatDays?: number | null; reminderNote?: string | null; tags: WsTag[]; diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx index 0d0c6213a..8ab051217 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx @@ -212,7 +212,8 @@ export const SecretApprovalRequest = () => { createdAt, policy, reviewers, - status + status, + isReplication } = secretApproval; const isApprover = policy?.approvers?.indexOf(myMembershipId || "") !== -1; const isReviewed = @@ -240,8 +241,9 @@ export const SecretApprovalRequest = () => { Opened {formatDistance(new Date(createdAt), new Date())} ago by{" "} {membersGroupById?.[committerId]?.user?.firstName}{" "} {membersGroupById?.[committerId]?.user?.lastName} ( - {membersGroupById?.[committerId]?.user?.email}){" "} - {isApprover && !isReviewed && status === "open" && "- Review required"} + {membersGroupById?.[committerId]?.user?.email}) + {isReplication && "via replication"} + {isApprover && !isReviewed && status === "open" && " - Review required"} ); diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 80dbe9f73..449d0b774 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -185,6 +185,7 @@ export const SecretApprovalRequestChanges = ({
{generateCommitText(secretApprovalRequestDetails.commits)} + {secretApprovalRequestDetails.isReplication && (replication)}
{committer?.user?.firstName} diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index dd0f5fbec..744a4be54 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -11,7 +11,7 @@ import { import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { ProjectPermissionCan } from "@app/components/permissions"; -import { EmptyState, IconButton, SecretInput, TableContainer } from "@app/components/v2"; +import { EmptyState, IconButton, SecretInput, TableContainer, Tag } from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; import { useToggle } from "@app/hooks"; @@ -19,6 +19,7 @@ type Props = { onDelete: () => void; environment: string; secretPath?: string; + isReplication?: boolean; importEnvName: string; importEnvPath: string; importedSecrets: { key: string; value: string; overriden: { env: string; secretPath: string } }[]; @@ -27,11 +28,20 @@ type Props = { }; // to show the environment and folder icon -export const EnvFolderIcon = ({ env, secretPath }: { env: string; secretPath: string }) => ( +export const EnvFolderIcon = ({ + env, + secretPath, + isReplication +}: { + env: string; + secretPath: string; + isReplication?: boolean; +}) => (
{env || "-"}
{secretPath && (
+ {isReplication && Replication Mode} {secretPath}
@@ -44,6 +54,7 @@ export const SecretImportItem = ({ id, importEnvName, importEnvPath, + isReplication, importedSecrets = [], searchTerm = "", secretPath, @@ -92,7 +103,11 @@ export const SecretImportItem = ({
- +
{items?.map((item) => { - const { importPath, importEnv, id } = item; + const { importPath, importEnv, id, isReplication } = item; return ( handlePopUpToggle("deleteSecretImport", isOpen)} onDeleteApproved={handleSecretImportDelete} /> diff --git a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx index 8b7077417..e8901b8cc 100644 --- a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx @@ -420,9 +420,8 @@ export const SecretItem = memo( 0 - ? `Every ${secretReminderRepeatDays} day${ - Number(secretReminderRepeatDays) > 1 ? "s" : "" - } + ? `Every ${secretReminderRepeatDays} day${Number(secretReminderRepeatDays) > 1 ? "s" : "" + } ` : "Reminder" }