awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added user status signs for replication failure etc

Browse Source
This commit is contained in:
=
2024-05-29 17:29:47 +05:30
parent 25a615cbb3
commit b0c472b5e1
6 changed files with 292 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/db/migrations/20240430024805_secret-replication.ts → backend/src/db/migrations/20240529111503_secret-replication.ts
View File

@@ -6,6 +6,9 @@ export async function up(knex: Knex): Promise<void> {
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
t.boolean("isReplication").defaultTo(false);
t.boolean("isReplicationSuccess").nullable();
t.text("replicationStatus").nullable();
t.datetime("lastReplicated").nullable();
});
}
@@ -38,6 +41,9 @@ export async function down(knex: Knex): Promise<void> {
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
t.dropColumns("isReplication");
t.dropColumns("isReplicationSuccess");
t.dropColumns("replicationStatus");
t.dropColumns("lastReplicated");
});
}

5
backend/src/db/schemas/secret-imports.ts
View File

@@ -16,7 +16,10 @@ export const SecretImportsSchema = z.object({
createdAt: z.date(),
updatedAt: z.date(),
folderId: z.string().uuid(),
isReplication: z.boolean().default(false).nullable().optional()
isReplication: z.boolean().default(false).nullable().optional(),
isReplicationSuccess: z.boolean().nullable().optional(),
replicationStatus: z.string().nullable().optional(),
lastReplicated: z.date().nullable().optional()
});
export type TSecretImports = z.infer<typeof SecretImportsSchema>;

396
backend/src/services/secret-replication/secret-replication-service.ts
View File

@@ -30,7 +30,7 @@ type TSecretReplicationServiceFactoryDep = {
"find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences"
>;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
@@ -115,131 +115,92 @@ export const secretReplicationServiceFactory = ({
try {
/* eslint-disable no-await-in-loop */
for (const secretImport of secretImports) {
const hasJobCompleted = await keyStore.getItem(
keystoreReplicationSuccessKey(job.id as string, secretImport.id),
KeyStorePrefixes.SecretReplication
);
if (hasJobCompleted) {
logger.info(
{ jobId: job.id, importId: secretImport.id },
"Skipping this job as this has been successfully replicated."
try {
const hasJobCompleted = await keyStore.getItem(
keystoreReplicationSuccessKey(job.id as string, secretImport.id),
KeyStorePrefixes.SecretReplication
);
// eslint-disable-next-line
continue;
}
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" });
const importFolderId = importedFolder.id;
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
folderId: importFolderId
});
const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string);
const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => {
return (
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
});
const locallyUpdatedSecrets = sanitizedSecrets.filter(
({ operation, id }) =>
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
const locallyDeletedSecrets = sanitizedSecrets.filter(
({ operation, id }) =>
operation === SecretOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
projectId,
importedFolder.environmentSlug,
importedFolder.path
);
// this means it should be a approval request rather than direct replication
if (policy && actor === ActorType.USER) {
const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId });
if (!membership) {
logger.error("Project membership not found in %s for user %s", projectId, actorId);
return;
if (hasJobCompleted) {
logger.info(
{ jobId: job.id, importId: secretImport.id },
"Skipping this job as this has been successfully replicated."
);
// eslint-disable-next-line
continue;
}
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
importFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: importFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerId: membership.id,
isReplicated: true
},
tx
);
const commits = locallyCreatedSecrets
.concat(locallyUpdatedSecrets)
.concat(locallyDeletedSecrets)
.map(({ id, operation }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
return {
op: operation,
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
requestId: approvalRequestDoc.id,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding,
// except create operation other two needs the secret id and version id
...(operation !== SecretOperations.Create
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
: {})
};
});
const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" });
const importFolderId = importedFolder.id;
return { ...approvalRequestDoc, commits: approvalCommits };
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
folderId: importFolderId
});
} else {
let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string);
const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => {
return (
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
});
const locallyUpdatedSecrets = sanitizedSecrets.filter(
({ operation, id }) =>
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
const locallyDeletedSecrets = sanitizedSecrets.filter(
({ operation, id }) =>
operation === SecretOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
projectId,
importedFolder.environmentSlug,
importedFolder.path
);
// this means it should be a approval request rather than direct replication
if (policy && actor === ActorType.USER) {
const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId });
if (!membership) {
logger.error("Project membership not found in %s for user %s", projectId, actorId);
return;
}
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
importFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: importFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerId: membership.id,
isReplicated: true
},
tx
);
const commits = locallyCreatedSecrets
.concat(locallyUpdatedSecrets)
.concat(locallyDeletedSecrets)
.map(({ id, operation }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
return {
op: operation,
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
requestId: approvalRequestDoc.id,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
@@ -252,31 +213,31 @@ export const secretReplicationServiceFactory = ({
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
skipMultilineEncoding: doc.skipMultilineEncoding,
// except create operation other two needs the secret id and version id
...(operation !== SecretOperations.Create
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
: {})
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
});
const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
return { ...approvalRequestDoc, commits: approvalCommits };
});
} else {
let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
@@ -293,60 +254,119 @@ export const secretReplicationServiceFactory = ({
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
},
isReplicated: true,
folderId: importFolderId
},
isReplicated: true,
folderId: importFolderId
},
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
);
}
});
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
);
}
});
const folderLock = await keyStore
.acquireLock([`secret-replication-${importFolderId}`], 5000)
.catch(() => null);
if (folderLock) {
await snapshotService.performSnapshot(importFolderId);
await folderLock.release();
const folderLock = await keyStore
.acquireLock([`secret-replication-${importFolderId}`], 5000)
.catch(() => null);
if (folderLock) {
await snapshotService.performSnapshot(importFolderId);
await folderLock.release();
}
await secretQueueService.syncSecrets({
projectId,
secretPath: importedFolder.path,
_deDupeReplicationQueue: deDupeReplicationQueue,
_deDupeQueue: deDupeQueue,
environmentSlug: importedFolder.environmentSlug,
actorId,
actor,
secrets: nestedImportSecrets,
folderId: importedFolder.id,
environmentId: importedFolder.envId
});
}
await secretQueueService.syncSecrets({
projectId,
secretPath: importedFolder.path,
_deDupeReplicationQueue: deDupeReplicationQueue,
_deDupeQueue: deDupeQueue,
environmentSlug: importedFolder.environmentSlug,
actorId,
actor,
secrets: nestedImportSecrets,
folderId: importedFolder.id,
environmentId: importedFolder.envId
// this is used to avoid multiple times generating secret approval by failed one
await keyStore.setItemWithExpiry(
keystoreReplicationSuccessKey(job.id as string, secretImport.id),
SECRET_IMPORT_SUCCESS_LOCK,
1,
KeyStorePrefixes.SecretReplication
);
await secretImportDAL.updateById(secretImport.id, {
lastReplicated: new Date(),
replicationStatus: null,
isReplicationSuccess: true
});
} catch (err) {
logger.error(
err,
`Failed to replicate secret with import id=[${secretImport.id}] env=[${secretImport.importEnv.slug}] path=[${secretImport.importPath}]`
);
await secretImportDAL.updateById(secretImport.id, {
lastReplicated: new Date(),
replicationStatus: (err as Error)?.message.slice(0, 500),
isReplicationSuccess: false
});
}
await keyStore.setItemWithExpiry(
keystoreReplicationSuccessKey(job.id as string, secretImport.id),
SECRET_IMPORT_SUCCESS_LOCK,
1,
KeyStorePrefixes.SecretReplication
);
}
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
/* eslint-enable no-await-in-loop */
} finally {

3
frontend/src/hooks/api/secretImports/types.ts
View File

@@ -11,6 +11,9 @@ export type TSecretImport = {
createdAt: string;
updatedAt: string;
isReplication?: boolean;
isReplicationSuccess?: boolean;
replicationStatus?: string;
lastReplicated?: string;
};
export type TGetImportedFoldersByEnvDTO = {

63
frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx
View File

@@ -2,19 +2,31 @@ import { useEffect } from "react";
import { subject } from "@casl/ability";
import { useSortable } from "@dnd-kit/sortable";
import {
faCalendarCheck,
faClose,
faFileImport,
faFolder,
faInfoCircle,
faKey,
faRotate,
faUpDown
faUpDown,
faWarning,
faXmark
} from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { format } from "date-fns";
import { twMerge } from "tailwind-merge";
import { createNotification } from "@app/components/notifications";
import { ProjectPermissionCan } from "@app/components/permissions";
import { EmptyState, IconButton, SecretInput, TableContainer, Tag } from "@app/components/v2";
import {
EmptyState,
IconButton,
SecretInput,
TableContainer,
Tag,
Tooltip
} from "@app/components/v2";
import { ProjectPermissionActions, ProjectPermissionSub, useWorkspace } from "@app/context";
import { useToggle } from "@app/hooks";
import { useResyncSecretReplication } from "@app/hooks/api";
@@ -24,6 +36,9 @@ type Props = {
environment: string;
secretPath?: string;
isReplication?: boolean;
isReplicationSuccess?: boolean;
replicationStatus?: string;
lastReplicated?: string;
importEnvName: string;
importEnvPath: string;
importedSecrets: { key: string; value: string; overriden: { env: string; secretPath: string } }[];
@@ -62,7 +77,10 @@ export const SecretImportItem = ({
importedSecrets = [],
searchTerm = "",
secretPath,
environment
environment,
isReplicationSuccess,
replicationStatus,
lastReplicated
}: Props) => {
const { currentWorkspace } = useWorkspace();
const [isExpanded, setIsExpanded] = useToggle();
@@ -137,7 +155,44 @@ export const SecretImportItem = ({
isReplication={isReplication}
/>
</div>
<div className="flex items-center space-x-2 px-4 py-2">
<div className="flex items-center space-x-4 px-4 py-2">
{lastReplicated && (
<Tooltip
position="left"
className="max-w-md whitespace-normal break-words"
content={
<div className="flex max-h-[10rem] flex-col overflow-auto ">
<div className="flex self-start">
<FontAwesomeIcon icon={faCalendarCheck} className="pt-0.5 pr-2 text-sm" />
<div className="text-sm">Last Replication</div>
</div>
<div className="pl-5 text-left text-xs">
{lastReplicated
? format(new Date(lastReplicated), "yyyy-MM-dd, hh:mm aaa")
: "-"}
</div>
{!isReplicationSuccess && (
<>
<div className="mt-2 flex self-start">
<FontAwesomeIcon icon={faXmark} className="pt-1 pr-2 text-sm" />
<div className="text-sm">Fail reason</div>
</div>
<div className="pl-5 text-left text-xs">{replicationStatus}</div>
</>
)}
</div>
}
>
<div
className={twMerge(
"opacity-0 group-hover:opacity-100",
!isReplicationSuccess && "text-red-600"
)}
>
<FontAwesomeIcon icon={isReplicationSuccess ? faInfoCircle : faWarning} />
</div>
</Tooltip>
)}
<ProjectPermissionCan
I={ProjectPermissionActions.Edit}
a={subject(ProjectPermissionSub.Secrets, { environment, secretPath })}

13
frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx
View File

@@ -159,7 +159,15 @@ export const SecretImportListView = ({
>
<SortableContext items={items} strategy={verticalListSortingStrategy}>
{items?.map((item) => {
const { importPath, importEnv, id, isReplication } = item;
const {
importPath,
importEnv,
id,
isReplication,
replicationStatus,
lastReplicated,
isReplicationSuccess
} = item;
return (
<SecretImportItem
searchTerm={searchTerm}
@@ -168,6 +176,9 @@ export const SecretImportListView = ({
isReplication={isReplication}
importEnvPath={importPath}
importEnvName={importEnv.name}
lastReplicated={lastReplicated}
replicationStatus={replicationStatus}
isReplicationSuccess={isReplicationSuccess}
importedSecrets={computeImportedSecretRows(
importEnv.slug,
importPath,