feat: added secret approval policy in replication

This commit is contained in:
Akhil Mohan
2024-05-01 22:46:13 +05:30
committed by =
parent 36b7911bcc
commit 0ae286a80e
16 changed files with 530 additions and 360 deletions

View File

@@ -25,6 +25,9 @@ export const mockKeyStore = (): TKeyStoreFactory => {
},
incrementBy: async () => {
return 1;
},
acquireLock: () => {
throw new Error("Not implemented");
}
};
};

View File

@@ -20,6 +20,12 @@ export async function up(knex: Knex): Promise<void> {
t.boolean("isReplicated");
});
}
if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
t.boolean("isReplicated");
});
}
}
export async function down(knex: Knex): Promise<void> {
@@ -36,7 +42,13 @@ export async function down(knex: Knex): Promise<void> {
}
if (await knex.schema.hasTable(TableName.SecretVersion)) {
await knex.schema.alterTable(TableName.Secret, (t) => {
await knex.schema.alterTable(TableName.SecretVersion, (t) => {
t.dropColumns("isReplicated");
});
}
if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) {
await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => {
t.dropColumns("isReplicated");
});
}

View File

@@ -31,7 +31,8 @@ export const SecretApprovalRequestsSecretsSchema = z.object({
requestId: z.string().uuid(),
op: z.string(),
secretId: z.string().uuid().nullable().optional(),
secretVersion: z.string().uuid().nullable().optional()
secretVersion: z.string().uuid().nullable().optional(),
isReplicated: z.boolean().nullable().optional()
});
export type TSecretApprovalRequestsSecrets = z.infer<typeof SecretApprovalRequestsSecretsSchema>;

View File

@@ -16,8 +16,15 @@ import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
import {
fnSecretBlindIndexCheck,
fnSecretBlindIndexCheckV2,
fnSecretBulkDelete,
fnSecretBulkInsert,
fnSecretBulkUpdate
} from "@app/services/secret/secret-fns";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { SecretOperations } from "@app/services/secret/secret-types";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
@@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ
import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal";
import {
ApprovalStatus,
CommitType,
RequestState,
TApprovalRequestCountDTO,
TGenerateSecretApprovalRequestDTO,
@@ -48,7 +54,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
secretDAL: TSecretDALFactory;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
@@ -56,16 +62,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretService: Pick<
TSecretServiceFactory,
| "fnSecretBulkInsert"
| "fnSecretBulkUpdate"
| "fnSecretBlindIndexCheck"
| "fnSecretBulkDelete"
| "fnSecretBlindIndexCheckV2"
>;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
};
export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -82,7 +79,6 @@ export const secretApprovalRequestServiceFactory = ({
projectDAL,
permissionService,
snapshotService,
secretService,
secretVersionDAL,
secretQueueService,
projectBotService
@@ -302,11 +298,12 @@ export const secretApprovalRequestServiceFactory = ({
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
const conflicts: Array<{ secretId: string; op: CommitType }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create);
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
@@ -319,17 +316,19 @@ export const secretApprovalRequestServiceFactory = ({
secretCreationCommits
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
.forEach((el) => {
conflicts.push({ op: CommitType.Create, secretId: el.id });
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
);
}
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update);
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
userId: "",
inputSecrets: secretUpdationCommits
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
.map(({ secretBlindIndex }) => {
@@ -347,7 +346,7 @@ export const secretApprovalRequestServiceFactory = ({
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
)
.forEach((el) => {
conflicts.push({ op: CommitType.Update, secretId: el.id });
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
secretUpdationCommits = secretUpdationCommits.filter(
@@ -356,11 +355,11 @@ export const secretApprovalRequestServiceFactory = ({
);
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await secretService.fnSecretBulkInsert({
? await fnSecretBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
@@ -380,7 +379,8 @@ export const secretApprovalRequestServiceFactory = ({
"secretReminderRepeatDays",
"algorithm",
"keyEncoding",
"secretBlindIndex"
"secretBlindIndex",
"isReplicated"
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
@@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({
})
: [];
const updatedSecrets = secretUpdationCommits.length
? await secretService.fnSecretBulkUpdate({
? await fnSecretBulkUpdate({
folderId,
projectId,
tx,
@@ -425,6 +425,7 @@ export const secretApprovalRequestServiceFactory = ({
"secretKeyTag",
"secretKeyIV",
"metadata",
"isReplicated",
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
@@ -449,11 +450,13 @@ export const secretApprovalRequestServiceFactory = ({
})
: [];
const deletedSecret = secretDeletionCommits.length
? await secretService.fnSecretBulkDelete({
? await fnSecretBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
@@ -480,12 +483,33 @@ export const secretApprovalRequestServiceFactory = ({
};
});
await snapshotService.performSnapshot(folderId);
const folder = await folderDAL.findById(folderId);
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
// TODO(akhilmhdh-pg): change query to do secret path from folder
await secretQueueService.syncSecrets({
projectId,
secretPath: "/",
environment: folder?.environment.envSlug as string
secretPath: folder.path,
environmentSlug: folder.environmentSlug,
folderId: folder.id,
membershipId: membership.id,
environmentId: folder.envId,
secrets: mergeStatus.secrets.created
.map(({ id, version }) => ({
operation: SecretOperations.Create,
version,
id
}))
.concat(
mergeStatus.secrets.updated.map(({ id, version }) => ({
operation: SecretOperations.Update,
version,
id
})),
mergeStatus.secrets.deleted.map(({ id, version }) => ({
operation: SecretOperations.Delete,
version,
id
}))
)
});
return mergeStatus;
};
@@ -533,9 +557,9 @@ export const secretApprovalRequestServiceFactory = ({
const commits: Omit<TSecretApprovalRequestsSecretsInsert, "requestId">[] = [];
const commitTagIds: Record<string, string[]> = {};
// for created secret approval change
const createdSecrets = data[CommitType.Create];
const createdSecrets = data[SecretOperations.Create];
if (createdSecrets && createdSecrets?.length) {
const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: createdSecrets,
folderId,
isNew: true,
@@ -546,7 +570,7 @@ export const secretApprovalRequestServiceFactory = ({
commits.push(
...createdSecrets.map(({ secretName, ...el }) => ({
...el,
op: CommitType.Create as const,
op: SecretOperations.Create as const,
version: 1,
secretBlindIndex: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
@@ -558,12 +582,12 @@ export const secretApprovalRequestServiceFactory = ({
});
}
// not secret approval for update operations
const updatedSecrets = data[CommitType.Update];
const updatedSecrets = data[SecretOperations.Update];
if (updatedSecrets && updatedSecrets?.length) {
// get all blind index
// Find all those secrets
// if not throw not found
const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({
const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({
inputSecrets: updatedSecrets,
folderId,
isNew: false,
@@ -574,7 +598,7 @@ export const secretApprovalRequestServiceFactory = ({
// now find any secret that needs to update its name
// same process as above
const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: nameUpdatedSecrets,
folderId,
isNew: true,
@@ -599,7 +623,7 @@ export const secretApprovalRequestServiceFactory = ({
return {
...latestSecretVersions[secretId],
...el,
op: CommitType.Update as const,
op: SecretOperations.Update as const,
secret: secretId,
secretVersion: latestSecretVersions[secretId].id,
secretBlindIndex,
@@ -609,12 +633,12 @@ export const secretApprovalRequestServiceFactory = ({
);
}
// deleted secrets
const deletedSecrets = data[CommitType.Delete];
const deletedSecrets = data[SecretOperations.Delete];
if (deletedSecrets && deletedSecrets.length) {
// get all blind index
// Find all those secrets
// if not throw not found
const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({
const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({
inputSecrets: deletedSecrets,
folderId,
isNew: false,
@@ -635,7 +659,7 @@ export const secretApprovalRequestServiceFactory = ({
if (!latestSecretVersions[secretId].secretBlindIndex)
throw new BadRequestError({ message: "Failed to find secret blind index" });
return {
op: CommitType.Delete as const,
op: SecretOperations.Delete as const,
...latestSecretVersions[secretId],
secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string,
secret: secretId,

View File

@@ -1,11 +1,6 @@
import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas";
import { TProjectPermission } from "@app/lib/types";
export enum CommitType {
Create = "create",
Update = "update",
Delete = "delete"
}
import { SecretOperations } from "@app/services/secret/secret-types";
export enum RequestState {
Open = "open",
@@ -18,14 +13,14 @@ export enum ApprovalStatus {
REJECTED = "rejected"
}
type TApprovalCreateSecret = Omit<
export type TApprovalCreateSecret = Omit<
TSecretApprovalRequestsSecrets,
TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex"
> & {
secretName: string;
tagIds?: string[];
};
type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
secretName: string;
newSecretName?: string;
tagIds?: string[];
@@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = {
secretPath: string;
policy: TSecretApprovalPolicies;
data: {
[CommitType.Create]?: TApprovalCreateSecret[];
[CommitType.Update]?: TApprovalUpdateSecret[];
[CommitType.Delete]?: { secretName: string }[];
[SecretOperations.Create]?: TApprovalCreateSecret[];
[SecretOperations.Update]?: TApprovalUpdateSecret[];
[SecretOperations.Delete]?: { secretName: string }[];
};
} & TProjectPermission;

View File

@@ -7,7 +7,7 @@ import {
TScanFullRepoEventPayload,
TScanPushEventPayload
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types";
import { TSyncSecretsDTO } from "@app/services/secret/secret-types";
export enum QueueName {
SecretRotation = "secret-rotation",
@@ -23,7 +23,8 @@ export enum QueueName {
SecretPushEventScan = "secret-push-event-scan",
UpgradeProjectToGhost = "upgrade-project-to-ghost",
DynamicSecretRevocation = "dynamic-secret-revocation",
SecretReplication = "secret-replication"
SecretReplication = "secret-replication",
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
}
export enum QueueJobs {
@@ -40,7 +41,8 @@ export enum QueueJobs {
UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
DynamicSecretRevocation = "dynamic-secret-revocation",
DynamicSecretPruning = "dynamic-secret-pruning",
SecretReplication = "secret-replication"
SecretReplication = "secret-replication",
SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
}
export type TQueueJobTypes = {
@@ -121,7 +123,11 @@ export type TQueueJobTypes = {
};
[QueueName.SecretReplication]: {
name: QueueJobs.SecretReplication;
payload: TSyncSecretReplicationDTO;
payload: Omit<TSyncSecretsDTO, "environmentSlug">;
};
[QueueName.SecretSync]: {
name: QueueJobs.SecretSync;
payload: TSyncSecretsDTO;
};
};

View File

@@ -243,8 +243,8 @@ export const registerRoutes = async (
const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db);
const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db);
const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db);
const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
const sarSecretDAL = secretApprovalRequestSecretDALFactory(db);
const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db);
const secretRotationDAL = secretRotationDALFactory(db);
const snapshotDAL = snapshotDALFactory(db);
@@ -492,7 +492,7 @@ export const registerRoutes = async (
projectBotDAL,
projectMembershipDAL,
secretApprovalRequestDAL,
secretApprovalSecretDAL: sarSecretDAL,
secretApprovalSecretDAL: secretApprovalRequestSecretDAL,
projectUserMembershipRoleDAL
});
@@ -603,17 +603,6 @@ export const registerRoutes = async (
secretDAL,
secretBlindIndexDAL
});
const secretReplicationService = secretReplicationServiceFactory({
secretTagDAL,
secretVersionTagDAL,
secretDAL,
secretVersionDAL,
secretImportDAL,
keyStore,
queueService,
secretReplicationDAL,
folderDAL
});
const secretService = secretServiceFactory({
folderDAL,
secretVersionDAL,
@@ -625,7 +614,6 @@ export const registerRoutes = async (
secretTagDAL,
snapshotService,
secretQueueService,
secretReplicationService,
secretImportDAL,
projectEnvDAL,
projectBotService
@@ -636,19 +624,18 @@ export const registerRoutes = async (
secretSharingDAL
});
const sarService = secretApprovalRequestServiceFactory({
const secretApprovalRequestService = secretApprovalRequestServiceFactory({
permissionService,
projectBotService,
folderDAL,
secretDAL,
secretTagDAL,
secretApprovalRequestSecretDAL: sarSecretDAL,
secretApprovalRequestReviewerDAL: sarReviewerDAL,
secretApprovalRequestSecretDAL,
secretApprovalRequestReviewerDAL,
projectDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretApprovalRequestDAL,
secretService,
snapshotService,
secretVersionTagDAL,
secretQueueService
@@ -677,6 +664,21 @@ export const registerRoutes = async (
accessApprovalPolicyApproverDAL
});
const secretReplicationService = secretReplicationServiceFactory({
secretTagDAL,
secretVersionTagDAL,
secretDAL,
secretVersionDAL,
secretImportDAL,
keyStore,
queueService,
secretReplicationDAL,
folderDAL,
secretApprovalPolicyService,
secretBlindIndexDAL,
secretApprovalRequestDAL,
secretApprovalRequestSecretDAL
});
const secretRotationQueue = secretRotationQueueFactory({
telemetryService,
secretRotationDAL,
@@ -858,10 +860,10 @@ export const registerRoutes = async (
identityGcpAuth: identityGcpAuthService,
identityAwsAuth: identityAwsAuthService,
identityAzureAuth: identityAzureAuthService,
secretApprovalPolicy: sapService,
accessApprovalPolicy: accessApprovalPolicyService,
accessApprovalRequest: accessApprovalRequestService,
secretApprovalRequest: sarService,
secretApprovalPolicy: secretApprovalPolicyService,
secretApprovalRequest: secretApprovalRequestService,
secretRotation: secretRotationService,
dynamicSecret: dynamicSecretService,
dynamicSecretLease: dynamicSecretLeaseService,

View File

@@ -9,7 +9,6 @@ import {
ServiceTokenScopes
} from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs";
import { BadRequestError } from "@app/lib/errors";
import { removeTrailingSlash } from "@app/lib/fn";
@@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
import { ProjectFilterType } from "@app/services/project/project-types";
import { SecretOperations } from "@app/services/secret/secret-types";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
import { secretRawSchema } from "../sanitizedSchemas";
@@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Create]: [
[SecretOperations.Create]: [
{
secretName: req.params.secretName,
secretValueCiphertext,
@@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Update]: [
[SecretOperations.Update]: [
{
secretName: req.params.secretName,
newSecretName,
@@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Delete]: [
[SecretOperations.Delete]: [
{
secretName: req.params.secretName
}
@@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Create]: inputSecrets
[SecretOperations.Create]: inputSecrets
}
});
@@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared")
[SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared")
}
});
@@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId,
policy,
data: {
[CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared")
[SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared")
}
});
await server.services.auditLog.createAuditLog({

View File

@@ -111,7 +111,8 @@ export const secretImportServiceFactory = ({
await secretQueueService.syncSecrets({
secretPath: secImport.importPath,
projectId,
environment: importEnv.slug
environmentSlug: importEnv.slug,
excludeReplication: true
});
return { ...secImport, importEnv };
@@ -223,7 +224,8 @@ export const secretImportServiceFactory = ({
await secretQueueService.syncSecrets({
secretPath: path,
projectId,
environment
environmentSlug: environment,
excludeReplication: true
});
return secImport;

View File

@@ -1,28 +1,40 @@
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretReplicationDALFactory } from "./secret-replication-dal";
import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
type TSecretReplicationServiceFactoryDep = {
secretReplicationDAL: TSecretReplicationDALFactory;
secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
secretApprovalRequestSecretDAL: Pick<
TSecretApprovalRequestSecretDALFactory,
"insertMany" | "insertApprovalSecretTags"
>;
};
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
@@ -44,18 +56,21 @@ export const secretReplicationServiceFactory = ({
keyStore,
secretVersionTagDAL,
secretTagDAL,
folderDAL
folderDAL,
secretApprovalPolicyService,
secretApprovalRequestSecretDAL,
secretApprovalRequestDAL
}: TSecretReplicationServiceFactoryDep) => {
queueService.start(QueueName.SecretReplication, async (job) => {
logger.info(job.data, "Replication started");
const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data;
const secretImports = await secretImportDAL.find({
importPath: secretPath,
importEnv: environmentId,
isReplication: true
});
console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
if (!secretImports.length) return;
if (!secretImports.length || !secrets.length) return;
// unfiltered secrets to be replicated
console.log(secrets.length);
@@ -75,7 +90,8 @@ export const secretReplicationServiceFactory = ({
try {
/* eslint-disable no-await-in-loop */
for (const secretImport of secretImports) {
const importFolderId = secretImport.folderId;
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
const importFolderId = importedFolder.id;
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
@@ -86,14 +102,14 @@ export const secretReplicationServiceFactory = ({
const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
return (
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
});
const locallyUpdatedSecrets = secrets.filter(
({ operation, id }) =>
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
(operation === SecretOperations.Create || operation === SecretOperations.Update) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
@@ -102,35 +118,48 @@ export const secretReplicationServiceFactory = ({
console.log("update", locallyUpdatedSecrets);
console.log("local board", localSecrets);
const locallyDeletedSecrets = secrets
.filter(
({ operation, id }) =>
operation === SecretReplicationOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
)
.map(
({ id }) =>
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
);
const locallyDeletedSecrets = secrets.filter(
({ operation, id }) =>
operation === SecretOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
projectId,
importedFolder.environmentSlug,
importedFolder.path
);
// this means it should be a approval request rather than direct replication
if (policy) {
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
importFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: importFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerId: membershipId
},
tx
);
const commits = locallyCreatedSecrets
.concat(locallyUpdatedSecrets)
.concat(locallyDeletedSecrets)
.map(({ id, operation }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
return {
op: operation,
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
folderId,
type: doc.type,
requestId: approvalRequestDoc.id,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
@@ -143,31 +172,31 @@ export const secretReplicationServiceFactory = ({
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
skipMultilineEncoding: doc.skipMultilineEncoding,
// except create operation other two needs the secret id and version id
...(operation !== SecretOperations.Create
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
: {})
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
});
const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
return { ...approvalRequestDoc, commits: approvalCommits };
});
} else {
let nestedImportSecrets: TSyncSecretsDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
@@ -184,39 +213,80 @@ export const secretReplicationServiceFactory = ({
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
},
isReplicated: true,
folderId: importFolderId
},
isReplicated: true,
folderId: importFolderId
},
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
);
}
});
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
folderId: folder.id,
projectId,
secrets: nestedImportSecrets,
secretPath: folder.path,
environmentId: folder.envId
});
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id }))
);
}
});
console.log("Environment ID -> slug", importedFolder.envId, importedFolder.environmentSlug);
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
folderId: importedFolder.id,
projectId,
secrets: nestedImportSecrets,
secretPath: importedFolder.path,
environmentId: importedFolder.envId,
membershipId
});
}
}
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
/* eslint-enable no-await-in-loop */
@@ -228,20 +298,4 @@ export const secretReplicationServiceFactory = ({
queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
logger.error(err, "Failed to replicate secret", job?.data);
});
const replicate = async (data: TSyncSecretReplicationDTO) => {
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
},
removeOnComplete: true,
removeOnFail: true
});
};
return {
replicate
};
};

View File

@@ -1,17 +1,3 @@
export enum SecretReplicationOperations {
Create = "create",
Update = "update",
Delete = "delete"
}
export type TSyncSecretReplicationDTO = {
secretPath: string;
projectId: string;
environmentId: string;
folderId: string;
secrets: {
operation: SecretReplicationOperations;
id: string;
version: number;
}[];
id: string;
};

View File

@@ -32,6 +32,8 @@ import {
TCreateManySecretsRawFn,
TCreateManySecretsRawFnFactory,
TFnSecretBlindIndexCheck,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TFnSecretBulkInsert,
TFnSecretBulkUpdate,
TUpdateManySecretsRawFn,
@@ -396,6 +398,30 @@ export const decryptSecretRaw = (
};
};
// this is used when secret blind index already exist
// mainly for secret approval
export const fnSecretBlindIndexCheckV2 = async ({
inputSecrets,
folderId,
userId,
secretDAL
}: TFnSecretBlindIndexCheckV2) => {
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretBlindIndex, type }) => ({
blindIndex: secretBlindIndex,
type: type || SecretType.Shared
})),
userId
);
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
return { secsGroupedByBlindIndex, secrets };
};
/**
* Grabs and processes nested secret references from a string
*
@@ -598,6 +624,35 @@ export const fnSecretBulkUpdate = async ({
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const fnSecretBulkDelete = async ({
folderId,
inputSecrets,
tx,
actorId,
secretDAL,
secretQueueService
}: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
blindIndex: secretBlindIndex,
type
})),
folderId,
actorId,
tx
);
await Promise.allSettled(
deletedSecrets
.filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
.map(({ id, secretReminderRepeatDays }) =>
secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
)
);
return deletedSecrets;
};
export const createManySecretsRawFnFactory = ({
projectDAL,
projectBotDAL,

View File

@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
import { fnTriggerWebhook } from "../webhook/webhook-fns";
import { TSecretDALFactory } from "./secret-dal";
import { interpolateSecrets } from "./secret-fns";
import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
import {
TCreateSecretReminderDTO,
THandleReminderDTO,
TRemoveSecretReminderDTO,
TSyncSecretsDTO
} from "./secret-types";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
@@ -116,21 +121,17 @@ export const secretQueueFactory = ({
});
};
const syncSecrets = async ({
deDupeQueue = {},
...dto
}: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
const syncSecrets = async <T extends boolean = false>({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO<T>) => {
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
);
const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath);
if (deDupeQueue?.[deDuplicationKey]) {
return;
}
// eslint-disable-next-line
deDupeQueue[deDuplicationKey] = true;
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]`
);
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, {
removeOnFail: true,
removeOnComplete: true,
delay: 1000,
@@ -140,7 +141,6 @@ export const secretQueueFactory = ({
delay: 3000
}
});
await syncIntegrations({ ...dto, deDupeQueue });
};
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -333,6 +333,52 @@ export const secretQueueFactory = ({
return content;
};
queueService.start(QueueName.SecretSync, async (job) => {
const {
secretPath,
environmentId,
projectId,
environmentSlug: environment,
secrets,
folderId,
excludeReplication,
membershipId
} = job.data;
await queueService.queue(
QueueName.SecretWebhook,
QueueJobs.SecWebhook,
{ environment, projectId, secretPath },
{
jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`,
removeOnFail: { count: 5 },
removeOnComplete: true,
delay: 1000,
attempts: 5,
backoff: {
type: "exponential",
delay: 3000
}
}
);
await syncIntegrations({ secretPath, projectId, environment });
if (!excludeReplication) {
await queueService.queue(
QueueName.SecretReplication,
QueueJobs.SecretReplication,
{ environmentId, projectId, secretPath, folderId, secrets, membershipId },
{
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
},
removeOnComplete: true,
removeOnFail: true
}
);
}
});
queueService.start(QueueName.IntegrationSync, async (job) => {
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
@@ -375,11 +421,12 @@ export const secretQueueFactory = ({
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
_depth: depth + 1,
projectId,
secretPath: foldersGroupedById[folderId][0].path,
environment: foldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
environmentSlug: foldersGroupedById[folderId][0].environmentSlug,
_deDupeQueue: deDupeQueue,
excludeReplication: true
})
)
);
@@ -412,11 +459,12 @@ export const secretQueueFactory = ({
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
_depth: depth + 1,
projectId,
secretPath: referencedFoldersGroupedById[folderId][0].path,
environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
environmentSlug: referencedFoldersGroupedById[folderId][0].environmentSlug,
_deDupeQueue: deDupeQueue,
excludeReplication: true
})
)
);
@@ -546,7 +594,7 @@ export const secretQueueFactory = ({
return {
// depth is internal only field thus no need to make it available outside
syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
syncSecrets,
syncIntegrations,
addSecretReminder,
removeSecretReminder,

View File

@@ -30,13 +30,12 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
decryptSecretRaw,
fnSecretBlindIndexCheck,
fnSecretBulkDelete,
fnSecretBulkInsert,
fnSecretBulkUpdate,
getAllNestedSecretReferences,
@@ -45,6 +44,7 @@ import {
} from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
SecretOperations,
TAttachSecretTagsDTO,
TBackFillSecretReferencesDTO,
TCreateBulkSecretDTO,
@@ -55,8 +55,6 @@ import {
TDeleteManySecretRawDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TGetASecretDTO,
TGetASecretRawDTO,
TGetSecretsDTO,
@@ -87,7 +85,6 @@ type TSecretServiceFactoryDep = {
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
};
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -104,8 +101,7 @@ export const secretServiceFactory = ({
projectDAL,
projectBotService,
secretImportDAL,
secretVersionTagDAL,
secretReplicationService
secretVersionTagDAL
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
@@ -143,53 +139,6 @@ export const secretServiceFactory = ({
return secretBlindIndex;
};
const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
blindIndex: secretBlindIndex,
type
})),
folderId,
actorId,
tx
);
for (const s of deletedSecrets) {
if (s.secretReminderRepeatDays) {
// eslint-disable-next-line no-await-in-loop
await secretQueueService
.removeSecretReminder({
secretId: s.id,
repeatDays: s.secretReminderRepeatDays
})
.catch((err) => {
logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
});
}
}
return deletedSecrets;
};
// this is used when secret blind index already exist
// mainly for secret approval
const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretBlindIndex, type }) => ({
blindIndex: secretBlindIndex,
type: type || SecretType.Shared
})),
userId
);
const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
return { secsGroupedByBlindIndex, secrets };
};
const createSecret = async ({
path,
actor,
@@ -200,7 +149,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TCreateSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -287,16 +236,17 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
secretPath: path,
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Create,
operation: SecretOperations.Create,
id: secret[0].id,
version: 1
}
@@ -315,7 +265,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TUpdateSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -430,15 +380,18 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Update,
operation: SecretOperations.Update,
id: updatedSecret[0].id,
version: updatedSecret[0].version
}
@@ -457,7 +410,7 @@ export const secretServiceFactory = ({
projectId,
...inputSecret
}: TDeleteSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -499,6 +452,8 @@ export const secretServiceFactory = ({
projectId,
folderId,
actorId,
secretDAL,
secretQueueService,
inputSecrets: [
{
type: inputSecret.type as SecretType,
@@ -510,15 +465,18 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: [
{
operation: SecretReplicationOperations.Delete,
operation: SecretOperations.Delete,
id: deletedSecret[0].id,
version: deletedSecret[0].version
}
@@ -744,7 +702,7 @@ export const secretServiceFactory = ({
projectId,
secrets: inputSecrets
}: TCreateBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -808,13 +766,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create }))
});
return newSecrets;
@@ -830,7 +791,7 @@ export const secretServiceFactory = ({
projectId,
secrets: inputSecrets
}: TUpdateBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -915,13 +876,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update }))
});
return secrets;
@@ -937,7 +901,7 @@ export const secretServiceFactory = ({
actorAuthMethod,
actorOrgId
}: TDeleteBulkSecretDTO) => {
const { permission } = await permissionService.getProjectPermission(
const { permission, membership } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
@@ -972,6 +936,8 @@ export const secretServiceFactory = ({
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
secretDAL,
secretQueueService,
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
secretBlindIndex: keyName2BlindIndex[secretName],
type
@@ -984,13 +950,16 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
await secretQueueService.syncSecrets({
// if secret service reached means there was no secret policy
// TODO(akhilmhdh): The policy based replication will fail if machine identity is used.
membershipId: membership?.id as string,
secretPath: path,
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
folderId: folder.id,
projectId,
environmentSlug: folder.environment.slug,
environmentId: folder.envId,
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete }))
});
return secretsDeleted;
@@ -1171,9 +1140,6 @@ export const secretServiceFactory = ({
skipMultilineEncoding
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1212,8 +1178,6 @@ export const secretServiceFactory = ({
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1243,9 +1207,6 @@ export const secretServiceFactory = ({
actorAuthMethod
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
@@ -1294,12 +1255,7 @@ export const secretServiceFactory = ({
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const updateManySecretsRaw = async ({
@@ -1348,12 +1304,7 @@ export const secretServiceFactory = ({
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const deleteManySecretsRaw = async ({
@@ -1384,12 +1335,7 @@ export const secretServiceFactory = ({
secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const getSecretVersions = async ({
@@ -1510,7 +1456,12 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
await secretQueueService.syncSecrets({
secretPath,
projectId: project.id,
environmentSlug: environment,
excludeReplication: true
});
return {
...updatedSecret[0],
@@ -1612,7 +1563,12 @@ export const secretServiceFactory = ({
);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
await secretQueueService.syncSecrets({
secretPath,
projectId: project.id,
environmentSlug: environment,
excludeReplication: true
});
return {
...updatedSecret[0],
@@ -1685,13 +1641,6 @@ export const secretServiceFactory = ({
createManySecretsRaw,
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
backfillSecretReferences,
// external services function
fnSecretBulkDelete,
fnSecretBulkUpdate,
fnSecretBlindIndexCheck,
fnSecretBulkInsert,
fnSecretBlindIndexCheckV2
getSecretVersions
};
};

View File

@@ -264,6 +264,10 @@ export type TFnSecretBulkDelete = {
inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
actorId: string;
tx?: Knex;
secretDAL: Pick<TSecretDALFactory, "deleteMany">;
secretQueueService: {
removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
};
};
export type TFnSecretBlindIndexCheck = {
@@ -277,6 +281,7 @@ export type TFnSecretBlindIndexCheck = {
// when blind index is already present
export type TFnSecretBlindIndexCheckV2 = {
secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
folderId: string;
userId?: string;
inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
@@ -363,3 +368,30 @@ export type TUpdateManySecretsRawFn = {
}[];
userId?: string;
};
export enum SecretOperations {
Create = "create",
Update = "update",
Delete = "delete"
}
export type TSyncSecretsDTO<T extends boolean = false> = {
_depth?: number;
_deDupeQueue?: Record<string, boolean>;
secretPath: string;
projectId: string;
environmentSlug: string;
// cases for just doing sync integration and webhook
excludeReplication?: T;
} & (T extends true
? object
: {
environmentId: string;
folderId: string;
membershipId: string;
secrets: {
operation: SecretOperations;
id: string;
version: number;
}[];
});

View File

@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
try {
if (!secretIds.length) return {};
const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersion}.secretId`, secretIds)