feat: implemented replication to a folder strategy

This commit is contained in:
=
2024-05-30 20:40:36 +05:30
parent f426025fd5
commit 5db6ac711c
12 changed files with 147 additions and 56 deletions

View File

@@ -13,6 +13,7 @@ export async function up(knex: Knex): Promise<void> {
"replicationStatus"
);
const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated");
const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved");
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
@@ -20,6 +21,14 @@ export async function up(knex: Knex): Promise<void> {
if (!doesSecretImportIsReplicationSuccessExist) t.boolean("isReplicationSuccess").nullable();
if (!doesSecretImportReplicationStatusExist) t.text("replicationStatus").nullable();
if (!doesSecretImportLastReplicatedExist) t.datetime("lastReplicated").nullable();
if (!doesSecretImportIsReservedExist) t.boolean("isReserved").defaultTo(false);
});
}
const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved");
if (await knex.schema.hasTable(TableName.SecretFolder)) {
await knex.schema.alterTable(TableName.SecretFolder, (t) => {
if (!doesSecretFolderReservedExist) t.boolean("isReserved").defaultTo(false);
});
}
@@ -69,12 +78,22 @@ export async function down(knex: Knex): Promise<void> {
"replicationStatus"
);
const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated");
const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved");
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
if (doesSecretImportIsReplicationExist) t.dropColumn("isReplication");
if (doesSecretImportIsReplicationSuccessExist) t.dropColumn("isReplicationSuccess");
if (doesSecretImportReplicationStatusExist) t.dropColumn("replicationStatus");
if (doesSecretImportLastReplicatedExist) t.dropColumn("lastReplicated");
if (doesSecretImportIsReservedExist) t.dropColumn("isReserved");
});
}
const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved");
if (await knex.schema.hasTable(TableName.SecretFolder)) {
await knex.schema.alterTable(TableName.SecretFolder, (t) => {
if (doesSecretFolderReservedExist) t.dropColumn("isReserved");
});
}

View File

@@ -14,7 +14,8 @@ export const SecretFoldersSchema = z.object({
createdAt: z.date(),
updatedAt: z.date(),
envId: z.string().uuid(),
parentId: z.string().uuid().nullable().optional()
parentId: z.string().uuid().nullable().optional(),
isReserved: z.boolean().default(false).nullable().optional()
});
export type TSecretFolders = z.infer<typeof SecretFoldersSchema>;

View File

@@ -19,7 +19,8 @@ export const SecretImportsSchema = z.object({
isReplication: z.boolean().default(false).nullable().optional(),
isReplicationSuccess: z.boolean().nullable().optional(),
replicationStatus: z.string().nullable().optional(),
lastReplicated: z.date().nullable().optional()
lastReplicated: z.date().nullable().optional(),
isReserved: z.boolean().default(false).nullable().optional()
});
export type TSecretImports = z.infer<typeof SecretImportsSchema>;

View File

@@ -601,7 +601,7 @@ export const secretApprovalRequestServiceFactory = ({
// same process as above
const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: nameUpdatedSecrets,
inputSecrets: nameUpdatedSecrets.map(({ newSecretName }) => ({ secretName: newSecretName as string })),
folderId,
isNew: true,
blindIndexCfg,
@@ -618,7 +618,7 @@ export const secretApprovalRequestServiceFactory = ({
const secretId = secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0].id;
const secretBlindIndex =
newSecretName && newKeyName2BlindIndex[newSecretName]
? newKeyName2BlindIndex?.[secretName]
? newKeyName2BlindIndex?.[newSecretName]
: keyName2BlindIndex[secretName];
// add tags
if (tagIds?.length) commitTagIds[keyName2BlindIndex[secretName]] = tagIds;

View File

@@ -18,6 +18,7 @@ import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-da
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types";
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
@@ -31,7 +32,7 @@ type TSecretReplicationServiceFactoryDep = {
>;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
@@ -52,6 +53,7 @@ export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicati
const SECRET_IMPORT_SUCCESS_LOCK = 10;
const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`;
const getReplicationKeyLockPrefix = (keyName: string) => `REPLICATION_SECRET_${keyName}`;
export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`;
export const secretReplicationServiceFactory = ({
secretReplicationDAL,
@@ -67,7 +69,6 @@ export const secretReplicationServiceFactory = ({
secretApprovalRequestSecretDAL,
secretApprovalRequestDAL,
secretQueueService,
snapshotService,
projectMembershipDAL
}: TSecretReplicationServiceFactoryDep) => {
queueService.start(QueueName.SecretReplication, async (job) => {
@@ -131,11 +132,25 @@ export const secretReplicationServiceFactory = ({
const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" });
const importFolderId = importedFolder.id;
let replicationFolder = await folderDAL.findOne({
parentId: importedFolder.id,
name: getReplicationFolderName(secretImport.id),
isReserved: true
});
if (!replicationFolder) {
replicationFolder = await folderDAL.create({
parentId: importedFolder.id,
name: getReplicationFolderName(secretImport.id),
envId: importedFolder.envId,
isReserved: true
});
}
const replicationFolderId = replicationFolder.id;
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
folderId: importFolderId
folderId: replicationFolderId
});
const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string);
@@ -181,13 +196,13 @@ export const secretReplicationServiceFactory = ({
const localSecretsLatestVersions = localSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
importFolderId,
replicationFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: importFolderId,
folderId: replicationFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
@@ -237,7 +252,7 @@ export const secretReplicationServiceFactory = ({
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
folderId: replicationFolderId,
secretVersionDAL,
secretDAL,
tx,
@@ -272,7 +287,7 @@ export const secretReplicationServiceFactory = ({
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
folderId: replicationFolderId,
secretVersionDAL,
secretDAL,
tx,
@@ -282,7 +297,7 @@ export const secretReplicationServiceFactory = ({
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
folderId: replicationFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
@@ -317,7 +332,7 @@ export const secretReplicationServiceFactory = ({
id: locallyDeletedSecrets.map(({ id }) => id)
},
isReplicated: true,
folderId: importFolderId
folderId: replicationFolderId
},
tx
);
@@ -327,14 +342,6 @@ export const secretReplicationServiceFactory = ({
}
});
const folderLock = await keyStore
.acquireLock([`secret-replication-${importFolderId}`], 5000)
.catch(() => null);
if (folderLock) {
await snapshotService.performSnapshot(importFolderId);
await folderLock.release();
}
await secretQueueService.syncSecrets({
projectId,
secretPath: importedFolder.path,

View File

@@ -220,7 +220,7 @@ export const secretSnapshotServiceFactory = ({
const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
// this will remove all secrets and folders on child
// due to sql foreign key and link list connection removing the folders removes everything below too
const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx);
const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx);
const deletedTopLevelFolders = groupBy(
deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
(item) => item.id

View File

@@ -276,7 +276,11 @@ export const secretFolderServiceFactory = ({
}
const newFolder = await folderDAL.transaction(async (tx) => {
const [doc] = await folderDAL.update({ envId: env.id, id: folder.id, parentId: parentFolder.id }, { name }, tx);
const [doc] = await folderDAL.update(
{ envId: env.id, id: folder.id, parentId: parentFolder.id, isReserved: false },
{ name },
tx
);
await folderVersionDAL.create(
{
name: doc.name,
@@ -354,7 +358,7 @@ export const secretFolderServiceFactory = ({
const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!parentFolder) return [];
const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id });
const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false });
return folders;
};

View File

@@ -1,5 +1,9 @@
import { TProjectPermission } from "@app/lib/types";
export enum ReservedFolders {
SecretReplication = "__reserve_replication_"
}
export type TCreateFolderDTO = {
environment: string;
path: string;

View File

@@ -15,7 +15,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
try {
const docs = await (tx || db)(TableName.SecretFolderVersion)
.join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`)
.where({ parentId: folderId })
.where({ parentId: folderId, isReserved: false })
.join<TSecretFolderVersions>(
(tx || db)(TableName.SecretFolderVersion)
.groupBy("envId", "folderId")

View File

@@ -20,14 +20,14 @@ export const secretImportDALFactory = (db: TDbClient) => {
return lastPos?.position || 0;
};
const updateAllPosition = async (folderId: string, pos: number, targetPos: number, tx?: Knex) => {
const updateAllPosition = async (folderId: string, pos: number, targetPos: number, positionInc = 1, tx?: Knex) => {
try {
if (targetPos === -1) {
// this means delete
await (tx || db)(TableName.SecretImport)
.where({ folderId })
.andWhere("position", ">", pos)
.decrement("position", 1);
.decrement("position", positionInc);
return;
}
@@ -36,13 +36,13 @@ export const secretImportDALFactory = (db: TDbClient) => {
.where({ folderId })
.where("position", "<=", targetPos)
.andWhere("position", ">", pos)
.decrement("position", 1);
.decrement("position", positionInc);
} else {
await (tx || db)(TableName.SecretImport)
.where({ folderId })
.where("position", ">=", targetPos)
.andWhere("position", "<", pos)
.increment("position", 1);
.increment("position", positionInc);
}
} catch (error) {
throw new DatabaseError({ error, name: "Update position" });
@@ -74,6 +74,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
try {
const docs = await (tx || db)(TableName.SecretImport)
.whereIn("folderId", folderIds)
.where("isReplication", false)
.join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
.select(
db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,

View File

@@ -1,9 +1,12 @@
import path from "node:path";
import { ForbiddenError, subject } from "@casl/ability";
import { TableName } from "@app/db/schemas";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service";
import { BadRequestError } from "@app/lib/errors";
import { TProjectDALFactory } from "../project/project-dal";
@@ -57,7 +60,7 @@ export const secretImportServiceFactory = ({
actorAuthMethod,
projectId,
isReplication,
path
path: secretPath
}: TCreateSecretImportDTO) => {
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -70,7 +73,7 @@ export const secretImportServiceFactory = ({
// check if user has permission to import into destination path
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Create,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
// check if user has permission to import from target path
@@ -92,7 +95,7 @@ export const secretImportServiceFactory = ({
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" });
const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]);
@@ -103,14 +106,14 @@ export const secretImportServiceFactory = ({
const existingImport = await secretImportDAL.findOne({
folderId: sourceFolder.id,
importEnv: folder.environment.id,
importPath: path
importPath: secretPath
});
if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
}
const secImport = await secretImportDAL.transaction(async (tx) => {
const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx);
return secretImportDAL.create(
const doc = await secretImportDAL.create(
{
folderId: folder.id,
position: lastPos + 1,
@@ -120,6 +123,19 @@ export const secretImportServiceFactory = ({
},
tx
);
if (doc.isReplication) {
await secretImportDAL.create(
{
folderId: folder.id,
position: lastPos + 2,
isReserved: true,
importEnv: folder.environment.id,
importPath: path.join(secretPath, getReplicationFolderName(doc.id))
},
tx
);
}
return doc;
});
if (secImport.isReplication && sourceFolder) {
@@ -148,7 +164,7 @@ export const secretImportServiceFactory = ({
};
const updateImport = async ({
path,
path: secretPath,
environment,
projectId,
actor,
@@ -167,10 +183,10 @@ export const secretImportServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Edit,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
const secImpDoc = await secretImportDAL.findOne({ folderId: folder.id, id });
@@ -190,7 +206,7 @@ export const secretImportServiceFactory = ({
const existingImport = await secretImportDAL.findOne({
folderId: sourceFolder.id,
importEnv: folder.environment.id,
importPath: path
importPath: secretPath
});
if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
}
@@ -199,12 +215,31 @@ export const secretImportServiceFactory = ({
const secImp = await secretImportDAL.findOne({ folderId: folder.id, id });
if (!secImp) throw ERR_SEC_IMP_NOT_FOUND;
if (data.position) {
await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, tx);
if (secImp.isReplication) {
await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 2, tx);
} else {
await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 1, tx);
}
}
if (secImp.isReplication) {
const replicationFolderPath = path.join(secretPath, getReplicationFolderName(secImp.id));
await secretImportDAL.update(
{
folderId: folder.id,
importEnv: folder.environment.id,
importPath: replicationFolderPath,
isReserved: true
},
{ position: data?.position ? data.position + 1 : undefined },
tx
);
}
const [doc] = await secretImportDAL.update(
{ id, folderId: folder.id },
{
position: data?.position,
// when moving replicated import, the position is meant for reserved import
// replicated one should always be behind the reserved import
position: data.position,
importEnv: data?.environment ? importedEnv.id : undefined,
importPath: data?.path
},
@@ -216,7 +251,7 @@ export const secretImportServiceFactory = ({
};
const deleteImport = async ({
path,
path: secretPath,
environment,
projectId,
actor,
@@ -234,16 +269,34 @@ export const secretImportServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Delete,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Delete import" });
const secImport = await secretImportDAL.transaction(async (tx) => {
const [doc] = await secretImportDAL.delete({ folderId: folder.id, id }, tx);
if (!doc) throw new BadRequestError({ name: "Sec imp del", message: "Secret import doc not found" });
await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, tx);
if (doc.isReplication) {
const replicationFolderPath = path.join(secretPath, getReplicationFolderName(doc.id));
const replicatedFolder = await folderDAL.findBySecretPath(projectId, environment, replicationFolderPath, tx);
if (replicatedFolder) {
await secretImportDAL.delete(
{
folderId: folder.id,
importEnv: folder.environment.id,
importPath: replicationFolderPath,
isReserved: true
},
tx
);
await folderDAL.deleteById(replicatedFolder.id, tx);
}
await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 2, tx);
} else {
await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 1, tx);
}
const importEnv = await projectEnvDAL.findById(doc.importEnv);
if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" });
@@ -251,7 +304,7 @@ export const secretImportServiceFactory = ({
});
await secretQueueService.syncSecrets({
secretPath: path,
secretPath,
projectId,
environmentSlug: environment,
excludeReplication: true
@@ -267,7 +320,7 @@ export const secretImportServiceFactory = ({
actorOrgId,
actorAuthMethod,
projectId,
path,
path: secretPath,
id: secretImportDocId
}: TResyncSecretImportReplicationDTO) => {
const { permission, membership } = await permissionService.getProjectPermission(
@@ -281,7 +334,7 @@ export const secretImportServiceFactory = ({
// check if user has permission to import into destination path
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Create,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const plan = await licenseService.getPlan(actorOrgId);
@@ -291,7 +344,7 @@ export const secretImportServiceFactory = ({
});
}
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
const [secretImportDoc] = await secretImportDAL.find({
@@ -338,7 +391,7 @@ export const secretImportServiceFactory = ({
};
const getImports = async ({
path,
path: secretPath,
environment,
projectId,
actor,
@@ -355,10 +408,10 @@ export const secretImportServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
const secImports = await secretImportDAL.find({ folderId: folder.id });
@@ -366,7 +419,7 @@ export const secretImportServiceFactory = ({
};
const getSecretsFromImports = async ({
path,
path: secretPath,
environment,
projectId,
actor,
@@ -383,9 +436,9 @@ export const secretImportServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return [];
// this will already order by position
// so anything based on this order will also be in right position

View File

@@ -151,7 +151,8 @@ export const recursivelyGetSecretPaths = ({
// Fetch all folders in env once with a single query
const folders = await folderDAL.find({
envId: env.id
envId: env.id,
isReserved: false
});
// Build the folder hierarchy map