feat: added bridge logic in secret replication, snapshot and approval for raw endpoint

This commit is contained in:
=
2024-07-30 23:11:04 +05:30
parent 9e314d7a09
commit ef2816b2ee
14 changed files with 2205 additions and 283 deletions
@@ -1,6 +1,6 @@
import { ForbiddenError, subject } from "@casl/ability";
import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas";
import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@@ -24,16 +24,24 @@ import { TSnapshotDALFactory } from "./snapshot-dal";
import { TSnapshotFolderDALFactory } from "./snapshot-folder-dal";
import { TSnapshotSecretDALFactory } from "./snapshot-secret-dal";
import { getFullFolderPath } from "./snapshot-service-fns";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSnapshotSecretV2DALFactory } from "./snapshot-secret-v2-dal";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
type TSecretSnapshotServiceFactoryDep = {
snapshotDAL: TSnapshotDALFactory;
snapshotSecretDAL: TSnapshotSecretDALFactory;
snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory;
snapshotFolderDAL: TSnapshotFolderDALFactory;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionByFolderId">;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionByFolderId">;
folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "findLatestVersionByFolderId" | "insertMany">;
secretDAL: Pick<TSecretDALFactory, "delete" | "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "delete" | "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "saveTagsToSecretV2">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretVersionV2TagBridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
folderDAL: Pick<TSecretFolderDALFactory, "findById" | "findBySecretPath" | "delete" | "insertMany" | "find">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
licenseService: Pick<TLicenseServiceFactory, "isValidLicense">;
@@ -52,7 +60,11 @@ export const secretSnapshotServiceFactory = ({
permissionService,
licenseService,
secretTagDAL,
secretVersionTagDAL
secretVersionTagDAL,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
snapshotSecretV2BridgeDAL,
secretVersionV2TagBridgeDAL
}: TSecretSnapshotServiceFactoryDep) => {
const projectSecretSnapshotCount = async ({
environment,
@@ -118,7 +130,7 @@ export const secretSnapshotServiceFactory = ({
};
const getSnapshotData = async ({ actorId, actor, actorOrgId, actorAuthMethod, id }: TGetSnapshotDataDTO) => {
const snapshot = await snapshotDAL.findSecretSnapshotDataById(id);
const snapshot = await snapshotDAL.findById(id);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -127,26 +139,73 @@ export const secretSnapshotServiceFactory = ({
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback);
const shouldUseBridge = snapshot.projectVersion === 3;
let snapshotDetails;
if (shouldUseBridge) {
snapshotDetails = await snapshotDAL.findSecretSnapshotV2DataById(id);
} else {
snapshotDetails = await snapshotDAL.findSecretSnapshotDataById(id);
}
const fullFolderPath = await getFullFolderPath({
folderDAL,
folderId: snapshot.folderId,
envId: snapshot.environment.id
folderId: snapshotDetails.folderId,
envId: snapshotDetails.environment.id
});
// We need to check if the user has access to the secrets in the folder. If we don't do this, a user could theoretically access snapshot secret values even if they don't have read access to the secrets in the folder.
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment: snapshot.environment.slug, secretPath: fullFolderPath })
subject(ProjectPermissionSub.Secrets, {
environment: snapshotDetails.environment.slug,
secretPath: fullFolderPath
})
);
return snapshot;
return snapshotDetails;
};
const performSnapshot = async (folderId: string) => {
const performSnapshot = async (folderId: string, shouldUseSecretV2Bridge: boolean) => {
try {
if (!licenseService.isValidLicense) throw new InternalServerError({ message: "Invalid license" });
if (shouldUseSecretV2Bridge) {
const snapshot = await snapshotDAL.transaction(async (tx) => {
const folder = await folderDAL.findById(folderId, tx);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx);
const folderVersions = await folderVersionDAL.findLatestVersionByFolderId(folderId, tx);
const newSnapshot = await snapshotDAL.create(
{
folderId,
envId: folder.environment.envId,
parentFolderId: folder.parentId
},
tx
);
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
secretVersions.map(({ id }) => ({
secretVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions.map(({ id }) => ({
folderVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
});
return snapshot;
}
const snapshot = await snapshotDAL.transaction(async (tx) => {
const folder = await folderDAL.findById(folderId, tx);
@@ -199,6 +258,7 @@ export const secretSnapshotServiceFactory = ({
}: TRollbackSnapshotDTO) => {
const snapshot = await snapshotDAL.findById(snapshotId);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const shouldUseBridge = snapshot.projectVersion === 3;
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -212,6 +272,117 @@ export const secretSnapshotServiceFactory = ({
ProjectPermissionSub.SecretRollback
);
if (shouldUseBridge) {
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshotsV2Bridge(snapshot.id, tx);
// this will remove all secrets in current folder
const deletedTopLevelSecs = await secretV2BridgeDAL.delete({ folderId: snapshot.folderId }, tx);
const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
// this will remove all secrets and folders on child
// due to sql foreign key and link list connection removing the folders removes everything below too
const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx);
const deletedTopLevelFolders = groupBy(
deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
(item) => item.id
);
const folders = await folderDAL.insertMany(
rollbackSnaps.flatMap(({ folderVersion, folderId }) =>
folderVersion.map(({ name, id, latestFolderVersion }) => ({
envId: snapshot.envId,
id,
// this means don't bump up the version if not root folder
// because below ones can be same version as nothing changed
version: deletedTopLevelFolders[folderId] ? latestFolderVersion + 1 : latestFolderVersion,
name,
parentId: folderId
}))
),
tx
);
const secrets = await secretV2BridgeDAL.insertMany(
rollbackSnaps.flatMap(({ secretVersions, folderId }) =>
secretVersions.map(
({ latestSecretVersion, version, updatedAt, createdAt, secretId, envId, id, tags, ...el }) => ({
...el,
id: secretId,
version: deletedTopLevelSecsGroupById[secretId] ? latestSecretVersion + 1 : latestSecretVersion,
folderId
})
)
),
tx
);
const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = [];
const secretVerTagToBeInsert: Record<string, string[]> = {};
rollbackSnaps.forEach(({ secretVersions }) => {
secretVersions.forEach((secVer) => {
secVer.tags.forEach((tag) => {
secretTagsToBeInsert.push({ secrets_v2Id: secVer.secretId, secret_tagsId: tag.id });
if (!secretVerTagToBeInsert?.[secVer.secretId]) secretVerTagToBeInsert[secVer.secretId] = [];
secretVerTagToBeInsert[secVer.secretId].push(tag.id);
});
});
});
await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx);
const folderVersions = await folderVersionDAL.insertMany(
folders.map(({ version, name, id, envId }) => ({
name,
version,
folderId: id,
envId
})),
tx
);
const secretVersions = await secretVersionV2BridgeDAL.insertMany(
secrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, secretId: id })),
tx
);
await secretVersionV2TagBridgeDAL.insertMany(
secretVersions.flatMap(({ secretId, id }) =>
secretVerTagToBeInsert?.[secretId]?.length
? secretVerTagToBeInsert[secretId].map((tagId) => ({
[`${TableName.SecretTag}Id` as const]: tagId,
[`${TableName.SecretVersionV2}Id` as const]: id
}))
: []
),
tx
);
const newSnapshot = await snapshotDAL.create(
{
folderId: snapshot.folderId,
envId: snapshot.envId,
parentFolderId: snapshot.parentFolderId
},
tx
);
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
secretVersions
.filter(({ secretId }) => Boolean(deletedTopLevelSecsGroupById?.[secretId]))
.map(({ id }) => ({
secretVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions
.filter(({ folderId }) => Boolean(deletedTopLevelFolders?.[folderId]))
.map(({ id }) => ({
folderVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, snapshotSecrets, snapshotFolders };
});
return rollback;
}
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshots(snapshot.id, tx);
// this will remove all secrets in current folder
@@ -4,11 +4,13 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretVersionsSchema,
SecretVersionsV2Schema,
TableName,
TSecretFolderVersions,
TSecretSnapshotFolders,
TSecretSnapshots,
TSecretVersions
TSecretVersions,
TSecretVersionsV2
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
@@ -24,12 +26,14 @@ export const snapshotDALFactory = (db: TDbClient) => {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, id)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.select(selectAllTableCols(TableName.Snapshot))
.select(
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug")
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("version").withSchema(TableName.Project).as("projectVersion")
)
.first();
if (data) {
@@ -149,6 +153,101 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
const findSecretSnapshotV2DataById = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, snapshotId)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SnapshotSecretV2, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(TableName.SnapshotFolder, `${TableName.SnapshotFolder}.snapshotId`, `${TableName.Snapshot}.id`)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"),
db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"),
db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
return sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({
snapshotId: id,
folderId,
projectId,
envId,
envSlug,
envName,
snapshotCreatedAt: createdAt,
snapshotUpdatedAt: updatedAt
}) => ({
id,
folderId,
projectId,
createdAt,
updatedAt,
environment: { id: envId, slug: envSlug, name: envName }
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => SecretVersionsV2Schema.parse(el),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name })
}
]
})?.[0];
} catch (error) {
throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" });
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
@@ -304,6 +403,161 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
// the recursive part find all snapshot id
// then joins with respective secrets and folder
const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)
.withRecursive("parent", (qb) => {
void qb
.from(TableName.Snapshot)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: 1 })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.where(`${TableName.Snapshot}.id`, snapshotId)
.union(
(cb) =>
void cb
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: db.raw("parent.depth + 1") })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.from(TableName.Snapshot)
.join<TSecretSnapshots, TSecretSnapshots & { secretId: string; max: number }>(
db(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"),
`${TableName.Snapshot}.createdAt`,
"latestVersion.max"
)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`)
);
})
.orderBy("depth", "asc")
.from<TSecretSnapshots & { folderVerId: string; folderVerName: string }>("parent")
.leftJoin<TSecretSnapshots>(TableName.SnapshotSecretV2, `parent.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin<TSecretVersionsV2>(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin<{ latestSecretVersion: number }>(
(tx || db)(TableName.SecretVersionV2)
.groupBy("secretId")
.select("secretId")
.max("version")
.as("secGroupByMaxVersion"),
`${TableName.SecretVersionV2}.secretId`,
"secGroupByMaxVersion.secretId"
)
.leftJoin<{ latestFolderVersion: number }>(
(tx || db)(TableName.SecretFolderVersion)
.groupBy("folderId")
.select("folderId")
.max("version")
.as("folderGroupByMaxVersion"),
`parent.folderId`,
"folderGroupByMaxVersion.folderId"
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema("parent").as("snapshotId"),
db.ref("folderId").withSchema("parent").as("snapshotFolderId"),
db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"),
db.ref("folderVerName").withSchema("parent"),
db.ref("folderVerId").withSchema("parent"),
db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"),
db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
const formated = sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({ snapshotId: id, snapshotFolderId: folderId, snapshotParentFolderId: parentFolderId }) => ({
id,
folderId,
parentFolderId
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => ({
...SecretVersionsV2Schema.parse(el),
latestSecretVersion: el.latestSecretVersion as number
}),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({
id,
name,
latestFolderVersion: latestFolderVersion as number
})
}
]
});
return formated;
} catch (error) {
throw new DatabaseError({ error, name: "FindRecursivelySnapshots" });
}
};
// instead of copying all child folders
// we will take the latest snapshot of those folders
// when we need to rollback we will pull from these snapshots
@@ -470,8 +724,10 @@ export const snapshotDALFactory = (db: TDbClient) => {
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
findRecursivelySnapshotsV2Bridge,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById,
findSecretSnapshotV2DataById,
pruneExcessSnapshots
};
};
@@ -0,0 +1,10 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TSnapshotSecretV2DALFactory = ReturnType<typeof snapshotSecretV2DALFactory>;
export const snapshotSecretV2DALFactory = (db: TDbClient) => {
const snapshotSecretOrm = ormify(db, TableName.SnapshotSecretV2);
return snapshotSecretOrm;
};