feat: changed backend-pg to backend

This commit is contained in:
Akhil Mohan
2024-01-27 19:14:33 +05:30
parent d13eafcef7
commit 66d258f02b
792 changed files with 40298 additions and 40607 deletions
@@ -0,0 +1,315 @@
import { ForbiddenError } from "@casl/ability";
import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { TPermissionServiceFactory } from "../permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission";
import {
TGetSnapshotDataDTO,
TProjectSnapshotCountDTO,
TProjectSnapshotListDTO,
TRollbackSnapshotDTO
} from "./secret-snapshot-types";
import { TSnapshotDALFactory } from "./snapshot-dal";
import { TSnapshotFolderDALFactory } from "./snapshot-folder-dal";
import { TSnapshotSecretDALFactory } from "./snapshot-secret-dal";
type TSecretSnapshotServiceFactoryDep = {
snapshotDAL: TSnapshotDALFactory;
snapshotSecretDAL: TSnapshotSecretDALFactory;
snapshotFolderDAL: TSnapshotFolderDALFactory;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionByFolderId">;
folderVersionDAL: Pick<
TSecretFolderVersionDALFactory,
"findLatestVersionByFolderId" | "insertMany"
>;
secretDAL: Pick<TSecretDALFactory, "delete" | "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
folderDAL: Pick<
TSecretFolderDALFactory,
"findById" | "findBySecretPath" | "delete" | "insertMany"
>;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
licenseService: Pick<TLicenseServiceFactory, "isValidLicense">;
};
export type TSecretSnapshotServiceFactory = ReturnType<typeof secretSnapshotServiceFactory>;
export const secretSnapshotServiceFactory = ({
snapshotDAL,
folderVersionDAL,
secretVersionDAL,
snapshotSecretDAL,
snapshotFolderDAL,
folderDAL,
secretDAL,
permissionService,
licenseService,
secretTagDAL,
secretVersionTagDAL
}: TSecretSnapshotServiceFactoryDep) => {
const projectSecretSnapshotCount = async ({
environment,
projectId,
actorId,
actor,
path
}: TProjectSnapshotCountDTO) => {
const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
ProjectPermissionSub.SecretRollback
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id);
return count;
};
const listSnapshots = async ({
environment,
projectId,
actorId,
actor,
path,
limit = 20,
offset = 0
}: TProjectSnapshotListDTO) => {
const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
ProjectPermissionSub.SecretRollback
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const snapshots = await snapshotDAL.find(
{ folderId: folder.id },
{ limit, offset, sort: [["createdAt", "desc"]] }
);
return snapshots;
};
const getSnapshotData = async ({ actorId, actor, id }: TGetSnapshotDataDTO) => {
const snapshot = await snapshotDAL.findSecretSnapshotDataById(id);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
snapshot.projectId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
ProjectPermissionSub.SecretRollback
);
return snapshot;
};
const performSnapshot = async (folderId: string) => {
try {
if (!licenseService.isValidLicense)
throw new InternalServerError({ message: "Invalid license" });
const snapshot = await snapshotDAL.transaction(async (tx) => {
const folder = await folderDAL.findById(folderId, tx);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const secretVersions = await secretVersionDAL.findLatestVersionByFolderId(folderId, tx);
const folderVersions = await folderVersionDAL.findLatestVersionByFolderId(folderId, tx);
const newSnapshot = await snapshotDAL.create(
{
folderId,
envId: folder.environment.envId,
parentFolderId: folder.parentId
},
tx
);
const snapshotSecrets = await snapshotSecretDAL.insertMany(
secretVersions.map(({ id }) => ({
secretVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions.map(({ id }) => ({
folderVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
});
return snapshot;
} catch (error) {
// this to avoid snapshot errors
logger.error("Failed to perform snasphot");
logger.error(error);
}
};
const rollbackSnapshot = async ({ id: snapshotId, actor, actorId }: TRollbackSnapshotDTO) => {
const snapshot = await snapshotDAL.findById(snapshotId);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
snapshot.projectId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Create,
ProjectPermissionSub.SecretRollback
);
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshots(snapshot.id, tx);
// this will remove all secrets in current folder
const deletedTopLevelSecs = await secretDAL.delete({ folderId: snapshot.folderId }, tx);
const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
// this will remove all secrets and folders on child
// due to sql foreign key and link list connection removing the folders removes everything below too
const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx);
const deletedTopLevelFolders = groupBy(
deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
(item) => item.id
);
const folders = await folderDAL.insertMany(
rollbackSnaps.flatMap(({ folderVersion, folderId }) =>
folderVersion.map(({ name, id, latestFolderVersion }) => ({
envId: snapshot.envId,
id,
// this means don't bump up the version if not root folder
// because below ones can be same version as nothing changed
version: deletedTopLevelFolders[folderId]
? latestFolderVersion + 1
: latestFolderVersion,
name,
parentId: folderId
}))
),
tx
);
const secrets = await secretDAL.insertMany(
rollbackSnaps.flatMap(({ secretVersions, folderId }) =>
secretVersions.map(
({
latestSecretVersion,
version,
updatedAt,
createdAt,
secretId,
envId,
id,
tags,
...el
}) => ({
...el,
id: secretId,
version: deletedTopLevelSecsGroupById[secretId]
? latestSecretVersion + 1
: latestSecretVersion,
folderId
})
)
),
tx
);
const secretTagsToBeInsert: TSecretTagJunctionInsert[] = [];
const secretVerTagToBeInsert: Record<string, string[]> = {};
rollbackSnaps.forEach(({ secretVersions }) => {
secretVersions.forEach((secVer) => {
secVer.tags.forEach((tag) => {
secretTagsToBeInsert.push({ secretsId: secVer.secretId, secret_tagsId: tag.id });
if (!secretVerTagToBeInsert?.[secVer.secretId])
secretVerTagToBeInsert[secVer.secretId] = [];
secretVerTagToBeInsert[secVer.secretId].push(tag.id);
});
});
});
await secretTagDAL.saveTagsToSecret(secretTagsToBeInsert, tx);
const folderVersions = await folderVersionDAL.insertMany(
folders.map(({ version, name, id, envId }) => ({
name,
version,
folderId: id,
envId
})),
tx
);
const secretVersions = await secretVersionDAL.insertMany(
secrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, secretId: id })),
tx
);
await secretVersionTagDAL.insertMany(
secretVersions.flatMap(({ secretId, id }) =>
secretVerTagToBeInsert?.[secretId]?.length
? secretVerTagToBeInsert[secretId].map((tagId) => ({
[`${TableName.SecretTag}Id` as const]: tagId,
[`${TableName.SecretVersion}Id` as const]: id
}))
: []
),
tx
);
const newSnapshot = await snapshotDAL.create(
{
folderId: snapshot.folderId,
envId: snapshot.envId,
parentFolderId: snapshot.parentFolderId
},
tx
);
const snapshotSecrets = await snapshotSecretDAL.insertMany(
secretVersions
.filter(({ secretId }) => Boolean(deletedTopLevelSecsGroupById?.[secretId]))
.map(({ id }) => ({
secretVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions
.filter(({ folderId }) => Boolean(deletedTopLevelFolders?.[folderId]))
.map(({ id }) => ({
folderVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, snapshotSecrets, snapshotFolders };
});
return rollback;
};
return {
performSnapshot,
projectSecretSnapshotCount,
listSnapshots,
getSnapshotData,
rollbackSnapshot
};
};
@@ -0,0 +1,21 @@
import { TProjectPermission } from "@app/lib/types";
export type TProjectSnapshotCountDTO = {
environment: string;
path: string;
} & TProjectPermission;
export type TProjectSnapshotListDTO = {
environment: string;
path: string;
offset?: number;
limit?: number;
} & TProjectPermission;
export type TGetSnapshotDataDTO = {
id: string;
} & Omit<TProjectPermission, "projectId">;
export type TRollbackSnapshotDTO = {
id: string;
} & Omit<TProjectPermission, "projectId">;
@@ -0,0 +1,356 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretVersionsSchema,
TableName,
TSecretFolderVersions,
TSecretSnapshotFolders,
TSecretSnapshots,
TSecretVersions
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
export type TSnapshotDALFactory = ReturnType<typeof snapshotDALFactory>;
export const snapshotDALFactory = (db: TDbClient) => {
const secretSnapshotOrm = ormify(db, TableName.Snapshot);
const findById = async (id: string, tx?: Knex) => {
try {
const data = await (tx || db)(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, id)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.select(selectAllTableCols(TableName.Snapshot))
.select(
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug")
)
.first();
if (data) {
const { envId, envName, envSlug } = data;
return { ...data, envId, enviroment: { id: envId, name: envName, slug: envSlug } };
}
} catch (error) {
throw new DatabaseError({ error, name: "FindById" });
}
};
const countOfSnapshotsByFolderId = async (folderId: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.Snapshot)
.where({ folderId })
.groupBy(["folderId"])
.count("folderId")
.first();
return parseInt((doc?.count as string) || "0", 10);
} catch (error) {
throw new DatabaseError({ error, name: "CountOfProjectSnapshot" });
}
};
const findSecretSnapshotDataById = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, snapshotId)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.leftJoin(
TableName.SnapshotSecret,
`${TableName.Snapshot}.id`,
`${TableName.SnapshotSecret}.snapshotId`
)
.leftJoin(
TableName.SecretVersion,
`${TableName.SnapshotSecret}.secretVersionId`,
`${TableName.SecretVersion}.id`
)
.leftJoin(
TableName.SecretVersionTag,
`${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`,
`${TableName.SecretVersion}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionTag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.SecretVersion))
.select(
db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"),
db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"),
db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
return sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({
snapshotId: id,
projectId,
envId,
envSlug,
envName,
snapshotCreatedAt: createdAt,
snapshotUpdatedAt: updatedAt
}) => ({
id,
projectId,
createdAt,
updatedAt,
environment: { id: envId, slug: envSlug, name: envName }
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => SecretVersionsSchema.parse(el),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({
tagId: id,
tagName: name,
tagSlug: slug,
tagColor: color,
tagVersionId: vId
}) => ({ id, name, slug, color, vId })
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name })
}
]
})?.[0];
} catch (error) {
throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" });
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
// the recursive part find all snapshot id
// then joins with respective secrets and folder
const findRecursivelySnapshots = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)
.withRecursive("parent", (qb) => {
qb.from(TableName.Snapshot)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: 1 })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.where(`${TableName.Snapshot}.id`, snapshotId)
.union((cb) =>
cb
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: db.raw("parent.depth + 1") })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.from(TableName.Snapshot)
.join<TSecretSnapshots, TSecretSnapshots & { secretId: string; max: number }>(
db(TableName.Snapshot)
.groupBy("folderId")
.max("createdAt")
.select("folderId")
.as("latestVersion"),
`${TableName.Snapshot}.createdAt`,
"latestVersion.max"
)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`)
);
})
.orderBy("depth", "asc")
.from<TSecretSnapshots & { folderVerId: string; folderVerName: string }>("parent")
.leftJoin<TSecretSnapshots>(
TableName.SnapshotSecret,
`parent.id`,
`${TableName.SnapshotSecret}.snapshotId`
)
.leftJoin<TSecretVersions>(
TableName.SecretVersion,
`${TableName.SnapshotSecret}.secretVersionId`,
`${TableName.SecretVersion}.id`
)
.leftJoin(
TableName.SecretVersionTag,
`${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`,
`${TableName.SecretVersion}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionTag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin<{ latestSecretVersion: number }>(
(tx || db)(TableName.SecretVersion)
.groupBy("secretId")
.select("secretId")
.max("version")
.as("secGroupByMaxVersion"),
`${TableName.SecretVersion}.secretId`,
"secGroupByMaxVersion.secretId"
)
.leftJoin<{ latestFolderVersion: number }>(
(tx || db)(TableName.SecretFolderVersion)
.groupBy("folderId")
.select("folderId")
.max("version")
.as("folderGroupByMaxVersion"),
`parent.folderId`,
"folderGroupByMaxVersion.folderId"
)
.select(selectAllTableCols(TableName.SecretVersion))
.select(
db.ref("id").withSchema("parent").as("snapshotId"),
db.ref("folderId").withSchema("parent").as("snapshotFolderId"),
db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"),
db.ref("folderVerName").withSchema("parent"),
db.ref("folderVerId").withSchema("parent"),
db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"),
db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
const formated = sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({
snapshotId: id,
snapshotFolderId: folderId,
snapshotParentFolderId: parentFolderId
}) => ({
id,
folderId,
parentFolderId
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => ({
...SecretVersionsSchema.parse(el),
latestSecretVersion: el.latestSecretVersion
}),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({
tagId: id,
tagName: name,
tagSlug: slug,
tagColor: color,
tagVersionId: vId
}) => ({ id, name, slug, color, vId })
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({
id,
name,
latestFolderVersion
})
}
]
});
return formated;
} catch (error) {
throw new DatabaseError({ error, name: "FindRecursivelySnapshots" });
}
};
// instead of copying all child folders
// we will take the latest snapshot of those folders
// when we need to rollback we will pull from these snapshots
const findLatestSnapshotByFolderId = async (folderId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Snapshot)
.where(`${TableName.Snapshot}.folderId`, folderId)
.join<TSecretSnapshots>(
(tx || db)(TableName.Snapshot)
.groupBy("folderId")
.max("createdAt")
.select("folderId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.Snapshot}.folderId`, "latestVersion.folderId").andOn(
`${TableName.Snapshot}.createdAt`,
"latestVersion.max"
);
}
)
.first();
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" });
}
};
return {
...secretSnapshotOrm,
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById
};
};
@@ -0,0 +1,11 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TSnapshotFolderDALFactory = ReturnType<typeof snapshotFolderDALFactory>;
export const snapshotFolderDALFactory = (db: TDbClient) => {
const snapshotFolderOrm = ormify(db, TableName.SnapshotFolder);
return snapshotFolderOrm;
};
@@ -0,0 +1,10 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TSnapshotSecretDALFactory = ReturnType<typeof snapshotSecretDALFactory>;
export const snapshotSecretDALFactory = (db: TDbClient) => {
const snapshotSecretOrm = ormify(db, TableName.SnapshotSecret);
return snapshotSecretOrm;
};