awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-18 06:58:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added bridge logic in secret replication, snapshot and approval for raw endpoint

Browse Source
This commit is contained in:
=
2024-07-30 23:11:04 +05:30
parent 9e314d7a09
commit ef2816b2ee
14 changed files with 2205 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+258 -2
View File
@@ -4,11 +4,13 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretVersionsSchema,
SecretVersionsV2Schema,
TableName,
TSecretFolderVersions,
TSecretSnapshotFolders,
TSecretSnapshots,
TSecretVersions
TSecretVersions,
TSecretVersionsV2
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
@@ -24,12 +26,14 @@ export const snapshotDALFactory = (db: TDbClient) => {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, id)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.select(selectAllTableCols(TableName.Snapshot))
.select(
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug")
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("version").withSchema(TableName.Project).as("projectVersion")
)
.first();
if (data) {
@@ -149,6 +153,101 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
const findSecretSnapshotV2DataById = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, snapshotId)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SnapshotSecretV2, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(TableName.SnapshotFolder, `${TableName.SnapshotFolder}.snapshotId`, `${TableName.Snapshot}.id`)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"),
db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"),
db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
return sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({
snapshotId: id,
folderId,
projectId,
envId,
envSlug,
envName,
snapshotCreatedAt: createdAt,
snapshotUpdatedAt: updatedAt
}) => ({
id,
folderId,
projectId,
createdAt,
updatedAt,
environment: { id: envId, slug: envSlug, name: envName }
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => SecretVersionsV2Schema.parse(el),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name })
}
]
})?.[0];
} catch (error) {
throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" });
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
@@ -304,6 +403,161 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
// the recursive part find all snapshot id
// then joins with respective secrets and folder
const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)
.withRecursive("parent", (qb) => {
void qb
.from(TableName.Snapshot)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: 1 })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.where(`${TableName.Snapshot}.id`, snapshotId)
.union(
(cb) =>
void cb
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: db.raw("parent.depth + 1") })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.from(TableName.Snapshot)
.join<TSecretSnapshots, TSecretSnapshots & { secretId: string; max: number }>(
db(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"),
`${TableName.Snapshot}.createdAt`,
"latestVersion.max"
)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`)
);
})
.orderBy("depth", "asc")
.from<TSecretSnapshots & { folderVerId: string; folderVerName: string }>("parent")
.leftJoin<TSecretSnapshots>(TableName.SnapshotSecretV2, `parent.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin<TSecretVersionsV2>(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin<{ latestSecretVersion: number }>(
(tx || db)(TableName.SecretVersionV2)
.groupBy("secretId")
.select("secretId")
.max("version")
.as("secGroupByMaxVersion"),
`${TableName.SecretVersionV2}.secretId`,
"secGroupByMaxVersion.secretId"
)
.leftJoin<{ latestFolderVersion: number }>(
(tx || db)(TableName.SecretFolderVersion)
.groupBy("folderId")
.select("folderId")
.max("version")
.as("folderGroupByMaxVersion"),
`parent.folderId`,
"folderGroupByMaxVersion.folderId"
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema("parent").as("snapshotId"),
db.ref("folderId").withSchema("parent").as("snapshotFolderId"),
db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"),
db.ref("folderVerName").withSchema("parent"),
db.ref("folderVerId").withSchema("parent"),
db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"),
db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
const formated = sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({ snapshotId: id, snapshotFolderId: folderId, snapshotParentFolderId: parentFolderId }) => ({
id,
folderId,
parentFolderId
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => ({
...SecretVersionsV2Schema.parse(el),
latestSecretVersion: el.latestSecretVersion as number
}),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({
id,
name,
latestFolderVersion: latestFolderVersion as number
})
}
]
});
return formated;
} catch (error) {
throw new DatabaseError({ error, name: "FindRecursivelySnapshots" });
}
};
// instead of copying all child folders
// we will take the latest snapshot of those folders
// when we need to rollback we will pull from these snapshots
@@ -470,8 +724,10 @@ export const snapshotDALFactory = (db: TDbClient) => {
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
findRecursivelySnapshotsV2Bridge,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById,
findSecretSnapshotV2DataById,
pruneExcessSnapshots
};
};