feature: server-side pagination/filtering/sorting for secrets overview and main pages

This commit is contained in:
Scott Wilson
2024-09-18 21:17:48 -07:00
parent d6b7045461
commit d0656358a2
43 changed files with 2899 additions and 721 deletions
@@ -1,10 +1,67 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
export type TDynamicSecretDALFactory = ReturnType<typeof dynamicSecretDALFactory>;
export const dynamicSecretDALFactory = (db: TDbClient) => {
const orm = ormify(db, TableName.DynamicSecret);
return orm;
// find dynamic secrets for multiple environments (folder IDs are cross env, thus need to rank for pagination)
const findMultiEnv = async (
{
folderIds,
search,
limit,
offset = 0,
orderBy = SecretsOrderBy.Name,
orderDirection = OrderByDirection.ASC
}: {
folderIds: string[];
search?: string;
limit?: number;
offset?: number;
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
},
tx?: Knex
) => {
try {
const query = (tx || db.replicaNode())(TableName.DynamicSecret)
.whereIn("folderId", folderIds)
.where((bd) => {
if (search) {
void bd.whereILike("name", `%${search}%`);
}
})
.select(
selectAllTableCols(TableName.DynamicSecret),
db.raw(`DENSE_RANK() OVER (ORDER BY "name" ${orderDirection ?? OrderByDirection.ASC}) as rank`)
)
.orderBy(orderBy, orderDirection);
if (limit) {
const rankOffset = offset + 1;
return await (tx || db)
.with("w", query)
.select("*")
.from<Awaited<typeof query>[number]>("w")
.where("w.rank", ">=", rankOffset)
.andWhere("w.rank", "<", rankOffset + limit);
}
const dynamicSecrets = await query;
return dynamicSecrets;
} catch (error) {
throw new DatabaseError({ error, name: "List dynamic secret multi env" });
}
};
return { ...orm, findMultiEnv };
};
@@ -6,6 +6,7 @@ import { TPermissionServiceFactory } from "@app/ee/services/permission/permissio
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { BadRequestError } from "@app/lib/errors";
import { OrderByDirection } from "@app/lib/types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
@@ -31,7 +32,7 @@ type TDynamicSecretServiceFactoryDep = {
"pruneDynamicSecret" | "unsetLeaseRevocation"
>;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findBySecretPathMultiEnv">;
projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
};
@@ -300,19 +301,75 @@ export const dynamicSecretServiceFactory = ({
return { ...dynamicSecretCfg, inputs: providerInputs };
};
const list = async ({
// get unique dynamic secret count across multiple envs
const getCountMultiEnv = async ({
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectSlug,
path,
environmentSlug
environmentSlugs,
search,
...params
}: Omit<TListDynamicSecretsDTO, "environmentSlug"> & { environmentSlugs: string[] }) => {
let { projectId } = params;
if (!projectId) {
if (!projectSlug) throw new BadRequestError({ message: "Project ID or slug required" });
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
projectId = project.id;
}
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
// verify user has access to each env in request
environmentSlugs.forEach((environmentSlug) =>
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment: environmentSlug, secretPath: path })
)
);
const folders = await folderDAL.findBySecretPathMultiEnv(projectId, environmentSlugs, path);
if (!folders.length) throw new BadRequestError({ message: "Folders not found" });
const dynamicSecretCfg = await dynamicSecretDAL.find(
{ $in: { folderId: folders.map((folder) => folder.id) }, $search: search ? { name: `%${search}%` } : undefined },
{ countDistinct: "name" }
);
return Number(dynamicSecretCfg[0]?.count ?? 0);
};
// get dynamic secret count for a single env
const getCount = async ({
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectSlug,
path,
environmentSlug,
search,
...params
}: TListDynamicSecretsDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
let { projectId } = params;
if (!projectId) {
if (!projectSlug) throw new BadRequestError({ message: "Project ID or slug required" });
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
projectId = project.id;
}
const projectId = project.id;
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
@@ -328,15 +385,126 @@ export const dynamicSecretServiceFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const dynamicSecretCfg = await dynamicSecretDAL.find({ folderId: folder.id });
const dynamicSecretCfg = await dynamicSecretDAL.find(
{ folderId: folder.id, $search: search ? { name: `%${search}%` } : undefined },
{ count: true }
);
return Number(dynamicSecretCfg[0]?.count ?? 0);
};
const list = async ({
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectSlug,
path,
environmentSlug,
limit,
offset,
orderBy,
orderDirection = OrderByDirection.ASC,
search,
...params
}: TListDynamicSecretsDTO) => {
let { projectId } = params;
if (!projectId) {
if (!projectSlug) throw new BadRequestError({ message: "Project ID or slug required" });
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
projectId = project.id;
}
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment: environmentSlug, secretPath: path })
);
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const dynamicSecretCfg = await dynamicSecretDAL.find(
{ folderId: folder.id, $search: search ? { name: `%${search}%` } : undefined },
{
limit,
offset,
sort: orderBy ? [[orderBy, orderDirection]] : undefined
}
);
return dynamicSecretCfg;
};
// get dynamic secrets for multiple envs
const listMultiEnv = async ({
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectSlug,
path,
environmentSlugs,
...params
}: Omit<TListDynamicSecretsDTO, "environmentSlug"> & { environmentSlugs: string[] }) => {
let { projectId } = params;
if (!projectId) {
if (!projectSlug) throw new BadRequestError({ message: "Project ID or slug required" });
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
projectId = project.id;
}
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
// verify user has access to each env in request
environmentSlugs.forEach((environmentSlug) =>
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment: environmentSlug, secretPath: path })
)
);
const folders = await folderDAL.findBySecretPathMultiEnv(projectId, environmentSlugs, path);
if (!folders.length) throw new BadRequestError({ message: "Folders not found" });
const dynamicSecretCfg = await dynamicSecretDAL.findMultiEnv({
folderIds: folders.map((folder) => folder.id),
...params
});
// map dynamic secrets to their respective env
const folderEnvMap: Map<string, string> = new Map(folders.map((folder) => [folder.id, folder.environment.slug]));
const data: { [key: string]: Array<(typeof dynamicSecretCfg)[number]> } = {};
dynamicSecretCfg.forEach((secret) => {
const slug = folderEnvMap.get(secret.folderId);
data[slug!] = [...(data[slug!] ?? []), secret];
});
return data;
};
return {
create,
updateByName,
deleteByName,
getDetails,
list
list,
listMultiEnv,
getCount,
getCountMultiEnv
};
};
@@ -1,6 +1,7 @@
import { z } from "zod";
import { TProjectPermission } from "@app/lib/types";
import { OrderByDirection, TProjectPermission } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { DynamicSecretProviderSchema } from "./providers/models";
@@ -50,5 +51,11 @@ export type TDetailsDynamicSecretDTO = {
export type TListDynamicSecretsDTO = {
path: string;
environmentSlug: string;
projectSlug: string;
projectSlug?: string;
projectId?: string;
offset?: number;
limit?: number;
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
search?: string;
} & Omit<TProjectPermission, "projectId">;
+32
View File
@@ -697,6 +697,38 @@ export const SECRET_IMPORTS = {
}
} as const;
export const DASHBOARD = {
SECRET_OVERVIEW_LIST: {
projectId: "The ID of the project to list secrets/folders from.",
environments:
"The slugs of the environments to list secrets/folders from (comma separated, ie 'environments=dev,staging,prod').",
secretPath: "The secret path to list secrets/folders from.",
offset: "The offset to start from. If you enter 10, it will start from the 10th secret/folder.",
limit: "The number of secrets/folders to return.",
orderBy: "The column to order secrets/folders by.",
orderDirection: "The direction to order secrets/folders in.",
search: "The text string to filter secret keys and folder names by.",
includeSecrets: "Whether to include project secrets in the response.",
includeFolders: "Whether to include project folders in the response.",
includeDynamicSecrets: "Whether to include dynamic project secrets in the response."
},
SECRET_DETAILS_LIST: {
projectId: "The ID of the project to list secrets/folders from.",
environment: "The slug of the environment to list secrets/folders from.",
secretPath: "The secret path to list secrets/folders from.",
offset: "The offset to start from. If you enter 10, it will start from the 10th secret/folder.",
limit: "The number of secrets/folders to return.",
orderBy: "The column to order secrets/folders by.",
orderDirection: "The direction to order secrets/folders in.",
search: "The text string to filter secret keys and folder names by.",
tags: "The tags to filter secrets by (comma separated, ie 'tags=billing,engineering').",
includeSecrets: "Whether to include project secrets in the response.",
includeFolders: "Whether to include project folders in the response.",
includeImports: "Whether to include project secret imports in the response.",
includeDynamicSecrets: "Whether to include dynamic project secrets in the response."
}
} as const;
export const AUDIT_LOGS = {
EXPORT: {
workspaceId: "The ID of the project to export audit logs from.",
+17 -5
View File
@@ -51,11 +51,17 @@ export type TFindReturn<TQuery extends Knex.QueryBuilder, TCount extends boolean
: unknown)
>;
export type TFindOpt<R extends object = object, TCount extends boolean = boolean> = {
export type TFindOpt<
R extends object = object,
TCount extends boolean = boolean,
TCountDistinct extends keyof R | undefined = undefined
> = {
limit?: number;
offset?: number;
sort?: Array<[keyof R, "asc" | "desc"] | [keyof R, "asc" | "desc", "first" | "last"]>;
groupBy?: keyof R;
count?: TCount;
countDistinct?: TCountDistinct;
tx?: Knex;
};
@@ -86,13 +92,18 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
throw new DatabaseError({ error, name: "Find one" });
}
},
find: async <TCount extends boolean = false>(
find: async <
TCount extends boolean = false,
TCountDistinct extends keyof Tables[Tname]["base"] | undefined = undefined
>(
filter: TFindFilter<Tables[Tname]["base"]>,
{ offset, limit, sort, count, tx }: TFindOpt<Tables[Tname]["base"], TCount> = {}
{ offset, limit, sort, count, tx, countDistinct }: TFindOpt<Tables[Tname]["base"], TCount, TCountDistinct> = {}
) => {
try {
const query = (tx || db.replicaNode())(tableName).where(buildFindFilter(filter));
if (count) {
if (countDistinct) {
void query.countDistinct(countDistinct);
} else if (count) {
void query.select(db.raw("COUNT(*) OVER() AS count"));
void query.select("*");
}
@@ -101,7 +112,8 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
if (sort) {
void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls })));
}
const res = (await query) as TFindReturn<typeof query, TCount>;
const res = (await query) as TFindReturn<typeof query, TCountDistinct extends undefined ? TCount : true>;
return res;
} catch (error) {
throw new DatabaseError({ error, name: "Find one" });
@@ -0,0 +1,620 @@
import { z } from "zod";
import { SecretFoldersSchema, SecretImportsSchema, SecretTagsSchema } from "@app/db/schemas";
import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types";
import { DASHBOARD } from "@app/lib/api-docs";
import { BadRequestError } from "@app/lib/errors";
import { removeTrailingSlash } from "@app/lib/fn";
import { OrderByDirection } from "@app/lib/types";
import { secretsLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { getUserAgentType } from "@app/server/plugins/audit-log";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { SanitizedDynamicSecretSchema, secretRawSchema } from "@app/server/routes/sanitizedSchemas";
import { AuthMode } from "@app/services/auth/auth-type";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
export const registerDashboardRouter = async (server: FastifyZodProvider) => {
server.route({
method: "GET",
url: "/secrets-overview",
config: {
rateLimit: secretsLimit
},
schema: {
description: "List project secrets overview",
security: [
{
bearerAuth: []
}
],
querystring: z.object({
projectId: z.string().trim().describe(DASHBOARD.SECRET_OVERVIEW_LIST.projectId),
environments: z
.string()
.trim()
.transform(decodeURIComponent)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.environments),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.secretPath),
offset: z.coerce.number().min(0).optional().default(0).describe(DASHBOARD.SECRET_OVERVIEW_LIST.offset),
limit: z.coerce.number().min(1).max(100).optional().default(100).describe(DASHBOARD.SECRET_OVERVIEW_LIST.limit),
orderBy: z
.nativeEnum(SecretsOrderBy)
.default(SecretsOrderBy.Name)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.orderBy)
.optional(),
orderDirection: z
.nativeEnum(OrderByDirection)
.default(OrderByDirection.ASC)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.orderDirection)
.optional(),
search: z.string().trim().describe(DASHBOARD.SECRET_OVERVIEW_LIST.search).optional(),
includeSecrets: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeSecrets),
includeFolders: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeFolders),
includeDynamicSecrets: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeDynamicSecrets)
}),
response: {
200: z.object({
folders: z.record(z.string(), SecretFoldersSchema.array()).optional(),
dynamicSecrets: z.record(z.string(), SanitizedDynamicSecretSchema.array()).optional(),
secrets: z
.record(
z.string(),
secretRawSchema
.extend({
secretPath: z.string().optional(),
tags: SecretTagsSchema.pick({
id: true,
slug: true,
color: true
})
.extend({ name: z.string() })
.array()
.optional()
})
.array()
)
.optional(),
totalFolderCount: z.number().optional(),
totalDynamicSecretCount: z.number().optional(),
totalSecretCount: z.number().optional(),
totalCount: z.number()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const {
secretPath,
projectId,
limit,
offset,
search,
orderBy,
orderDirection,
includeFolders,
includeSecrets,
includeDynamicSecrets
} = req.query;
const environments = req.query.environments.split(",");
if (!projectId || environments.length === 0)
throw new BadRequestError({ message: "Missing workspace id or environment(s)" });
const { shouldUseSecretV2Bridge } = await server.services.projectBot.getBotKey(projectId);
// prevent older projects from accessing endpoint
if (!shouldUseSecretV2Bridge) throw new BadRequestError({ message: "Project version not supported" });
let remainingLimit = limit;
let adjustedOffset = offset;
let folders: Awaited<ReturnType<typeof server.services.folder.getFoldersMultiEnv>> | undefined;
let secrets: Awaited<ReturnType<typeof server.services.secret.getSecretsRawMultiEnv>> | undefined;
let dynamicSecrets: Awaited<ReturnType<typeof server.services.dynamicSecret.listMultiEnv>> | undefined;
let totalFolderCount: number | undefined;
let totalDynamicSecretCount: number | undefined;
let totalSecretCount: number | undefined;
if (includeFolders) {
// this is the unique count, ie duplicate folders across envs only count as 1
totalFolderCount = await server.services.folder.getProjectFolderCount({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId: req.query.projectId,
path: secretPath,
environments,
search
});
if (remainingLimit > 0 && totalFolderCount > adjustedOffset) {
folders = await server.services.folder.getFoldersMultiEnv({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
environments,
path: secretPath,
orderBy,
orderDirection,
search,
limit: remainingLimit,
offset: adjustedOffset
});
// get the count of unique folder names to properly adjust remaining limit
const uniqueFolderCount = new Set(
Object.values(folders).flatMap((folderGroup) => folderGroup.flatMap((folder) => folder.name))
).size;
remainingLimit -= uniqueFolderCount;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalFolderCount);
}
}
if (includeDynamicSecrets) {
// this is the unique count, ie duplicate secrets across envs only count as 1
totalDynamicSecretCount = await server.services.dynamicSecret.getCountMultiEnv({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
search,
environmentSlugs: environments,
path: secretPath
});
if (remainingLimit > 0 && totalDynamicSecretCount > adjustedOffset) {
dynamicSecrets = await server.services.dynamicSecret.listMultiEnv({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
search,
orderBy,
orderDirection,
environmentSlugs: environments,
path: secretPath,
limit: remainingLimit,
offset: adjustedOffset
});
// get the count of unique dynamic secret names to properly adjust remaining limit
const uniqueDynamicSecretsCount = new Set(
Object.values(dynamicSecrets).flatMap((dynamicSecretGroup) =>
dynamicSecretGroup.flatMap((dynamicSecret) => dynamicSecret.name)
)
).size;
remainingLimit -= uniqueDynamicSecretsCount;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalDynamicSecretCount);
}
}
if (includeSecrets) {
// this is the unique count, ie duplicate secrets across envs only count as 1
totalSecretCount = await server.services.secret.getSecretsCountMultiEnv({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
environments,
actorAuthMethod: req.permission.authMethod,
projectId,
path: secretPath,
search
});
if (remainingLimit > 0 && totalSecretCount > adjustedOffset) {
secrets = await server.services.secret.getSecretsRawMultiEnv({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
environments,
actorAuthMethod: req.permission.authMethod,
projectId,
path: secretPath,
orderBy,
orderDirection,
search,
limit: remainingLimit,
offset: adjustedOffset
});
for await (const environment of environments) {
const secretCountFromEnv = secrets[environment]?.length;
if (secretCountFromEnv) {
await server.services.auditLog.createAuditLog({
projectId,
...req.auditLogInfo,
event: {
type: EventType.GET_SECRETS,
metadata: {
environment,
secretPath,
numberOfSecrets: secretCountFromEnv
}
}
});
if (getUserAgentType(req.headers["user-agent"]) !== UserAgentType.K8_OPERATOR) {
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secretCountFromEnv,
workspaceId: projectId,
environment,
secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
}
}
}
}
}
return {
folders,
dynamicSecrets,
secrets,
totalFolderCount,
totalDynamicSecretCount,
totalSecretCount,
totalCount: (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0)
};
}
});
server.route({
method: "GET",
url: "/secrets-details",
config: {
rateLimit: secretsLimit
},
schema: {
description: "List project secrets details",
security: [
{
bearerAuth: []
}
],
querystring: z.object({
projectId: z.string().trim().describe(DASHBOARD.SECRET_DETAILS_LIST.projectId),
environment: z.string().trim().describe(DASHBOARD.SECRET_DETAILS_LIST.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(DASHBOARD.SECRET_DETAILS_LIST.secretPath),
offset: z.coerce.number().min(0).optional().default(0).describe(DASHBOARD.SECRET_DETAILS_LIST.offset),
limit: z.coerce.number().min(1).max(100).optional().default(100).describe(DASHBOARD.SECRET_DETAILS_LIST.limit),
orderBy: z
.nativeEnum(SecretsOrderBy)
.default(SecretsOrderBy.Name)
.describe(DASHBOARD.SECRET_DETAILS_LIST.orderBy)
.optional(),
orderDirection: z
.nativeEnum(OrderByDirection)
.default(OrderByDirection.ASC)
.describe(DASHBOARD.SECRET_DETAILS_LIST.orderDirection)
.optional(),
search: z.string().trim().describe(DASHBOARD.SECRET_DETAILS_LIST.search).optional(),
tags: z.string().trim().transform(decodeURIComponent).describe(DASHBOARD.SECRET_DETAILS_LIST.tags).optional(),
includeSecrets: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_DETAILS_LIST.includeSecrets),
includeFolders: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_DETAILS_LIST.includeFolders),
includeDynamicSecrets: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_DETAILS_LIST.includeDynamicSecrets),
includeImports: z.coerce
.boolean()
.optional()
.default(true)
.describe(DASHBOARD.SECRET_DETAILS_LIST.includeImports)
}),
response: {
200: z.object({
imports: SecretImportsSchema.omit({ importEnv: true })
.extend({
importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
})
.array()
.optional(),
folders: SecretFoldersSchema.array().optional(),
dynamicSecrets: SanitizedDynamicSecretSchema.array().optional(),
secrets: secretRawSchema
.extend({
secretPath: z.string().optional(),
tags: SecretTagsSchema.pick({
id: true,
slug: true,
color: true
})
.extend({ name: z.string() })
.array()
.optional()
})
.array()
.optional(),
totalImportCount: z.number().optional(),
totalFolderCount: z.number().optional(),
totalDynamicSecretCount: z.number().optional(),
totalSecretCount: z.number().optional(),
totalCount: z.number()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const {
secretPath,
environment,
projectId,
limit,
offset,
search,
orderBy,
orderDirection,
includeFolders,
includeSecrets,
includeDynamicSecrets,
includeImports
} = req.query;
if (!projectId || !environment) throw new BadRequestError({ message: "Missing workspace id or environment" });
const { shouldUseSecretV2Bridge } = await server.services.projectBot.getBotKey(projectId);
// prevent older projects from accessing endpoint
if (!shouldUseSecretV2Bridge) throw new BadRequestError({ message: "Project version not supported" });
const tags = req.query.tags?.split(",") ?? [];
let remainingLimit = limit;
let adjustedOffset = offset;
let imports: Awaited<ReturnType<typeof server.services.secretImport.getImports>> | undefined;
let folders: Awaited<ReturnType<typeof server.services.folder.getFolders>> | undefined;
let secrets: Awaited<ReturnType<typeof server.services.secret.getSecretsRaw>>["secrets"] | undefined;
let dynamicSecrets: Awaited<ReturnType<typeof server.services.dynamicSecret.list>> | undefined;
let totalImportCount: number | undefined;
let totalFolderCount: number | undefined;
let totalDynamicSecretCount: number | undefined;
let totalSecretCount: number | undefined;
if (includeImports) {
totalImportCount = await server.services.secretImport.getProjectImportCount({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
environment,
path: secretPath,
search
});
if (remainingLimit > 0 && totalImportCount > adjustedOffset) {
imports = await server.services.secretImport.getImports({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
environment,
path: secretPath,
search,
limit: remainingLimit,
offset: adjustedOffset
});
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
projectId: req.query.projectId,
event: {
type: EventType.GET_SECRET_IMPORTS,
metadata: {
environment,
folderId: imports?.[0]?.folderId,
numberOfImports: imports.length
}
}
});
remainingLimit -= imports.length;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalImportCount);
}
}
if (includeFolders) {
totalFolderCount = await server.services.folder.getProjectFolderCount({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
path: secretPath,
environments: [environment],
search
});
if (remainingLimit > 0 && totalFolderCount > adjustedOffset) {
folders = await server.services.folder.getFolders({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
environment,
path: secretPath,
orderBy,
orderDirection,
search,
limit: remainingLimit,
offset: adjustedOffset
});
remainingLimit -= folders.length;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalFolderCount);
}
}
if (includeDynamicSecrets) {
totalDynamicSecretCount = await server.services.dynamicSecret.getCount({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
search,
environmentSlug: environment,
path: secretPath
});
if (remainingLimit > 0 && totalDynamicSecretCount > adjustedOffset) {
dynamicSecrets = await server.services.dynamicSecret.list({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId,
search,
orderBy,
orderDirection,
environmentSlug: environment,
path: secretPath,
limit: remainingLimit,
offset: adjustedOffset
});
remainingLimit -= dynamicSecrets.length;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalDynamicSecretCount);
}
}
if (includeSecrets) {
totalSecretCount = await server.services.secret.getSecretsCount({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
environment,
actorAuthMethod: req.permission.authMethod,
projectId,
path: secretPath,
search,
tagSlugs: tags
});
if (remainingLimit > 0 && totalSecretCount > adjustedOffset) {
const secretsRaw = await server.services.secret.getSecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
environment,
actorAuthMethod: req.permission.authMethod,
projectId,
path: secretPath,
orderBy,
orderDirection,
search,
limit: remainingLimit,
offset: adjustedOffset,
tagSlugs: tags
});
secrets = secretsRaw.secrets;
await server.services.auditLog.createAuditLog({
projectId,
...req.auditLogInfo,
event: {
type: EventType.GET_SECRETS,
metadata: {
environment,
secretPath,
numberOfSecrets: secrets.length
}
}
});
if (getUserAgentType(req.headers["user-agent"]) !== UserAgentType.K8_OPERATOR) {
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretPulled,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: projectId,
environment,
secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
}
}
}
return {
imports,
folders,
dynamicSecrets,
secrets,
totalImportCount,
totalFolderCount,
totalDynamicSecretCount,
totalSecretCount,
totalCount:
(totalImportCount ?? 0) + (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0)
};
}
});
};
+2
View File
@@ -1,3 +1,4 @@
import { registerDashboardRouter } from "./dashboard-router";
import { registerLoginRouter } from "./login-router";
import { registerSecretBlindIndexRouter } from "./secret-blind-index-router";
import { registerSecretRouter } from "./secret-router";
@@ -10,4 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => {
await server.register(registerUserRouter, { prefix: "/users" });
await server.register(registerSecretRouter, { prefix: "/secrets" });
await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" });
await server.register(registerDashboardRouter, { prefix: "/dashboard" });
};
@@ -5,6 +5,8 @@ import { TableName, TProjectEnvironments, TSecretFolders, TSecretFoldersUpdate }
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { groupBy, removeTrailingSlash } from "@app/lib/fn";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
export const validateFolderName = (folderName: string) => {
const validNameRegex = /^[a-zA-Z0-9-_]+$/;
@@ -83,7 +85,7 @@ const sqlFindMultipleFolderByEnvPathQuery = (db: Knex, query: Array<{ envId: str
.from<TSecretFolders & { depth: number; path: string }>("parent");
};
const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: string, secretPath: string) => {
const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environments: string[], secretPath: string) => {
// this is removing an trailing slash like /folder1/folder2/ -> /folder1/folder2
const formatedPath = secretPath.at(-1) === "/" && secretPath.length > 1 ? secretPath.slice(0, -1) : secretPath;
// next goal to sanitize saw the raw sql query is safe
@@ -111,7 +113,7 @@ const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: stri
projectId,
parentId: null
})
.where(`${TableName.Environment}.slug`, environment)
.whereIn(`${TableName.Environment}.slug`, environments)
.select(selectAllTableCols(TableName.SecretFolder))
.union(
(qb) =>
@@ -139,14 +141,14 @@ const sqlFindFolderByPathQuery = (db: Knex, projectId: string, environment: stri
.from<TSecretFolders & { depth: number; path: string }>("parent")
.leftJoin<TProjectEnvironments>(TableName.Environment, `${TableName.Environment}.id`, "parent.envId")
.select<
TSecretFolders & {
(TSecretFolders & {
depth: number;
path: string;
envId: string;
envSlug: string;
envName: string;
projectId: string;
}
})[]
>(
selectAllTableCols("parent" as TableName.SecretFolder),
db.ref("id").withSchema(TableName.Environment).as("envId"),
@@ -214,7 +216,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
const folder = await sqlFindFolderByPathQuery(
tx || db.replicaNode(),
projectId,
environment,
[environment],
removeTrailingSlash(path)
)
.orderBy("depth", "desc")
@@ -230,6 +232,35 @@ export const secretFolderDALFactory = (db: TDbClient) => {
}
};
// finds folders by path for multiple envs
const findBySecretPathMultiEnv = async (projectId: string, environments: string[], path: string, tx?: Knex) => {
try {
const pathDepth = removeTrailingSlash(path).split("/").filter(Boolean).length + 1;
const folders = await sqlFindFolderByPathQuery(
tx || db.replicaNode(),
projectId,
environments,
removeTrailingSlash(path)
)
.orderBy("depth", "desc")
.where("depth", pathDepth);
const firstFolder = folders[0];
if (firstFolder && firstFolder.path !== removeTrailingSlash(path)) {
return [];
}
return folders.map((folder) => {
const { envId: id, envName: name, envSlug: slug, ...el } = folder;
return { ...el, envId: id, environment: { id, name, slug } };
});
} catch (error) {
throw new DatabaseError({ error, name: "Find folders by secret path multi env" });
}
};
// used in folder creation
// even if its the original given /path1/path2
// it will stop automatically at /path2
@@ -238,7 +269,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
const folder = await sqlFindFolderByPathQuery(
tx || db.replicaNode(),
projectId,
environment,
[environment],
removeTrailingSlash(path)
)
.orderBy("depth", "desc")
@@ -352,14 +383,71 @@ export const secretFolderDALFactory = (db: TDbClient) => {
}
};
// find project folders for multiple envs
const findByMultiEnv = async (
{
environmentIds,
parentIds,
search,
limit,
offset = 0,
orderBy = SecretsOrderBy.Name,
orderDirection = OrderByDirection.ASC
}: {
environmentIds: string[];
parentIds: string[];
search?: string;
limit?: number;
offset?: number;
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
},
tx?: Knex
) => {
try {
const query = (tx || db.replicaNode())(TableName.SecretFolder)
.whereIn("parentId", parentIds)
.whereIn("envId", environmentIds)
.where("isReserved", false)
.where((bd) => {
if (search) {
void bd.whereILike("name", `%${search}%`);
}
})
.select(
selectAllTableCols(TableName.SecretFolder),
db.raw(`DENSE_RANK() OVER (ORDER BY "name" ${orderDirection ?? OrderByDirection.ASC}) as rank`)
)
.orderBy(orderBy, orderDirection);
if (limit) {
const rankOffset = offset + 1; // ranks start from 1
return await (tx || db)
.with("w", query)
.select("*")
.from<Awaited<typeof query>[number]>("w")
.where("w.rank", ">=", rankOffset)
.andWhere("w.rank", "<", rankOffset + limit);
}
const folders = await query;
return folders;
} catch (error) {
throw new DatabaseError({ error, name: "Find folders multi env" });
}
};
return {
...secretFolderOrm,
update,
findBySecretPath,
findBySecretPathMultiEnv,
findById,
findByManySecretPath,
findSecretPathByFolderIds,
findClosestFolder,
findByProjectId
findByProjectId,
findByMultiEnv
};
};
@@ -2,11 +2,12 @@ import { ForbiddenError, subject } from "@casl/ability";
import path from "path";
import { v4 as uuidv4, validate as uuidValidate } from "uuid";
import { TSecretFoldersInsert } from "@app/db/schemas";
import { TSecretFolders, TSecretFoldersInsert } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrderByDirection } from "@app/lib/types";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
@@ -26,7 +27,7 @@ type TSecretFolderServiceFactoryDep = {
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
folderDAL: TSecretFolderDALFactory;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "findBySlugs">;
folderVersionDAL: TSecretFolderVersionDALFactory;
projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
};
@@ -396,7 +397,12 @@ export const secretFolderServiceFactory = ({
actorOrgId,
actorAuthMethod,
environment,
path: secretPath
path: secretPath,
search,
orderBy,
orderDirection,
limit,
offset
}: TGetFolderDTO) => {
// folder list is allowed to be read by anyone
// permission to check does user has access
@@ -408,11 +414,100 @@ export const secretFolderServiceFactory = ({
const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!parentFolder) return [];
const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false });
const folders = await folderDAL.find(
{
envId: env.id,
parentId: parentFolder.id,
isReserved: false,
$search: search ? { name: `%${search}%` } : undefined
},
{
sort: orderBy ? [[orderBy, orderDirection ?? OrderByDirection.ASC]] : undefined,
limit,
offset
}
);
return folders;
};
// get folders for multiple envs
const getFoldersMultiEnv = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
environments,
path: secretPath,
...params
}: Omit<TGetFolderDTO, "environment"> & { environments: string[] }) => {
// folder list is allowed to be read by anyone
// permission to check does user has access
await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
const envs = await projectEnvDAL.findBySlugs(projectId, environments);
const data: { [key: string]: TSecretFolders[] } = {};
if (!envs.length)
throw new BadRequestError({ message: "Environment(s) not found", name: "get project folder count" });
const parentFolders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, secretPath);
if (!parentFolders.length) return data;
const folders = await folderDAL.findByMultiEnv({
environmentIds: envs.map((env) => env.id),
parentIds: parentFolders.map((folder) => folder.id),
...params
});
// reorganize folders into respective envs
const envMap: Map<string, string> = new Map(envs.map((env) => [env.id, env.slug]));
folders.forEach((folder) => {
const slug = envMap.get(folder.envId);
data[slug!] = [...(data[slug!] ?? []), folder];
});
return data;
};
// get the unique count of folders within a project path
const getProjectFolderCount = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
environments,
path: secretPath,
search
}: Omit<TGetFolderDTO, "environment"> & { environments: string[] }) => {
// folder list is allowed to be read by anyone
// permission to check does user has access
await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
const envs = await projectEnvDAL.findBySlugs(projectId, environments);
if (!envs.length)
throw new BadRequestError({ message: "Environment(s) not found", name: "get project folder count" });
const parentFolders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, secretPath);
if (!parentFolders.length) return 0;
const folders = await folderDAL.find(
{
$in: {
envId: envs.map((env) => env.id),
parentId: parentFolders.map((folder) => folder.id)
},
isReserved: false,
$search: search ? { name: `%${search}%` } : undefined
},
{ countDistinct: "name" }
);
return Number(folders[0]?.count ?? 0);
};
const getFolderById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetFolderByIdDTO) => {
const folder = await folderDAL.findById(id);
if (!folder) throw new NotFoundError({ message: "folder not found" });
@@ -429,6 +524,8 @@ export const secretFolderServiceFactory = ({
updateManyFolders,
deleteFolder,
getFolders,
getFolderById
getFolderById,
getProjectFolderCount,
getFoldersMultiEnv
};
};
@@ -1,4 +1,5 @@
import { TProjectPermission } from "@app/lib/types";
import { OrderByDirection, TProjectPermission } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
export enum ReservedFolders {
SecretReplication = "__reserve_replication_"
@@ -36,6 +37,11 @@ export type TDeleteFolderDTO = {
export type TGetFolderDTO = {
environment: string;
path: string;
search?: string;
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
limit?: number;
offset?: number;
} & TProjectPermission;
export type TGetFolderByIdDTO = {
@@ -49,10 +49,30 @@ export const secretImportDALFactory = (db: TDbClient) => {
}
};
const find = async (filter: Partial<TSecretImports & { projectId: string }>, tx?: Knex) => {
const find = async (
{
search,
limit,
offset,
...filter
}: Partial<
TSecretImports & {
projectId: string;
search?: string;
limit?: number;
offset?: number;
}
>,
tx?: Knex
) => {
try {
const docs = await (tx || db.replicaNode())(TableName.SecretImport)
const query = (tx || db.replicaNode())(TableName.SecretImport)
.where(filter)
.where((bd) => {
if (search) {
void bd.whereILike("importPath", `%${search}%`);
}
})
.join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
.select(
db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,
@@ -61,6 +81,13 @@ export const secretImportDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.Environment).as("envId")
)
.orderBy("position", "asc");
if (limit) {
void query.limit(limit).offset(offset ?? 0);
}
const docs = await query;
return docs.map(({ envId, slug, name, ...el }) => ({
...el,
importEnv: { id: envId, slug, name }
@@ -70,6 +97,27 @@ export const secretImportDALFactory = (db: TDbClient) => {
}
};
const getProjectImportCount = async (
{ search, ...filter }: Partial<TSecretImports & { projectId: string; search?: string }>,
tx?: Knex
) => {
try {
const docs = await (tx || db.replicaNode())(TableName.SecretImport)
.where(filter)
.where((bd) => {
if (search) {
void bd.whereILike("importPath", `%${search}%`);
}
})
.join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
.count();
return Number(docs[0]?.count ?? 0);
} catch (error) {
throw new DatabaseError({ error, name: "get secret imports count" });
}
};
const findByFolderIds = async (folderIds: string[], tx?: Knex) => {
try {
const docs = await (tx || db.replicaNode())(TableName.SecretImport)
@@ -97,6 +145,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
find,
findByFolderIds,
findLastImportPosition,
updateAllPosition
updateAllPosition,
getProjectImportCount
};
};
@@ -394,14 +394,15 @@ export const secretImportServiceFactory = ({
return { message: "replication started" };
};
const getImports = async ({
const getProjectImportCount = async ({
path: secretPath,
environment,
projectId,
actor,
actorId,
actorAuthMethod,
actorOrgId
actorOrgId,
search
}: TGetSecretImportsDTO) => {
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -418,7 +419,39 @@ export const secretImportServiceFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
const secImports = await secretImportDAL.find({ folderId: folder.id });
const count = await secretImportDAL.getProjectImportCount({ folderId: folder.id, search });
return count;
};
const getImports = async ({
path: secretPath,
environment,
projectId,
actor,
actorId,
actorAuthMethod,
actorOrgId,
search,
limit,
offset
}: TGetSecretImportsDTO) => {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
const secImports = await secretImportDAL.find({ folderId: folder.id, search, limit, offset });
return secImports;
};
@@ -531,6 +564,7 @@ export const secretImportServiceFactory = ({
getSecretsFromImports,
getRawSecretsFromImports,
resyncSecretImportReplication,
getProjectImportCount,
fnSecretsFromImports
};
};
@@ -32,6 +32,9 @@ export type TDeleteSecretImportDTO = {
export type TGetSecretImportsDTO = {
environment: string;
path: string;
search?: string;
limit?: number;
offset?: number;
} & TProjectPermission;
export type TGetSecretsFromImportDTO = {
@@ -5,6 +5,8 @@ import { TDbClient } from "@app/db";
import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
@@ -181,7 +183,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderIds = async (folderIds: string[], userId?: string, tx?: Knex) => {
// get unique secret count by folder IDs
const countByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: {
search?: string;
tagSlugs?: string[];
}
) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
@@ -189,8 +200,70 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId = undefined;
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn("folderId", folderIds)
.where((bd) => {
if (filters?.search) {
void bd.whereILike("key", `%${filters?.search}%`);
}
})
.where((bd) => {
void bd.whereNull("userId").orWhere({ userId: userId || null });
})
.countDistinct("key");
// only need to join tags if filtering by tag slugs
const slugs = filters?.tagSlugs?.filter(Boolean);
if (slugs && slugs.length > 0) {
void query
.leftJoin(
TableName.SecretV2JnTag,
`${TableName.SecretV2}.id`,
`${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.whereIn("slug", slugs);
}
const secrets = await query;
return Number(secrets[0]?.count ?? 0);
} catch (error) {
throw new DatabaseError({ error, name: "get folder secret count" });
}
};
const findByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: {
limit?: number;
offset?: number;
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
search?: string;
tagSlugs?: string[];
}
) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
// eslint-disable-next-line no-param-reassign
userId = undefined;
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn("folderId", folderIds)
.where((bd) => {
if (filters?.search) {
void bd.whereILike("key", `%${filters?.search}%`);
}
})
.where((bd) => {
void bd.whereNull("userId").orWhere({ userId: userId || null });
})
@@ -204,11 +277,37 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
`${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.select(selectAllTableCols(TableName.SecretV2))
.select(
selectAllTableCols(TableName.SecretV2),
db.raw(`DENSE_RANK() OVER (ORDER BY "key" ${filters?.orderDirection ?? OrderByDirection.ASC}) as rank`)
)
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.orderBy("id", "asc");
.where((bd) => {
const slugs = filters?.tagSlugs?.filter(Boolean);
if (slugs && slugs.length > 0) {
void bd.whereIn("slug", slugs);
}
})
.orderBy(
filters?.orderBy === SecretsOrderBy.Name ? "key" : "id",
filters?.orderDirection ?? OrderByDirection.ASC
);
let secs: Awaited<typeof query>;
if (filters?.limit) {
const rankOffset = (filters?.offset ?? 0) + 1; // ranks start at 1
secs = await (tx || db)
.with("w", query)
.select("*")
.from<Awaited<typeof query>[number]>("w")
.where("w.rank", ">=", rankOffset)
.andWhere("w.rank", "<", rankOffset + filters.limit);
} else {
secs = await query;
}
const data = sqlNestRelationships({
data: secs,
@@ -384,6 +483,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
findBySecretKeys,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
findAllProjectSecretValues,
countByFolderIds
};
};
@@ -59,7 +59,7 @@ type TSecretV2BridgeServiceFactoryDep = {
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
folderDAL: Pick<
TSecretFolderDALFactory,
"findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find"
"findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find" | "findBySecretPathMultiEnv"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "handleSecretReminder" | "removeSecretReminder">;
@@ -431,6 +431,175 @@ export const secretV2BridgeServiceFactory = ({
});
};
// get unique secrets count for multiple envs
const getSecretsCountMultiEnv = async ({
actorId,
path,
projectId,
actor,
actorOrgId,
actorAuthMethod,
environments,
...params
}: Pick<TGetSecretsDTO, "actorId" | "actor" | "path" | "projectId" | "actorOrgId" | "actorAuthMethod" | "search"> & {
environments: string[];
}) => {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
// verify user has access to all environments
environments.forEach((environment) =>
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
)
);
const folders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, path);
if (!folders.length) return 0;
const count = await secretDAL.countByFolderIds(
folders.map((folder) => folder.id),
actorId,
undefined,
params
);
return count;
};
// get secret count for individual env
const getSecretsCount = async ({
actorId,
path,
environment,
projectId,
actor,
actorOrgId,
actorAuthMethod,
...params
}: Pick<
TGetSecretsDTO,
| "actorId"
| "actor"
| "path"
| "projectId"
| "actorOrgId"
| "actorAuthMethod"
| "tagSlugs"
| "environment"
| "search"
>) => {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) return 0;
const count = await secretDAL.countByFolderIds([folder.id], actorId, undefined, params);
return count;
};
// get secrets for multiple envs
const getSecretsMultiEnv = async ({
actorId,
path,
environments,
projectId,
actor,
actorOrgId,
actorAuthMethod,
...params
}: Pick<TGetSecretsDTO, "actorId" | "actor" | "path" | "projectId" | "actorOrgId" | "actorAuthMethod" | "search"> & {
environments: string[];
}) => {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
let paths: { folderId: string; path: string; environment: string }[] = [];
// verify user has access to all environments
environments.forEach((environment) =>
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
)
);
const folders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, path);
const data: { [key: string]: typeof decryptedSecrets } = {};
if (!folders.length) {
environments.forEach((env) => {
data[env] = [];
});
return data;
}
paths = folders.map((folder) => ({ folderId: folder.id, path, environment: folder.environment.slug }));
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId,
undefined,
params
);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const decryptedSecrets = secrets.map((secret) =>
reshapeBridgeSecret(
projectId,
groupedPaths[secret.folderId][0].environment,
groupedPaths[secret.folderId][0].path,
{
...secret,
value: secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString()
: "",
comment: secret.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: secret.encryptedComment }).toString()
: ""
}
)
);
decryptedSecrets.forEach((secret) => {
data[secret.environment] = [...(data[secret.environment] ?? []), secret];
});
return data;
};
const getSecrets = async ({
actorId,
path,
@@ -441,8 +610,8 @@ export const secretV2BridgeServiceFactory = ({
actorAuthMethod,
includeImports,
recursive,
tagSlugs = [],
expandSecretReferences: shouldExpandSecretReferences
expandSecretReferences: shouldExpandSecretReferences,
...params
}: TGetSecretsDTO) => {
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -490,7 +659,9 @@ export const secretV2BridgeServiceFactory = ({
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId
actorId,
undefined,
params
);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
@@ -509,9 +680,7 @@ export const secretV2BridgeServiceFactory = ({
: ""
})
);
const filteredSecrets = tagSlugs.length
? decryptedSecrets.filter((secret) => Boolean(secret.tags?.find((el) => tagSlugs.includes(el.slug))))
: decryptedSecrets;
const expandSecretReferences = expandSecretReferencesFactory({
projectId,
folderDAL,
@@ -520,7 +689,7 @@ export const secretV2BridgeServiceFactory = ({
});
if (shouldExpandSecretReferences) {
const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
const secretsGroupByPath = groupBy(decryptedSecrets, (i) => i.secretPath);
await Promise.allSettled(
Object.keys(secretsGroupByPath).map((groupedPath) =>
Promise.allSettled(
@@ -541,7 +710,7 @@ export const secretV2BridgeServiceFactory = ({
if (!includeImports) {
return {
secrets: filteredSecrets
secrets: decryptedSecrets
};
}
@@ -569,7 +738,7 @@ export const secretV2BridgeServiceFactory = ({
});
return {
secrets: filteredSecrets,
secrets: decryptedSecrets,
imports: importedSecrets
};
};
@@ -1416,6 +1585,9 @@ export const secretV2BridgeServiceFactory = ({
getSecrets,
getSecretVersions,
backfillSecretReferences,
moveSecrets
moveSecrets,
getSecretsCount,
getSecretsCountMultiEnv,
getSecretsMultiEnv
};
};
@@ -1,8 +1,9 @@
import { Knex } from "knex";
import { SecretType, TSecretsV2, TSecretsV2Insert, TSecretsV2Update } from "@app/db/schemas";
import { TProjectPermission } from "@app/lib/types";
import { OrderByDirection, TProjectPermission } from "@app/lib/types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
@@ -21,6 +22,11 @@ export type TGetSecretsDTO = {
includeImports?: boolean;
recursive?: boolean;
tagSlugs?: string[];
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
offset?: number;
limit?: number;
search?: string;
} & TProjectPermission;
export type TGetASecretDTO = {
+122 -3
View File
@@ -954,6 +954,120 @@ export const secretServiceFactory = ({
return secretsDeleted;
};
const getSecretsCount = async ({
projectId,
path,
actor,
actorId,
actorOrgId,
actorAuthMethod,
environment,
tagSlugs = [],
...v2Params
}: Pick<
TGetSecretsRawDTO,
| "projectId"
| "path"
| "actor"
| "actorId"
| "actorOrgId"
| "actorAuthMethod"
| "environment"
| "tagSlugs"
| "search"
>) => {
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (!shouldUseSecretV2Bridge)
throw new BadRequestError({
message: "Project version does not support pagination",
name: "pagination_not_supported"
});
const count = await secretV2BridgeService.getSecretsCount({
projectId,
actorId,
actor,
actorOrgId,
environment,
path,
actorAuthMethod,
tagSlugs,
...v2Params
});
return count;
};
const getSecretsCountMultiEnv = async ({
projectId,
path,
actor,
actorId,
actorOrgId,
actorAuthMethod,
environments,
...v2Params
}: Pick<
TGetSecretsRawDTO,
"projectId" | "path" | "actor" | "actorId" | "actorOrgId" | "actorAuthMethod" | "search"
> & { environments: string[] }) => {
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (!shouldUseSecretV2Bridge)
throw new BadRequestError({
message: "Project version does not support pagination",
name: "pagination_not_supported"
});
const count = await secretV2BridgeService.getSecretsCountMultiEnv({
projectId,
actorId,
actor,
actorOrgId,
environments,
path,
actorAuthMethod,
...v2Params
});
return count;
};
const getSecretsRawMultiEnv = async ({
projectId,
path,
actor,
actorId,
actorOrgId,
actorAuthMethod,
environments,
...params
}: Omit<TGetSecretsRawDTO, "environment" | "includeImports" | "expandSecretReferences" | "recursive" | "tagSlugs"> & {
environments: string[];
}) => {
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (!shouldUseSecretV2Bridge)
throw new BadRequestError({
message: "Project version does not support pagination",
name: "pagination_not_supported"
});
const secrets = await secretV2BridgeService.getSecretsMultiEnv({
projectId,
actorId,
actor,
actorOrgId,
environments,
path,
actorAuthMethod,
...params
});
return secrets;
};
const getSecretsRaw = async ({
projectId,
path,
@@ -965,7 +1079,8 @@ export const secretServiceFactory = ({
includeImports,
expandSecretReferences,
recursive,
tagSlugs = []
tagSlugs = [],
...paramsV2
}: TGetSecretsRawDTO) => {
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (shouldUseSecretV2Bridge) {
@@ -980,7 +1095,8 @@ export const secretServiceFactory = ({
recursive,
actorAuthMethod,
includeImports,
tagSlugs
tagSlugs,
...paramsV2
});
return { secrets, imports };
}
@@ -2656,6 +2772,9 @@ export const secretServiceFactory = ({
getSecretVersions,
backfillSecretReferences,
moveSecrets,
startSecretV2Migration
startSecretV2Migration,
getSecretsCount,
getSecretsCountMultiEnv,
getSecretsRawMultiEnv
};
};
+12 -1
View File
@@ -1,7 +1,7 @@
import { Knex } from "knex";
import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas";
import { TProjectPermission } from "@app/lib/types";
import { OrderByDirection, TProjectPermission } from "@app/lib/types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
@@ -81,6 +81,8 @@ export type TGetSecretsDTO = {
environment: string;
includeImports?: boolean;
recursive?: boolean;
limit?: number;
offset?: number;
} & TProjectPermission;
export type TGetASecretDTO = {
@@ -143,6 +145,10 @@ export type TDeleteBulkSecretDTO = {
}>;
} & TProjectPermission;
export enum SecretsOrderBy {
Name = "name" // "key" for secrets but using name for use across resources
}
export type TGetSecretsRawDTO = {
expandSecretReferences?: boolean;
path: string;
@@ -150,6 +156,11 @@ export type TGetSecretsRawDTO = {
includeImports?: boolean;
recursive?: boolean;
tagSlugs?: string[];
orderBy?: SecretsOrderBy;
orderDirection?: OrderByDirection;
offset?: number;
limit?: number;
search?: string;
} & TProjectPermission;
export type TGetASecretRawDTO = {