awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-06 17:27:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvements: address feedback

Browse Source
This commit is contained in:
Scott Wilson
2025-04-02 14:11:59 -07:00
parent 766c1242fd
commit 577c81be65
44 changed files with 453 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/db/migrations/20250329002640_secrets-v2-unique-key-index.ts
-29
View File
@@ -1,29 +0,0 @@
import { Knex } from "knex";
import { TableName } from "@app/db/schemas";
const INDEX_NAME = "idx_unique_secret_v2_key";
export async function up(knex: Knex): Promise<void> {
const hasKeyCol = await knex.schema.hasColumn(TableName.SecretV2, "key");
const hasFolderIdCol = await knex.schema.hasColumn(TableName.SecretV2, "folderId");
const hasTypeCol = await knex.schema.hasColumn(TableName.SecretV2, "type");
if (hasKeyCol && hasFolderIdCol && hasTypeCol) {
await knex.raw(`
CREATE UNIQUE INDEX ${INDEX_NAME}
ON ${TableName.SecretV2} ("key", "folderId")
WHERE type = 'shared'
`);
}
}
export async function down(knex: Knex): Promise<void> {
const hasKeyCol = await knex.schema.hasColumn(TableName.SecretV2, "key");
const hasFolderIdCol = await knex.schema.hasColumn(TableName.SecretV2, "folderId");
const hasTypeCol = await knex.schema.hasColumn(TableName.SecretV2, "type");
if (hasKeyCol && hasFolderIdCol && hasTypeCol) {
await knex.raw(`DROP INDEX IF EXISTS ${INDEX_NAME}`);
}
}
backend/src/ee/routes/v1/secret-approval-request-router.ts
+3 -1
View File
@@ -277,8 +277,10 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
reviewers: approvalRequestUser.extend({ status: z.string(), comment: z.string().optional() }).array(),
secretPath: z.string(),
commits: secretRawSchema
.omit({ _id: true, environment: true, workspace: true, type: true, version: true })
.omit({ _id: true, environment: true, workspace: true, type: true, version: true, secretValue: true })
.extend({
secretValue: z.string().optional(),
isRotatedSecret: z.boolean().optional(),
op: z.string(),
tags: SanitizedTagSchema.array().optional(),
secretMetadata: ResourceMetadataSchema.nullish(),
backend/src/ee/routes/v1/snapshot-router.ts
+2 -1
View File
@@ -33,7 +33,8 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => {
.extend({
secretValueHidden: z.boolean(),
secretId: z.string(),
tags: SanitizedTagSchema.array()
tags: SanitizedTagSchema.array(),
isRotatedSecret: z.boolean().optional()
})
.array(),
folderVersion: z.object({ id: z.string(), name: z.string() }).array(),
backend/src/ee/routes/v2/secret-rotation-v2-routers/secret-rotation-v2-endpoints.ts
+1 -1
View File
@@ -151,7 +151,7 @@ export const registerSecretRotationEndpoints = <
rateLimit: readLimit
},
schema: {
description: `Get the specified ${rotationType} Rotation by name and project ID.`,
description: `Get the specified ${rotationType} Rotation by name, secret path, environment and project ID.`,
params: z.object({
rotationName: z
.string()
backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts
+2 -1
View File
@@ -13,7 +13,8 @@ export const verifyHostInputValidity = async (host: string, isGateway = false) =
const reservedHosts = [appCfg.DB_HOST || getDbConnectionHost(appCfg.DB_CONNECTION_URI)].concat(
(appCfg.DB_READ_REPLICAS || []).map((el) => getDbConnectionHost(el.DB_CONNECTION_URI)),
getDbConnectionHost(appCfg.REDIS_URL)
getDbConnectionHost(appCfg.REDIS_URL),
getDbConnectionHost(appCfg.AUDIT_LOGS_DB_CONNECTION_URI)
);
// get host db ip
backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts
+11 -3
View File
@@ -257,6 +257,11 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
db.ref("id").withSchema("secVerTag")
)
.leftJoin(TableName.ResourceMetadata, `${TableName.SecretV2}.id`, `${TableName.ResourceMetadata}.secretId`)
.leftJoin(
TableName.SecretRotationV2SecretMapping,
`${TableName.SecretV2}.id`,
`${TableName.SecretRotationV2SecretMapping}.secretId`
)
.select(selectAllTableCols(TableName.SecretApprovalRequestSecretV2))
.select({
secVerTagId: "secVerTag.id",
@@ -285,7 +290,8 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.ResourceMetadata).as("metadataId"),
db.ref("key").withSchema(TableName.ResourceMetadata).as("metadataKey"),
db.ref("value").withSchema(TableName.ResourceMetadata).as("metadataValue")
);
)
.select(db.ref("rotationId").withSchema(TableName.SecretRotationV2SecretMapping));
const formatedDoc = sqlNestRelationships({
data: doc,
key: "id",
@@ -304,14 +310,16 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
{
key: "secretId",
label: "secret" as const,
mapper: ({ orgSecVersion, orgSecKey, orgSecValue, orgSecComment, secretId }) =>
mapper: ({ orgSecVersion, orgSecKey, orgSecValue, orgSecComment, secretId, rotationId }) =>
secretId
? {
id: secretId,
version: orgSecVersion,
key: orgSecKey,
encryptedValue: orgSecValue,
encryptedComment: orgSecComment
encryptedComment: orgSecComment,
isRotatedSecret: Boolean(rotationId),
rotationId
}
: undefined
},
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
+8 -2
View File
@@ -262,7 +262,13 @@ export const secretApprovalRequestServiceFactory = ({
id: el.id,
version: el.version,
secretMetadata: el.secretMetadata as ResourceMetadataDTO,
secretValue: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
isRotatedSecret: el.secret.isRotatedSecret,
// eslint-disable-next-line no-nested-ternary
secretValue: el.secret.isRotatedSecret
? undefined
: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: "",
secretComment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: "",
@@ -609,7 +615,7 @@ export const secretApprovalRequestServiceFactory = ({
tx,
inputSecrets: secretUpdationCommits.map((el) => {
const encryptedValue =
typeof el.encryptedValue !== "undefined"
!el.secret.isRotatedSecret && typeof el.encryptedValue !== "undefined"
? {
encryptedValue: el.encryptedValue as Buffer,
references: el.encryptedValue
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-dal.ts
+5 -3
View File
@@ -189,9 +189,11 @@ export const secretRotationV2DALFactory = (
.countDistinct(`${TableName.SecretRotationV2}.name`);
if (search) {
void query
.whereILike(`${TableName.SecretV2}.key`, `%${search}%`)
.orWhereILike(`${TableName.SecretRotationV2}.name`, `%${search}%`);
void query.where((qb) => {
void qb
.whereILike(`${TableName.SecretV2}.key`, `%${search}%`)
.orWhereILike(`${TableName.SecretRotationV2}.name`, `%${search}%`);
});
}
const result = await query;
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts
+4 -4
View File
@@ -23,7 +23,7 @@ export const listSecretRotationOptions = () => {
return Object.values(SECRET_ROTATION_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
};
const getNextUTCMidnight = ({ hours, minutes }: TSecretRotationV2["rotateAtUtc"] = { hours: 0, minutes: 0 }) => {
const getNextUTCDayInterval = ({ hours, minutes }: TSecretRotationV2["rotateAtUtc"] = { hours: 0, minutes: 0 }) => {
const now = new Date();
return new Date(
@@ -39,7 +39,7 @@ const getNextUTCMidnight = ({ hours, minutes }: TSecretRotationV2["rotateAtUtc"]
);
};
const getNextUTCMinute = ({ minutes }: TSecretRotationV2["rotateAtUtc"] = { hours: 0, minutes: 0 }) => {
const getNextUTCMinuteInterval = ({ minutes }: TSecretRotationV2["rotateAtUtc"] = { hours: 0, minutes: 0 }) => {
const now = new Date();
return new Date(
Date.UTC(
@@ -58,10 +58,10 @@ export const getNextUtcRotationInterval = (rotateAtUtc?: TSecretRotationV2["rota
const appCfg = getConfig();
if (appCfg.isRotationDevelopmentMode) {
return getNextUTCMinute(rotateAtUtc);
return getNextUTCMinuteInterval(rotateAtUtc);
}
return getNextUTCMidnight(rotateAtUtc);
return getNextUTCDayInterval(rotateAtUtc);
};
export const encryptSecretRotationCredentials = async ({
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts
+4 -4
View File
@@ -80,7 +80,7 @@ export const secretRotationV2QueueServiceFactory = async ({
{
batchSize: 1,
workerCount: 1,
pollingIntervalSeconds: 0.5
pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
}
);
@@ -122,7 +122,7 @@ export const secretRotationV2QueueServiceFactory = async ({
},
{
batchSize: 1,
workerCount: 30,
workerCount: 2,
pollingIntervalSeconds: 0.5
}
);
@@ -179,8 +179,8 @@ export const secretRotationV2QueueServiceFactory = async ({
},
{
batchSize: 1,
workerCount: 5,
pollingIntervalSeconds: 30
workerCount: 2,
pollingIntervalSeconds: 1
}
);
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts
+100 -47
View File
@@ -1,4 +1,5 @@
import { ForbiddenError, subject } from "@casl/ability";
import { Knex } from "knex";
import isEqual from "lodash.isequal";
import { ActionProjectType, SecretType, TableName } from "@app/db/schemas";
@@ -46,7 +47,7 @@ import {
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import { sqlCredentialsRotationFactory } from "@app/ee/services/secret-rotation-v2/shared/sql-credentials";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { KeyStorePrefixes, PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, InternalServerError, NotFoundError } from "@app/lib/errors";
@@ -141,6 +142,37 @@ export const secretRotationV2ServiceFactory = ({
);
};
const $throwOnConflictingSecrets = async ({
secretKeys,
folderId,
tx,
secretPath
}: {
secretKeys: string[];
folderId: string;
tx: Knex;
secretPath: string;
}) => {
const conflictingSecrets = await secretV2BridgeDAL.find(
{
$in: {
[`${TableName.SecretV2}.key` as "key"]: secretKeys
},
[`${TableName.SecretV2}.folderId` as "folderId"]: folderId,
[`${TableName.SecretV2}.type` as "type"]: SecretType.Shared
},
{ tx }
);
if (conflictingSecrets.length) {
throw new BadRequestError({
message: `The following secrets already exist at the path "${secretPath}": ${conflictingSecrets
.map(({ key }) => key)
.join(", ")}`
});
}
};
const listSecretRotationsByProjectId = async (
{ projectId, type }: TListSecretRotationsV2ByProjectId,
actor: OrgServiceActor
@@ -345,6 +377,7 @@ export const secretRotationV2ServiceFactory = ({
secretPath,
environment,
rotateAtUtc = { hours: 0, minutes: 0 },
secretsMapping,
...payload
}: TCreateSecretRotationV2DTO,
actor: OrgServiceActor
@@ -368,7 +401,10 @@ export const secretRotationV2ServiceFactory = ({
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (!shouldUseSecretV2Bridge)
throw new BadRequestError({ message: "Project version does not support Secret Rotation V2" });
throw new BadRequestError({
message:
"Project version does not support Secret Rotation V2. Please upgrade your project via the Infiscal Dashboard to gain access."
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretRotationActions.Create,
@@ -389,7 +425,7 @@ export const secretRotationV2ServiceFactory = ({
const rotationFactory = SECRET_ROTATION_FACTORY_MAP[payload.type]({
parameters: payload.parameters,
secretsMapping: payload.secretsMapping,
secretsMapping,
connection
} as TSecretRotationV2WithConnection);
@@ -405,9 +441,19 @@ export const secretRotationV2ServiceFactory = ({
});
return secretRotationV2DAL.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.SecretRotationV2Creation(folder.id)]);
await $throwOnConflictingSecrets({
secretPath,
secretKeys: Object.values(secretsMapping),
tx,
folderId: folder.id
});
const createdRotation = await secretRotationV2DAL.create(
{
folderId: folder.id,
secretsMapping,
...payload,
encryptedGeneratedCredentials,
rotateAtUtc,
@@ -483,12 +529,6 @@ export const secretRotationV2ServiceFactory = ({
throw new BadRequestError({
message: `A Secret Rotation with the name "${payload.name}" already exists at the secret path "${secretPath}"`
});
case TableName.SecretV2:
throw new BadRequestError({
message: `One or more of the following secrets already exists at the secret path "${secretPath}": ${Object.values(
payload.secretsMapping
).join(", ")}`
});
default:
throw err;
}
@@ -497,6 +537,8 @@ export const secretRotationV2ServiceFactory = ({
throw err;
}
if (err instanceof BadRequestError) throw err;
throw new BadRequestError({
message: parseRotationErrorMessage(err)
});
@@ -521,7 +563,8 @@ export const secretRotationV2ServiceFactory = ({
message: `Could not find ${SECRET_ROTATION_NAME_MAP[type]} Rotation with ID ${rotationId}`
});
const { folder, environment, projectId, folderId, connection, secretsMapping } = secretRotation;
const { folder, environment, projectId, folderId, connection } = secretRotation;
const secretsMapping = secretRotation.secretsMapping as TSecretRotationV2["secretsMapping"];
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
@@ -551,26 +594,36 @@ export const secretRotationV2ServiceFactory = ({
isManualRotation: false
});
let secretsMappingUpdated = false;
try {
const updatedSecretRotation = await secretRotationV2DAL.transaction(async (tx) => {
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.SecretRotationV2Creation(folder.id)]);
if (payload.secretsMapping && !isEqual(payload.secretsMapping, secretsMapping)) {
const currentMappingKeys = Object.values(secretsMapping);
await $throwOnConflictingSecrets({
secretPath: folder.path,
secretKeys: Object.values(payload.secretsMapping).filter((key) => !currentMappingKeys.includes(key)),
tx,
folderId: folder.id
});
// update mapped secrets names
await fnSecretBulkUpdate({
folderId,
orgId: connection.orgId,
tx,
inputSecrets: Object.entries(secretsMapping as TSecretRotationV2["secretsMapping"]).map(
([mappingKey, secretKey]) => ({
filter: {
key: secretKey,
folderId,
type: SecretType.Shared
},
data: {
key: payload.secretsMapping![mappingKey as keyof TSecretRotationV2["secretsMapping"]]
}
})
),
inputSecrets: Object.entries(secretsMapping).map(([mappingKey, secretKey]) => ({
filter: {
key: secretKey,
folderId,
type: SecretType.Shared
},
data: {
key: payload.secretsMapping![mappingKey as keyof TSecretRotationV2["secretsMapping"]]
}
})),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL,
@@ -578,14 +631,7 @@ export const secretRotationV2ServiceFactory = ({
resourceMetadataDAL
});
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
secretPath: folder.path,
projectId,
environmentSlug: environment.slug,
excludeReplication: true
});
secretsMappingUpdated = true;
}
return secretRotationV2DAL.updateById(
@@ -598,6 +644,17 @@ export const secretRotationV2ServiceFactory = ({
);
});
if (secretsMappingUpdated) {
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
secretPath: folder.path,
projectId,
environmentSlug: environment.slug,
excludeReplication: true
});
}
// queue for rotation if adjusted time falls before next cron
if (nextRotationAt && nextRotationAt.getTime() < getNextUtcRotationInterval().getTime()) {
await queueService.queuePg(
@@ -620,20 +677,14 @@ export const secretRotationV2ServiceFactory = ({
message: `A Secret Rotation with the name "${payload.name}" already exists at the secret path "${folder.path}"`
});
break;
case TableName.SecretV2:
if (payload.secretsMapping)
throw new BadRequestError({
message: `One or more of the following secrets already exists at the secret path "${
folder.path
}": ${Object.values(payload.secretsMapping).join(", ")}`
});
break;
default:
throw err;
}
}
}
if (err instanceof BadRequestError) throw err;
throw err;
}
};
@@ -695,15 +746,6 @@ export const secretRotationV2ServiceFactory = ({
actorId: actor.id, // not actually used since rotated secrets are shared
tx
});
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
secretPath: folder.path,
projectId,
environmentSlug: environment.slug,
excludeReplication: true
});
}
return secretRotationV2DAL.deleteById(rotationId, tx);
@@ -728,6 +770,17 @@ export const secretRotationV2ServiceFactory = ({
await deleteTransaction;
}
if (deleteSecrets) {
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
secretPath: folder.path,
projectId,
environmentSlug: environment.slug,
excludeReplication: true
});
}
return expandSecretRotation(secretRotation, kmsService);
};
backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts
+51 -24
View File
@@ -398,8 +398,32 @@ export const secretSnapshotServiceFactory = ({
if (shouldUseBridge) {
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshotsV2Bridge(snapshot.id, tx);
// this will remove all secrets in current folder
const deletedTopLevelSecs = await secretV2BridgeDAL.delete({ folderId: snapshot.folderId }, tx);
const secretRotationIds = rollbackSnaps
.flatMap((snap) => snap.secretVersions)
.filter((el) => el.isRotatedSecret)
.map((el) => el.secretId);
// this will remove all secrets in current folder except rotated secrets which we ignore
const deletedTopLevelSecs = await secretV2BridgeDAL.delete(
{
$complex: {
operator: "and",
value: [
{
operator: "eq",
field: "folderId",
value: snapshot.folderId
},
{
operator: "notIn",
field: "id",
value: secretRotationIds
}
]
}
},
tx
);
const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
// this will remove all secrets and folders on child
// due to sql foreign key and link list connection removing the folders removes everything below too
@@ -424,28 +448,31 @@ export const secretSnapshotServiceFactory = ({
);
const secrets = await secretV2BridgeDAL.insertMany(
rollbackSnaps.flatMap(({ secretVersions, folderId }) =>
secretVersions.map(
({
latestSecretVersion,
version,
updatedAt,
createdAt,
secretId,
envId,
id,
tags,
// exclude the bottom fields from the secret - they are for versioning only.
userActorId,
identityActorId,
actorType,
...el
}) => ({
...el,
id: secretId,
version: deletedTopLevelSecsGroupById[secretId] ? latestSecretVersion + 1 : latestSecretVersion,
folderId
})
)
secretVersions
.filter((v) => !v.isRotatedSecret)
.map(
({
latestSecretVersion,
version,
updatedAt,
createdAt,
secretId,
envId,
id,
tags,
// exclude the bottom fields from the secret - they are for versioning only.
userActorId,
identityActorId,
actorType,
isRotatedSecret,
...el
}) => ({
...el,
id: secretId,
version: deletedTopLevelSecsGroupById[secretId] ? latestSecretVersion + 1 : latestSecretVersion,
folderId
})
)
),
tx
);
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+21 -4
View File
@@ -181,6 +181,11 @@ export const snapshotDALFactory = (db: TDbClient) => {
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.leftJoin(
TableName.SecretRotationV2SecretMapping,
`${TableName.SecretRotationV2SecretMapping}.secretId`,
`${TableName.SecretVersionV2}.secretId`
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"),
@@ -195,7 +200,8 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("rotationId").withSchema(TableName.SecretRotationV2SecretMapping)
);
return sqlNestRelationships({
data,
@@ -221,7 +227,11 @@ export const snapshotDALFactory = (db: TDbClient) => {
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => SecretVersionsV2Schema.parse(el),
mapper: (el) => ({
...SecretVersionsV2Schema.parse(el),
isRotatedSecret: Boolean(el.rotationId),
rotationId: el.rotationId
}),
childrenMapper: [
{
key: "tagVersionId",
@@ -476,6 +486,11 @@ export const snapshotDALFactory = (db: TDbClient) => {
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(
TableName.SecretRotationV2SecretMapping,
`${TableName.SecretVersionV2}.secretId`,
`${TableName.SecretRotationV2SecretMapping}.secretId`
)
.leftJoin<{ latestSecretVersion: number }>(
(tx || db)(TableName.SecretVersionV2)
.groupBy("secretId")
@@ -506,7 +521,8 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("rotationId").withSchema(TableName.SecretRotationV2SecretMapping)
);
const formated = sqlNestRelationships({
@@ -523,7 +539,8 @@ export const snapshotDALFactory = (db: TDbClient) => {
label: "secretVersions" as const,
mapper: (el) => ({
...SecretVersionsV2Schema.parse(el),
latestSecretVersion: el.latestSecretVersion as number
latestSecretVersion: el.latestSecretVersion as number,
isRotatedSecret: Boolean(el.rotationId)
}),
childrenMapper: [
{
backend/src/keystore/keystore.ts
+2 -1
View File
@@ -8,7 +8,8 @@ export const PgSqlLock = {
SuperAdminInit: 2024,
KmsRootKeyInit: 2025,
OrgGatewayRootCaInit: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-root-ca:${orgId}`),
OrgGatewayCertExchange: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-cert-exchange:${orgId}`)
OrgGatewayCertExchange: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-cert-exchange:${orgId}`),
SecretRotationV2Creation: (folderId: string) => pgAdvisoryLockHashText(`secret-rotation-v2-creation:${folderId}`)
} as const;
export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
backend/src/lib/knex/connection.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { URL } from "url"; // Import the URL class
export const getDbConnectionHost = (urlString: string) => {
export const getDbConnectionHost = (urlString?: string) => {
if (!urlString) return null;
try {
const url = new URL(urlString);
// Split hostname and port (if provided)
backend/src/lib/knex/dynamic.ts
+15 -5
View File
@@ -2,11 +2,17 @@ import { Knex } from "knex";
import { UnauthorizedError } from "../errors";
type TKnexDynamicPrimitiveOperator<T extends object> = {
operator: "eq" | "ne" | "startsWith" | "endsWith";
value: string;
field: Extract<keyof T, string>;
};
type TKnexDynamicPrimitiveOperator<T extends object> =
| {
operator: "eq" | "ne" | "startsWith" | "endsWith";
value: string;
field: Extract<keyof T, string>;
}
| {
operator: "notIn";
value: string[];
field: Extract<keyof T, string>;
};
type TKnexDynamicInOperator<T extends object> = {
operator: "in";
@@ -48,6 +54,10 @@ export const buildDynamicKnexQuery = <T extends object>(
void queryBuilder.whereILike(filterAst.field, `%${filterAst.value}`);
break;
}
case "notIn": {
void queryBuilder.whereNotIn(filterAst.field, filterAst.value);
break;
}
case "and": {
filterAst.value.forEach((el) => {
void queryBuilder.andWhere((subQueryBuilder) => {
backend/src/services/app-connection/app-connection-service.ts
+33 -42
View File
@@ -25,7 +25,7 @@ import {
TAppConnectionRaw,
TCreateAppConnectionDTO,
TUpdateAppConnectionDTO,
TValidateAppConnectionCredentials
TValidateAppConnectionCredentialsSchema
} from "./app-connection-types";
import { ValidateAwsConnectionCredentialsSchema } from "./aws";
import { awsConnectionService } from "./aws/aws-connection-service";
@@ -50,7 +50,7 @@ export type TAppConnectionServiceFactoryDep = {
export type TAppConnectionServiceFactory = ReturnType<typeof appConnectionServiceFactory>;
const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAppConnectionCredentials> = {
const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAppConnectionCredentialsSchema> = {
[AppConnection.AWS]: ValidateAwsConnectionCredentialsSchema,
[AppConnection.GitHub]: ValidateGitHubConnectionCredentialsSchema,
[AppConnection.GCP]: ValidateGcpConnectionCredentialsSchema,
@@ -170,26 +170,22 @@ export const appConnectionServiceFactory = ({
} as TAppConnectionConfig);
try {
const createTransaction = (connectionCredentials: TAppConnection["credentials"]) =>
appConnectionDAL.transaction(async (tx) => {
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: connectionCredentials,
orgId: actor.orgId,
kmsService
});
return appConnectionDAL.create(
{
orgId: actor.orgId,
encryptedCredentials,
method,
app,
...params
},
tx
);
const createConnection = async (connectionCredentials: TAppConnection["credentials"]) => {
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: connectionCredentials,
orgId: actor.orgId,
kmsService
});
return appConnectionDAL.create({
orgId: actor.orgId,
encryptedCredentials,
method,
app,
...params
});
};
let connection: TAppConnectionRaw;
if (params.isPlatformManagedCredentials) {
@@ -200,10 +196,10 @@ export const appConnectionServiceFactory = ({
credentials: validatedCredentials,
method
} as TAppConnectionConfig,
(platformCredentials) => createTransaction(platformCredentials)
(platformCredentials) => createConnection(platformCredentials)
);
} else {
connection = await createTransaction(validatedCredentials);
connection = await createConnection(validatedCredentials);
}
return {
@@ -277,26 +273,21 @@ export const appConnectionServiceFactory = ({
}
try {
const updateTransaction = (connectionCredentials: TAppConnection["credentials"] | undefined) =>
appConnectionDAL.transaction(async (tx) => {
const encryptedCredentials = connectionCredentials
? await encryptAppConnectionCredentials({
credentials: connectionCredentials,
orgId: actor.orgId,
kmsService
})
: undefined;
return appConnectionDAL.updateById(
connectionId,
{
const updateConnection = async (connectionCredentials: TAppConnection["credentials"] | undefined) => {
const encryptedCredentials = connectionCredentials
? await encryptAppConnectionCredentials({
credentials: connectionCredentials,
orgId: actor.orgId,
encryptedCredentials,
...params
},
tx
);
kmsService
})
: undefined;
return appConnectionDAL.updateById(connectionId, {
orgId: actor.orgId,
encryptedCredentials,
...params
});
};
let updatedConnection: TAppConnectionRaw;
@@ -312,10 +303,10 @@ export const appConnectionServiceFactory = ({
credentials: updatedCredentials,
method
} as TAppConnectionConfig,
(platformCredentials) => updateTransaction(platformCredentials)
(platformCredentials) => updateConnection(platformCredentials)
);
} else {
updatedConnection = await updateTransaction(updatedCredentials);
updatedConnection = await updateConnection(updatedCredentials);
}
return await decryptAppConnection(updatedConnection, kmsService);
backend/src/services/app-connection/app-connection-types.ts
+33 -19
View File
@@ -3,40 +3,54 @@ import { TSqlConnectionConfig } from "@app/services/app-connection/shared/sql/sq
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { AWSRegion } from "./app-connection-enums";
import { TAwsConnection, TAwsConnectionConfig, TAwsConnectionInput, TValidateAwsConnectionCredentials } from "./aws";
import {
TAwsConnection,
TAwsConnectionConfig,
TAwsConnectionInput,
TValidateAwsConnectionCredentialsSchema
} from "./aws";
import {
TAzureAppConfigurationConnection,
TAzureAppConfigurationConnectionConfig,
TAzureAppConfigurationConnectionInput,
TValidateAzureAppConfigurationConnectionCredentials
TValidateAzureAppConfigurationConnectionCredentialsSchema
} from "./azure-app-configuration";
import {
TAzureKeyVaultConnection,
TAzureKeyVaultConnectionConfig,
TAzureKeyVaultConnectionInput,
TValidateAzureKeyVaultConnectionCredentials
TValidateAzureKeyVaultConnectionCredentialsSchema
} from "./azure-key-vault";
import {
TDatabricksConnection,
TDatabricksConnectionConfig,
TDatabricksConnectionInput,
TValidateDatabricksConnectionCredentials
TValidateDatabricksConnectionCredentialsSchema
} from "./databricks";
import { TGcpConnection, TGcpConnectionConfig, TGcpConnectionInput, TValidateGcpConnectionCredentials } from "./gcp";
import {
TGcpConnection,
TGcpConnectionConfig,
TGcpConnectionInput,
TValidateGcpConnectionCredentialsSchema
} from "./gcp";
import {
TGitHubConnection,
TGitHubConnectionConfig,
TGitHubConnectionInput,
TValidateGitHubConnectionCredentials
TValidateGitHubConnectionCredentialsSchema
} from "./github";
import {
THumanitecConnection,
THumanitecConnectionConfig,
THumanitecConnectionInput,
TValidateHumanitecConnectionCredentials
TValidateHumanitecConnectionCredentialsSchema
} from "./humanitec";
import { TMsSqlConnection, TMsSqlConnectionInput, TValidateMsSqlConnectionCredentials } from "./mssql";
import { TPostgresConnection, TPostgresConnectionInput, TValidatePostgresConnectionCredentials } from "./postgres";
import { TMsSqlConnection, TMsSqlConnectionInput, TValidateMsSqlConnectionCredentialsSchema } from "./mssql";
import {
TPostgresConnection,
TPostgresConnectionInput,
TValidatePostgresConnectionCredentialsSchema
} from "./postgres";
export type TAppConnection = { id: string } & (
| TAwsConnection
@@ -87,16 +101,16 @@ export type TAppConnectionConfig =
| THumanitecConnectionConfig
| TSqlConnectionConfig;
export type TValidateAppConnectionCredentials =
| TValidateAwsConnectionCredentials
| TValidateGitHubConnectionCredentials
| TValidateGcpConnectionCredentials
| TValidateAzureKeyVaultConnectionCredentials
| TValidateAzureAppConfigurationConnectionCredentials
| TValidateDatabricksConnectionCredentials
| TValidateHumanitecConnectionCredentials
| TValidatePostgresConnectionCredentials
| TValidateMsSqlConnectionCredentials;
export type TValidateAppConnectionCredentialsSchema =
| TValidateAwsConnectionCredentialsSchema
| TValidateGitHubConnectionCredentialsSchema
| TValidateGcpConnectionCredentialsSchema
| TValidateAzureKeyVaultConnectionCredentialsSchema
| TValidateAzureAppConfigurationConnectionCredentialsSchema
| TValidateDatabricksConnectionCredentialsSchema
| TValidateHumanitecConnectionCredentialsSchema
| TValidatePostgresConnectionCredentialsSchema
| TValidateMsSqlConnectionCredentialsSchema;
export type TListAwsConnectionKmsKeys = {
connectionId: string;
backend/src/services/app-connection/aws/aws-connection-types.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export type TAwsConnectionInput = z.infer<typeof CreateAwsConnectionSchema> & {
app: AppConnection.AWS;
};
export type TValidateAwsConnectionCredentials = typeof ValidateAwsConnectionCredentialsSchema;
export type TValidateAwsConnectionCredentialsSchema = typeof ValidateAwsConnectionCredentialsSchema;
export type TAwsConnectionConfig = DiscriminativePick<TAwsConnectionInput, "method" | "app" | "credentials"> & {
orgId: string;
backend/src/services/app-connection/azure-app-configuration/azure-app-configuration-connection-types.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export type TAzureAppConfigurationConnectionInput = z.infer<typeof CreateAzureAp
app: AppConnection.AzureAppConfiguration;
};
export type TValidateAzureAppConfigurationConnectionCredentials =
export type TValidateAzureAppConfigurationConnectionCredentialsSchema =
typeof ValidateAzureAppConfigurationConnectionCredentialsSchema;
export type TAzureAppConfigurationConnectionConfig = DiscriminativePick<
backend/src/services/app-connection/azure-key-vault/azure-key-vault-connection-types.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export type TAzureKeyVaultConnectionInput = z.infer<typeof CreateAzureKeyVaultCo
app: AppConnection.AzureKeyVault;
};
export type TValidateAzureKeyVaultConnectionCredentials = typeof ValidateAzureKeyVaultConnectionCredentialsSchema;
export type TValidateAzureKeyVaultConnectionCredentialsSchema = typeof ValidateAzureKeyVaultConnectionCredentialsSchema;
export type TAzureKeyVaultConnectionConfig = DiscriminativePick<
TAzureKeyVaultConnectionInput,
backend/src/services/app-connection/databricks/databricks-connection-types.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export type TDatabricksConnectionInput = z.infer<typeof CreateDatabricksConnecti
app: AppConnection.Databricks;
};
export type TValidateDatabricksConnectionCredentials = typeof ValidateDatabricksConnectionCredentialsSchema;
export type TValidateDatabricksConnectionCredentialsSchema = typeof ValidateDatabricksConnectionCredentialsSchema;
export type TDatabricksConnectionConfig = DiscriminativePick<
TDatabricksConnection,
backend/src/services/app-connection/gcp/gcp-connection-types.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export type TGcpConnectionInput = z.infer<typeof CreateGcpConnectionSchema> & {
app: AppConnection.GCP;
};
export type TValidateGcpConnectionCredentials = typeof ValidateGcpConnectionCredentialsSchema;
export type TValidateGcpConnectionCredentialsSchema = typeof ValidateGcpConnectionCredentialsSchema;
export type TGcpConnectionConfig = DiscriminativePick<TGcpConnectionInput, "method" | "app" | "credentials"> & {
orgId: string;
backend/src/services/app-connection/github/github-connection-types.ts
+1 -1
View File
@@ -15,6 +15,6 @@ export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema
app: AppConnection.GitHub;
};
export type TValidateGitHubConnectionCredentials = typeof ValidateGitHubConnectionCredentialsSchema;
export type TValidateGitHubConnectionCredentialsSchema = typeof ValidateGitHubConnectionCredentialsSchema;
export type TGitHubConnectionConfig = DiscriminativePick<TGitHubConnectionInput, "method" | "app" | "credentials">;
backend/src/services/app-connection/humanitec/humanitec-connection-types.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export type THumanitecConnectionInput = z.infer<typeof CreateHumanitecConnection
app: AppConnection.Humanitec;
};
export type TValidateHumanitecConnectionCredentials = typeof ValidateHumanitecConnectionCredentialsSchema;
export type TValidateHumanitecConnectionCredentialsSchema = typeof ValidateHumanitecConnectionCredentialsSchema;
export type THumanitecConnectionConfig = DiscriminativePick<
THumanitecConnectionInput,
backend/src/services/app-connection/mssql/mssql-connection-types.ts
+1 -1
View File
@@ -13,4 +13,4 @@ export type TMsSqlConnectionInput = z.infer<typeof CreateMsSqlConnectionSchema>
app: AppConnection.MsSql;
};
export type TValidateMsSqlConnectionCredentials = typeof ValidateMsSqlConnectionCredentialsSchema;
export type TValidateMsSqlConnectionCredentialsSchema = typeof ValidateMsSqlConnectionCredentialsSchema;
backend/src/services/app-connection/postgres/postgres-connection-types.ts
+1 -1
View File
@@ -13,4 +13,4 @@ export type TPostgresConnectionInput = z.infer<typeof CreatePostgresConnectionSc
app: AppConnection.Postgres;
};
export type TValidatePostgresConnectionCredentials = typeof ValidatePostgresConnectionCredentialsSchema;
export type TValidatePostgresConnectionCredentialsSchema = typeof ValidatePostgresConnectionCredentialsSchema;
backend/src/services/app-connection/shared/sql/sql-connection-fns.ts
+10 -5
View File
@@ -18,10 +18,7 @@ const SQL_CONNECTION_CLIENT_MAP = {
[AppConnection.MsSql]: "mssql"
};
export const getSqlConnectionClient = async (
appConnection: Pick<TSqlConnection, "credentials" | "app">,
options?: Record<string, unknown>
) => {
export const getSqlConnectionClient = async (appConnection: Pick<TSqlConnection, "credentials" | "app">) => {
const {
app,
credentials: { host: baseHost, database, port, sslCertificate, password, username }
@@ -41,7 +38,15 @@ export const getSqlConnectionClient = async (
password,
connectionTimeoutMillis: EXTERNAL_REQUEST_TIMEOUT,
ssl,
options
// following dynamic secret mssql driver requirements (see sql-database.ts)
// @ts-expect-error this is because of knexjs type signature issue. This is directly passed to driver
options:
app === AppConnection.MsSql
? {
trustServerCertificate: !sslCertificate,
cryptoCredentialsDetails: sslCertificate ? { ca: sslCertificate } : {}
}
: undefined
}
});
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+14 -3
View File
@@ -548,6 +548,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
try {
const secrets = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
query.forEach((el) => {
if (el.type === SecretType.Personal && !el.userId) {
@@ -559,10 +560,20 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId: el.type === SecretType.Personal ? el.userId : null
});
});
});
return secrets;
})
.leftJoin(
TableName.SecretRotationV2SecretMapping,
`${TableName.SecretV2}.id`,
`${TableName.SecretRotationV2SecretMapping}.secretId`
)
.select(selectAllTableCols(TableName.SecretV2))
.select(db.ref("rotationId").withSchema(TableName.SecretRotationV2SecretMapping));
return secrets.map((secret) => ({
...secret,
isRotatedSecret: Boolean(secret.rotationId)
}));
} catch (error) {
throw new DatabaseError({ error, name: "find by blind indexes" });
throw new DatabaseError({ error, name: "find by secret keys" });
}
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+4
View File
@@ -2234,6 +2234,10 @@ export const secretV2BridgeServiceFactory = ({
const destinationActions = [ProjectPermissionSecretActions.Create, ProjectPermissionSecretActions.Edit] as const;
sourceSecrets.forEach((secret) => {
if (secret.isRotatedSecret) {
throw new BadRequestError({ message: `Cannot move rotated secret: ${secret.key}` });
}
for (const sourceAction of sourceActions) {
if (
sourceAction === ProjectPermissionSecretActions.DescribeSecret ||