awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 07:30:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into shubham/eng-993-allow-self-hosted-users-to-set-their-own-rate-limits-in

Browse Source
This commit is contained in:
Sheen Capadngan
2024-06-13 15:44:50 +08:00
parent 9e24050f17 81513e4a75
commit 810a58c836
92 changed files with 2041 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/db/migrations/20240610181521_add-consecutive-failed-password-attempts-user.ts
+29
View File
@@ -0,0 +1,29 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasConsecutiveFailedPasswordAttempts = await knex.schema.hasColumn(
TableName.Users,
"consecutiveFailedPasswordAttempts"
);
await knex.schema.alterTable(TableName.Users, (tb) => {
if (!hasConsecutiveFailedPasswordAttempts) {
tb.integer("consecutiveFailedPasswordAttempts").defaultTo(0);
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasConsecutiveFailedPasswordAttempts = await knex.schema.hasColumn(
TableName.Users,
"consecutiveFailedPasswordAttempts"
);
await knex.schema.alterTable(TableName.Users, (tb) => {
if (hasConsecutiveFailedPasswordAttempts) {
tb.dropColumn("consecutiveFailedPasswordAttempts");
}
});
}
backend/src/db/migrations/20240612200518_add-pit-version-limit.ts
+21
View File
@@ -0,0 +1,21 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit");
await knex.schema.alterTable(TableName.Project, (tb) => {
if (!hasPitVersionLimitColumn) {
tb.integer("pitVersionLimit").notNullable().defaultTo(10);
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit");
await knex.schema.alterTable(TableName.Project, (tb) => {
if (hasPitVersionLimitColumn) {
tb.dropColumn("pitVersionLimit");
}
});
}
backend/src/db/schemas/projects.ts
+2 -1
View File
@@ -16,7 +16,8 @@ export const ProjectsSchema = z.object({
createdAt: z.date(),
updatedAt: z.date(),
version: z.number().default(1),
upgradeStatus: z.string().nullable().optional()
upgradeStatus: z.string().nullable().optional(),
pitVersionLimit: z.number().default(10)
});
export type TProjects = z.infer<typeof ProjectsSchema>;
backend/src/db/schemas/users.ts
+2 -1
View File
@@ -25,7 +25,8 @@ export const UsersSchema = z.object({
isEmailVerified: z.boolean().default(false).nullable().optional(),
consecutiveFailedMfaAttempts: z.number().default(0).nullable().optional(),
isLocked: z.boolean().default(false).nullable().optional(),
temporaryLockDateEnd: z.date().nullable().optional()
temporaryLockDateEnd: z.date().nullable().optional(),
consecutiveFailedPasswordAttempts: z.number().default(0).nullable().optional()
});
export type TUsers = z.infer<typeof UsersSchema>;
backend/src/ee/routes/v1/scim-router.ts
+1
View File
@@ -362,6 +362,7 @@ export const registerScimRouter = async (server: FastifyZodProvider) => {
const groups = await req.server.services.scim.listScimGroups({
orgId: req.permission.orgId,
startIndex: req.query.startIndex,
filter: req.query.filter,
limit: req.query.count
});
backend/src/ee/routes/v1/secret-approval-policy-router.ts
+12 -3
View File
@@ -1,6 +1,7 @@
import { nanoid } from "nanoid";
import { z } from "zod";
import { removeTrailingSlash } from "@app/lib/fn";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { sapPubSchema } from "@app/server/routes/sanitizedSchemas";
@@ -19,7 +20,11 @@ export const registerSecretApprovalPolicyRouter = async (server: FastifyZodProvi
workspaceId: z.string(),
name: z.string().optional(),
environment: z.string(),
secretPath: z.string().optional().nullable(),
secretPath: z
.string()
.optional()
.nullable()
.transform((val) => (val ? removeTrailingSlash(val) : val)),
approvers: z.string().array().min(1),
approvals: z.number().min(1).default(1)
})
@@ -63,7 +68,11 @@ export const registerSecretApprovalPolicyRouter = async (server: FastifyZodProvi
name: z.string().optional(),
approvers: z.string().array().min(1),
approvals: z.number().min(1).default(1),
secretPath: z.string().optional().nullable()
secretPath: z
.string()
.optional()
.nullable()
.transform((val) => (val ? removeTrailingSlash(val) : val))
})
.refine((data) => data.approvals <= data.approvers.length, {
path: ["approvals"],
@@ -157,7 +166,7 @@ export const registerSecretApprovalPolicyRouter = async (server: FastifyZodProvi
querystring: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretPath: z.string().trim()
secretPath: z.string().trim().transform(removeTrailingSlash)
}),
response: {
200: z.object({
backend/src/ee/services/ldap-config/ldap-config-service.ts
+2 -1
View File
@@ -77,7 +77,7 @@ type TLdapConfigServiceFactoryDep = {
>;
userAliasDAL: Pick<TUserAliasDALFactory, "create" | "findOne">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
licenseService: Pick<TLicenseServiceFactory, "getPlan" | "updateSubscriptionOrgMemberCount">;
};
export type TLdapConfigServiceFactory = ReturnType<typeof ldapConfigServiceFactory>;
@@ -510,6 +510,7 @@ export const ldapConfigServiceFactory = ({
return newUserAlias;
});
}
await licenseService.updateSubscriptionOrgMemberCount(organization.id);
const user = await userDAL.transaction(async (tx) => {
const newUser = await userDAL.findOne({ id: userAlias.userId }, tx);
backend/src/ee/services/saml-config/saml-config-service.ts
+2 -1
View File
@@ -50,7 +50,7 @@ type TSamlConfigServiceFactoryDep = {
orgMembershipDAL: Pick<TOrgMembershipDALFactory, "create">;
orgBotDAL: Pick<TOrgBotDALFactory, "findOne" | "create" | "transaction">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
licenseService: Pick<TLicenseServiceFactory, "getPlan" | "updateSubscriptionOrgMemberCount">;
tokenService: Pick<TAuthTokenServiceFactory, "createTokenForUser">;
smtpService: Pick<TSmtpService, "sendMail">;
};
@@ -449,6 +449,7 @@ export const samlConfigServiceFactory = ({
return newUser;
});
}
await licenseService.updateSubscriptionOrgMemberCount(organization.id);
const isUserCompleted = Boolean(user.isAccepted);
const providerAuthToken = jwt.sign(
backend/src/ee/services/scim/scim-fns.ts
+14
View File
@@ -18,6 +18,20 @@ export const buildScimUserList = ({
};
};
export const parseScimFilter = (filterToParse: string | undefined) => {
if (!filterToParse) return {};
const [parsedName, parsedValue] = filterToParse.split("eq").map((s) => s.trim());
let attributeName = parsedName;
if (parsedName === "userName") {
attributeName = "email";
} else if (parsedName === "displayName") {
attributeName = "name";
}
return { [attributeName]: parsedValue.replace(/"/g, "") };
};
export const buildScimUser = ({
orgMembershipId,
username,
backend/src/ee/services/scim/scim-service.ts
+6 -17
View File
@@ -30,7 +30,7 @@ import { UserAliasType } from "@app/services/user-alias/user-alias-types";
import { TLicenseServiceFactory } from "../license/license-service";
import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
import { TPermissionServiceFactory } from "../permission/permission-service";
import { buildScimGroup, buildScimGroupList, buildScimUser, buildScimUserList } from "./scim-fns";
import { buildScimGroup, buildScimGroupList, buildScimUser, buildScimUserList, parseScimFilter } from "./scim-fns";
import {
TCreateScimGroupDTO,
TCreateScimTokenDTO,
@@ -184,18 +184,6 @@ export const scimServiceFactory = ({
status: 403
});
const parseFilter = (filterToParse: string | undefined) => {
if (!filterToParse) return {};
const [parsedName, parsedValue] = filterToParse.split("eq").map((s) => s.trim());
let attributeName = parsedName;
if (parsedName === "userName") {
attributeName = "email";
}
return { [attributeName]: parsedValue.replace(/"/g, "") };
};
const findOpts = {
...(startIndex && { offset: startIndex - 1 }),
...(limit && { limit })
@@ -204,7 +192,7 @@ export const scimServiceFactory = ({
const users = await orgDAL.findMembership(
{
[`${TableName.OrgMembership}.orgId` as "id"]: orgId,
...parseFilter(filter)
...parseScimFilter(filter)
},
findOpts
);
@@ -391,7 +379,7 @@ export const scimServiceFactory = ({
);
}
}
await licenseService.updateSubscriptionOrgMemberCount(org.id);
return { user, orgMembership };
});
@@ -557,7 +545,7 @@ export const scimServiceFactory = ({
return {}; // intentionally return empty object upon success
};
const listScimGroups = async ({ orgId, startIndex, limit }: TListScimGroupsDTO) => {
const listScimGroups = async ({ orgId, startIndex, limit, filter }: TListScimGroupsDTO) => {
const plan = await licenseService.getPlan(orgId);
if (!plan.groups)
throw new BadRequestError({
@@ -580,7 +568,8 @@ export const scimServiceFactory = ({
const groups = await groupDAL.findGroups(
{
orgId
orgId,
...(filter && parseScimFilter(filter))
},
{
offset: startIndex - 1,
backend/src/ee/services/scim/scim-types.ts
+1
View File
@@ -66,6 +66,7 @@ export type TDeleteScimUserDTO = {
export type TListScimGroupsDTO = {
startIndex: number;
filter?: string;
limit: number;
orgId: string;
};
backend/src/ee/services/secret-approval-policy/secret-approval-policy-service.ts
+3 -1
View File
@@ -4,6 +4,7 @@ import picomatch from "picomatch";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { BadRequestError } from "@app/lib/errors";
import { removeTrailingSlash } from "@app/lib/fn";
import { containsGlobPatterns } from "@app/lib/picomatch";
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
@@ -207,7 +208,8 @@ export const secretApprovalPolicyServiceFactory = ({
return sapPolicies;
};
const getSecretApprovalPolicy = async (projectId: string, environment: string, secretPath: string) => {
const getSecretApprovalPolicy = async (projectId: string, environment: string, path: string) => {
const secretPath = removeTrailingSlash(path);
const env = await projectEnvDAL.findOne({ slug: environment, projectId });
if (!env) throw new BadRequestError({ message: "Environment not found" });
backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts
+1 -2
View File
@@ -81,8 +81,7 @@ export const secretSnapshotServiceFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id);
return count;
return snapshotDAL.countOfSnapshotsByFolderId(folder.id);
};
const listSnapshots = async ({
backend/src/ee/services/secret-snapshot/snapshot-dal.ts
+143 -1
View File
@@ -1,3 +1,4 @@
/* eslint-disable no-await-in-loop */
import { Knex } from "knex";
import { TDbClient } from "@app/db";
@@ -11,6 +12,7 @@ import {
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
export type TSnapshotDALFactory = ReturnType<typeof snapshotDALFactory>;
@@ -325,12 +327,152 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
/**
* Prunes excess snapshots from the database to ensure only a specified number of recent snapshots are retained for each folder.
*
* This function operates in three main steps:
* 1. Pruning snapshots from root/non-versioned folders.
* 2. Pruning snapshots from versioned folders.
* 3. Removing orphaned snapshots that do not belong to any existing folder or folder version.
*
* The function processes snapshots in batches, determined by the `PRUNE_FOLDER_BATCH_SIZE` constant,
* to manage the large datasets without overwhelming the DB.
*
* Steps:
* - Fetch a batch of folder IDs.
* - For each batch, use a Common Table Expression (CTE) to rank snapshots within each folder by their creation date.
* - Identify and delete snapshots that exceed the project's point-in-time version limit (`pitVersionLimit`).
* - Repeat the process for versioned folders.
* - Finally, delete orphaned snapshots that do not have an associated folder.
*/
const pruneExcessSnapshots = async () => {
const PRUNE_FOLDER_BATCH_SIZE = 10000;
try {
let uuidOffset = "00000000-0000-0000-0000-000000000000";
// cleanup snapshots from root/non-versioned folders
// eslint-disable-next-line no-constant-condition, no-unreachable-loop
while (true) {
const folderBatch = await db(TableName.SecretFolder)
.where("id", ">", uuidOffset)
.where("isReserved", false)
.orderBy("id", "asc")
.limit(PRUNE_FOLDER_BATCH_SIZE)
.select("id");
const batchEntries = folderBatch.map((folder) => folder.id);
if (folderBatch.length) {
try {
logger.info(`Pruning snapshots in [range=${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}]`);
await db(TableName.Snapshot)
.with("snapshot_cte", (qb) => {
void qb
.from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, batchEntries)
.select(
"folderId",
`${TableName.Snapshot}.id as id`,
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereNull(`${TableName.SecretFolder}.parentId`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (err) {
logger.error(
`Failed to prune snapshots from root/non-versioned folders in range ${batchEntries[0]}:${
batchEntries[batchEntries.length - 1]
}`
);
} finally {
uuidOffset = batchEntries[batchEntries.length - 1];
}
} else {
break;
}
}
// cleanup snapshots from versioned folders
uuidOffset = "00000000-0000-0000-0000-000000000000";
// eslint-disable-next-line no-constant-condition
while (true) {
const folderBatch = await db(TableName.SecretFolderVersion)
.select("folderId")
.distinct("folderId")
.where("folderId", ">", uuidOffset)
.orderBy("folderId", "asc")
.limit(PRUNE_FOLDER_BATCH_SIZE);
const batchEntries = folderBatch.map((folder) => folder.folderId);
if (folderBatch.length) {
try {
logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`);
await db(TableName.Snapshot)
.with("snapshot_cte", (qb) => {
void qb
.from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, batchEntries)
.select(
"folderId",
`${TableName.Snapshot}.id as id`,
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
);
})
.join(
TableName.SecretFolderVersion,
`${TableName.SecretFolderVersion}.folderId`,
`${TableName.Snapshot}.folderId`
)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (err) {
logger.error(
`Failed to prune snapshots from versioned folders in range ${batchEntries[0]}:${
batchEntries[batchEntries.length - 1]
}`
);
} finally {
uuidOffset = batchEntries[batchEntries.length - 1];
}
} else {
break;
}
}
// cleanup orphaned snapshots (those that don't belong to an existing folder and folder version)
await db(TableName.Snapshot)
.whereNotIn("folderId", (qb) => {
void qb
.select("folderId")
.from(TableName.SecretFolderVersion)
.union((qb1) => void qb1.select("id").from(TableName.SecretFolder));
})
.delete();
} catch (error) {
throw new DatabaseError({ error, name: "SnapshotPrune" });
}
};
return {
...secretSnapshotOrm,
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById
findSecretSnapshotDataById,
pruneExcessSnapshots
};
};
backend/src/lib/api-docs/constants.ts
+6 -1
View File
@@ -386,6 +386,8 @@ export const SECRET_IMPORTS = {
environment: "The slug of the environment to import into.",
path: "The path to import into.",
workspaceId: "The ID of the project you are working in.",
isReplication:
"When true, secrets from the source will be automatically sent to the destination. If approval policies exist at the destination, the secrets will be sent as approval requests instead of being applied immediately.",
import: {
environment: "The slug of the environment to import from.",
path: "The path to import from."
@@ -674,7 +676,10 @@ export const INTEGRATION = {
secretGCPLabel: "The label for GCP secrets.",
secretAWSTag: "The tags for AWS secrets.",
kmsKeyId: "The ID of the encryption key from AWS KMS.",
shouldDisableDelete: "The flag to disable deletion of secrets in AWS Parameter Store."
shouldDisableDelete: "The flag to disable deletion of secrets in AWS Parameter Store.",
shouldMaskSecrets: "Specifies if the secrets synced from Infisical to Gitlab should be marked as 'Masked'.",
shouldProtectSecrets: "Specifies if the secrets synced from Infisical to Gitlab should be marked as 'Protected'.",
shouldEnableDelete: "The flag to enable deletion of secrets"
}
},
UPDATE: {
backend/src/lib/config/env.ts
+22 -12
View File
@@ -39,7 +39,9 @@ const envSchema = z
HTTPS_ENABLED: zodStrBool,
// smtp options
SMTP_HOST: zpStr(z.string().optional()),
SMTP_SECURE: zodStrBool,
SMTP_IGNORE_TLS: zodStrBool.default("false"),
SMTP_REQUIRE_TLS: zodStrBool.default("true"),
SMTP_TLS_REJECT_UNAUTHORIZED: zodStrBool.default("true"),
SMTP_PORT: z.coerce.number().default(587),
SMTP_USERNAME: zpStr(z.string().optional()),
SMTP_PASSWORD: zpStr(z.string().optional()),
@@ -120,7 +122,8 @@ const envSchema = z
.transform((val) => val === "true")
.optional(),
INFISICAL_CLOUD: zodStrBool.default("false"),
MAINTENANCE_MODE: zodStrBool.default("false")
MAINTENANCE_MODE: zodStrBool.default("false"),
CAPTCHA_SECRET: zpStr(z.string().optional())
})
.transform((data) => ({
...data,
@@ -152,13 +155,20 @@ export const initEnvConfig = (logger: Logger) => {
return envCfg;
};
export const formatSmtpConfig = () => ({
host: envCfg.SMTP_HOST,
port: envCfg.SMTP_PORT,
auth:
envCfg.SMTP_USERNAME && envCfg.SMTP_PASSWORD
? { user: envCfg.SMTP_USERNAME, pass: envCfg.SMTP_PASSWORD }
: undefined,
secure: envCfg.SMTP_SECURE,
from: `"${envCfg.SMTP_FROM_NAME}" <${envCfg.SMTP_FROM_ADDRESS}>`
});
export const formatSmtpConfig = () => {
return {
host: envCfg.SMTP_HOST,
port: envCfg.SMTP_PORT,
auth:
envCfg.SMTP_USERNAME && envCfg.SMTP_PASSWORD
? { user: envCfg.SMTP_USERNAME, pass: envCfg.SMTP_PASSWORD }
: undefined,
secure: envCfg.SMTP_PORT === 465,
from: `"${envCfg.SMTP_FROM_NAME}" <${envCfg.SMTP_FROM_ADDRESS}>`,
ignoreTLS: envCfg.SMTP_IGNORE_TLS,
requireTLS: envCfg.SMTP_REQUIRE_TLS,
tls: {
rejectUnauthorized: envCfg.SMTP_TLS_REJECT_UNAUTHORIZED
}
};
};
backend/src/server/boot-strap-check.ts
+1 -2
View File
@@ -5,7 +5,6 @@ import { createTransport } from "nodemailer";
import { formatSmtpConfig, getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { getTlsOption } from "@app/services/smtp/smtp-service";
import { getServerCfg } from "@app/services/super-admin/super-admin-service";
type BootstrapOpt = {
@@ -44,7 +43,7 @@ export const bootstrapCheck = async ({ db }: BootstrapOpt) => {
console.info("Testing smtp connection");
const smtpCfg = formatSmtpConfig();
await createTransport({ ...smtpCfg, ...getTlsOption(smtpCfg.host, smtpCfg.secure) })
await createTransport(smtpCfg)
.verify()
.then(async () => {
console.info("SMTP successfully connected");
backend/src/server/routes/index.ts
+3
View File
@@ -830,6 +830,9 @@ export const registerRoutes = async (
const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({
auditLogDAL,
queueService,
secretVersionDAL,
secretFolderVersionDAL: folderVersionDAL,
snapshotDAL,
identityAccessTokenDAL,
secretSharingDAL
});
backend/src/server/routes/v1/integration-router.ts
+3 -58
View File
@@ -8,7 +8,7 @@ import { writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
import { IntegrationMappingBehavior } from "@app/services/integration-auth/integration-list";
import { IntegrationMetadataSchema } from "@app/services/integration/integration-schema";
import { PostHogEventTypes, TIntegrationCreatedEvent } from "@app/services/telemetry/telemetry-types";
export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
@@ -46,36 +46,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
path: z.string().trim().optional().describe(INTEGRATION.CREATE.path),
region: z.string().trim().optional().describe(INTEGRATION.CREATE.region),
scope: z.string().trim().optional().describe(INTEGRATION.CREATE.scope),
metadata: z
.object({
secretPrefix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretPrefix),
secretSuffix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretSuffix),
initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
mappingBehavior: z
.nativeEnum(IntegrationMappingBehavior)
.optional()
.describe(INTEGRATION.CREATE.metadata.mappingBehavior),
shouldAutoRedeploy: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldAutoRedeploy),
secretGCPLabel: z
.object({
labelName: z.string(),
labelValue: z.string()
})
.optional()
.describe(INTEGRATION.CREATE.metadata.secretGCPLabel),
secretAWSTag: z
.array(
z.object({
key: z.string(),
value: z.string()
})
)
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
kmsKeyId: z.string().optional().describe(INTEGRATION.CREATE.metadata.kmsKeyId),
shouldDisableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldDisableDelete)
})
.default({})
metadata: IntegrationMetadataSchema.default({})
}),
response: {
200: z.object({
@@ -161,33 +132,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
targetEnvironment: z.string().trim().describe(INTEGRATION.UPDATE.targetEnvironment),
owner: z.string().trim().describe(INTEGRATION.UPDATE.owner),
environment: z.string().trim().describe(INTEGRATION.UPDATE.environment),
metadata: z
.object({
secretPrefix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretPrefix),
secretSuffix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretSuffix),
initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
mappingBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.mappingBehavior),
shouldAutoRedeploy: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldAutoRedeploy),
secretGCPLabel: z
.object({
labelName: z.string(),
labelValue: z.string()
})
.optional()
.describe(INTEGRATION.CREATE.metadata.secretGCPLabel),
secretAWSTag: z
.array(
z.object({
key: z.string(),
value: z.string()
})
)
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
kmsKeyId: z.string().optional().describe(INTEGRATION.CREATE.metadata.kmsKeyId),
shouldDisableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldDisableDelete)
})
.optional()
metadata: IntegrationMetadataSchema.optional()
}),
response: {
200: z.object({
backend/src/server/routes/v1/project-router.ts
+38
View File
@@ -334,6 +334,44 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
}
});
server.route({
method: "PUT",
url: "/:workspaceSlug/version-limit",
config: {
rateLimit: writeLimit
},
schema: {
params: z.object({
workspaceSlug: z.string().trim()
}),
body: z.object({
pitVersionLimit: z.number().min(1).max(100)
}),
response: {
200: z.object({
message: z.string(),
workspace: ProjectsSchema
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const workspace = await server.services.project.updateVersionLimit({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
pitVersionLimit: req.body.pitVersionLimit,
workspaceSlug: req.params.workspaceSlug
});
return {
message: "Successfully changed workspace version limit",
workspace
};
}
});
server.route({
method: "GET",
url: "/:workspaceId/integrations",
backend/src/server/routes/v1/secret-import-router.ts
+1 -1
View File
@@ -30,7 +30,7 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment),
path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path)
}),
isReplication: z.boolean().default(false)
isReplication: z.boolean().default(false).describe(SECRET_IMPORTS.CREATE.isReplication)
}),
response: {
200: z.object({
backend/src/server/routes/v3/login-router.ts
+3 -1
View File
@@ -80,7 +80,8 @@ export const registerLoginRouter = async (server: FastifyZodProvider) => {
body: z.object({
email: z.string().trim(),
providerAuthToken: z.string().trim().optional(),
clientProof: z.string().trim()
clientProof: z.string().trim(),
captchaToken: z.string().trim().optional()
}),
response: {
200: z.discriminatedUnion("mfaEnabled", [
@@ -106,6 +107,7 @@ export const registerLoginRouter = async (server: FastifyZodProvider) => {
const appCfg = getConfig();
const data = await server.services.login.loginExchangeClientProof({
captchaToken: req.body.captchaToken,
email: req.body.email,
ip: req.realIp,
userAgent,
backend/src/services/auth/auth-login-service.ts
+49 -3
View File
@@ -3,6 +3,7 @@ import jwt from "jsonwebtoken";
import { TUsers, UserDeviceSchema } from "@app/db/schemas";
import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { generateSrpServerKey, srpCheckClientProof } from "@app/lib/crypto";
import { BadRequestError, DatabaseError, UnauthorizedError } from "@app/lib/errors";
import { getServerCfg } from "@app/services/super-admin/super-admin-service";
@@ -176,12 +177,16 @@ export const authLoginServiceFactory = ({
clientProof,
ip,
userAgent,
providerAuthToken
providerAuthToken,
captchaToken
}: TLoginClientProofDTO) => {
const appCfg = getConfig();
const userEnc = await userDAL.findUserEncKeyByUsername({
username: email
});
if (!userEnc) throw new Error("Failed to find user");
const user = await userDAL.findById(userEnc.userId);
const cfg = getConfig();
let authMethod = AuthMethod.EMAIL;
@@ -196,6 +201,31 @@ export const authLoginServiceFactory = ({
}
}
if (
user.consecutiveFailedPasswordAttempts &&
user.consecutiveFailedPasswordAttempts >= 10 &&
Boolean(appCfg.CAPTCHA_SECRET)
) {
if (!captchaToken) {
throw new BadRequestError({
name: "Captcha Required",
message: "Accomplish the required captcha by logging in via Web"
});
}
// validate captcha token
const response = await request.postForm<{ success: boolean }>("https://api.hcaptcha.com/siteverify", {
response: captchaToken,
secret: appCfg.CAPTCHA_SECRET
});
if (!response.data.success) {
throw new BadRequestError({
name: "Invalid Captcha"
});
}
}
if (!userEnc.serverPrivateKey || !userEnc.clientPublicKey) throw new Error("Failed to authenticate. Try again?");
const isValidClientProof = await srpCheckClientProof(
userEnc.salt,
@@ -204,15 +234,31 @@ export const authLoginServiceFactory = ({
userEnc.clientPublicKey,
clientProof
);
if (!isValidClientProof) throw new Error("Failed to authenticate. Try again?");
if (!isValidClientProof) {
await userDAL.update(
{ id: userEnc.userId },
{
$incr: {
consecutiveFailedPasswordAttempts: 1
}
}
);
throw new Error("Failed to authenticate. Try again?");
}
await userDAL.updateUserEncryptionByUserId(userEnc.userId, {
serverPrivateKey: null,
clientPublicKey: null
});
await userDAL.updateById(userEnc.userId, {
consecutiveFailedPasswordAttempts: 0
});
// send multi factor auth token if they it enabled
if (userEnc.isMfaEnabled && userEnc.email) {
const user = await userDAL.findById(userEnc.userId);
enforceUserLockStatus(Boolean(user.isLocked), user.temporaryLockDateEnd);
const mfaToken = jwt.sign(
backend/src/services/auth/auth-login-type.ts
+1
View File
@@ -12,6 +12,7 @@ export type TLoginClientProofDTO = {
providerAuthToken?: string;
ip: string;
userAgent: string;
captchaToken?: string;
};
export type TVerifyMfaTokenDTO = {
backend/src/services/auth/auth-signup-service.ts
+2 -2
View File
@@ -231,7 +231,7 @@ export const authSignupServiceFactory = ({
const accessToken = jwt.sign(
{
authMethod: AuthMethod.EMAIL,
authMethod: authMethod || AuthMethod.EMAIL,
authTokenType: AuthTokenType.ACCESS_TOKEN,
userId: updateduser.info.id,
tokenVersionId: tokenSession.id,
@@ -244,7 +244,7 @@ export const authSignupServiceFactory = ({
const refreshToken = jwt.sign(
{
authMethod: AuthMethod.EMAIL,
authMethod: authMethod || AuthMethod.EMAIL,
authTokenType: AuthTokenType.REFRESH_TOKEN,
userId: updateduser.info.id,
tokenVersionId: tokenSession.id,
backend/src/services/integration-auth/integration-sync-secret.ts
+41 -35
View File
@@ -31,6 +31,7 @@ import { logger } from "@app/lib/logger";
import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/secret/secret-types";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { IntegrationMetadataSchema } from "../integration/integration-schema";
import {
IntegrationInitialSyncBehavior,
IntegrationMappingBehavior,
@@ -1363,38 +1364,41 @@ const syncSecretsGitHub = async ({
}
}
for await (const encryptedSecret of encryptedSecrets) {
if (
!(encryptedSecret.name in secrets) &&
!(appendices?.prefix !== undefined && !encryptedSecret.name.startsWith(appendices?.prefix)) &&
!(appendices?.suffix !== undefined && !encryptedSecret.name.endsWith(appendices?.suffix))
) {
switch (integration.scope) {
case GithubScope.Org: {
await octokit.request("DELETE /orgs/{org}/actions/secrets/{secret_name}", {
org: integration.owner as string,
secret_name: encryptedSecret.name
});
break;
}
case GithubScope.Env: {
await octokit.request(
"DELETE /repositories/{repository_id}/environments/{environment_name}/secrets/{secret_name}",
{
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string,
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
if (metadata.shouldEnableDelete) {
for await (const encryptedSecret of encryptedSecrets) {
if (
!(encryptedSecret.name in secrets) &&
!(appendices?.prefix !== undefined && !encryptedSecret.name.startsWith(appendices?.prefix)) &&
!(appendices?.suffix !== undefined && !encryptedSecret.name.endsWith(appendices?.suffix))
) {
switch (integration.scope) {
case GithubScope.Org: {
await octokit.request("DELETE /orgs/{org}/actions/secrets/{secret_name}", {
org: integration.owner as string,
secret_name: encryptedSecret.name
}
);
break;
}
default: {
await octokit.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: integration.owner as string,
repo: integration.app as string,
secret_name: encryptedSecret.name
});
break;
});
break;
}
case GithubScope.Env: {
await octokit.request(
"DELETE /repositories/{repository_id}/environments/{environment_name}/secrets/{secret_name}",
{
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string,
secret_name: encryptedSecret.name
}
);
break;
}
default: {
await octokit.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: integration.owner as string,
repo: integration.app as string,
secret_name: encryptedSecret.name
});
break;
}
}
}
}
@@ -1917,13 +1921,13 @@ const syncSecretsGitLab = async ({
return allEnvVariables;
};
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
const allEnvVariables = await getAllEnvVariables(integration?.appId as string, accessToken);
const getSecretsRes: GitLabSecret[] = allEnvVariables
.filter((secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment)
.filter((gitLabSecret) => {
let isValid = true;
const metadata = z.record(z.any()).parse(integration.metadata);
if (metadata.secretPrefix && !gitLabSecret.key.startsWith(metadata.secretPrefix)) {
isValid = false;
}
@@ -1943,8 +1947,8 @@ const syncSecretsGitLab = async ({
{
key,
value: secrets[key].value,
protected: false,
masked: false,
protected: Boolean(metadata.shouldProtectSecrets),
masked: Boolean(metadata.shouldMaskSecrets),
raw: false,
environment_scope: integration.targetEnvironment
},
@@ -1961,7 +1965,9 @@ const syncSecretsGitLab = async ({
`${gitLabApiUrl}/v4/projects/${integration?.appId}/variables/${existingSecret.key}?filter[environment_scope]=${integration.targetEnvironment}`,
{
...existingSecret,
value: secrets[existingSecret.key].value
value: secrets[existingSecret.key].value,
protected: Boolean(metadata.shouldProtectSecrets),
masked: Boolean(metadata.shouldMaskSecrets)
},
{
headers: {
backend/src/services/integration/integration-schema.ts
+37
View File
@@ -0,0 +1,37 @@
import { z } from "zod";
import { INTEGRATION } from "@app/lib/api-docs";
import { IntegrationMappingBehavior } from "../integration-auth/integration-list";
export const IntegrationMetadataSchema = z.object({
secretPrefix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretPrefix),
secretSuffix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretSuffix),
initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
mappingBehavior: z
.nativeEnum(IntegrationMappingBehavior)
.optional()
.describe(INTEGRATION.CREATE.metadata.mappingBehavior),
shouldAutoRedeploy: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldAutoRedeploy),
secretGCPLabel: z
.object({
labelName: z.string(),
labelValue: z.string()
})
.optional()
.describe(INTEGRATION.CREATE.metadata.secretGCPLabel),
secretAWSTag: z
.array(
z.object({
key: z.string(),
value: z.string()
})
)
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
kmsKeyId: z.string().optional().describe(INTEGRATION.CREATE.metadata.kmsKeyId),
shouldDisableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldDisableDelete),
shouldEnableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldEnableDelete),
shouldMaskSecrets: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldMaskSecrets),
shouldProtectSecrets: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldProtectSecrets)
});
backend/src/services/integration/integration-types.ts
+4
View File
@@ -29,6 +29,9 @@ export type TCreateIntegrationDTO = {
}[];
kmsKeyId?: string;
shouldDisableDelete?: boolean;
shouldMaskSecrets?: boolean;
shouldProtectSecrets?: boolean;
shouldEnableDelete?: boolean;
};
} & Omit<TProjectPermission, "projectId">;
@@ -54,6 +57,7 @@ export type TUpdateIntegrationDTO = {
}[];
kmsKeyId?: string;
shouldDisableDelete?: boolean;
shouldEnableDelete?: boolean;
};
} & Omit<TProjectPermission, "projectId">;
backend/src/services/org/org-service.ts
+1
View File
@@ -336,6 +336,7 @@ export const orgServiceFactory = ({
return org;
});
await licenseService.updateSubscriptionOrgMemberCount(organization.id);
return organization;
};
backend/src/services/project/project-service.ts
+34 -2
View File
@@ -39,6 +39,7 @@ import {
TToggleProjectAutoCapitalizationDTO,
TUpdateProjectDTO,
TUpdateProjectNameDTO,
TUpdateProjectVersionLimitDTO,
TUpgradeProjectDTO
} from "./project-types";
@@ -133,7 +134,8 @@ export const projectServiceFactory = ({
name: workspaceName,
orgId: organization.id,
slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`),
version: ProjectVersion.V2
version: ProjectVersion.V2,
pitVersionLimit: 10
},
tx
);
@@ -406,6 +408,35 @@ export const projectServiceFactory = ({
return updatedProject;
};
const updateVersionLimit = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
pitVersionLimit,
workspaceSlug
}: TUpdateProjectVersionLimitDTO) => {
const project = await projectDAL.findProjectBySlug(workspaceSlug, actorOrgId);
if (!project) {
throw new BadRequestError({
message: "Project not found"
});
}
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
project.id,
actorAuthMethod,
actorOrgId
);
if (!hasRole(ProjectMembershipRole.Admin))
throw new BadRequestError({ message: "Only admins are allowed to take this action" });
return projectDAL.updateById(project.id, { pitVersionLimit });
};
const updateName = async ({
projectId,
actor,
@@ -501,6 +532,7 @@ export const projectServiceFactory = ({
getAProject,
toggleAutoCapitalization,
updateName,
upgradeProject
upgradeProject,
updateVersionLimit
};
};
backend/src/services/project/project-types.ts
+5
View File
@@ -43,6 +43,11 @@ export type TToggleProjectAutoCapitalizationDTO = {
autoCapitalization: boolean;
} & TProjectPermission;
export type TUpdateProjectVersionLimitDTO = {
pitVersionLimit: number;
workspaceSlug: string;
} & Omit<TProjectPermission, "projectId">;
export type TUpdateProjectNameDTO = {
name: string;
} & TProjectPermission;
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
+12
View File
@@ -1,13 +1,19 @@
import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
type TDailyResourceCleanUpQueueServiceFactoryDep = {
auditLogDAL: Pick<TAuditLogDALFactory, "pruneAuditLog">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "removeExpiredTokens">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
queueService: TQueueServiceFactory;
};
@@ -17,6 +23,9 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyRe
export const dailyResourceCleanUpQueueServiceFactory = ({
auditLogDAL,
queueService,
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
identityAccessTokenDAL,
secretSharingDAL
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
@@ -25,6 +34,9 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await secretSharingDAL.pruneExpiredSharedSecrets();
await snapshotDAL.pruneExcessSnapshots();
await secretVersionDAL.pruneExcessVersions();
await secretFolderVersionDAL.pruneExcessVersions();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
});
backend/src/services/secret-folder/secret-folder-version-dal.ts
+28 -1
View File
@@ -62,5 +62,32 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
}
};
return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId };
const pruneExcessVersions = async () => {
try {
await db(TableName.SecretFolderVersion)
.with("folder_cte", (qb) => {
void qb
.from(TableName.SecretFolderVersion)
.select(
"id",
"folderId",
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`)
.whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (error) {
throw new DatabaseError({
error,
name: "Secret Folder Version Prune"
});
}
};
return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions };
};
backend/src/services/secret/secret-fns.ts
+6 -5
View File
@@ -309,7 +309,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
};
const expandSecrets = async (
secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>
secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
) => {
const expandedSec: Record<string, string> = {};
const interpolatedSec: Record<string, string> = {};
@@ -329,8 +329,8 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
// should not do multi line encoding if user has set it to skip
// eslint-disable-next-line
secrets[key].value = secrets[key].skipMultilineEncoding
? expandedSec[key]
: formatMultiValueEnv(expandedSec[key]);
? formatMultiValueEnv(expandedSec[key])
: expandedSec[key];
// eslint-disable-next-line
continue;
}
@@ -347,7 +347,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
);
// eslint-disable-next-line
secrets[key].value = secrets[key].skipMultilineEncoding ? expandedVal : formatMultiValueEnv(expandedVal);
secrets[key].value = secrets[key].skipMultilineEncoding ? formatMultiValueEnv(expandedVal) : expandedVal;
}
return secrets;
@@ -395,7 +395,8 @@ export const decryptSecretRaw = (
type: secret.type,
_id: secret.id,
id: secret.id,
user: secret.userId
user: secret.userId,
skipMultilineEncoding: secret.skipMultilineEncoding
};
};
backend/src/services/secret/secret-queue.ts
+11 -3
View File
@@ -1,4 +1,6 @@
/* eslint-disable no-await-in-loop */
import { AxiosError } from "axios";
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
@@ -67,7 +69,10 @@ const MAX_SYNC_SECRET_DEPTH = 5;
export const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
`secret-queue-dedupe-${environment}-${secretPath}`;
type TIntegrationSecret = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
type TIntegrationSecret = Record<
string,
{ value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined }
>;
export const secretQueueFactory = ({
queueService,
integrationDAL,
@@ -567,11 +572,14 @@ export const secretQueueFactory = ({
isSynced: true
});
} catch (err: unknown) {
logger.info("Secret integration sync error:", err);
logger.info("Secret integration sync error: %o", err);
const message =
err instanceof AxiosError ? JSON.stringify((err as AxiosError)?.response?.data) : (err as Error)?.message;
await integrationDAL.updateById(integration.id, {
lastSyncJobId: job.id,
lastUsed: new Date(),
syncMessage: (err as Error)?.message,
syncMessage: message,
isSynced: false
});
}
backend/src/services/secret/secret-service.ts
+63 -11
View File
@@ -952,15 +952,49 @@ export const secretServiceFactory = ({
});
const decryptedSecrets = secrets.map((el) => decryptSecretRaw(el, botKey));
const decryptedImports = (imports || [])?.map(({ secrets: importedSecrets, ...el }) => ({
...el,
secrets: importedSecrets.map((sec) =>
const processedImports = (imports || [])?.map(({ secrets: importedSecrets, ...el }) => {
const decryptedImportSecrets = importedSecrets.map((sec) =>
decryptSecretRaw(
{ ...sec, environment: el.environment, workspace: projectId, secretPath: el.secretPath },
botKey
)
)
}));
);
// secret-override to handle duplicate keys from different import levels
// this prioritizes secret values from direct imports
const importedKeys = new Set<string>();
const importedEntries = decryptedImportSecrets.reduce(
(
accum: {
secretKey: string;
secretPath: string;
workspace: string;
environment: string;
secretValue: string;
secretComment: string;
version: number;
type: string;
_id: string;
id: string;
user: string | null | undefined;
skipMultilineEncoding: boolean | null | undefined;
}[],
sec
) => {
if (!importedKeys.has(sec.secretKey)) {
importedKeys.add(sec.secretKey);
return [...accum, sec];
}
return accum;
},
[]
);
return {
...el,
secrets: importedEntries
};
});
if (expandSecretReferences) {
const expandSecrets = interpolateSecrets({
@@ -971,10 +1005,24 @@ export const secretServiceFactory = ({
});
const batchSecretsExpand = async (
secretBatch: { secretKey: string; secretValue: string; secretComment?: string; secretPath: string }[]
secretBatch: {
secretKey: string;
secretValue: string;
secretComment?: string;
secretPath: string;
skipMultilineEncoding: boolean | null | undefined;
}[]
) => {
// Group secrets by secretPath
const secretsByPath: Record<string, { secretKey: string; secretValue: string; secretComment?: string }[]> = {};
const secretsByPath: Record<
string,
{
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding: boolean | null | undefined;
}[]
> = {};
secretBatch.forEach((secret) => {
if (!secretsByPath[secret.secretPath]) {
@@ -990,11 +1038,15 @@ export const secretServiceFactory = ({
continue;
}
const secretRecord: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }> = {};
const secretRecord: Record<
string,
{ value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined }
> = {};
secretsByPath[secPath].forEach((decryptedSecret) => {
secretRecord[decryptedSecret.secretKey] = {
value: decryptedSecret.secretValue,
comment: decryptedSecret.secretComment
comment: decryptedSecret.secretComment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
};
});
@@ -1011,12 +1063,12 @@ export const secretServiceFactory = ({
await batchSecretsExpand(decryptedSecrets);
// expand imports by batch
await Promise.all(decryptedImports.map((decryptedImport) => batchSecretsExpand(decryptedImport.secrets)));
await Promise.all(processedImports.map((processedImport) => batchSecretsExpand(processedImport.secrets)));
}
return {
secrets: decryptedSecrets,
imports: decryptedImports
imports: processedImports
};
};
backend/src/services/secret/secret-version-dal.ts
+29
View File
@@ -111,8 +111,37 @@ export const secretVersionDALFactory = (db: TDbClient) => {
}
};
const pruneExcessVersions = async () => {
try {
await db(TableName.SecretVersion)
.with("version_cte", (qb) => {
void qb
.from(TableName.SecretVersion)
.select(
"id",
"folderId",
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`)
.whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (error) {
throw new DatabaseError({
error,
name: "Secret Version Prune"
});
}
};
return {
...secretVersionOrm,
pruneExcessVersions,
findLatestVersionMany,
bulkUpdate,
findLatestVersionByFolderId,
backend/src/services/smtp/smtp-service.ts
+1 -14
View File
@@ -41,21 +41,8 @@ export enum SmtpHost {
Office365 = "smtp.office365.com"
}
export const getTlsOption = (host?: SmtpHost | string, secure?: boolean) => {
if (!secure) return { secure: false };
if (!host) return { secure: true };
if ((host as SmtpHost) === SmtpHost.Sendgrid) {
return { secure: true, port: 465 }; // more details here https://nodemailer.com/smtp/
}
if (host.includes("amazonaws.com")) {
return { tls: { ciphers: "TLSv1.2" } };
}
return { requireTLS: true, tls: { ciphers: "TLSv1.2" } };
};
export const smtpServiceFactory = (cfg: TSmtpConfig) => {
const smtp = createTransport({ ...cfg, ...getTlsOption(cfg.host, cfg.secure) });
const smtp = createTransport(cfg);
const isSmtpOn = Boolean(cfg.host);
const sendMail = async ({ substitutions, recipients, template, subjectLine }: TSmtpSendMail) => {
backend/src/services/user/user-service.ts
+11 -1
View File
@@ -21,6 +21,7 @@ type TUserServiceFactoryDep = {
| "findOneUserAction"
| "createUserAction"
| "findUserEncKeyByUserId"
| "delete"
>;
userAliasDAL: Pick<TUserAliasDALFactory, "find" | "insertMany">;
orgMembershipDAL: Pick<TOrgMembershipDALFactory, "find" | "insertMany">;
@@ -85,7 +86,7 @@ export const userServiceFactory = ({
tx
);
// check if there are users with the same email.
// check if there are verified users with the same email.
const users = await userDAL.find(
{
email,
@@ -134,6 +135,15 @@ export const userServiceFactory = ({
);
}
} else {
await userDAL.delete(
{
email,
isAccepted: false,
isEmailVerified: false
},
tx
);
// update current user's username to [email]
await userDAL.updateById(
user.id,