Merge remote-tracking branch 'origin' into certificate-alerting

This commit is contained in:
Tuan Dang
2024-08-17 19:46:23 -07:00
109 changed files with 1518 additions and 565 deletions
+49 -6
View File
@@ -25,6 +25,7 @@
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@peculiar/asn1-schema": "^2.3.8",
@@ -7812,19 +7813,45 @@
}
},
"node_modules/@octokit/plugin-retry": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz",
"integrity": "sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==",
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-5.0.5.tgz",
"integrity": "sha512-sB1RWMhSrre02Atv95K6bhESlJ/sPdZkK/wE/w1IdSCe0yM6FxSjksLa6T7aAvxvxlLKzQEC4KIiqpqyov1Tbg==",
"dependencies": {
"@octokit/request-error": "^5.0.0",
"@octokit/types": "^12.0.0",
"@octokit/request-error": "^4.0.1",
"@octokit/types": "^10.0.0",
"bottleneck": "^2.15.3"
},
"engines": {
"node": ">= 18"
},
"peerDependencies": {
"@octokit/core": ">=5"
"@octokit/core": ">=3"
}
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": {
"version": "18.1.1",
"resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.1.1.tgz",
"integrity": "sha512-VRaeH8nCDtF5aXWnjPuEMIYf1itK/s3JYyJcWFJT8X9pSNnBtriDf7wlEWsGuhPLl4QIH4xM8fqTXDwJ3Mu6sw=="
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/request-error": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-4.0.2.tgz",
"integrity": "sha512-uqwUEmZw3x4I9DGYq9fODVAAvcLsPQv97NRycP6syEFu5916M189VnNBW2zANNwqg3OiligNcAey7P0SET843w==",
"dependencies": {
"@octokit/types": "^10.0.0",
"deprecation": "^2.0.0",
"once": "^1.4.0"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/types": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@octokit/types/-/types-10.0.0.tgz",
"integrity": "sha512-Vm8IddVmhCgU1fxC1eyinpwqzXPEYu0NrYzD3YZjlGjyftdLBTeqNblRC0jmJmgxbJIsQlyogVeGnrNaaMVzIg==",
"dependencies": {
"@octokit/openapi-types": "^18.0.0"
}
},
"node_modules/@octokit/plugin-throttling": {
@@ -17396,6 +17423,22 @@
"node": ">=18"
}
},
"node_modules/probot/node_modules/@octokit/plugin-retry": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz",
"integrity": "sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==",
"dependencies": {
"@octokit/request-error": "^5.0.0",
"@octokit/types": "^12.0.0",
"bottleneck": "^2.15.3"
},
"engines": {
"node": ">= 18"
},
"peerDependencies": {
"@octokit/core": ">=5"
}
},
"node_modules/probot/node_modules/commander": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz",
+1
View File
@@ -121,6 +121,7 @@
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@peculiar/asn1-schema": "^2.3.8",
+20 -1
View File
@@ -7,14 +7,33 @@ const prompt = promptSync({
sigint: true
});
type ComponentType = 1 | 2 | 3;
console.log(`
Component List
--------------
0. Exit
1. Service component
2. DAL component
3. Router component
`);
const componentType = parseInt(prompt("Select a component: "), 10);
function getComponentType(): ComponentType {
while (true) {
const input = prompt("Select a component (0-3): ");
const componentType = parseInt(input, 10);
if (componentType === 0) {
console.log("Exiting the program. Goodbye!");
process.exit(0);
} else if (componentType === 1 || componentType === 2 || componentType === 3) {
return componentType;
} else {
console.log("Invalid input. Please enter 0, 1, 2, or 3.");
}
}
}
const componentType = getComponentType();
if (componentType === 1) {
const componentName = prompt("Enter service name: ");
+2
View File
@@ -18,6 +18,7 @@ import { TOidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-ser
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service";
import { TRateLimitServiceFactory } from "@app/ee/services/rate-limit/rate-limit-service";
import { RateLimitConfiguration } from "@app/ee/services/rate-limit/rate-limit-types";
import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-config-service";
import { TScimServiceFactory } from "@app/ee/services/scim/scim-service";
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
@@ -91,6 +92,7 @@ declare module "fastify" {
id: string;
orgId: string;
};
rateLimits: RateLimitConfiguration;
// passport data
passportUser: {
isUserCompleted: string;
@@ -0,0 +1,21 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasCreationLimitCol = await knex.schema.hasColumn(TableName.RateLimit, "creationLimit");
await knex.schema.alterTable(TableName.RateLimit, (t) => {
if (hasCreationLimitCol) {
t.dropColumn("creationLimit");
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasCreationLimitCol = await knex.schema.hasColumn(TableName.RateLimit, "creationLimit");
await knex.schema.alterTable(TableName.RateLimit, (t) => {
if (!hasCreationLimitCol) {
t.integer("creationLimit").defaultTo(30).notNullable();
}
});
}
@@ -0,0 +1,21 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasNameField = await knex.schema.hasColumn(TableName.SecretTag, "name");
if (hasNameField) {
await knex.schema.alterTable(TableName.SecretTag, (t) => {
t.dropColumn("name");
});
}
}
export async function down(knex: Knex): Promise<void> {
const hasNameField = await knex.schema.hasColumn(TableName.SecretTag, "name");
if (!hasNameField) {
await knex.schema.alterTable(TableName.SecretTag, (t) => {
t.string("name");
});
}
}
-1
View File
@@ -15,7 +15,6 @@ export const RateLimitSchema = z.object({
authRateLimit: z.number().default(60),
inviteUserRateLimit: z.number().default(30),
mfaRateLimit: z.number().default(20),
creationLimit: z.number().default(30),
publicEndpointLimit: z.number().default(30),
createdAt: z.date(),
updatedAt: z.date()
-1
View File
@@ -9,7 +9,6 @@ import { TImmutableDBKeys } from "./models";
export const SecretTagsSchema = z.object({
id: z.string().uuid(),
name: z.string(),
slug: z.string(),
color: z.string().nullable().optional(),
createdAt: z.date(),
@@ -131,7 +131,7 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide
.default("/")
.transform(removeTrailingSlash)
.describe(DYNAMIC_SECRET_LEASES.RENEW.path),
environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.RENEW.ttl)
environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.RENEW.environmentSlug)
}),
response: {
200: z.object({
@@ -58,7 +58,6 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => {
authRateLimit: z.number(),
inviteUserRateLimit: z.number(),
mfaRateLimit: z.number(),
creationLimit: z.number(),
publicEndpointLimit: z.number()
}),
response: {
@@ -75,15 +75,16 @@ export const auditLogDALFactory = (db: TDbClient) => {
.del()
.returning("id");
numberOfRetryOnFailure = 0; // reset
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 100); // time to breathe for db
});
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, "Failed to delete audit log on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
} while (deletedAuditLogIds.length > 0 && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE);
} while (deletedAuditLogIds.length > 0 || numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE);
};
return { ...auditLogOrm, pruneAuditLog, find };
@@ -352,6 +352,7 @@ interface DeleteIntegrationEvent {
targetServiceId?: string;
path?: string;
region?: string;
shouldDeleteIntegrationSecrets?: boolean;
};
}
@@ -40,7 +40,12 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
secretRotation: true,
caCrl: false,
instanceUserManagement: false,
externalKms: false
externalKms: false,
rateLimits: {
readLimit: 60,
writeLimit: 200,
secretsLimit: 40
}
});
export const setupLicenceRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {
@@ -58,6 +58,11 @@ export type TFeatureSet = {
caCrl: false;
instanceUserManagement: false;
externalKms: false;
rateLimits: {
readLimit: number;
writeLimit: number;
secretsLimit: number;
};
};
export type TOrgPlansTableDTO = {
@@ -4,17 +4,16 @@ import { logger } from "@app/lib/logger";
import { TLicenseServiceFactory } from "../license/license-service";
import { TRateLimitDALFactory } from "./rate-limit-dal";
import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types";
import { RateLimitConfiguration, TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types";
let rateLimitMaxConfiguration = {
let rateLimitMaxConfiguration: RateLimitConfiguration = {
readLimit: 60,
publicEndpointLimit: 30,
writeLimit: 200,
secretsLimit: 60,
authRateLimit: 60,
inviteUserRateLimit: 30,
mfaRateLimit: 20,
creationLimit: 30
mfaRateLimit: 20
};
Object.freeze(rateLimitMaxConfiguration);
@@ -67,8 +66,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL, licenseService }: TRateL
secretsLimit: rateLimit.secretsRateLimit,
authRateLimit: rateLimit.authRateLimit,
inviteUserRateLimit: rateLimit.inviteUserRateLimit,
mfaRateLimit: rateLimit.mfaRateLimit,
creationLimit: rateLimit.creationLimit
mfaRateLimit: rateLimit.mfaRateLimit
};
logger.info(`syncRateLimitConfiguration: rate limit configuration: %o`, newRateLimitMaxConfiguration);
@@ -5,7 +5,6 @@ export type TRateLimitUpdateDTO = {
authRateLimit: number;
inviteUserRateLimit: number;
mfaRateLimit: number;
creationLimit: number;
publicEndpointLimit: number;
};
@@ -14,3 +13,13 @@ export type TRateLimit = {
createdAt: Date;
updatedAt: Date;
} & TRateLimitUpdateDTO;
export type RateLimitConfiguration = {
readLimit: number;
publicEndpointLimit: number;
writeLimit: number;
secretsLimit: number;
authRateLimit: number;
inviteUserRateLimit: number;
mfaRateLimit: number;
};
@@ -81,15 +81,13 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
.select({
secVerTagId: "secVerTag.id",
secVerTagColor: "secVerTag.color",
secVerTagSlug: "secVerTag.slug",
secVerTagName: "secVerTag.name"
secVerTagSlug: "secVerTag.slug"
})
.select(
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTag).as("tagJnId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
)
.select(
db.ref("secretBlindIndex").withSchema(TableName.Secret).as("orgSecBlindIndex"),
@@ -124,9 +122,9 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
{
key: "tagJnId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color }) => ({
id,
name,
name: slug,
slug,
color
})
@@ -200,11 +198,11 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
{
key: "secVerTagId",
label: "tags" as const,
mapper: ({ secVerTagId: id, secVerTagName: name, secVerTagSlug: slug, secVerTagColor: color }) => ({
mapper: ({ secVerTagId: id, secVerTagSlug: slug, secVerTagColor: color }) => ({
// eslint-disable-next-line
id,
// eslint-disable-next-line
name,
name: slug,
// eslint-disable-next-line
slug,
// eslint-disable-next-line
@@ -262,15 +260,13 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
.select({
secVerTagId: "secVerTag.id",
secVerTagColor: "secVerTag.color",
secVerTagSlug: "secVerTag.slug",
secVerTagName: "secVerTag.name"
secVerTagSlug: "secVerTag.slug"
})
.select(
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTagV2).as("tagJnId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
)
.select(
db.ref("version").withSchema(TableName.SecretV2).as("orgSecVersion"),
@@ -292,9 +288,9 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
{
key: "tagJnId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color }) => ({
id,
name,
name: slug,
slug,
color
})
@@ -330,11 +326,11 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
{
key: "secVerTagId",
label: "tags" as const,
mapper: ({ secVerTagId: id, secVerTagName: name, secVerTagSlug: slug, secVerTagColor: color }) => ({
mapper: ({ secVerTagId: id, secVerTagSlug: slug, secVerTagColor: color }) => ({
// eslint-disable-next-line
id,
// eslint-disable-next-line
name,
name: slug,
// eslint-disable-next-line
slug,
// eslint-disable-next-line
@@ -224,12 +224,10 @@ export const secretApprovalRequestServiceFactory = ({
secretKey: el.key,
id: el.id,
version: el.version,
secretValue: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: undefined,
secretValue: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
secretComment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined,
: "",
secret: el.secret
? {
secretKey: el.secret.key,
@@ -237,10 +235,10 @@ export const secretApprovalRequestServiceFactory = ({
version: el.secret.version,
secretValue: el.secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString()
: undefined,
: "",
secretComment: el.secret.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString()
: undefined
: ""
}
: undefined,
secretVersion: el.secretVersion
@@ -250,10 +248,10 @@ export const secretApprovalRequestServiceFactory = ({
version: el.secretVersion.version,
secretValue: el.secretVersion.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString()
: undefined,
: "",
secretComment: el.secretVersion.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString()
: undefined
: ""
}
: undefined
}));
@@ -257,7 +257,7 @@ export const secretReplicationServiceFactory = ({
secretDAL: secretV2BridgeDAL,
folderDAL,
secretImportDAL,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined)
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
});
// secrets that gets replicated across imports
const sourceDecryptedLocalSecrets = sourceLocalSecrets.map((el) => ({
@@ -449,7 +449,7 @@ export const secretReplicationServiceFactory = ({
});
}
if (locallyDeletedSecrets.length) {
await secretDAL.delete(
await secretV2BridgeDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
@@ -164,10 +164,10 @@ export const secretSnapshotServiceFactory = ({
secretKey: el.key,
secretValue: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: undefined,
: "",
secretComment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined
: ""
}))
};
} else {
@@ -100,8 +100,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
);
return sqlNestRelationships({
data,
@@ -132,9 +131,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
name: slug,
slug,
color,
vId
@@ -195,8 +194,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
);
return sqlNestRelationships({
data,
@@ -227,9 +225,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
name: slug,
slug,
color,
vId
@@ -353,8 +351,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
);
const formated = sqlNestRelationships({
@@ -377,9 +374,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
name: slug,
slug,
color,
vId
@@ -508,8 +505,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")
);
const formated = sqlNestRelationships({
@@ -532,9 +528,9 @@ export const snapshotDALFactory = (db: TDbClient) => {
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
mapper: ({ tagId: id, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
name: slug,
slug,
color,
vId
+2 -1
View File
@@ -596,7 +596,8 @@ export const RAW_SECRETS = {
"The slug of the project to list secrets from. This parameter is only applicable by machine identities.",
environment: "The slug of the environment to list secrets from.",
secretPath: "The secret path to list secrets from.",
includeImports: "Weather to include imported secrets or not."
includeImports: "Weather to include imported secrets or not.",
tagSlugs: "The comma separated tag slugs to filter secrets"
},
CREATE: {
secretName: "The name of the secret to create.",
+10
View File
@@ -128,6 +128,16 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
throw new DatabaseError({ error, name: "Create" });
}
},
// This spilit the insert into multiple chunk
batchInsert: async (data: readonly Tables[Tname]["insert"][], tx?: Knex) => {
try {
if (!data.length) return [];
const res = await (tx || db).batchInsert(tableName, data as never).returning("*");
return res as Tables[Tname]["base"][];
} catch (error) {
throw new DatabaseError({ error, name: "batchInsert" });
}
},
upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
try {
if (!data.length) return [];
+14 -15
View File
@@ -1,7 +1,6 @@
import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit";
import { Redis } from "ioredis";
import { getRateLimiterConfig } from "@app/ee/services/rate-limit/rate-limit-service";
import { getConfig } from "@app/lib/config/env";
export const globalRateLimiterCfg = (): RateLimitPluginOptions => {
@@ -22,14 +21,16 @@ export const globalRateLimiterCfg = (): RateLimitPluginOptions => {
// GET endpoints
export const readLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().readLimit,
hook: "preValidation",
max: (req) => req.rateLimits.readLimit,
keyGenerator: (req) => req.realIp
};
// POST, PATCH, PUT, DELETE endpoints
export const writeLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().writeLimit,
hook: "preValidation",
max: (req) => req.rateLimits.writeLimit,
keyGenerator: (req) => req.realIp
};
@@ -37,42 +38,40 @@ export const writeLimit: RateLimitOptions = {
export const secretsLimit: RateLimitOptions = {
// secrets, folders, secret imports
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().secretsLimit,
hook: "preValidation",
max: (req) => req.rateLimits.secretsLimit,
keyGenerator: (req) => req.realIp
};
export const authRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().authRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.authRateLimit,
keyGenerator: (req) => req.realIp
};
export const inviteUserRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().inviteUserRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.inviteUserRateLimit,
keyGenerator: (req) => req.realIp
};
export const mfaRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().mfaRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.mfaRateLimit,
keyGenerator: (req) => {
return req.headers.authorization?.split(" ")[1] || req.realIp;
}
};
export const creationLimit: RateLimitOptions = {
// identity, project, org
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().creationLimit,
keyGenerator: (req) => req.realIp
};
// Public endpoints to avoid brute force attacks
export const publicEndpointLimit: RateLimitOptions = {
// Read Shared Secrets
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().publicEndpointLimit,
hook: "preValidation",
max: (req) => req.rateLimits.publicEndpointLimit,
keyGenerator: (req) => req.realIp
};
@@ -0,0 +1,38 @@
import fp from "fastify-plugin";
import { getRateLimiterConfig } from "@app/ee/services/rate-limit/rate-limit-service";
import { getConfig } from "@app/lib/config/env";
export const injectRateLimits = fp(async (server) => {
server.decorateRequest("rateLimits", null);
server.addHook("onRequest", async (req) => {
const appCfg = getConfig();
const instanceRateLimiterConfig = getRateLimiterConfig();
if (!req.auth?.orgId) {
// for public endpoints, we always use the instance-wide default rate limits
req.rateLimits = instanceRateLimiterConfig;
return;
}
const { rateLimits, customRateLimits } = await server.services.license.getPlan(req.auth.orgId);
if (customRateLimits && !appCfg.isCloud) {
// we do this because for self-hosted/dedicated instances, we want custom rate limits to be based on admin configuration
// note that the syncing of custom rate limit happens on the instanceRateLimiterConfig object
req.rateLimits = instanceRateLimiterConfig;
return;
}
// we're using the null coalescing operator in order to handle outdated licenses
req.rateLimits = {
readLimit: rateLimits?.readLimit ?? instanceRateLimiterConfig.readLimit,
writeLimit: rateLimits?.writeLimit ?? instanceRateLimiterConfig.writeLimit,
secretsLimit: rateLimits?.secretsLimit ?? instanceRateLimiterConfig.secretsLimit,
publicEndpointLimit: instanceRateLimiterConfig.publicEndpointLimit,
authRateLimit: instanceRateLimiterConfig.authRateLimit,
inviteUserRateLimit: instanceRateLimiterConfig.inviteUserRateLimit,
mfaRateLimit: instanceRateLimiterConfig.mfaRateLimit
};
});
});
+12 -2
View File
@@ -189,6 +189,7 @@ import { webhookServiceFactory } from "@app/services/webhook/webhook-service";
import { injectAuditLogInfo } from "../plugins/audit-log";
import { injectIdentity } from "../plugins/auth/inject-identity";
import { injectPermission } from "../plugins/auth/inject-permission";
import { injectRateLimits } from "../plugins/inject-rate-limits";
import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
import { registerV1Routes } from "./v1";
import { registerV2Routes } from "./v2";
@@ -924,8 +925,15 @@ export const registerRoutes = async (
folderDAL,
integrationDAL,
integrationAuthDAL,
secretQueueService
secretQueueService,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
secretImportDAL,
secretDAL,
kmsService
});
const serviceTokenService = serviceTokenServiceFactory({
projectEnvDAL,
serviceTokenDAL,
@@ -1058,7 +1066,8 @@ export const registerRoutes = async (
snapshotDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL: secretVersionV2BridgeDAL
secretVersionV2DAL: secretVersionV2BridgeDAL,
identityUniversalAuthClientSecretDAL: identityUaClientSecretDAL
});
const oidcService = oidcConfigServiceFactory({
@@ -1173,6 +1182,7 @@ export const registerRoutes = async (
await server.register(injectIdentity, { userDAL, serviceTokenDAL });
await server.register(injectPermission);
await server.register(injectRateLimits);
await server.register(injectAuditLogInfo);
server.route({
@@ -63,8 +63,8 @@ export const secretRawSchema = z.object({
version: z.number(),
type: z.string(),
secretKey: z.string(),
secretValue: z.string().optional(),
secretComment: z.string().optional(),
secretValue: z.string(),
secretComment: z.string(),
secretReminderNote: z.string().nullable().optional(),
secretReminderRepeatDays: z.number().nullable().optional(),
skipMultilineEncoding: z.boolean().default(false).nullable().optional(),
@@ -3,7 +3,7 @@ import { z } from "zod";
import { IdentitiesSchema, IdentityOrgMembershipsSchema, OrgMembershipRole, OrgRolesSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { IDENTITIES } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@@ -16,7 +16,7 @@ export const registerIdentityRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
schema: {
@@ -170,6 +170,12 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
params: z.object({
integrationId: z.string().trim().describe(INTEGRATION.DELETE.integrationId)
}),
querystring: z.object({
shouldDeleteIntegrationSecrets: z
.enum(["true", "false"])
.optional()
.transform((val) => val === "true")
}),
response: {
200: z.object({
integration: IntegrationsSchema
@@ -183,7 +189,8 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
actorAuthMethod: req.permission.authMethod,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
id: req.params.integrationId
id: req.params.integrationId,
shouldDeleteIntegrationSecrets: req.query.shouldDeleteIntegrationSecrets
});
await server.services.auditLog.createAuditLog({
@@ -205,7 +212,8 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
targetService: integration.targetService,
targetServiceId: integration.targetServiceId,
path: integration.path,
region: integration.region
region: integration.region,
shouldDeleteIntegrationSecrets: req.query.shouldDeleteIntegrationSecrets
// eslint-disable-next-line
}) as any
}
@@ -1,3 +1,4 @@
import slugify from "@sindresorhus/slugify";
import { z } from "zod";
import { SecretTagsSchema } from "@app/db/schemas";
@@ -49,7 +50,8 @@ export const registerSecretTagRouter = async (server: FastifyZodProvider) => {
}),
response: {
200: z.object({
workspaceTag: SecretTagsSchema
// akhilmhdh: for terraform backward compatiability
workspaceTag: SecretTagsSchema.extend({ name: z.string() })
})
}
},
@@ -79,7 +81,8 @@ export const registerSecretTagRouter = async (server: FastifyZodProvider) => {
}),
response: {
200: z.object({
workspaceTag: SecretTagsSchema
// akhilmhdh: for terraform backward compatiability
workspaceTag: SecretTagsSchema.extend({ name: z.string() })
})
}
},
@@ -108,8 +111,14 @@ export const registerSecretTagRouter = async (server: FastifyZodProvider) => {
projectId: z.string().trim().describe(SECRET_TAGS.CREATE.projectId)
}),
body: z.object({
name: z.string().trim().describe(SECRET_TAGS.CREATE.name),
slug: z.string().trim().describe(SECRET_TAGS.CREATE.slug),
slug: z
.string()
.toLowerCase()
.trim()
.describe(SECRET_TAGS.CREATE.slug)
.refine((v) => slugify(v) === v, {
message: "Invalid slug. Slug can only contain alphanumeric characters and hyphens."
}),
color: z.string().trim().describe(SECRET_TAGS.CREATE.color)
}),
response: {
@@ -144,8 +153,14 @@ export const registerSecretTagRouter = async (server: FastifyZodProvider) => {
tagId: z.string().trim().describe(SECRET_TAGS.UPDATE.tagId)
}),
body: z.object({
name: z.string().trim().describe(SECRET_TAGS.UPDATE.name),
slug: z.string().trim().describe(SECRET_TAGS.UPDATE.slug),
slug: z
.string()
.toLowerCase()
.trim()
.describe(SECRET_TAGS.UPDATE.slug)
.refine((v) => slugify(v) === v, {
message: "Invalid slug. Slug can only contain alphanumeric characters and hyphens."
}),
color: z.string().trim().describe(SECRET_TAGS.UPDATE.color)
}),
response: {
@@ -9,7 +9,7 @@ import {
UsersSchema
} from "@app/db/schemas";
import { ORGANIZATIONS } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
@@ -307,7 +307,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
schema: {
body: z.object({
@@ -10,7 +10,7 @@ import {
} from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { PROJECTS } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@@ -148,7 +148,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
schema: {
description: "Create a new project",
+25 -17
View File
@@ -59,9 +59,10 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
tags: SecretTagsSchema.pick({
id: true,
slug: true,
name: true,
color: true
}).array()
})
.extend({ name: z.string() })
.array()
})
)
})
@@ -116,16 +117,15 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}),
response: {
200: z.object({
secret: SecretsSchema.omit({ secretBlindIndex: true }).merge(
z.object({
tags: SecretTagsSchema.pick({
id: true,
slug: true,
name: true,
color: true
}).array()
secret: SecretsSchema.omit({ secretBlindIndex: true }).extend({
tags: SecretTagsSchema.pick({
id: true,
slug: true,
color: true
})
)
.extend({ name: z.string() })
.array()
})
})
}
},
@@ -180,7 +180,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
.enum(["true", "false"])
.default("false")
.transform((value) => value === "true")
.describe(RAW_SECRETS.LIST.includeImports)
.describe(RAW_SECRETS.LIST.includeImports),
tagSlugs: z
.string()
.describe(RAW_SECRETS.LIST.tagSlugs)
.optional()
// split by comma and trim the strings
.transform((el) => (el ? el.split(",").map((i) => i.trim()) : []))
}),
response: {
200: z.object({
@@ -190,9 +196,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
tags: SecretTagsSchema.pick({
id: true,
slug: true,
name: true,
color: true
})
.extend({ name: z.string() })
.array()
.optional()
})
@@ -251,7 +257,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId: workspaceId,
path: secretPath,
includeImports: req.query.include_imports,
recursive: req.query.recursive
recursive: req.query.recursive,
tagSlugs: req.query.tagSlugs
});
await server.services.auditLog.createAuditLog({
@@ -325,9 +332,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
tags: SecretTagsSchema.pick({
id: true,
slug: true,
name: true,
color: true
})
.extend({ name: z.string() })
.array()
.optional()
})
@@ -731,9 +738,10 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
tags: SecretTagsSchema.pick({
id: true,
slug: true,
name: true,
color: true
}).array()
})
.extend({ name: z.string() })
.array()
})
.array(),
imports: z
@@ -4,6 +4,7 @@ import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
export type TIdentityUaClientSecretDALFactory = ReturnType<typeof identityUaClientSecretDALFactory>;
@@ -23,5 +24,55 @@ export const identityUaClientSecretDALFactory = (db: TDbClient) => {
}
};
return { ...uaClientSecretOrm, incrementUsage };
const removeExpiredClientSecrets = async (tx?: Knex) => {
const BATCH_SIZE = 10000;
const MAX_RETRY_ON_FAILURE = 3;
let deletedClientSecret: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
do {
try {
const findExpiredClientSecretQuery = (tx || db)(TableName.IdentityUaClientSecret)
.where({
isClientSecretRevoked: true
})
.orWhere((qb) => {
void qb
.where("clientSecretNumUses", ">", 0)
.andWhere(
"clientSecretNumUses",
">=",
db.ref("clientSecretNumUsesLimit").withSchema(TableName.IdentityUaClientSecret)
);
})
.orWhere((qb) => {
void qb
.where("clientSecretTTL", ">", 0)
.andWhereRaw(
`"${TableName.IdentityUaClientSecret}"."createdAt" + make_interval(secs => "${TableName.IdentityUaClientSecret}"."clientSecretTTL") < NOW()`
);
})
.select("id")
.limit(BATCH_SIZE);
// eslint-disable-next-line no-await-in-loop
deletedClientSecret = await (tx || db)(TableName.IdentityUaClientSecret)
.whereIn("id", findExpiredClientSecretQuery)
.del()
.returning("id");
numberOfRetryOnFailure = 0; // reset
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, "Failed to delete client secret on pruning");
} finally {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
} while (deletedClientSecret.length > 0 || numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE);
};
return { ...uaClientSecretOrm, incrementUsage, removeExpiredClientSecrets };
};
@@ -0,0 +1,357 @@
import { retry } from "@octokit/plugin-retry";
import { Octokit } from "@octokit/rest";
import { TIntegrationAuths, TIntegrations } from "@app/db/schemas";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { IntegrationMetadataSchema } from "../integration/integration-schema";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TIntegrationAuthServiceFactory } from "./integration-auth-service";
import { Integrations } from "./integration-list";
const MAX_SYNC_SECRET_DEPTH = 5;
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
*/
const getIntegrationSecretsV2 = async (
dto: {
projectId: string;
environment: string;
folderId: string;
depth: number;
decryptor: (value: Buffer | null | undefined) => string;
},
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">,
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">,
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">
) => {
const content: Record<string, boolean> = {};
if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
logger.info(
`getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
);
return content;
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = secret.key;
content[secretKey] = true;
});
// check if current folder has any imports from other folders
const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,
secretImportDAL,
allowedImports: secretImports
});
for (let i = importedSecrets.length - 1; i >= 0; i -= 1) {
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
const importedSecret = importedSecrets[i].secrets[j];
if (!content[importedSecret.key]) {
content[importedSecret.key] = true;
}
}
}
return content;
};
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
*/
const getIntegrationSecretsV1 = async (
dto: {
projectId: string;
environment: string;
folderId: string;
key: string;
depth: number;
},
secretDAL: Pick<TSecretDALFactory, "findByFolderId">,
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">,
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">
) => {
let content: Record<string, boolean> = {};
if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
logger.info(
`getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
);
return content;
}
// process secrets in current folder
const secrets = await secretDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: dto.key
});
content[secretKey] = true;
});
// check if current folder has any imports from other folders
const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
// if no imports then return secrets in the current folder
if (!secretImport) return content;
const importedFolders = await folderDAL.findByManySecretPath(
secretImport.map(({ importEnv, importPath }) => ({
envId: importEnv.id,
secretPath: importPath
}))
);
for await (const folder of importedFolders) {
if (folder) {
// get secrets contained in each imported folder by recursively calling
// this function against the imported folder
const importedSecrets = await getIntegrationSecretsV1(
{
environment: dto.environment,
projectId: dto.projectId,
folderId: folder.id,
key: dto.key,
depth: dto.depth + 1
},
secretDAL,
folderDAL,
secretImportDAL
);
// add the imported secrets to the current folder secrets
content = { ...importedSecrets, ...content };
}
}
return content;
};
export const deleteGithubSecrets = async ({
integration,
secrets,
accessToken
}: {
integration: Omit<TIntegrations, "envId">;
secrets: Record<string, boolean>;
accessToken: string;
}) => {
interface GitHubSecret {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
}
const OctokitWithRetry = Octokit.plugin(retry);
const octokit = new OctokitWithRetry({
auth: accessToken
});
enum GithubScope {
Repo = "github-repo",
Org = "github-org",
Env = "github-env"
}
let encryptedGithubSecrets: GitHubSecret[];
switch (integration.scope) {
case GithubScope.Org: {
encryptedGithubSecrets = (
await octokit.request("GET /orgs/{org}/actions/secrets", {
org: integration.owner as string
})
).data.secrets;
break;
}
case GithubScope.Env: {
encryptedGithubSecrets = (
await octokit.request("GET /repositories/{repository_id}/environments/{environment_name}/secrets", {
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string
})
).data.secrets;
break;
}
default: {
encryptedGithubSecrets = (
await octokit.request("GET /repos/{owner}/{repo}/actions/secrets", {
owner: integration.owner as string,
repo: integration.app as string
})
).data.secrets;
break;
}
}
for await (const encryptedSecret of encryptedGithubSecrets) {
if (encryptedSecret.name in secrets) {
switch (integration.scope) {
case GithubScope.Org: {
await octokit.request("DELETE /orgs/{org}/actions/secrets/{secret_name}", {
org: integration.owner as string,
secret_name: encryptedSecret.name
});
break;
}
case GithubScope.Env: {
await octokit.request(
"DELETE /repositories/{repository_id}/environments/{environment_name}/secrets/{secret_name}",
{
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string,
secret_name: encryptedSecret.name
}
);
break;
}
default: {
await octokit.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: integration.owner as string,
repo: integration.app as string,
secret_name: encryptedSecret.name
});
break;
}
}
// small delay to prevent hitting API rate limits
await new Promise((resolve) => {
setTimeout(resolve, 50);
});
}
}
};
export const deleteIntegrationSecrets = async ({
integration,
integrationAuth,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
folderDAL,
secretDAL,
secretImportDAL,
kmsService
}: {
integration: Omit<TIntegrations, "envId"> & {
projectId: string;
environment: {
id: string;
name: string;
slug: string;
};
secretPath: string;
};
integrationAuth: TIntegrationAuths;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken" | "getIntegrationAuth">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath" | "findBySecretPath">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
}) => {
const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integration.projectId);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: integration.projectId
});
const folder = await folderDAL.findBySecretPath(
integration.projectId,
integration.environment.slug,
integration.secretPath
);
if (!folder) {
throw new NotFoundError({
message: "Folder not found."
});
}
const { accessToken } = await integrationAuthService.getIntegrationAccessToken(
integrationAuth,
shouldUseSecretV2Bridge,
botKey
);
const secrets = shouldUseSecretV2Bridge
? await getIntegrationSecretsV2(
{
environment: integration.environment.id,
projectId: integration.projectId,
folderId: folder.id,
depth: 1,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
},
secretV2BridgeDAL,
folderDAL,
secretImportDAL
)
: await getIntegrationSecretsV1(
{
environment: integration.environment.id,
projectId: integration.projectId,
folderId: folder.id,
key: botKey as string,
depth: 1
},
secretDAL,
folderDAL,
secretImportDAL
);
const suffixedSecrets: typeof secrets = {};
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
if (metadata) {
Object.keys(secrets).forEach((key) => {
const prefix = metadata?.secretPrefix || "";
const suffix = metadata?.secretSuffix || "";
const newKey = prefix + key + suffix;
suffixedSecrets[newKey] = secrets[key];
});
}
switch (integration.integration) {
case Integrations.GITHUB: {
await deleteGithubSecrets({
integration,
accessToken,
secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets
});
break;
}
default:
throw new BadRequestError({
message: "Invalid integration"
});
}
};
@@ -538,19 +538,20 @@ const syncSecretsAWSParameterStore = async ({
integration,
secrets,
accessId,
accessToken
accessToken,
projectId
}: {
integration: TIntegrations;
integration: TIntegrations & { secretPath: string; environment: { slug: string } };
secrets: Record<string, { value: string; comment?: string }>;
accessId: string | null;
accessToken: string;
projectId?: string;
}) => {
let response: { isSynced: boolean; syncMessage: string } | null = null;
if (!accessId) {
throw new Error("AWS access ID is required");
}
const config = new AWS.Config({
region: integration.region as string,
credentials: {
@@ -567,7 +568,9 @@ const syncSecretsAWSParameterStore = async ({
const metadata = z.record(z.any()).parse(integration.metadata || {});
const awsParameterStoreSecretsObj: Record<string, AWS.SSM.Parameter> = {};
logger.info(
`getIntegrationSecrets: integration sync triggered for ssm with [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [shouldDisableDelete=${metadata.shouldDisableDelete}]`
);
// now fetch all aws parameter store secrets
let hasNext = true;
let nextToken: string | undefined;
@@ -594,6 +597,18 @@ const syncSecretsAWSParameterStore = async ({
nextToken = parameters.NextToken;
}
logger.info(
`getIntegrationSecrets: all fetched keys from AWS SSM [projectId=${projectId}] [environment=${
integration.environment.slug
}] [secretPath=${integration.secretPath}] [awsParameterStoreSecretsObj=${Object.keys(
awsParameterStoreSecretsObj
).join(",")}]`
);
logger.info(
`getIntegrationSecrets: all secrets from Infisical to send to AWS SSM [projectId=${projectId}] [environment=${
integration.environment.slug
}] [secretPath=${integration.secretPath}] [secrets=${Object.keys(secrets).join(",")}]`
);
// Identify secrets to create
// don't use Promise.all() and promise map here
// it will cause rate limit
@@ -603,24 +618,56 @@ const syncSecretsAWSParameterStore = async ({
// case: secret does not exist in AWS parameter store
// -> create secret
if (secrets[key].value) {
logger.info(
`getIntegrationSecrets: create secret in AWS SSM for [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}]`
);
await ssm
.putParameter({
Name: `${integration.path}${key}`,
Type: "SecureString",
Value: secrets[key].value,
...(metadata.kmsKeyId && { KeyId: metadata.kmsKeyId }),
// Overwrite: true,
Tags: metadata.secretAWSTag
? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
Key: tag.key,
Value: tag.value
}))
: []
Overwrite: true
})
.promise();
if (metadata.secretAWSTag?.length) {
try {
await ssm
.addTagsToResource({
ResourceType: "Parameter",
ResourceId: `${integration.path}${key}`,
Tags: metadata.secretAWSTag
? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
Key: tag.key,
Value: tag.value
}))
: []
})
.promise();
} catch (err) {
logger.error(
err,
`getIntegrationSecrets: create secret in AWS SSM for failed [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}]`
);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if ((err as any).code === "AccessDeniedException") {
logger.error(
`AWS Parameter Store Error [integration=${integration.id}]: double check AWS account permissions (refer to the Infisical docs)`
);
}
response = {
isSynced: false,
syncMessage: (err as AWSError)?.message || "Error syncing with AWS Parameter Store"
};
}
}
}
// case: secret exists in AWS parameter store
} else {
logger.info(
`getIntegrationSecrets: update secret in AWS SSM for [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}]`
);
// -> update secret
if (awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
await ssm
@@ -648,6 +695,10 @@ const syncSecretsAWSParameterStore = async ({
})
.promise();
} catch (err) {
logger.error(
err,
`getIntegrationSecrets: update secret in AWS SSM for failed [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}]`
);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if ((err as any).code === "AccessDeniedException") {
logger.error(
@@ -670,9 +721,18 @@ const syncSecretsAWSParameterStore = async ({
}
if (!metadata.shouldDisableDelete) {
logger.info(
`getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [step=1]`
);
for (const key in awsParameterStoreSecretsObj) {
if (Object.hasOwn(awsParameterStoreSecretsObj, key)) {
logger.info(
`getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}] [step=2]`
);
if (!(key in secrets)) {
logger.info(
`getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}] [step=3]`
);
// case:
// -> delete secret
await ssm
@@ -680,6 +740,9 @@ const syncSecretsAWSParameterStore = async ({
Name: awsParameterStoreSecretsObj[key].Name as string
})
.promise();
logger.info(
`getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}] [secretPath=${integration.secretPath}] [key=${key}] [step=4]`
);
}
await new Promise((resolve) => {
setTimeout(resolve, 50);
@@ -3656,7 +3719,8 @@ export const syncIntegrationSecrets = async ({
integration,
secrets,
accessId,
accessToken
accessToken,
projectId
});
break;
case Integrations.AWS_SECRET_MANAGER:
@@ -6,8 +6,15 @@ import { BadRequestError } from "@app/lib/errors";
import { TProjectPermission } from "@app/lib/types";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
import { deleteIntegrationSecrets } from "../integration-auth/integration-delete-secret";
import { TKmsServiceFactory } from "../kms/kms-service";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretQueueFactory } from "../secret/secret-queue";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TIntegrationDALFactory } from "./integration-dal";
import {
TCreateIntegrationDTO,
@@ -19,9 +26,15 @@ import {
type TIntegrationServiceFactoryDep = {
integrationDAL: TIntegrationDALFactory;
integrationAuthDAL: TIntegrationAuthDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
integrationAuthService: TIntegrationAuthServiceFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findByManySecretPath">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
projectBotService: TProjectBotServiceFactory;
secretQueueService: Pick<TSecretQueueFactory, "syncIntegrations">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
};
export type TIntegrationServiceFactory = ReturnType<typeof integrationServiceFactory>;
@@ -31,7 +44,13 @@ export const integrationServiceFactory = ({
integrationAuthDAL,
folderDAL,
permissionService,
secretQueueService
secretQueueService,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
secretImportDAL,
kmsService,
secretDAL
}: TIntegrationServiceFactoryDep) => {
const createIntegration = async ({
app,
@@ -161,7 +180,14 @@ export const integrationServiceFactory = ({
return updatedIntegration;
};
const deleteIntegration = async ({ actorId, id, actor, actorAuthMethod, actorOrgId }: TDeleteIntegrationDTO) => {
const deleteIntegration = async ({
actorId,
id,
actor,
actorAuthMethod,
actorOrgId,
shouldDeleteIntegrationSecrets
}: TDeleteIntegrationDTO) => {
const integration = await integrationDAL.findById(id);
if (!integration) throw new BadRequestError({ message: "Integration auth not found" });
@@ -174,6 +200,22 @@ export const integrationServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Integrations);
const integrationAuth = await integrationAuthDAL.findById(integration.integrationAuthId);
if (shouldDeleteIntegrationSecrets) {
await deleteIntegrationSecrets({
integration,
integrationAuth,
projectBotService,
integrationAuthService,
secretV2BridgeDAL,
folderDAL,
secretImportDAL,
secretDAL,
kmsService
});
}
const deletedIntegration = await integrationDAL.transaction(async (tx) => {
// delete integration
const deletedIntegrationResult = await integrationDAL.deleteById(id, tx);
@@ -63,6 +63,7 @@ export type TUpdateIntegrationDTO = {
export type TDeleteIntegrationDTO = {
id: string;
shouldDeleteIntegrationSecrets?: boolean;
} & Omit<TProjectPermission, "projectId">;
export type TSyncIntegrationDTO = {
@@ -46,6 +46,7 @@ export const projectBotDALFactory = (db: TDbClient) => {
const doc = await db
.replicaNode()(TableName.ProjectMembership)
.where(`${TableName.ProjectMembership}.projectId` as "projectId", projectId)
.where(`${TableName.ProjectKeys}.projectId` as "projectId", projectId)
.where(`${TableName.Users}.isGhost` as "isGhost", false)
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.join(TableName.ProjectKeys, `${TableName.ProjectMembership}.userId`, `${TableName.ProjectKeys}.receiverId`)
@@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TPkiAlertServiceFactory } from "@app/services/pki-alert/pki-alert-service";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
@@ -13,6 +14,7 @@ import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-d
type TDailyResourceCleanUpQueueServiceFactoryDep = {
auditLogDAL: Pick<TAuditLogDALFactory, "pruneAuditLog">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "removeExpiredTokens">;
identityUniversalAuthClientSecretDAL: Pick<TIdentityUaClientSecretDALFactory, "removeExpiredClientSecrets">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretVersionV2DAL: Pick<TSecretVersionV2DALFactory, "pruneExcessVersions">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
@@ -33,12 +35,14 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
secretFolderVersionDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL
secretVersionV2DAL,
identityUniversalAuthClientSecretDAL
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
await secretSharingDAL.pruneExpiredSharedSecrets();
await snapshotDAL.pruneExcessSnapshots();
await secretVersionDAL.pruneExcessVersions();
@@ -36,8 +36,8 @@ type TSecretImportSecretsV2 = {
secretKey: string;
// akhilmhdh: yes i know you can put ?.
// But for somereason ts consider ? and undefined explicit as different just ts things
secretValue: string | undefined;
secretComment: string | undefined;
secretValue: string;
secretComment: string;
})[];
};
@@ -157,7 +157,7 @@ export const fnSecretsV2FromImports = async ({
secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds">;
depth?: number;
cyclicDetector?: Set<string>;
decryptor: (value?: Buffer | null) => string | undefined;
decryptor: (value?: Buffer | null) => string;
expandSecretReferences?: (
secrets: Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
) => Promise<Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>>;
@@ -231,6 +231,7 @@ export const fnSecretsV2FromImports = async ({
_id: item.id // The old Python SDK depends on the _id field being returned. We return this to keep the older Python SDK versions backwards compatible with the new Postgres backend.
}))
.concat(folderDeeperImportSecrets);
return {
secretPath: importPath,
environment: importEnv.slug,
@@ -254,7 +255,7 @@ export const fnSecretsV2FromImports = async ({
};
return acc;
},
{} as Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
{} as Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
);
// eslint-disable-next-line
await expandSecretReferences(secretsGroupByKey);
@@ -507,7 +507,7 @@ export const secretImportServiceFactory = ({
folderDAL,
secretDAL: secretV2BridgeDAL,
secretImportDAL,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined)
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
});
return importedSecrets;
}
@@ -51,7 +51,7 @@ export const secretTagDALFactory = (db: TDbClient) => {
...secretTagOrm,
saveTagsToSecret: secretJnTagOrm.insertMany,
deleteTagsToSecret: secretJnTagOrm.delete,
saveTagsToSecretV2: secretV2JnTagOrm.insertMany,
saveTagsToSecretV2: secretV2JnTagOrm.batchInsert,
deleteTagsToSecretV2: secretV2JnTagOrm.delete,
findSecretTagsByProjectId,
deleteTagsManySecret,
@@ -22,16 +22,7 @@ type TSecretTagServiceFactoryDep = {
export type TSecretTagServiceFactory = ReturnType<typeof secretTagServiceFactory>;
export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSecretTagServiceFactoryDep) => {
const createTag = async ({
name,
slug,
actor,
color,
actorId,
actorOrgId,
actorAuthMethod,
projectId
}: TCreateTagDTO) => {
const createTag = async ({ slug, actor, color, actorId, actorOrgId, actorAuthMethod, projectId }: TCreateTagDTO) => {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
@@ -46,7 +37,6 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
const newTag = await secretTagDAL.create({
projectId,
name,
slug,
color,
createdBy: actorId,
@@ -55,7 +45,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
return newTag;
};
const updateTag = async ({ actorId, actor, actorOrgId, actorAuthMethod, id, name, color, slug }: TUpdateTagDTO) => {
const updateTag = async ({ actorId, actor, actorOrgId, actorAuthMethod, id, color, slug }: TUpdateTagDTO) => {
const tag = await secretTagDAL.findById(id);
if (!tag) throw new BadRequestError({ message: "Tag doesn't exist" });
@@ -73,7 +63,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Tags);
const updatedTag = await secretTagDAL.updateById(tag.id, { name, color, slug });
const updatedTag = await secretTagDAL.updateById(tag.id, { color, slug });
return updatedTag;
};
@@ -107,7 +97,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Tags);
return tag;
return { ...tag, name: tag.slug };
};
const getTagBySlug = async ({ actorId, actor, actorOrgId, actorAuthMethod, slug, projectId }: TGetTagBySlugDTO) => {
@@ -123,7 +113,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Tags);
return tag;
return { ...tag, name: tag.slug };
};
const getProjectTags = async ({ actor, actorId, actorOrgId, actorAuthMethod, projectId }: TListProjectTagsDTO) => {
@@ -1,14 +1,12 @@
import { TProjectPermission } from "@app/lib/types";
export type TCreateTagDTO = {
name: string;
color: string;
slug: string;
} & TProjectPermission;
export type TUpdateTagDTO = {
id: string;
name?: string;
slug?: string;
color?: string;
} & Omit<TProjectPermission, "projectId">;
@@ -136,7 +136,6 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
.orderBy("id", "asc");
const data = sqlNestRelationships({
@@ -147,11 +146,11 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
@@ -169,14 +168,13 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
.where({ [`${TableName.SecretV2}Id` as const]: secretId })
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"));
return tags.map((el) => ({
id: el.tagId,
color: el.tagColor,
slug: el.tagSlug,
name: el.tagName
name: el.tagSlug
}));
} catch (error) {
throw new DatabaseError({ error, name: "get secret tags" });
@@ -210,7 +208,6 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
.orderBy("id", "asc");
const data = sqlNestRelationships({
@@ -221,11 +218,11 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
@@ -290,7 +287,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}))
);
if (!newSecretReferences.length) return;
const secretReferences = await (tx || db)(TableName.SecretReferenceV2).insert(newSecretReferences);
const secretReferences = await (tx || db).batchInsert(TableName.SecretReferenceV2, newSecretReferences);
return secretReferences;
} catch (error) {
throw new DatabaseError({ error, name: "UpsertSecretReference" });
@@ -350,8 +347,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
.select(selectAllTableCols(TableName.SecretV2))
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"));
const docs = sqlNestRelationships({
data: rawDocs,
key: "id",
@@ -360,11 +356,11 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
@@ -528,8 +528,8 @@ export const reshapeBridgeSecret = (
environment: string,
secretPath: string,
secret: Omit<TSecretsV2, "encryptedValue" | "encryptedComment"> & {
value?: string;
comment?: string;
value: string;
comment: string;
tags?: {
id: string;
slug: string;
@@ -542,8 +542,8 @@ export const reshapeBridgeSecret = (
secretPath,
workspace: workspaceId,
environment,
secretValue: secret.value,
secretComment: secret.comment,
secretValue: secret.value || "",
secretComment: secret.comment || "",
version: secret.version,
type: secret.type,
_id: secret.id,
@@ -196,7 +196,7 @@ export const secretV2BridgeServiceFactory = ({
return reshapeBridgeSecret(projectId, environment, secretPath, {
...secret[0],
value: inputSecret.secretValue,
comment: inputSecret.secretComment
comment: inputSecret.secretComment || ""
});
};
@@ -339,8 +339,8 @@ export const secretV2BridgeServiceFactory = ({
});
return reshapeBridgeSecret(projectId, environment, secretPath, {
...updatedSecret[0],
value: inputSecret.secretValue,
comment: inputSecret.secretComment
value: inputSecret.secretValue || "",
comment: inputSecret.secretComment || ""
});
};
@@ -378,6 +378,18 @@ export const secretV2BridgeServiceFactory = ({
throw new BadRequestError({ message: "Must be user to delete personal secret" });
}
const secretToDelete = await secretDAL.findOne({
key: inputSecret.secretName,
folderId,
...(inputSecret.type === SecretType.Shared
? {}
: {
type: SecretType.Personal,
userId: actorId
})
});
if (!secretToDelete) throw new NotFoundError({ message: "Secret not found" });
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
projectId,
@@ -412,10 +424,10 @@ export const secretV2BridgeServiceFactory = ({
...deletedSecret[0],
value: deletedSecret[0].encryptedValue
? secretManagerDecryptor({ cipherTextBlob: deletedSecret[0].encryptedValue }).toString()
: undefined,
: "",
comment: deletedSecret[0].encryptedComment
? secretManagerDecryptor({ cipherTextBlob: deletedSecret[0].encryptedComment }).toString()
: undefined
: ""
});
};
@@ -429,6 +441,7 @@ export const secretV2BridgeServiceFactory = ({
actorAuthMethod,
includeImports,
recursive,
tagSlugs = [],
expandSecretReferences: shouldExpandSecretReferences
}: TGetSecretsDTO) => {
const { permission } = await permissionService.getProjectPermission(
@@ -496,6 +509,9 @@ export const secretV2BridgeServiceFactory = ({
: ""
})
);
const filteredSecrets = tagSlugs.length
? decryptedSecrets.filter((secret) => Boolean(secret.tags?.find((el) => tagSlugs.includes(el.slug))))
: decryptedSecrets;
const expandSecretReferences = expandSecretReferencesFactory({
projectId,
folderDAL,
@@ -504,7 +520,7 @@ export const secretV2BridgeServiceFactory = ({
});
if (shouldExpandSecretReferences) {
const secretsGroupByPath = groupBy(decryptedSecrets, (i) => i.secretPath);
const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
for (const secretPathKey in secretsGroupByPath) {
if (Object.hasOwn(secretsGroupByPath, secretPathKey)) {
const secretsGroupByKey = secretsGroupByPath[secretPathKey].reduce(
@@ -522,7 +538,7 @@ export const secretV2BridgeServiceFactory = ({
await expandSecretReferences(secretsGroupByKey);
secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value;
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value || "";
});
}
}
@@ -530,7 +546,7 @@ export const secretV2BridgeServiceFactory = ({
if (!includeImports) {
return {
secrets: decryptedSecrets
secrets: filteredSecrets
};
}
@@ -554,11 +570,11 @@ export const secretV2BridgeServiceFactory = ({
folderDAL,
secretImportDAL,
expandSecretReferences,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined)
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
});
return {
secrets: decryptedSecrets,
secrets: filteredSecrets,
imports: importedSecrets
};
};
@@ -654,7 +670,7 @@ export const secretV2BridgeServiceFactory = ({
secretDAL,
folderDAL,
secretImportDAL,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined),
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : ""),
expandSecretReferences: shouldExpandSecretReferences ? expandSecretReferences : undefined
});
@@ -662,12 +678,11 @@ export const secretV2BridgeServiceFactory = ({
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
const importedSecret = importedSecrets[i].secrets[j];
if (secretName === importedSecret.key) {
return reshapeBridgeSecret(
projectId,
importedSecrets[i].environment,
importedSecrets[i].secretPath,
importedSecret
);
return reshapeBridgeSecret(projectId, importedSecrets[i].environment, importedSecrets[i].secretPath, {
...importedSecret,
value: importedSecret.secretValue || "",
comment: importedSecret.secretComment || ""
});
}
}
}
@@ -676,7 +691,7 @@ export const secretV2BridgeServiceFactory = ({
let secretValue = secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString()
: undefined;
: "";
if (shouldExpandSecretReferences && secretValue) {
const secretReferenceExpandedRecord = {
[secret.key]: { value: secretValue }
@@ -691,7 +706,7 @@ export const secretV2BridgeServiceFactory = ({
value: secretValue,
comment: secret.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: secret.encryptedComment }).toString()
: undefined
: ""
});
};
@@ -781,10 +796,8 @@ export const secretV2BridgeServiceFactory = ({
return newSecrets.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined,
comment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : ""
})
);
};
@@ -902,10 +915,8 @@ export const secretV2BridgeServiceFactory = ({
return secrets.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined,
comment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : ""
})
);
};
@@ -981,10 +992,8 @@ export const secretV2BridgeServiceFactory = ({
return secretsDeleted.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined,
comment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : ""
})
);
};
@@ -1020,10 +1029,8 @@ export const secretV2BridgeServiceFactory = ({
return secretVersions.map((el) =>
reshapeBridgeSecret(folder.projectId, folder.environment.envSlug, "/", {
...el,
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined,
comment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: undefined
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : ""
})
);
};
@@ -20,6 +20,7 @@ export type TGetSecretsDTO = {
environment: string;
includeImports?: boolean;
recursive?: boolean;
tagSlugs?: string[];
} & TProjectPermission;
export type TGetASecretDTO = {
+9 -13
View File
@@ -123,7 +123,6 @@ export const secretDALFactory = (db: TDbClient) => {
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
.orderBy("id", "asc");
const data = sqlNestRelationships({
data: secs,
@@ -133,11 +132,11 @@ export const secretDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
@@ -155,14 +154,13 @@ export const secretDALFactory = (db: TDbClient) => {
.where({ [`${TableName.Secret}Id` as const]: secretId })
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"));
return tags.map((el) => ({
id: el.tagId,
color: el.tagColor,
slug: el.tagSlug,
name: el.tagName
name: el.tagSlug
}));
} catch (error) {
throw new DatabaseError({ error, name: "get secret tags" });
@@ -188,7 +186,6 @@ export const secretDALFactory = (db: TDbClient) => {
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
.orderBy("id", "asc");
const data = sqlNestRelationships({
data: secs,
@@ -198,11 +195,11 @@ export const secretDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
@@ -318,8 +315,7 @@ export const secretDALFactory = (db: TDbClient) => {
.select(selectAllTableCols(TableName.Secret))
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
.select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
.select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
.select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"));
const docs = sqlNestRelationships({
data: rawDocs,
key: "id",
@@ -328,11 +324,11 @@ export const secretDALFactory = (db: TDbClient) => {
{
key: "tagId",
label: "tags" as const,
mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({
id,
color,
slug,
name
name: slug
})
}
]
+1 -2
View File
@@ -370,7 +370,6 @@ export const decryptSecretRaw = (
id: string;
slug: string;
color?: string | null;
name: string;
}[];
},
key: string
@@ -412,7 +411,7 @@ export const decryptSecretRaw = (
_id: secret.id,
id: secret.id,
user: secret.userId,
tags: secret.tags,
tags: secret.tags?.map((el) => ({ ...el, name: el.slug })),
skipMultilineEncoding: secret.skipMultilineEncoding,
secretReminderRepeatDays: secret.secretReminderRepeatDays,
secretReminderNote: secret.secretReminderNote,
+11 -8
View File
@@ -73,12 +73,12 @@ type TSecretQueueFactoryDep = {
secretVersionTagDAL: TSecretVersionTagDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
secretRotationDAL: Pick<TSecretRotationDALFactory, "secretOutputV2InsertMany" | "find">;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "deleteByProjectId">;
snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">;
snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany">;
snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany" | "batchInsert">;
};
export type TGetSecrets = {
@@ -728,7 +728,10 @@ export const secretQueueFactory = ({
isSynced: response?.isSynced ?? true
});
} catch (err) {
logger.info("Secret integration sync error: %o", err);
logger.error(
err,
`Secret integration sync error [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}]`
);
const message =
(err instanceof AxiosError ? JSON.stringify(err?.response?.data) : (err as Error)?.message) ||
@@ -828,7 +831,7 @@ export const secretQueueFactory = ({
secretId: string;
references: { environment: string; secretPath: string; secretKey: string }[];
}[] = [];
await secretV2BridgeDAL.insertMany(
await secretV2BridgeDAL.batchInsert(
projectV1Secrets.map((el) => {
const key = decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretKeyCiphertext,
@@ -1004,14 +1007,14 @@ export const secretQueueFactory = ({
const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById);
if (projectV3SecretVersions.length) {
await secretVersionV2BridgeDAL.insertMany(projectV3SecretVersions, tx);
await secretVersionV2BridgeDAL.batchInsert(projectV3SecretVersions, tx);
}
if (projectV3SecretVersionTags.length) {
await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx);
await secretVersionTagV2BridgeDAL.batchInsert(projectV3SecretVersionTags, tx);
}
if (projectV3SnapshotSecrets.length) {
await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx);
await snapshotSecretV2BridgeDAL.batchInsert(projectV3SnapshotSecrets, tx);
}
await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx);
}
+11 -5
View File
@@ -964,7 +964,8 @@ export const secretServiceFactory = ({
environment,
includeImports,
expandSecretReferences,
recursive
recursive,
tagSlugs = []
}: TGetSecretsRawDTO) => {
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (shouldUseSecretV2Bridge) {
@@ -978,7 +979,8 @@ export const secretServiceFactory = ({
path,
recursive,
actorAuthMethod,
includeImports
includeImports,
tagSlugs
});
return { secrets, imports };
}
@@ -998,6 +1000,9 @@ export const secretServiceFactory = ({
});
const decryptedSecrets = secrets.map((el) => decryptSecretRaw(el, botKey));
const filteredSecrets = tagSlugs.length
? decryptedSecrets.filter((secret) => Boolean(secret.tags?.find((el) => tagSlugs.includes(el.slug))))
: decryptedSecrets;
const processedImports = (imports || [])?.map(({ secrets: importedSecrets, ...el }) => {
const decryptedImportSecrets = importedSecrets.map((sec) =>
decryptSecretRaw(
@@ -1106,14 +1111,14 @@ export const secretServiceFactory = ({
};
// expand secrets
await batchSecretsExpand(decryptedSecrets);
await batchSecretsExpand(filteredSecrets);
// expand imports by batch
await Promise.all(processedImports.map((processedImport) => batchSecretsExpand(processedImport.secrets)));
}
return {
secrets: decryptedSecrets,
secrets: filteredSecrets,
imports: processedImports
};
};
@@ -1149,6 +1154,7 @@ export const secretServiceFactory = ({
type,
secretName
});
return secret;
}
@@ -2081,7 +2087,7 @@ export const secretServiceFactory = ({
return {
...updatedSecret[0],
tags: [...existingSecretTags, ...tags].map((t) => ({ id: t.id, slug: t.slug, name: t.name, color: t.color }))
tags: [...existingSecretTags, ...tags].map((t) => ({ id: t.id, slug: t.slug, name: t.slug, color: t.color }))
};
};
@@ -149,6 +149,7 @@ export type TGetSecretsRawDTO = {
environment: string;
includeImports?: boolean;
recursive?: boolean;
tagSlugs?: string[];
} & TProjectPermission;
export type TGetASecretRawDTO = {