awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into k8s-auth

Browse Source
This commit is contained in:
Tuan Dang
2024-05-17 15:39:52 -07:00
parent ace008f44e 18aaa423a9
commit d79fd826a4
35 changed files with 790 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build-staging-and-deploy-aws.yml vendored
View File

@@ -122,13 +122,13 @@ jobs:
uses: pr-mpt/actions-commit-hash@v2
- name: Download task definition
run: |
aws ecs describe-task-definition --task-definition infisical-prod-platform --query taskDefinition > task-definition.json
aws ecs describe-task-definition --task-definition infisical-core-platform --query taskDefinition > task-definition.json
- name: Render Amazon ECS task definition
id: render-web-container
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: task-definition.json
container-name: infisical-prod-platform
container-name: infisical-core-platform
image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
environment-variables: "LOG_LEVEL=info"
- name: Deploy to Amazon ECS service

6
backend/src/@types/knex.d.ts vendored
View File

@@ -234,6 +234,7 @@ import {
TWebhooksInsert,
TWebhooksUpdate
} from "@app/db/schemas";
import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
declare module "knex/types/tables" {
interface Tables {
@@ -307,6 +308,11 @@ declare module "knex/types/tables" {
>;
[TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
[TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
[TableName.SecretReference]: Knex.CompositeTableType<
TSecretReferences,
TSecretReferencesInsert,
TSecretReferencesUpdate
>;
[TableName.SecretBlindIndex]: Knex.CompositeTableType<
TSecretBlindIndexes,
TSecretBlindIndexesInsert,

24
backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts Normal file
View File

@@ -0,0 +1,24 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasTable(TableName.SecretReference))) {
await knex.schema.createTable(TableName.SecretReference, (t) => {
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
t.string("environment").notNullable();
t.string("secretPath").notNullable();
t.uuid("secretId").notNullable();
t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE");
t.timestamps(true, true, true);
});
await createOnUpdateTrigger(knex, TableName.SecretReference);
}
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.dropTableIfExists(TableName.SecretReference);
await dropOnUpdateTrigger(knex, TableName.SecretReference);
}

1
backend/src/db/schemas/models.ts
View File

@@ -28,6 +28,7 @@ export enum TableName {
ProjectUserMembershipRole = "project_user_membership_roles",
ProjectKeys = "project_keys",
Secret = "secrets",
SecretReference = "secret_references",
SecretBlindIndex = "secret_blind_indexes",
SecretVersion = "secret_versions",
SecretFolder = "secret_folders",

21
backend/src/db/schemas/secret-references.ts Normal file
View File

@@ -0,0 +1,21 @@
// Code generated by automation script, DO NOT EDIT.
// Automated by pulling database and generating zod schema
// To update. Just run npm run generate:schema
// Written by akhilmhdh.
import { z } from "zod";
import { TImmutableDBKeys } from "./models";
export const SecretReferencesSchema = z.object({
id: z.string().uuid(),
environment: z.string(),
secretPath: z.string(),
secretId: z.string().uuid(),
createdAt: z.date(),
updatedAt: z.date()
});
export type TSecretReferences = z.infer<typeof SecretReferencesSchema>;
export type TSecretReferencesInsert = Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>;
export type TSecretReferencesUpdate = Partial<Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>>;

8
backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts
View File

@@ -8,7 +8,7 @@ import { IDENTITY_ADDITIONAL_PRIVILEGE } from "@app/lib/api-docs";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { PermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas";
import { ProjectPermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas";
import { AuthMode } from "@app/services/auth/auth-type";
export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: FastifyZodProvider) => {
@@ -39,7 +39,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
})
.optional()
.describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
}),
response: {
200: z.object({
@@ -90,7 +90,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
})
.optional()
.describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions),
permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions),
temporaryMode: z
.nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)
.describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.temporaryMode),
@@ -155,7 +155,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
message: "Slug must be a valid slug"
})
.describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.newSlug),
permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions),
permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions),
isTemporary: z.boolean().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.isTemporary),
temporaryMode: z
.nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)

33
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
View File

@@ -7,12 +7,15 @@ import {
SecretType,
TSecretApprovalRequestsSecretsInsert
} from "@app/db/schemas";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
import { groupBy, pick, unique } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { ActorType } from "@app/services/auth/auth-type";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -53,6 +56,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretService: Pick<
TSecretServiceFactory,
| "fnSecretBulkInsert"
@@ -80,7 +84,8 @@ export const secretApprovalRequestServiceFactory = ({
snapshotService,
secretService,
secretVersionDAL,
secretQueueService
secretQueueService,
projectBotService
}: TSecretApprovalRequestServiceFactoryDep) => {
const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => {
if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
@@ -352,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await secretService.fnSecretBulkInsert({
@@ -379,7 +384,17 @@ export const secretApprovalRequestServiceFactory = ({
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared
type: SecretType.Shared,
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
})),
secretDAL,
secretVersionDAL,
@@ -414,7 +429,17 @@ export const secretApprovalRequestServiceFactory = ({
"secretReminderNote",
"secretReminderRepeatDays",
"secretBlindIndex"
])
]),
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
}
})),
secretDAL,

3
backend/src/lib/api-docs/constants.ts
View File

@@ -89,6 +89,9 @@ export const UNIVERSAL_AUTH = {
},
RENEW_ACCESS_TOKEN: {
accessToken: "The access token to renew."
},
REVOKE_ACCESS_TOKEN: {
accessToken: "The access token to revoke."
}
} as const;

8
backend/src/queue/queue-service.ts
View File

@@ -65,7 +65,13 @@ export type TQueueJobTypes = {
};
[QueueName.IntegrationSync]: {
name: QueueJobs.IntegrationSync;
payload: { projectId: string; environment: string; secretPath: string; depth?: number };
payload: {
projectId: string;
environment: string;
secretPath: string;
depth?: number;
deDupeQueue?: Record<string, boolean>;
};
};
[QueueName.SecretFullRepoScan]: {
name: QueueJobs.SecretScan;

1
backend/src/server/routes/index.ts
View File

@@ -610,6 +610,7 @@ export const registerRoutes = async (
});
const sarService = secretApprovalRequestServiceFactory({
permissionService,
projectBotService,
folderDAL,
secretDAL,
secretTagDAL,

9
backend/src/server/routes/sanitizedSchemas.ts
View File

@@ -8,6 +8,7 @@ import {
UsersSchema
} from "@app/db/schemas";
import { UnpackedPermissionSchema } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
// sometimes the return data must be santizied to avoid leaking important values
// always prefer pick over omit in zod
@@ -64,14 +65,12 @@ export const secretRawSchema = z.object({
secretComment: z.string().optional()
});
export const PermissionSchema = z.object({
export const ProjectPermissionSchema = z.object({
action: z
.string()
.min(1)
.nativeEnum(ProjectPermissionActions)
.describe("Describe what action an entity can take. Possible actions: create, edit, delete, and read"),
subject: z
.string()
.min(1)
.nativeEnum(ProjectPermissionSub)
.describe("The entity this permission pertains to. Possible options: secrets, environments"),
conditions: z
.object({

25
backend/src/server/routes/v1/identity-access-token-router.ts
View File

@@ -36,4 +36,29 @@ export const registerIdentityAccessTokenRouter = async (server: FastifyZodProvid
};
}
});
server.route({
url: "/token/revoke",
method: "POST",
config: {
rateLimit: writeLimit
},
schema: {
description: "Revoke access token",
body: z.object({
accessToken: z.string().trim().describe(UNIVERSAL_AUTH.REVOKE_ACCESS_TOKEN.accessToken)
}),
response: {
200: z.object({
message: z.string()
})
}
},
handler: async (req) => {
await server.services.identityAccessToken.revokeAccessToken(req.body.accessToken);
return {
message: "Successfully revoked access token"
};
}
});
};

37
backend/src/server/routes/v3/secret-router.ts
View File

@@ -1926,4 +1926,41 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
return { secrets };
}
});
server.route({
method: "POST",
url: "/backfill-secret-references",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Backfill secret references",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectId: z.string().trim().min(1)
}),
response: {
200: z.object({
message: z.string()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const { projectId } = req.body;
const message = await server.services.secret.backfillSecretReferences({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
projectId
});
return message;
}
});
};

49
backend/src/services/identity-access-token/identity-access-token-dal.ts
View File

@@ -1,7 +1,7 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName, TIdentityAccessTokens } from "@app/db/schemas";
import { IdentityAuthMethod, TableName, TIdentityAccessTokens } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
@@ -15,23 +15,46 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
const doc = await (tx || db)(TableName.IdentityAccessToken)
.where(filter)
.join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`)
.leftJoin(
TableName.IdentityUaClientSecret,
`${TableName.IdentityAccessToken}.identityUAClientSecretId`,
`${TableName.IdentityUaClientSecret}.id`
)
.leftJoin(
TableName.IdentityUniversalAuth,
`${TableName.IdentityUaClientSecret}.identityUAId`,
`${TableName.IdentityUniversalAuth}.id`
)
.leftJoin(TableName.IdentityUaClientSecret, (qb) => {
qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn(
`${TableName.IdentityAccessToken}.identityUAClientSecretId`,
`${TableName.IdentityUaClientSecret}.id`
);
})
.leftJoin(TableName.IdentityUniversalAuth, (qb) => {
qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn(
`${TableName.IdentityUaClientSecret}.identityUAId`,
`${TableName.IdentityUniversalAuth}.id`
);
})
.leftJoin(TableName.IdentityGcpAuth, (qb) => {
qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.GCP_AUTH])).andOn(
`${TableName.Identity}.id`,
`${TableName.IdentityGcpAuth}.identityId`
);
})
.leftJoin(TableName.IdentityAwsAuth, (qb) => {
qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.AWS_AUTH])).andOn(
`${TableName.Identity}.id`,
`${TableName.IdentityAwsAuth}.identityId`
);
})
.select(selectAllTableCols(TableName.IdentityAccessToken))
.select(
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth),
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth).as("accessTokenTrustedIpsUa"),
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityGcpAuth).as("accessTokenTrustedIpsGcp"),
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityAwsAuth).as("accessTokenTrustedIpsAws"),
db.ref("name").withSchema(TableName.Identity)
)
.first();
return doc;
if (!doc) return;
return {
...doc,
accessTokenTrustedIps:
doc.accessTokenTrustedIpsUa || doc.accessTokenTrustedIpsGcp || doc.accessTokenTrustedIpsAws
};
} catch (error) {
throw new DatabaseError({ error, name: "IdAccessTokenFindOne" });
}

20
backend/src/services/identity-access-token/identity-access-token-service.ts
View File

@@ -106,6 +106,24 @@ export const identityAccessTokenServiceFactory = ({
return { accessToken, identityAccessToken: updatedIdentityAccessToken };
};
const revokeAccessToken = async (accessToken: string) => {
const appCfg = getConfig();
const decodedToken = jwt.verify(accessToken, appCfg.AUTH_SECRET) as JwtPayload & {
identityAccessTokenId: string;
};
if (decodedToken.authTokenType !== AuthTokenType.IDENTITY_ACCESS_TOKEN) throw new UnauthorizedError();
const identityAccessToken = await identityAccessTokenDAL.findOne({
[`${TableName.IdentityAccessToken}.id` as "id"]: decodedToken.identityAccessTokenId,
isAccessTokenRevoked: false
});
if (!identityAccessToken) throw new UnauthorizedError();
const revokedToken = await identityAccessTokenDAL.deleteById(identityAccessToken.id);
return { revokedToken };
};
const fnValidateIdentityAccessToken = async (token: TIdentityAccessTokenJwtPayload, ipAddress?: string) => {
const identityAccessToken = await identityAccessTokenDAL.findOne({
[`${TableName.IdentityAccessToken}.id` as "id"]: token.identityAccessTokenId,
@@ -132,5 +150,5 @@ export const identityAccessTokenServiceFactory = ({
return { ...identityAccessToken, orgId: identityOrgMembership.orgId };
};
return { renewAccessToken, fnValidateIdentityAccessToken };
return { renewAccessToken, revokeAccessToken, fnValidateIdentityAccessToken };
};

23
backend/src/services/identity-project/identity-project-service.ts
View File

@@ -82,6 +82,7 @@ export const identityProjectServiceFactory = ({
role,
project.id
);
const hasPriviledge = isAtLeastAsPrivileged(permission, rolePermission);
if (!hasPriviledge)
throw new ForbiddenRequestError({
@@ -135,16 +136,18 @@ export const identityProjectServiceFactory = ({
message: `Identity with id ${identityId} doesn't exists in project with id ${projectId}`
});
const { permission: identityRolePermission } = await permissionService.getProjectPermission(
ActorType.IDENTITY,
projectIdentity.identityId,
projectIdentity.projectId,
actorAuthMethod,
actorOrgId
);
const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, identityRolePermission);
if (!hasRequiredPriviledges)
throw new ForbiddenRequestError({ message: "Failed to delete more privileged identity" });
for await (const { role: requestedRoleChange } of roles) {
const { permission: rolePermission } = await permissionService.getProjectPermissionByRole(
requestedRoleChange,
projectId
);
const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, rolePermission);
if (!hasRequiredPriviledges) {
throw new ForbiddenRequestError({ message: "Failed to change to a more privileged role" });
}
}
// validate custom roles input
const customInputRoles = roles.filter(

66
backend/src/services/integration-auth/integration-sync-secret.ts
View File

@@ -462,27 +462,39 @@ const syncSecretsAWSParameterStore = async ({
ssm.config.update(config);
const metadata = z.record(z.any()).parse(integration.metadata || {});
const awsParameterStoreSecretsObj: Record<string, AWS.SSM.Parameter> = {};
const params = {
Path: integration.path as string,
Recursive: false,
WithDecryption: true
};
// now fetch all aws parameter store secrets
let hasNext = true;
let nextToken: string | undefined;
while (hasNext) {
const parameters = await ssm
.getParametersByPath({
Path: integration.path as string,
Recursive: false,
WithDecryption: true,
MaxResults: 10,
NextToken: nextToken
})
.promise();
const parameterList = (await ssm.getParametersByPath(params).promise()).Parameters;
if (parameters.Parameters) {
parameters.Parameters.forEach((parameter) => {
if (parameter.Name) {
const secKey = parameter.Name.substring((integration.path as string).length);
awsParameterStoreSecretsObj[secKey] = parameter;
}
});
}
hasNext = Boolean(parameters.NextToken);
nextToken = parameters.NextToken;
}
const awsParameterStoreSecretsObj = (parameterList || [])
.filter(({ Name }) => Boolean(Name))
.reduce(
(obj, secret) => ({
...obj,
[(secret.Name as string).substring((integration.path as string).length)]: secret
}),
{} as Record<string, AWS.SSM.Parameter>
);
// Identify secrets to create
await Promise.all(
Object.keys(secrets).map(async (key) => {
// don't use Promise.all() and promise map here
// it will cause rate limit
for (const key in secrets) {
if (Object.hasOwn(secrets, key)) {
if (!(key in awsParameterStoreSecretsObj)) {
// case: secret does not exist in AWS parameter store
// -> create secret
@@ -517,13 +529,16 @@ const syncSecretsAWSParameterStore = async ({
})
.promise();
}
})
);
await new Promise((resolve) => {
setTimeout(resolve, 50);
});
}
}
if (!metadata.shouldDisableDelete) {
// Identify secrets to delete
await Promise.all(
Object.keys(awsParameterStoreSecretsObj).map(async (key) => {
for (const key in awsParameterStoreSecretsObj) {
if (Object.hasOwn(awsParameterStoreSecretsObj, key)) {
if (!(key in secrets)) {
// case:
// -> delete secret
@@ -533,8 +548,11 @@ const syncSecretsAWSParameterStore = async ({
})
.promise();
}
})
);
await new Promise((resolve) => {
setTimeout(resolve, 50);
});
}
}
}
};

73
backend/src/services/secret/secret-dal.ts
View File

@@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const upsertSecretReferences = async (
data: {
secretId: string;
references: Array<{ environment: string; secretPath: string }>;
}[] = [],
tx?: Knex
) => {
try {
if (!data.length) return;
await (tx || db)(TableName.SecretReference)
.whereIn(
"secretId",
data.map(({ secretId }) => secretId)
)
.delete();
const newSecretReferences = data
.filter(({ references }) => references.length)
.flatMap(({ secretId, references }) =>
references.map(({ environment, secretPath }) => ({
secretPath,
secretId,
environment
}))
);
if (!newSecretReferences.length) return;
const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences);
return secretReferences;
} catch (error) {
throw new DatabaseError({ error, name: "UpsertSecretReference" });
}
};
const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretReference)
.where({
secretPath,
environment: envSlug
})
.join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`)
.join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where("projectId", projectId)
.select(selectAllTableCols(TableName.SecretReference))
.select("folderId");
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "FindReferencedSecretReferences" });
}
};
// special query to backfill secret value
const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Secret)
.join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where("projectId", projectId)
// not empty
.whereNotNull("secretValueCiphertext")
.select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id");
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "FindAllProjectSecretValues" });
}
};
return {
...secretOrm,
update,
@@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => {
getSecretTags,
findByFolderId,
findByFolderIds,
findByBlindIndexes
findByBlindIndexes,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
};
};

199
backend/src/services/secret/secret-fns.ts
View File

@@ -194,6 +194,7 @@ type TInterpolateSecretArg = {
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
};
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
const fetchSecretsCrossEnv = () => {
const fetchCache: Record<string, Record<string, string>> = {};
@@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
};
};
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
const recursivelyExpandSecret = async (
expandedSec: Record<string, string>,
interpolatedSec: Record<string, string>,
@@ -353,7 +353,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
};
export const decryptSecretRaw = (
secret: TSecrets & { workspace: string; environment: string; secretPath?: string },
secret: TSecrets & { workspace: string; environment: string; secretPath: string },
key: string
) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
@@ -396,6 +396,37 @@ export const decryptSecretRaw = (
};
};
/**
* Grabs and processes nested secret references from a string
*
* This function looks for patterns that match the interpolation syntax in the input string.
* It filters out references that include nested paths, splits them into environment and
* secret path parts, and then returns an array of objects with the environment and the
* joined secret path.
*
* @param {string} maybeSecretReference - The string that has the potential secret references.
* @returns {Array<{ environment: string, secretPath: string }>} - An array of objects
* with the environment and joined secret path.
*
* @example
* const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}";
* const result = getAllNestedSecretReferences(value);
* // result will be:
* // [
* // { environment: 'dev', secretPath: '/someFolder/OtherFolder' },
* // { environment: 'prod', secretPath: '/anotherFolder' }
* // ]
*/
export const getAllNestedSecretReferences = (maybeSecretReference: string) => {
const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]);
return references
.filter((el) => el.includes("."))
.map((el) => {
const [environment, ...secretPathList] = el.split(".");
return { environment, secretPath: path.join("/", ...secretPathList.slice(0, -1)) };
});
};
/**
* Checks and handles secrets using a blind index method.
* The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
@@ -467,7 +498,7 @@ export const fnSecretBulkInsert = async ({
tx
}: TFnSecretBulkInsert) => {
const newSecrets = await secretDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })),
tx
);
const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
@@ -478,13 +509,20 @@ export const fnSecretBulkInsert = async ({
}))
);
const secretVersions = await secretVersionDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({
inputSecrets.map(({ tags, references, ...el }) => ({
...el,
folderId,
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
})),
tx
);
await secretDAL.upsertSecretReferences(
inputSecrets.map(({ references = [], secretBlindIndex }) => ({
secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id,
references
})),
tx
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
@@ -509,7 +547,7 @@ export const fnSecretBulkUpdate = async ({
secretVersionTagDAL
}: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
@@ -522,6 +560,15 @@ export const fnSecretBulkUpdate = async ({
})),
tx
);
await secretDAL.upsertSecretReferences(
inputSecrets
.filter(({ data: { references } }) => Boolean(references))
.map(({ data: { references = [] } }, i) => ({
secretId: newSecrets[i].id,
references
})),
tx
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
@@ -591,50 +638,39 @@ export const createManySecretsRawFnFactory = ({
folderId,
isNew: true,
blindIndexCfg,
userId,
secretDAL
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
const inputSecrets = secrets.map((secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: secretReferences
};
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
// get all tags
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
@@ -703,56 +739,35 @@ export const updateManySecretsRawFnFactory = ({
userId
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const inputSecrets = secrets.map((secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to update personal secret override for no corresponding shared secret"
});
if (secret.newSecretName)
throw new BadRequestError({ message: "Personal secret cannot change the key name" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: secretReferences
};
});
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];

91
backend/src/services/secret/secret-queue.ts
View File

@@ -59,6 +59,7 @@ export type TGetSecrets = {
};
const MAX_SYNC_SECRET_DEPTH = 5;
const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
export const secretQueueFactory = ({
queueService,
@@ -102,28 +103,35 @@ export const secretQueueFactory = ({
folderDAL
});
const syncIntegrations = async (dto: TGetSecrets) => {
const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
attempts: 5,
attempts: 3,
delay: 1000,
backoff: {
type: "exponential",
delay: 3000
},
removeOnComplete: true,
removeOnFail: {
count: 5 // keep the most recent jobs
}
removeOnFail: true
});
};
const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => {
const syncSecrets = async ({
deDupeQueue = {},
...dto
}: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
if (deDupeQueue?.[deDuplicationKey]) {
return;
}
// eslint-disable-next-line
deDupeQueue[deDuplicationKey] = true;
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]`
);
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
removeOnFail: { count: 5 },
removeOnFail: true,
removeOnComplete: true,
delay: 1000,
attempts: 5,
@@ -132,7 +140,7 @@ export const secretQueueFactory = ({
delay: 3000
}
});
await syncIntegrations(dto);
await syncIntegrations({ ...dto, deDupeQueue });
};
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -326,7 +334,7 @@ export const secretQueueFactory = ({
};
queueService.start(QueueName.IntegrationSync, async (job) => {
const { environment, projectId, secretPath, depth = 1 } = job.data;
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) {
@@ -349,21 +357,68 @@ export const secretQueueFactory = ({
const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
logger.info(
`getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
await Promise.all(
imports
.filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
.map(({ folderId }) => {
const syncDto = {
// filter out already synced ones
.filter(
({ folderId }) =>
!deDupeQueue[
uniqueIntegrationKey(
foldersGroupedById[folderId][0].environmentSlug,
foldersGroupedById[folderId][0].path
)
]
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
projectId,
secretPath: foldersGroupedById[folderId][0].path,
environment: foldersGroupedById[folderId][0].environmentSlug
};
logger.info(
`getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
return syncSecrets(syncDto);
})
environment: foldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
})
)
);
}
const secretReferences = await secretDAL.findReferencedSecretReferences(
projectId,
folder.environment.slug,
secretPath
);
if (secretReferences.length) {
const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
logger.info(
`getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
await Promise.all(
secretReferences
.filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
// filter out already synced ones
.filter(
({ folderId }) =>
!deDupeQueue[
uniqueIntegrationKey(
referencedFoldersGroupedById[folderId][0].environmentSlug,
referencedFoldersGroupedById[folderId][0].path
)
]
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
projectId,
secretPath: referencedFoldersGroupedById[folderId][0].path,
environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
})
)
);
}
} else {

191
backend/src/services/secret/secret-service.ts
View File

@@ -2,12 +2,22 @@
/* eslint-disable no-await-in-loop */
import { ForbiddenError, subject } from "@casl/ability";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
import {
ProjectMembershipRole,
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretsSchema,
SecretType
} from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import {
buildSecretBlindIndexFromName,
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, pick } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@@ -27,12 +37,14 @@ import {
fnSecretBlindIndexCheck,
fnSecretBulkInsert,
fnSecretBulkUpdate,
getAllNestedSecretReferences,
interpolateSecrets,
recursivelyGetSecretPaths
} from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
TAttachSecretTagsDTO,
TBackFillSecretReferencesDTO,
TCreateBulkSecretDTO,
TCreateManySecretRawDTO,
TCreateSecretDTO,
@@ -91,6 +103,22 @@ export const secretServiceFactory = ({
secretImportDAL,
secretVersionTagDAL
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
return (el: { ciphertext?: string; iv: string; tag: string }) =>
botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.ciphertext || "",
iv: el.iv,
tag: el.tag,
key: botKey
})
)
: undefined;
};
// utility function to get secret blind index data
const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
const appCfg = getConfig();
@@ -225,6 +253,7 @@ export const secretServiceFactory = ({
if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const { secretName, type, ...el } = inputSecret;
const references = await getSecretReference(projectId);
const secret = await secretDAL.transaction((tx) =>
fnSecretBulkInsert({
folderId,
@@ -237,7 +266,12 @@ export const secretServiceFactory = ({
userId: inputSecret.type === SecretType.Personal ? actorId : null,
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8,
tags: inputSecret.tags
tags: inputSecret.tags,
references: references({
ciphertext: inputSecret.secretValueCiphertext,
iv: inputSecret.secretValueIV,
tag: inputSecret.secretValueTag
})
}
],
secretDAL,
@@ -251,7 +285,7 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
return { ...secret[0], environment, workspace: projectId, tags };
return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
};
const updateSecret = async ({
@@ -335,6 +369,7 @@ export const secretServiceFactory = ({
const { secretName, ...el } = inputSecret;
const references = await getSecretReference(projectId);
const updatedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
@@ -360,7 +395,12 @@ export const secretServiceFactory = ({
"secretReminderRepeatDays",
"tags"
]),
secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName]
secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName],
references: references({
ciphertext: inputSecret.secretValueCiphertext,
iv: inputSecret.secretValueIV,
tag: inputSecret.secretValueTag
})
}
}
],
@@ -375,7 +415,7 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
return { ...updatedSecret[0], workspace: projectId, environment };
return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
};
const deleteSecret = async ({
@@ -444,7 +484,7 @@ export const secretServiceFactory = ({
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment };
return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
};
const getSecrets = async ({
@@ -641,7 +681,8 @@ export const secretServiceFactory = ({
return {
...importedSecrets[i].secrets[j],
workspace: projectId,
environment: importedSecrets[i].environment
environment: importedSecrets[i].environment,
secretPath: importedSecrets[i].secretPath
};
}
}
@@ -649,7 +690,7 @@ export const secretServiceFactory = ({
}
if (!secret) throw new BadRequestError({ message: "Secret not found" });
return { ...secret, workspace: projectId, environment };
return { ...secret, workspace: projectId, environment, secretPath: path };
};
const createManySecret = async ({
@@ -700,6 +741,7 @@ export const secretServiceFactory = ({
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
const references = await getSecretReference(projectId);
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -708,7 +750,12 @@ export const secretServiceFactory = ({
secretBlindIndex: keyName2BlindIndex[secretName],
type: SecretType.Shared,
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
keyEncoding: SecretKeyEncoding.UTF8,
references: references({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag
})
})),
folderId,
secretDAL,
@@ -783,6 +830,8 @@ export const secretServiceFactory = ({
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const references = await getSecretReference(projectId);
const secrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
@@ -799,7 +848,15 @@ export const secretServiceFactory = ({
? newKeyName2BlindIndex[newSecretName]
: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
keyEncoding: SecretKeyEncoding.UTF8,
references:
el.secretValueIV && el.secretValueTag
? references({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag
})
: undefined
}
})),
secretDAL,
@@ -924,34 +981,40 @@ export const secretServiceFactory = ({
});
const batchSecretsExpand = async (
secretBatch: {
secretKey: string;
secretValue: string;
secretComment?: string;
}[]
secretBatch: { secretKey: string; secretValue: string; secretComment?: string; secretPath: string }[]
) => {
const secretRecord: Record<
string,
{
value: string;
comment?: string;
skipMultilineEncoding?: boolean;
// Group secrets by secretPath
const secretsByPath: Record<string, { secretKey: string; secretValue: string; secretComment?: string }[]> = {};
secretBatch.forEach((secret) => {
if (!secretsByPath[secret.secretPath]) {
secretsByPath[secret.secretPath] = [];
}
> = {};
secretBatch.forEach((decryptedSecret) => {
secretRecord[decryptedSecret.secretKey] = {
value: decryptedSecret.secretValue,
comment: decryptedSecret.secretComment
};
secretsByPath[secret.secretPath].push(secret);
});
await expandSecrets(secretRecord);
// Expand secrets for each group
for (const secPath in secretsByPath) {
if (!Object.hasOwn(secretsByPath, path)) {
// eslint-disable-next-line no-continue
continue;
}
secretBatch.forEach((decryptedSecret, index) => {
// eslint-disable-next-line no-param-reassign
secretBatch[index].secretValue = secretRecord[decryptedSecret.secretKey].value;
});
const secretRecord: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }> = {};
secretsByPath[secPath].forEach((decryptedSecret) => {
secretRecord[decryptedSecret.secretKey] = {
value: decryptedSecret.secretValue,
comment: decryptedSecret.secretComment
};
});
await expandSecrets(secretRecord);
secretsByPath[secPath].forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value;
});
}
};
// expand secrets
@@ -999,6 +1062,7 @@ export const secretServiceFactory = ({
includeImports,
version
});
return decryptSecretRaw(secret, botKey);
};
@@ -1171,7 +1235,9 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
};
const updateManySecretsRaw = async ({
@@ -1223,7 +1289,9 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
};
const deleteManySecretsRaw = async ({
@@ -1257,7 +1325,9 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
return secrets.map((secret) =>
decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
);
};
const getSecretVersions = async ({
@@ -1488,6 +1558,52 @@ export const secretServiceFactory = ({
};
};
// this is a backfilling API for secret references
// what it does is it will go through all the secret values and parse all references
// populate the secret reference to do sync integrations
const backfillSecretReferences = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: TBackFillSecretReferencesDTO) => {
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
if (!hasRole(ProjectMembershipRole.Admin))
throw new BadRequestError({ message: "Only admins are allowed to take this action" });
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey)
throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" });
await secretDAL.transaction(async (tx) => {
const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx);
await secretDAL.upsertSecretReferences(
secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({
secretId: id,
references: getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: secretValueCiphertext,
iv: secretValueIV,
tag: secretValueTag,
key: botKey
})
)
})),
tx
);
});
return { message: "Successfully backfilled secret references" };
};
return {
attachTags,
detachTags,
@@ -1508,6 +1624,7 @@ export const secretServiceFactory = ({
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
backfillSecretReferences,
// external services function
fnSecretBulkDelete,
fnSecretBulkUpdate,

15
backend/src/services/secret/secret-types.ts
View File

@@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
secretId: string;
};
export type TSecretReference = { environment: string; secretPath: string };
export type TFnSecretBulkInsert = {
folderId: string;
tx?: Knex;
inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
secretDAL: Pick<TSecretDALFactory, "insertMany">;
inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[]; references?: TSecretReference[] }>;
secretDAL: Pick<TSecretDALFactory, "insertMany" | "upsertSecretReferences">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = {
export type TFnSecretBulkUpdate = {
folderId: string;
projectId: string;
inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
inputSecrets: {
filter: Partial<TSecrets>;
data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] };
}[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "upsertSecretReferences">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = {
repeatDays: number;
};
export type TBackFillSecretReferencesDTO = TProjectPermission;
// ---
export type TCreateManySecretsRawFnFactory = {

4
docs/api-reference/endpoints/universal-auth/revoke-access-token.mdx Normal file
View File

@@ -0,0 +1,4 @@
---
title: "Revoke Access Token"
openapi: "POST /api/v1/auth/token/revoke"
---

6
docs/cli/commands/export.mdx
View File

@@ -128,6 +128,12 @@ infisical export --template=<path to template>
</Accordion>
<Accordion title="--include-imports">
By default imported secrets are available, you can disable it by setting this option to false.
Default value: `true`
</Accordion>
<Accordion title="--format">
Format of the output file. Accepted values: `dotenv`, `dotenv-export`, `csv`, `json` and `yaml`

6
docs/cli/commands/run.mdx
View File

@@ -126,6 +126,12 @@ $ infisical run -- npm run dev
</Accordion>
<Accordion title="--include-imports">
By default imported secrets are available, you can disable it by setting this option to false.
Default value: `true`
</Accordion>
{" "}
<Accordion title="--env">

1
docs/cli/faq.mdx
View File

@@ -13,6 +13,7 @@ If none of the available stores work for you, you can try using the `file` store
If you are still experiencing trouble, please seek support.
[Learn more about vault command](./commands/vault)
</Accordion>
<Accordion title="Can I fetch secrets with Infisical if I am offline?">

2
docs/documentation/platform/identities/gcp-auth.mdx
View File

@@ -123,7 +123,7 @@ access the Infisical API using the GCP ID Token authentication method.
<CodeGroup>
```bash curl
curl -H "Metadata-Flavor: Google" \
'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=<identityId>'
'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=<identityId>&format=full'
```
</CodeGroup>

2
docs/documentation/platform/secret-versioning.mdx
View File

@@ -3,7 +3,7 @@ title: "Secret Versioning"
description: "Learn how secret versioning works in Infisical."
---
Every time a secret change is persformed, a new version of the same secret is created.
Every time a secret change is performed, a new version of the same secret is created.
Such versions can be accessed visually by opening up the [secret sidebar](/documentation/platform/project#drawer) (as seen below) or [retrieved via API](/api-reference/endpoints/secrets/read)
by specifying the `version` query parameter.

3
docs/mint.json
View File

@@ -417,7 +417,8 @@
"api-reference/endpoints/universal-auth/create-client-secret",
"api-reference/endpoints/universal-auth/list-client-secrets",
"api-reference/endpoints/universal-auth/revoke-client-secret",
"api-reference/endpoints/universal-auth/renew-access-token"
"api-reference/endpoints/universal-auth/renew-access-token",
"api-reference/endpoints/universal-auth/revoke-access-token"
]
},
{

1
frontend/src/hooks/api/secrets/index.ts
View File

@@ -1,4 +1,5 @@
export {
useBackfillSecretReference,
useCreateSecretBatch,
useCreateSecretV3,
useDeleteSecretBatch,

50
frontend/src/hooks/api/secrets/mutations.tsx
View File

@@ -87,11 +87,11 @@ export const useCreateSecretV3 = ({
const randomBytes = latestFileKey
? decryptAssymmetric({
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
: crypto.randomBytes(16).toString("hex");
const reqBody = {
@@ -148,11 +148,11 @@ export const useUpdateSecretV3 = ({
const randomBytes = latestFileKey
? decryptAssymmetric({
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
: crypto.randomBytes(16).toString("hex");
const reqBody = {
@@ -244,11 +244,11 @@ export const useCreateSecretBatch = ({
const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
const randomBytes = latestFileKey
? decryptAssymmetric({
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
: crypto.randomBytes(16).toString("hex");
const reqBody = {
@@ -297,11 +297,11 @@ export const useUpdateSecretBatch = ({
const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
const randomBytes = latestFileKey
? decryptAssymmetric({
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
ciphertext: latestFileKey.encryptedKey,
nonce: latestFileKey.nonce,
publicKey: latestFileKey.sender.publicKey,
privateKey: PRIVATE_KEY
})
: crypto.randomBytes(16).toString("hex");
const reqBody = {
@@ -379,3 +379,13 @@ export const createSecret = async (dto: CreateSecretDTO) => {
const { data } = await apiRequest.post(`/api/v3/secrets/${dto.secretKey}`, dto);
return data;
};
export const useBackfillSecretReference = () =>
useMutation<{ message: string }, {}, { projectId: string }>({
mutationFn: async ({ projectId }) => {
const { data } = await apiRequest.post("/api/v3/secrets/backfill-secret-references", {
projectId
});
return data.message;
}
});

43
frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx Normal file
View File

@@ -0,0 +1,43 @@
import { createNotification } from "@app/components/notifications";
import { Button } from "@app/components/v2";
import { useProjectPermission, useWorkspace } from "@app/context";
import { useBackfillSecretReference } from "@app/hooks/api";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
export const BackfillSecretReferenceSecretion = () => {
const { currentWorkspace } = useWorkspace();
const { membership } = useProjectPermission();
const backfillSecretReferences = useBackfillSecretReference();
if (!currentWorkspace) return null;
const handleBackfill = async () => {
if (backfillSecretReferences.isLoading) return;
try {
await backfillSecretReferences.mutateAsync({ projectId: currentWorkspace.id || "" });
createNotification({ text: "Successfully re-indexed secret references", type: "success" });
} catch {
createNotification({ text: "Failed to re-index secret references", type: "error" });
}
};
const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin);
return (
<div className="mb-6 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
<div className="flex w-full items-center justify-between">
<p className="text-xl font-semibold">Index Secret References</p>
</div>
<p className="mb-4 mt-2 max-w-2xl text-sm text-gray-400">
This will index all secret references, enabling integrations to be triggered when their values change going forward.
</p>
<Button
variant="outline_bg"
isLoading={backfillSecretReferences.isLoading}
onClick={handleBackfill}
isDisabled={!isAdmin}
>
Index Secret References
</Button>
</div>
);
};

1
frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx Normal file
View File

@@ -0,0 +1 @@
export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";

2
frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx
View File

@@ -1,4 +1,5 @@
import { AutoCapitalizationSection } from "../AutoCapitalizationSection";
import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSection";
import { DeleteProjectSection } from "../DeleteProjectSection";
import { E2EESection } from "../E2EESection";
import { EnvironmentSection } from "../EnvironmentSection";
@@ -13,6 +14,7 @@ export const ProjectGeneralTab = () => {
<SecretTagsSection />
<AutoCapitalizationSection />
<E2EESection />
<BackfillSecretReferenceSecretion />
<DeleteProjectSection />
</div>
);

1
frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx
View File

@@ -1,4 +1,5 @@
export { AutoCapitalizationSection } from "./AutoCapitalizationSection";
export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";
export { DeleteProjectSection } from "./DeleteProjectSection";
export { E2EESection } from "./E2EESection";
export { EnvironmentSection } from "./EnvironmentSection";