awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make sync behavior apply on first sync only, finish MVP create/update bidirectional sync for Heroku

Browse Source
This commit is contained in:
Tuan Dang
2024-03-07 18:06:56 -08:00
parent 43d40d7475
commit 0b98feea50
8 changed files with 429 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/db/migrations/20240307232900_integration-last-used.ts Normal file
View File

@@ -0,0 +1,15 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable(TableName.Integration, (t) => {
t.datetime("lastUsed");
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.alterTable(TableName.Integration, (t) => {
t.dropColumn("lastUsed");
});
}

3
backend/src/db/schemas/integrations.ts
View File

@@ -27,7 +27,8 @@ export const IntegrationsSchema = z.object({
envId: z.string().uuid(),
secretPath: z.string().default("/"),
createdAt: z.date(),
updatedAt: z.date()
updatedAt: z.date(),
lastUsed: z.date().nullable().optional()
});
export type TIntegrations = z.infer<typeof IntegrationsSchema>;

12
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
View File

@@ -12,14 +12,14 @@ import { groupBy, pick, unique } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { ActorType } from "@app/services/auth/auth-type";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TPermissionServiceFactory } from "../permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission";
@@ -47,7 +47,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
secretDAL: TSecretDALFactory;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret">;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
@@ -377,7 +377,11 @@ export const secretApprovalRequestServiceFactory = ({
"secretBlindIndex"
])
}
}))
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length

154
backend/src/services/integration-auth/integration-sync-secret.ts
View File

@@ -20,20 +20,20 @@ import sodium from "libsodium-wrappers";
import isEqual from "lodash.isequal";
import { z } from "zod";
import { TIntegrationAuths, TIntegrations, SecretType } from "@app/db/schemas";
import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas";
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
import { Integrations, IntegrationUrls, IntegrationSyncBehavior } from "./integration-list";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { createManySecretsRawHelper } from "../secret/secret-fns";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { createManySecretsRawHelper, updateManySecretsRawHelper } from "../secret/secret-fns";
import { Integrations, IntegrationSyncBehavior, IntegrationUrls } from "./integration-list";
const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -589,10 +589,11 @@ const syncSecretsAWSSecretManager = async ({
/**
* Sync/push [secrets] to Heroku app named [integration.app]
* server.services...
* server.services...
*/
const syncSecretsHeroku = async ({
projectDAL,
integrationDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
@@ -608,6 +609,7 @@ const syncSecretsHeroku = async ({
accessToken
}: {
projectDAL: TProjectDALFactory;
integrationDAL: TIntegrationDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
@@ -622,7 +624,6 @@ const syncSecretsHeroku = async ({
secrets: Record<string, { value: string; comment?: string } | null>;
accessToken: string;
}) => {
const herokuSecrets = (
await request.get(`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`, {
headers: {
@@ -632,62 +633,90 @@ const syncSecretsHeroku = async ({
}
})
).data;
let secretsToAdd: { [key: string]: string } = {};
let secretsToUpdate: { [key: string]: string } = {};
const secretsToAdd: { [key: string]: string } = {};
const secretsToUpdate: { [key: string]: string } = {};
const metadata = z.record(z.any()).parse(integration.metadata);
Object.keys(herokuSecrets).forEach((key) => {
switch (metadata.syncBehavior) {
case IntegrationSyncBehavior.OVERWRITE_TARGET: {
if (!(key in secrets)) secrets[key] = null;
break;
};
case IntegrationSyncBehavior.PREFER_TARGET: {
secrets[key] = herokuSecrets[key];
if (!(key in secrets)) {
secretsToAdd[key] = herokuSecrets[key];
} else {
if (secrets[key] !== herokuSecrets[key]) {
secretsToUpdate[key] = secrets[key]?.value as string;
if (!integration.lastUsed) {
// first time using integration
// -> apply initial sync behavior rule
switch (metadata.syncBehavior) {
case IntegrationSyncBehavior.OVERWRITE_TARGET: {
if (!(key in secrets)) secrets[key] = null;
break;
}
case IntegrationSyncBehavior.PREFER_TARGET: {
if (!(key in secrets)) {
secretsToAdd[key] = herokuSecrets[key];
} else if (secrets[key]?.value !== herokuSecrets[key]) {
secretsToUpdate[key] = herokuSecrets[key];
}
secrets[key] = {
value: herokuSecrets[key]
};
break;
}
break;
};
case IntegrationSyncBehavior.PREFER_SOURCE: {
if(!(key in secrets)) {
secrets[key] = herokuSecrets[key];
secretsToAdd[key] = herokuSecrets[key];
case IntegrationSyncBehavior.PREFER_SOURCE: {
if (!(key in secrets)) {
secrets[key] = herokuSecrets[key];
secretsToAdd[key] = herokuSecrets[key];
}
break;
}
break;
};
default: {
if (!(key in secrets)) secrets[key] = null;
break;
};
}
default: {
if (!(key in secrets)) secrets[key] = null;
break;
}
}
} else if (!(key in secrets)) secrets[key] = null;
});
await createManySecretsRawHelper({
botKey,
projectDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
projectId,
environment,
path: secretPath,
secrets: Object.keys(secretsToAdd).map((key) => ({
secretName: key,
secretValue: secretsToAdd[key],
type: SecretType.Shared,
secretComment: ""
}))
});
if (Object.keys(secretsToAdd).length) {
await createManySecretsRawHelper({
botKey,
projectDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
projectId,
environment,
path: secretPath,
secrets: Object.keys(secretsToAdd).map((key) => ({
secretName: key,
secretValue: secretsToAdd[key],
type: SecretType.Shared,
secretComment: ""
}))
});
}
if (Object.keys(secretsToUpdate).length) {
await updateManySecretsRawHelper({
projectId,
environment,
path: secretPath,
secrets: Object.keys(secretsToUpdate).map((key) => ({
secretName: key,
secretValue: secretsToUpdate[key],
type: SecretType.Shared,
secretComment: ""
})),
botKey, // TODO: consider getting botKey inside this fn
projectDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
});
}
await request.patch(
`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`,
@@ -700,6 +729,10 @@ const syncSecretsHeroku = async ({
}
}
);
await integrationDAL.updateById(integration.id, {
lastUsed: new Date()
});
};
/**
@@ -3013,12 +3046,13 @@ const syncSecretsHasuraCloud = async ({
/**
* Sync/push [secrets] to [app] in integration named [integration]
*
*
* Do this in terms of DAL
*
*
*/
export const syncIntegrationSecrets = async ({
projectDAL,
integrationDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
@@ -3037,6 +3071,7 @@ export const syncIntegrationSecrets = async ({
appendices
}: {
projectDAL: TProjectDALFactory;
integrationDAL: TIntegrationDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
@@ -3088,6 +3123,7 @@ export const syncIntegrationSecrets = async ({
case Integrations.HEROKU:
await syncSecretsHeroku({
projectDAL,
integrationDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
@@ -3100,7 +3136,7 @@ export const syncIntegrationSecrets = async ({
secretPath,
integration,
secrets,
accessToken,
accessToken
});
break;
case Integrations.VERCEL:

322
backend/src/services/secret/secret-fns.ts
View File

@@ -1,32 +1,33 @@
/* eslint-disable no-await-in-loop */
import path from "path";
import {
SecretKeyEncoding,
TSecretBlindIndexes,
TSecrets,
import {
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretType,
TableName
TableName,
TSecretBlindIndexes,
TSecrets
} from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import {
buildSecretBlindIndexFromName,
encryptSymmetric128BitHexKeyUTF8,
decryptSymmetric128BitHexKeyUTF8
import {
buildSecretBlindIndexFromName,
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
import {
TCreateManySecretsRawHelper,
TFnSecretBlindIndexCheck,
TFnSecretBulkInsert,
TFnSecretBlindIndexCheck
TFnSecretBulkUpdate,
TUpdateManySecretsRawHelper
} from "./secret-types";
import { groupBy, unique } from "@app/lib/fn";
import { BadRequestError } from "@app/lib/errors";
export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
const appCfg = getConfig();
const secretBlindIndex = await buildSecretBlindIndexFromName({
@@ -308,14 +309,15 @@ export const fnSecretBlindIndexCheck = async ({
// these functions are special functions shared by a couple of resources
// used by secret approval, rotation or anywhere in which secret needs to modified
export const fnSecretBulkInsert = async ({ // TODO: Pick types here
folderId,
inputSecrets,
export const fnSecretBulkInsert = async ({
// TODO: Pick types here
folderId,
inputSecrets,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
tx
}: TFnSecretBulkInsert) => {
const newSecrets = await secretDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
@@ -349,10 +351,63 @@ export const fnSecretBulkInsert = async ({ // TODO: Pick types here
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const createManySecretsRawHelper = async ({ // TODO: place on top
export const fnSecretBulkUpdate = async ({
tx,
inputSecrets,
folderId,
projectId,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
}: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
tx
);
const secretVersions = await secretVersionDAL.insertMany(
newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
...el,
secretId: id
})),
tx
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
if (secsUpdatedTag.length) {
await secretTagDAL.deleteTagsManySecret(
projectId,
secsUpdatedTag.map(({ secretId }) => secretId),
tx
);
const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: secretId
}))
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const createManySecretsRawHelper = async ({
projectId,
environment,
path,
path: secretPath,
secrets,
userId,
botKey, // TODO: consider getting botKey inside this fn
@@ -364,67 +419,66 @@ export const createManySecretsRawHelper = async ({ // TODO: place on top
secretVersionTagDAL,
folderDAL
}: TCreateManySecretsRawHelper) => {
await projectDAL.checkProjectUpgradeStatus(projectId);
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
const folderId = folder.id;
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return ({
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
});
})
);
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
// insert operation
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets,
inputSecrets: secrets,
folderId,
isNew: true,
blindIndexCfg,
secretDAL
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -444,4 +498,130 @@ const inputSecrets = await Promise.all(
);
return newSecrets;
}
};
export const updateManySecretsRawHelper = async ({
projectId,
environment,
path: secretPath,
secrets,
userId,
botKey, // TODO: consider getting botKey inside this fn
projectDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
}: TUpdateManySecretsRawHelper) => {
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: secrets,
folderId,
isNew: false,
blindIndexCfg,
secretDAL,
userId
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to update personal secret override for no corresponding shared secret"
});
if (secret.newSecretName) throw new BadRequestError({ message: "Personal secret cannot change the key name" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
// now find any secret that needs to update its name
// same process as above
const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: nameUpdatedSecrets,
folderId,
isNew: true,
blindIndexCfg,
secretDAL
});
const updatedSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => ({
filter: { secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared },
data: {
...el,
folderId,
secretBlindIndex:
newSecretName && newKeyName2BlindIndex[newSecretName]
? newKeyName2BlindIndex[newSecretName]
: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
);
return updatedSecrets;
};

14
backend/src/services/secret/secret-queue.ts
View File

@@ -6,6 +6,10 @@ import { BadRequestError } from "@app/lib/errors";
import { isSamePath } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
@@ -25,16 +29,11 @@ import { TSecretDALFactory } from "./secret-dal";
import { interpolateSecrets } from "./secret-fns";
import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
queueService: TQueueServiceFactory;
integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2">;
integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
folderDAL: TSecretFolderDALFactory;
@@ -75,7 +74,7 @@ export const secretQueueFactory = ({
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
secretVersionTagDAL
}: TSecretQueueFactoryDep) => {
const syncIntegrations = async (dto: TGetSecrets) => {
await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
@@ -321,6 +320,7 @@ export const secretQueueFactory = ({
await syncIntegrationSecrets({
projectDAL,
integrationDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,

160
backend/src/services/secret/secret-service.ts
View File

@@ -1,6 +1,6 @@
import { ForbiddenError, subject } from "@casl/ability";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType, TableName } from "@app/db/schemas";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
@@ -19,14 +19,9 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
decryptSecretRaw,
fnSecretBlindIndexCheck,
fnSecretBulkInsert
} from "./secret-fns";
import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
TCreateManySecretsRawHelper,
TCreateBulkSecretDTO,
TCreateSecretDTO,
TCreateSecretRawDTO,
@@ -35,7 +30,6 @@ import {
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TFnSecretBulkUpdate,
TGetASecretDTO,
TGetASecretRawDTO,
TGetSecretsDTO,
@@ -98,50 +92,6 @@ export const secretServiceFactory = ({
return secretBlindIndex;
};
const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
tx
);
const secretVersions = await secretVersionDAL.insertMany(
newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
...el,
secretId: id
})),
tx
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
if (secsUpdatedTag.length) {
await secretTagDAL.deleteTagsManySecret(
projectId,
secsUpdatedTag.map(({ secretId }) => secretId),
tx
);
const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: secretId
}))
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
@@ -372,6 +322,10 @@ export const secretServiceFactory = ({
}
}
],
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
@@ -598,7 +552,7 @@ export const secretServiceFactory = ({
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets,
@@ -656,7 +610,7 @@ export const secretServiceFactory = ({
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
@@ -703,7 +657,11 @@ export const secretServiceFactory = ({
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
}
}))
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
);
@@ -826,98 +784,6 @@ export const secretServiceFactory = ({
});
return decryptSecretRaw(secret, botKey);
};
const createManySecretsRawHelper = async ({ // TODO: place on top
projectId,
environment,
path,
secrets,
userId
}: TCreateManySecretsRawHelper) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return ({
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
});
})
);
// insert operation
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets,
folderId,
isNew: true,
blindIndexCfg,
secretDAL
});
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
...el,
version: 0,
secretBlindIndex: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
})),
folderId,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
return newSecrets;
}
const createSecretRaw = async ({
secretName,

43
backend/src/services/secret/secret-types.ts
View File

@@ -4,11 +4,11 @@ import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpda
import { TProjectPermission } from "@app/lib/types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
@@ -198,6 +198,10 @@ export type TFnSecretBulkUpdate = {
folderId: string;
projectId: string;
inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
tx?: Knex;
};
@@ -257,7 +261,7 @@ export type TCreateManySecretsRawHelper = {
tags?: string[];
metadata?: {
source?: string;
}
};
}[];
userId?: string; // only relevant for personal secret(s)
botKey: string;
@@ -268,4 +272,33 @@ export type TCreateManySecretsRawHelper = {
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
}
};
export type TUpdateManySecretsRawHelper = {
projectId: string;
environment: string;
path: string;
secrets: {
secretName: string;
newSecretName?: string;
secretValue: string;
type: SecretType;
secretComment?: string;
skipMultilineEncoding?: boolean;
secretReminderRepeatDays?: number | null;
secretReminderNote?: string | null;
tags?: string[];
metadata?: {
source?: string;
};
}[];
userId?: string; // only relevant for personal secret(s)
botKey: string;
projectDAL: TProjectDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
};