awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/multipleApprovalEnvs

Browse Source
This commit is contained in:
Carlos Monastyrski
2025-07-25 16:29:04 -03:00
parent 99e8bdef58 929822514e
commit 608979efa7
74 changed files with 2315 additions and 784 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/src/@types/fastify.d.ts vendored
View File

@@ -126,6 +126,15 @@ declare module "@fastify/request-context" {
namespace: string;
name: string;
};
aws?: {
accountId: string;
arn: string;
userId: string;
partition: string;
service: string;
resourceType: string;
resourceName: string;
};
};
identityPermissionMetadata?: Record<string, unknown>; // filled by permission service
assumedPrivilegeDetails?: { requesterId: string; actorId: string; actorType: ActorType; projectId: string };

6
backend/src/server/plugins/auth/inject-identity.ts
View File

@@ -162,6 +162,12 @@ export const injectIdentity = fp(async (server: FastifyZodProvider) => {
kubernetes: token?.identityAuth?.kubernetes
});
}
if (token?.identityAuth?.aws) {
requestContext.set("identityAuthInfo", {
identityId: identity.identityId,
aws: token?.identityAuth?.aws
});
}
break;
}
case AuthMode.SERVICE_TOKEN: {

8
backend/src/server/routes/v1/dashboard-router.ts
View File

@@ -270,11 +270,6 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
}
}
});
remainingLimit -= imports.length;
adjustedOffset = 0;
} else {
adjustedOffset = Math.max(0, adjustedOffset - totalImportCount);
}
}
@@ -317,7 +312,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
}
}
if (!includeDynamicSecrets && !includeSecrets)
if (!includeDynamicSecrets && !includeSecrets && !includeSecretRotations)
return {
folders,
totalFolderCount,
@@ -547,7 +542,6 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
(totalFolderCount ?? 0) +
(totalDynamicSecretCount ?? 0) +
(totalSecretCount ?? 0) +
(totalImportCount ?? 0) +
(totalSecretRotationCount ?? 0)
};
}

32
backend/src/server/routes/v3/external-migration-router.ts
View File

@@ -1,9 +1,11 @@
import fastifyMultipart from "@fastify/multipart";
import { z } from "zod";
import { BadRequestError } from "@app/lib/errors";
import { readLimit } from "@app/server/config/rateLimiter";
import { writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
import { VaultMappingType } from "@app/services/external-migration/external-migration-types";
const MB25_IN_BYTES = 26214400;
@@ -15,7 +17,7 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
bodyLimit: MB25_IN_BYTES,
url: "/env-key",
config: {
rateLimit: readLimit
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
@@ -52,4 +54,30 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
});
}
});
server.route({
method: "POST",
url: "/vault",
config: {
rateLimit: writeLimit
},
schema: {
body: z.object({
vaultAccessToken: z.string(),
vaultNamespace: z.string().trim().optional(),
vaultUrl: z.string(),
mappingType: z.nativeEnum(VaultMappingType)
})
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
await server.services.migration.importVaultData({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
...req.body
});
}
});
};

2
backend/src/server/routes/v3/index.ts
View File

@@ -11,5 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => {
await server.register(registerUserRouter, { prefix: "/users" });
await server.register(registerSecretRouter, { prefix: "/secrets" });
await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" });
await server.register(registerExternalMigrationRouter, { prefix: "/migrate" });
await server.register(registerExternalMigrationRouter, { prefix: "/external-migration" });
};

2
backend/src/services/app-connection/shared/sql/sql-connection-fns.ts
View File

@@ -164,7 +164,7 @@ export const validateSqlConnectionCredentials = async (
) => {
try {
await executeWithPotentialGateway(config, gatewayService, async (client) => {
await client.raw(`Select 1`);
await client.raw(config.app === AppConnection.OracleDB ? `SELECT 1 FROM DUAL` : `Select 1`);
});
return config.credentials;
} catch (error) {

361
backend/src/services/external-migration/external-migration-fns.ts → backend/src/services/external-migration/external-migration-fns/envkey.ts
View File

@@ -1,32 +1,26 @@
import slugify from "@sindresorhus/slugify";
import sjcl from "sjcl";
import tweetnacl from "tweetnacl";
import tweetnaclUtil from "tweetnacl-util";
import { SecretType, TSecretFolders } from "@app/db/schemas";
import { crypto } from "@app/lib/crypto/cryptography";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { chunkArray } from "@app/lib/fn";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectServiceFactory } from "../project/project-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../../kms/kms-service";
import { TProjectDALFactory } from "../../project/project-dal";
import { TProjectServiceFactory } from "../../project/project-service";
import { TProjectEnvDALFactory } from "../../project-env/project-env-dal";
import { TProjectEnvServiceFactory } from "../../project-env/project-env-service";
import { TResourceMetadataDALFactory } from "../../resource-metadata/resource-metadata-dal";
import { TSecretFolderDALFactory } from "../../secret-folder/secret-folder-dal";
import { TSecretFolderVersionDALFactory } from "../../secret-folder/secret-folder-version-dal";
import { TSecretTagDALFactory } from "../../secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "../../secret-v2-bridge/secret-v2-bridge-dal";
import type { TSecretV2BridgeServiceFactory } from "../../secret-v2-bridge/secret-v2-bridge-service";
import { TSecretVersionV2DALFactory } from "../../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../../secret-v2-bridge/secret-version-tag-dal";
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "../external-migration-types";
export type TImportDataIntoInfisicalDTO = {
projectDAL: Pick<TProjectDALFactory, "transaction">;
@@ -499,326 +493,3 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
return infisicalImportData;
};
export const importDataIntoInfisicalFn = async ({
projectService,
projectEnvDAL,
projectDAL,
secretDAL,
kmsService,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
resourceMetadataDAL,
folderVersionDAL,
folderCommitService,
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
}: TImportDataIntoInfisicalDTO) => {
// Import data to infisical
if (!data || !data.projects) {
throw new BadRequestError({ message: "No projects found in data" });
}
const originalToNewProjectId = new Map<string, string>();
const originalToNewEnvironmentId = new Map<
string,
{ envId: string; envSlug: string; rootFolderId: string; projectId: string }
>();
const originalToNewFolderId = new Map<
string,
{
folderId: string;
projectId: string;
}
>();
const projectsNotImported: string[] = [];
await projectDAL.transaction(async (tx) => {
for await (const project of data.projects) {
const newProject = await projectService
.createProject({
actor,
actorId,
actorOrgId,
actorAuthMethod,
workspaceName: project.name,
createDefaultEnvs: false,
tx
})
.catch((e) => {
logger.error(e, `Failed to import to project [name:${project.name}]`);
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
});
originalToNewProjectId.set(project.id, newProject.id);
}
// Import environments
if (data.environments) {
for await (const environment of data.environments) {
const projectId = originalToNewProjectId.get(environment.projectId);
const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
if (!projectId) {
projectsNotImported.push(environment.projectId);
// eslint-disable-next-line no-continue
continue;
}
const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
if (existingEnv) {
throw new BadRequestError({
message: `Environment with slug '${slug}' already exist`,
name: "CreateEnvironment"
});
}
const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
originalToNewEnvironmentId.set(environment.id, {
envSlug: doc.slug,
envId: doc.id,
rootFolderId: folder.id,
projectId
});
}
}
if (data.folders) {
for await (const folder of data.folders) {
const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
if (!parentEnv) {
// eslint-disable-next-line no-continue
continue;
}
const newFolder = await folderDAL.create(
{
name: folder.name,
envId: parentEnv.envId,
parentId: parentEnv.rootFolderId
},
tx
);
const newFolderVersion = await folderVersionDAL.create(
{
name: newFolder.name,
envId: newFolder.envId,
version: newFolder.version,
folderId: newFolder.id
},
tx
);
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Changed by external migration",
folderId: parentEnv.rootFolderId,
changes: [
{
type: CommitType.ADD,
folderVersionId: newFolderVersion.id
}
]
},
tx
);
originalToNewFolderId.set(folder.id, {
folderId: newFolder.id,
projectId: parentEnv.projectId
});
}
}
// Useful for debugging:
// console.log("data.secrets", data.secrets);
// console.log("data.folders", data.folders);
// console.log("data.environment", data.environments);
if (data.secrets && data.secrets.length > 0) {
const mappedToEnvironmentId = new Map<
string,
{
secretKey: string;
secretValue: string;
folderId?: string;
isFromBlock?: boolean;
}[]
>();
for (const secret of data.secrets) {
const targetId = secret.folderId || secret.environmentId;
// Skip if we can't find either an environment or folder mapping for this secret
if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret");
// eslint-disable-next-line no-continue
continue;
}
if (!mappedToEnvironmentId.has(targetId)) {
mappedToEnvironmentId.set(targetId, []);
}
const alreadyHasSecret = mappedToEnvironmentId
.get(targetId)!
.find((el) => el.secretKey === secret.name && el.folderId === secret.folderId);
if (alreadyHasSecret && alreadyHasSecret.isFromBlock) {
// remove the existing secret if any
mappedToEnvironmentId
.get(targetId)!
.splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1);
}
mappedToEnvironmentId.get(targetId)!.push({
secretKey: secret.name,
secretValue: secret.value || "",
folderId: secret.folderId,
isFromBlock: secret.appBlockOrderIndex !== undefined
});
}
// for each of the mappedEnvironmentId
for await (const [targetId, secrets] of mappedToEnvironmentId) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
let selectedFolder: TSecretFolders | undefined;
let selectedProjectId: string | undefined;
// Case 1: Secret belongs to a folder / branch / branch of a block
const foundFolder = originalToNewFolderId.get(targetId);
if (foundFolder) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
selectedProjectId = foundFolder.projectId;
} else {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
const environment = data.environments.find((env) => env.id === targetId);
if (!environment) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const projectId = originalToNewProjectId.get(environment.projectId)!;
if (!projectId) {
throw new BadRequestError({ message: `Failed to import secret, project not found` });
}
const env = originalToNewEnvironmentId.get(targetId);
if (!env) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
if (!folder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
name: "Create secret"
});
}
selectedFolder = folder;
selectedProjectId = projectId;
}
if (!selectedFolder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
if (!selectedProjectId) {
throw new NotFoundError({
message: `Project not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
{
type: KmsDataKey.SecretManager,
projectId: selectedProjectId
},
tx
);
const secretBatches = chunkArray(secrets, 2500);
for await (const secretBatch of secretBatches) {
const secretsByKeys = await secretDAL.findBySecretKeys(
selectedFolder.id,
secretBatch.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
})),
tx
);
if (secretsByKeys.length) {
throw new BadRequestError({
message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
});
}
await fnSecretBulkInsert({
inputSecrets: secretBatch.map((el) => {
const references = getAllSecretReferences(el.secretValue).nestedReferences;
return {
version: 1,
encryptedValue: el.secretValue
? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
: undefined,
key: el.secretKey,
references,
type: SecretType.Shared
};
}),
folderId: selectedFolder.id,
orgId: actorOrgId,
resourceMetadataDAL,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
folderCommitService,
actor: {
type: actor,
actorId
},
tx
});
}
}
}
});
return { projectsNotImported };
};

352
backend/src/services/external-migration/external-migration-fns/import.ts Normal file
View File

@@ -0,0 +1,352 @@
import slugify from "@sindresorhus/slugify";
import { SecretType, TSecretFolders } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { chunkArray } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { CommitType } from "@app/services/folder-commit/folder-commit-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { fnSecretBulkInsert, getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
import { TImportDataIntoInfisicalDTO } from "./envkey";
export const importDataIntoInfisicalFn = async ({
projectService,
projectEnvDAL,
projectDAL,
secretDAL,
kmsService,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
resourceMetadataDAL,
folderVersionDAL,
folderCommitService,
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
}: TImportDataIntoInfisicalDTO) => {
// Import data to infisical
if (!data || !data.projects) {
throw new BadRequestError({ message: "No projects found in data" });
}
const originalToNewProjectId = new Map<string, string>();
const originalToNewEnvironmentId = new Map<
string,
{ envId: string; envSlug: string; rootFolderId?: string; projectId: string }
>();
const originalToNewFolderId = new Map<
string,
{
envId: string;
envSlug: string;
folderId: string;
projectId: string;
}
>();
const projectsNotImported: string[] = [];
await projectDAL.transaction(async (tx) => {
for await (const project of data.projects) {
const newProject = await projectService
.createProject({
actor,
actorId,
actorOrgId,
actorAuthMethod,
workspaceName: project.name,
createDefaultEnvs: false,
tx
})
.catch((e) => {
logger.error(e, `Failed to import to project [name:${project.name}]`);
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
});
originalToNewProjectId.set(project.id, newProject.id);
}
// Import environments
if (data.environments) {
for await (const environment of data.environments) {
const projectId = originalToNewProjectId.get(environment.projectId);
const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
if (!projectId) {
projectsNotImported.push(environment.projectId);
// eslint-disable-next-line no-continue
continue;
}
const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
if (existingEnv) {
throw new BadRequestError({
message: `Environment with slug '${slug}' already exist`,
name: "CreateEnvironment"
});
}
const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
originalToNewEnvironmentId.set(environment.id, {
envSlug: doc.slug,
envId: doc.id,
rootFolderId: folder.id,
projectId
});
}
}
if (data.folders) {
for await (const folder of data.folders) {
const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
const parentFolder = originalToNewFolderId.get(folder.parentFolderId as string);
let newFolder: TSecretFolders;
if (parentEnv?.rootFolderId) {
newFolder = await folderDAL.create(
{
name: folder.name,
envId: parentEnv.envId,
parentId: parentEnv.rootFolderId
},
tx
);
} else if (parentFolder) {
newFolder = await folderDAL.create(
{
name: folder.name,
envId: parentFolder.envId,
parentId: parentFolder.folderId
},
tx
);
} else {
logger.info({ folder }, "No parent environment found for folder");
// eslint-disable-next-line no-continue
continue;
}
const newFolderVersion = await folderVersionDAL.create(
{
name: newFolder.name,
envId: newFolder.envId,
version: newFolder.version,
folderId: newFolder.id
},
tx
);
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Changed by external migration",
folderId: parentEnv?.rootFolderId || parentFolder?.folderId || "",
changes: [
{
type: CommitType.ADD,
folderVersionId: newFolderVersion.id
}
]
},
tx
);
originalToNewFolderId.set(folder.id, {
folderId: newFolder.id,
envId: parentEnv?.envId || parentFolder?.envId || "",
envSlug: parentEnv?.envSlug || parentFolder?.envSlug || "",
projectId: parentEnv?.projectId || parentFolder?.projectId || ""
});
}
}
// Useful for debugging:
// console.log("data.secrets", data.secrets);
// console.log("data.folders", data.folders);
// console.log("data.environment", data.environments);
if (data.secrets && data.secrets.length > 0) {
const mappedToEnvironmentId = new Map<
string,
{
secretKey: string;
secretValue: string;
folderId?: string;
isFromBlock?: boolean;
}[]
>();
for (const secret of data.secrets) {
const targetId = secret.folderId || secret.environmentId;
// Skip if we can't find either an environment or folder mapping for this secret
if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret");
// eslint-disable-next-line no-continue
continue;
}
if (!mappedToEnvironmentId.has(targetId)) {
mappedToEnvironmentId.set(targetId, []);
}
const alreadyHasSecret = mappedToEnvironmentId
.get(targetId)!
.find((el) => el.secretKey === secret.name && el.folderId === secret.folderId);
if (alreadyHasSecret && alreadyHasSecret.isFromBlock) {
// remove the existing secret if any
mappedToEnvironmentId
.get(targetId)!
.splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1);
}
mappedToEnvironmentId.get(targetId)!.push({
secretKey: secret.name,
secretValue: secret.value || "",
folderId: secret.folderId,
isFromBlock: secret.appBlockOrderIndex !== undefined
});
}
// for each of the mappedEnvironmentId
for await (const [targetId, secrets] of mappedToEnvironmentId) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
let selectedFolder: TSecretFolders | undefined;
let selectedProjectId: string | undefined;
// Case 1: Secret belongs to a folder / branch / branch of a block
const foundFolder = originalToNewFolderId.get(targetId);
if (foundFolder) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
selectedProjectId = foundFolder.projectId;
} else {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
const environment = data.environments.find((env) => env.id === targetId);
if (!environment) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const projectId = originalToNewProjectId.get(environment.projectId)!;
if (!projectId) {
throw new BadRequestError({ message: `Failed to import secret, project not found` });
}
const env = originalToNewEnvironmentId.get(targetId);
if (!env) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
if (!folder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
name: "Create secret"
});
}
selectedFolder = folder;
selectedProjectId = projectId;
}
if (!selectedFolder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
if (!selectedProjectId) {
throw new NotFoundError({
message: `Project not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
{
type: KmsDataKey.SecretManager,
projectId: selectedProjectId
},
tx
);
const secretBatches = chunkArray(secrets, 2500);
for await (const secretBatch of secretBatches) {
const secretsByKeys = await secretDAL.findBySecretKeys(
selectedFolder.id,
secretBatch.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
})),
tx
);
if (secretsByKeys.length) {
throw new BadRequestError({
message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
});
}
await fnSecretBulkInsert({
inputSecrets: secretBatch.map((el) => {
const references = getAllSecretReferences(el.secretValue).nestedReferences;
return {
version: 1,
encryptedValue: el.secretValue
? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
: undefined,
key: el.secretKey,
references,
type: SecretType.Shared
};
}),
folderId: selectedFolder.id,
orgId: actorOrgId,
resourceMetadataDAL,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
folderCommitService,
actor: {
type: actor,
actorId
},
tx
});
}
}
}
});
return { projectsNotImported };
};

3
backend/src/services/external-migration/external-migration-fns/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from "./envkey";
export * from "./import";
export * from "./vault";

341
backend/src/services/external-migration/external-migration-fns/vault.ts Normal file
View File

@@ -0,0 +1,341 @@
import axios, { AxiosInstance } from "axios";
import { v4 as uuidv4 } from "uuid";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
type VaultData = {
namespace: string;
mount: string;
path: string;
secretData: Record<string, string>;
};
const vaultFactory = () => {
const getMounts = async (request: AxiosInstance) => {
const response = await request
.get<
Record<
string,
{
accessor: string;
options: {
version?: string;
} | null;
type: string;
}
>
>("/v1/sys/mounts")
.catch((err) => {
if (axios.isAxiosError(err)) {
logger.error(err.response?.data, "External migration: Failed to get Vault mounts");
}
throw err;
});
return response.data;
};
const getPaths = async (
request: AxiosInstance,
{ mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
) => {
try {
// For KV v2: /v1/{mount}/metadata/{path}?list=true
const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
const response = await request.get<{
data: {
keys: string[];
};
}>(`/v1/${path}?list=true`);
return response.data.data.keys;
} catch (err) {
if (axios.isAxiosError(err)) {
logger.error(err.response?.data, "External migration: Failed to get Vault paths");
if (err.response?.status === 404) {
return null;
}
}
throw err;
}
};
const getSecrets = async (
request: AxiosInstance,
{ mountPath, secretPath }: { mountPath: string; secretPath: string }
) => {
// For KV v2: /v1/{mount}/data/{path}
const response = await request
.get<{
data: {
data: Record<string, string>; // KV v2 has nested data structure
metadata: {
created_time: string;
deletion_time: string;
destroyed: boolean;
version: number;
};
};
}>(`/v1/${mountPath}/data/${secretPath}`)
.catch((err) => {
if (axios.isAxiosError(err)) {
logger.error(err.response?.data, "External migration: Failed to get Vault secret");
}
throw err;
});
return response.data.data.data;
};
// helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
// const isKvV2Mount = (mountInfo: { type: string; options?: { version?: string } | null }) => {
// return mountInfo.type === "kv" && mountInfo.options?.version === "2";
// };
const recursivelyGetAllPaths = async (
request: AxiosInstance,
mountPath: string,
currentPath: string = ""
): Promise<string[]> => {
const paths = await getPaths(request, { mountPath, secretPath: currentPath });
if (paths === null || paths.length === 0) {
return [];
}
const allSecrets: string[] = [];
for await (const path of paths) {
const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path;
const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath;
if (path.endsWith("/")) {
// it's a folder so we recurse into it
const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
allSecrets.push(...subSecrets);
} else {
// it's a secret so we add it to our results
allSecrets.push(`${mountPath}/${fullItemPath}`);
}
}
return allSecrets;
};
async function collectVaultData({
baseUrl,
namespace,
accessToken
}: {
baseUrl: string;
namespace?: string;
accessToken: string;
}): Promise<VaultData[]> {
const request = axios.create({
baseURL: baseUrl,
headers: {
"X-Vault-Token": accessToken,
...(namespace ? { "X-Vault-Namespace": namespace } : {})
}
});
const allData: VaultData[] = [];
// Get all mounts in this namespace
const mounts = await getMounts(request);
for (const mount of Object.keys(mounts)) {
if (!mount.endsWith("/")) {
delete mounts[mount];
}
}
for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
// skip non-KV mounts
if (!mountInfo.type.startsWith("kv")) {
// eslint-disable-next-line no-continue
continue;
}
// get all paths in this mount
const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
const cleanMountPath = mountPath.replace(/\/$/, "");
for await (const secretPath of paths) {
// get the actual secret data
const secretData = await getSecrets(request, {
mountPath: cleanMountPath,
secretPath: secretPath.replace(`${cleanMountPath}/`, "")
});
allData.push({
namespace: namespace || "",
mount: mountPath.replace(/\/$/, ""),
path: secretPath.replace(`${cleanMountPath}/`, ""),
secretData
});
}
}
return allData;
}
return {
collectVaultData,
getMounts,
getPaths,
getSecrets,
recursivelyGetAllPaths
};
};
export const transformToInfisicalFormatNamespaceToProjects = (
vaultData: VaultData[],
mappingType: VaultMappingType
): InfisicalImportData => {
const projects: Array<{ name: string; id: string }> = [];
const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = [];
const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = [];
const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = [];
// track created entities to avoid duplicates
const projectMap = new Map<string, string>(); // namespace -> projectId
const environmentMap = new Map<string, string>(); // namespace:mount -> environmentId
const folderMap = new Map<string, string>(); // namespace:mount:folderPath -> folderId
let environmentId: string = "";
for (const data of vaultData) {
const { namespace, mount, path, secretData } = data;
if (mappingType === VaultMappingType.Namespace) {
// create project (namespace)
if (!projectMap.has(namespace)) {
const projectId = uuidv4();
projectMap.set(namespace, projectId);
projects.push({
name: namespace,
id: projectId
});
}
const projectId = projectMap.get(namespace)!;
// create environment (mount)
const envKey = `${namespace}:${mount}`;
if (!environmentMap.has(envKey)) {
environmentId = uuidv4();
environmentMap.set(envKey, environmentId);
environments.push({
name: mount,
id: environmentId,
projectId
});
}
environmentId = environmentMap.get(envKey)!;
} else if (mappingType === VaultMappingType.KeyVault) {
if (!projectMap.has(mount)) {
const projectId = uuidv4();
projectMap.set(mount, projectId);
projects.push({
name: mount,
id: projectId
});
}
const projectId = projectMap.get(mount)!;
// create single "Production" environment per project, because we have no good way of determining environments from vault
if (!environmentMap.has(mount)) {
environmentId = uuidv4();
environmentMap.set(mount, environmentId);
environments.push({
name: "Production",
id: environmentId,
projectId
});
}
environmentId = environmentMap.get(mount)!;
}
// create folder structure
let currentFolderId: string | undefined;
let currentPath = "";
if (path.includes("/")) {
const pathParts = path.split("/").filter(Boolean);
const folderParts = pathParts;
// create nested folder structure for the entire path
for (const folderName of folderParts) {
currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
const folderKey = `${namespace}:${mount}:${currentPath}`;
if (!folderMap.has(folderKey)) {
const folderId = uuidv4();
folderMap.set(folderKey, folderId);
folders.push({
id: folderId,
name: folderName,
environmentId,
parentFolderId: currentFolderId || environmentId
});
currentFolderId = folderId;
} else {
currentFolderId = folderMap.get(folderKey)!;
}
}
}
for (const [key, value] of Object.entries(secretData)) {
secrets.push({
id: uuidv4(),
name: key,
environmentId,
value: String(value),
folderId: currentFolderId
});
}
}
return {
projects,
environments,
folders,
secrets
};
};
export const importVaultDataFn = async ({
vaultAccessToken,
vaultNamespace,
vaultUrl,
mappingType
}: {
vaultAccessToken: string;
vaultNamespace?: string;
vaultUrl: string;
mappingType: VaultMappingType;
}) => {
await blockLocalAndPrivateIpAddresses(vaultUrl);
if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
throw new BadRequestError({
message: "Vault namespace is required when project mapping type is set to namespace."
});
}
const vaultApi = vaultFactory();
const vaultData = await vaultApi.collectVaultData({
accessToken: vaultAccessToken,
baseUrl: vaultUrl,
namespace: vaultNamespace
});
const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);
return infisicalData;
};

3
backend/src/services/external-migration/external-migration-queue.ts
View File

@@ -19,7 +19,7 @@ import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-d
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { importDataIntoInfisicalFn } from "./external-migration-fns";
import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
import { ExternalPlatforms, ImportType, TImportInfisicalDataCreate } from "./external-migration-types";
export type TExternalMigrationQueueFactoryDep = {
smtpService: TSmtpService;
@@ -67,6 +67,7 @@ export const externalMigrationQueueFactory = ({
const startImport = async (dto: {
actorEmail: string;
data: {
importType: ImportType;
iv: string;
tag: string;
ciphertext: string;

64
backend/src/services/external-migration/external-migration-service.ts
View File

@@ -4,9 +4,9 @@ import { crypto } from "@app/lib/crypto/cryptography";
import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
import { TUserDALFactory } from "../user/user-dal";
import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
import { decryptEnvKeyDataFn, importVaultDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
import { TExternalMigrationQueueFactory } from "./external-migration-queue";
import { TImportEnvKeyDataCreate } from "./external-migration-types";
import { ImportType, TImportEnvKeyDataDTO, TImportVaultDataDTO } from "./external-migration-types";
type TExternalMigrationServiceFactoryDep = {
permissionService: TPermissionServiceFactory;
@@ -28,7 +28,7 @@ export const externalMigrationServiceFactory = ({
actorId,
actorOrgId,
actorAuthMethod
}: TImportEnvKeyDataCreate) => {
}: TImportEnvKeyDataDTO) => {
if (crypto.isFipsModeEnabled()) {
throw new BadRequestError({ message: "EnvKey migration is not supported when running in FIPS mode." });
}
@@ -60,11 +60,65 @@ export const externalMigrationServiceFactory = ({
await externalMigrationQueue.startImport({
actorEmail: user.email!,
data: encrypted
data: {
importType: ImportType.EnvKey,
...encrypted
}
});
};
const importVaultData = async ({
vaultAccessToken,
vaultNamespace,
mappingType,
vaultUrl,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: TImportVaultDataDTO) => {
const { membership } = await permissionService.getOrgPermission(
actor,
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
if (membership.role !== OrgMembershipRole.Admin) {
throw new ForbiddenRequestError({ message: "Only admins can import data" });
}
const user = await userDAL.findById(actorId);
const vaultData = await importVaultDataFn({
vaultAccessToken,
vaultNamespace,
vaultUrl,
mappingType
});
const stringifiedJson = JSON.stringify({
data: vaultData,
actor,
actorId,
actorOrgId,
actorAuthMethod
});
const encrypted = crypto.encryption().symmetric().encryptWithRootEncryptionKey(stringifiedJson);
await externalMigrationQueue.startImport({
actorEmail: user.email!,
data: {
importType: ImportType.Vault,
...encrypted
}
});
};
return {
importEnvKeyData
importEnvKeyData,
importVaultData
};
};

27
backend/src/services/external-migration/external-migration-types.ts
View File

@@ -1,5 +1,17 @@
import { TOrgPermission } from "@app/lib/types";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export enum ImportType {
EnvKey = "envkey",
Vault = "vault"
}
export enum VaultMappingType {
Namespace = "namespace",
KeyVault = "key-vault"
}
export type InfisicalImportData = {
projects: Array<{ name: string; id: string }>;
environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>;
@@ -14,14 +26,17 @@ export type InfisicalImportData = {
}>;
};
export type TImportEnvKeyDataCreate = {
export type TImportEnvKeyDataDTO = {
decryptionKey: string;
encryptedJson: { nonce: string; data: string };
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
};
} & Omit<TOrgPermission, "orgId">;
export type TImportVaultDataDTO = {
vaultAccessToken: string;
vaultNamespace?: string;
mappingType: VaultMappingType;
vaultUrl: string;
} & Omit<TOrgPermission, "orgId">;
export type TImportInfisicalDataCreate = {
data: InfisicalImportData;

11
backend/src/services/identity-access-token/identity-access-token-types.ts
View File

@@ -15,5 +15,16 @@ export type TIdentityAccessTokenJwtPayload = {
namespace: string;
name: string;
};
aws?: {
accountId: string;
arn: string;
userId: string;
// Derived from ARN
partition: string; // "aws", "aws-gov", "aws-cn"
service: string; // "iam", "sts"
resourceType: string; // "user" or "role"
resourceName: string;
};
};
};

140
backend/src/services/identity-aws-auth/identity-aws-auth-fns.ts
View File

@@ -1,67 +1,91 @@
interface PrincipalArnEntity {
Partition: string;
Service: "iam" | "sts";
AccountNumber: string;
Type: "user" | "role" | "instance-profile";
Path: string;
FriendlyName: string;
SessionInfo: string; // Only populated for assumed-role
}
export const extractPrincipalArnEntity = (arn: string): PrincipalArnEntity => {
// split the ARN into parts using ":" as the delimiter
const fullParts = arn.split(":");
if (fullParts.length !== 6) {
throw new Error(`Unrecognized ARN: "${arn}" contains ${fullParts.length} colon-separated parts, expected 6`);
}
const [prefix, partition, service, , accountNumber, resource] = fullParts;
if (prefix !== "arn") {
throw new Error(`Unrecognized ARN: "${arn}" does not begin with "arn:"`);
}
// validate the service is either 'iam' or 'sts'
if (service !== "iam" && service !== "sts") {
throw new Error(`Unrecognized service: "${service}" in ARN "${arn}", expected "iam" or "sts"`);
}
// parse the last part of the ARN which describes the resource
const parts = resource.split("/");
if (parts.length < 2) {
throw new Error(
`Unrecognized ARN: "${resource}" in ARN "${arn}" contains fewer than 2 slash-separated parts (expected type/name)`
);
}
const [rawType, ...rest] = parts;
let finalType: PrincipalArnEntity["Type"];
let friendlyName: string = parts[parts.length - 1];
let path: string = "";
let sessionInfo: string = "";
// handle different types of resources
switch (rawType) {
case "assumed-role": {
if (rest.length < 2) {
throw new Error(
`Unrecognized ARN: "${resource}" for assumed-role in ARN "${arn}" contains fewer than 3 slash-separated parts (type/roleName/sessionId)`
);
}
// assumed roles use a special format where the friendly name is the role name
const [roleName, sessionId] = rest;
finalType = "role"; // treat assumed role case as role
friendlyName = roleName;
sessionInfo = sessionId;
break;
}
case "user":
case "role":
case "instance-profile":
finalType = rawType;
path = rest.slice(0, -1).join("/");
break;
default:
throw new Error(
`Unrecognized principal type: "${rawType}" in ARN "${arn}". Expected "user", "role", "instance-profile", or "assumed-role".`
);
}
const entity: PrincipalArnEntity = {
Partition: partition,
Service: service,
AccountNumber: accountNumber,
Type: finalType,
Path: path,
FriendlyName: friendlyName,
SessionInfo: sessionInfo
};
return entity;
};
/**
* Extracts the identity ARN from the GetCallerIdentity response to one of the following formats:
* - arn:aws:iam::123456789012:user/MyUserName
* - arn:aws:iam::123456789012:role/MyRoleName
*/
export const extractPrincipalArn = (arn: string) => {
// split the ARN into parts using ":" as the delimiter
const fullParts = arn.split(":");
if (fullParts.length !== 6) {
throw new Error(`Unrecognized ARN: contains ${fullParts.length} colon-separated parts, expected 6`);
}
const [prefix, partition, service, , accountNumber, resource] = fullParts;
if (prefix !== "arn") {
throw new Error('Unrecognized ARN: does not begin with "arn:"');
}
// structure to hold the parsed data
const entity = {
Partition: partition,
Service: service,
AccountNumber: accountNumber,
Type: "",
Path: "",
FriendlyName: "",
SessionInfo: ""
};
// validate the service is either 'iam' or 'sts'
if (entity.Service !== "iam" && entity.Service !== "sts") {
throw new Error(`Unrecognized service: ${entity.Service}, not one of iam or sts`);
}
// parse the last part of the ARN which describes the resource
const parts = resource.split("/");
if (parts.length < 2) {
throw new Error(`Unrecognized ARN: "${resource}" contains fewer than 2 slash-separated parts`);
}
const [type, ...rest] = parts;
entity.Type = type;
entity.FriendlyName = parts[parts.length - 1];
// handle different types of resources
switch (entity.Type) {
case "assumed-role": {
if (rest.length < 2) {
throw new Error(`Unrecognized ARN: "${resource}" contains fewer than 3 slash-separated parts`);
}
// assumed roles use a special format where the friendly name is the role name
const [roleName, sessionId] = rest;
entity.Type = "role"; // treat assumed role case as role
entity.FriendlyName = roleName;
entity.SessionInfo = sessionId;
break;
}
case "user":
case "role":
case "instance-profile":
// standard cases: just join back the path if there's any
entity.Path = rest.slice(0, -1).join("/");
break;
default:
throw new Error(`Unrecognized principal type: "${entity.Type}"`);
}
const entity = extractPrincipalArnEntity(arn);
return `arn:aws:iam::${entity.AccountNumber}:${entity.Type}/${entity.FriendlyName}`;
};

20
backend/src/services/identity-aws-auth/identity-aws-auth-service.ts
View File

@@ -22,7 +22,7 @@ import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identit
import { TIdentityAccessTokenJwtPayload } from "../identity-access-token/identity-access-token-types";
import { validateIdentityUpdateForSuperAdminPrivileges } from "../super-admin/super-admin-fns";
import { TIdentityAwsAuthDALFactory } from "./identity-aws-auth-dal";
import { extractPrincipalArn } from "./identity-aws-auth-fns";
import { extractPrincipalArn, extractPrincipalArnEntity } from "./identity-aws-auth-fns";
import {
TAttachAwsAuthDTO,
TAwsGetCallerIdentityHeaders,
@@ -107,7 +107,7 @@ export const identityAwsAuthServiceFactory = ({
const {
data: {
GetCallerIdentityResponse: {
GetCallerIdentityResult: { Account, Arn }
GetCallerIdentityResult: { Account, Arn, UserId }
}
}
}: { data: TGetCallerIdentityResponse } = await axios({
@@ -168,11 +168,25 @@ export const identityAwsAuthServiceFactory = ({
});
const appCfg = getConfig();
const splitArn = extractPrincipalArnEntity(Arn);
const accessToken = crypto.jwt().sign(
{
identityId: identityAwsAuth.identityId,
identityAccessTokenId: identityAccessToken.id,
authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN,
identityAuth: {
aws: {
accountId: Account,
arn: Arn,
userId: UserId,
// Derived from ARN
partition: splitArn.Partition,
service: splitArn.Service,
resourceType: splitArn.Type,
resourceName: splitArn.FriendlyName
}
}
} as TIdentityAccessTokenJwtPayload,
appCfg.AUTH_SECRET,
// akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error

4
backend/src/services/reminder/reminder-queue.ts
View File

@@ -11,7 +11,7 @@ import { TReminderServiceFactory } from "./reminder-types";
type TDailyReminderQueueServiceFactoryDep = {
reminderService: TReminderServiceFactory;
queueService: TQueueServiceFactory;
secretDAL: Pick<TSecretV2BridgeDALFactory, "transaction" | "findSecretsWithReminderRecipients">;
secretDAL: Pick<TSecretV2BridgeDALFactory, "transaction" | "findSecretsWithReminderRecipientsOld">;
secretReminderRecipientsDAL: Pick<TSecretReminderRecipientsDALFactory, "delete">;
};
@@ -69,7 +69,7 @@ export const dailyReminderQueueServiceFactory = ({
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const secrets = await secretDAL.findSecretsWithReminderRecipients(batchIds, REMINDER_PRUNE_BATCH_SIZE);
const secrets = await secretDAL.findSecretsWithReminderRecipientsOld(batchIds, REMINDER_PRUNE_BATCH_SIZE);
const secretsWithReminder = secrets.filter((secret) => secret.reminderRepeatDays);
const foundSecretIds = new Set(secretsWithReminder.map((secret) => secret.id));

5
backend/src/services/reminder/reminder-service.ts
View File

@@ -308,12 +308,11 @@ export const reminderServiceFactory = ({
);
const newReminders = await reminderDAL.insertMany(
processedReminders.map(({ secretId, message, repeatDays, nextReminderDate, projectId }) => ({
processedReminders.map(({ secretId, message, repeatDays, nextReminderDate }) => ({
secretId,
message,
repeatDays,
nextReminderDate,
projectId
nextReminderDate
})),
tx
);

165
backend/src/services/secret-sync/render/render-sync-fns.ts
View File

@@ -8,7 +8,26 @@ import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TRenderSecret, TRenderSyncWithCredentials } from "./render-sync-types";
const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredentials) => {
const MAX_RETRIES = 5;
const retrySleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 60000);
});
const makeRequestWithRetry = async <T>(requestFn: () => Promise<T>, attempt = 0): Promise<T> => {
try {
return await requestFn();
} catch (error) {
if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
await retrySleep();
return await makeRequestWithRetry(requestFn, attempt + 1);
}
throw error;
}
};
const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredentials): Promise<TRenderSecret[]> => {
const {
destinationConfig,
connection: {
@@ -22,20 +41,23 @@ const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredential
do {
const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
const { data } = await request.get<
{
envVar: {
key: string;
value: string;
};
cursor: string;
}[]
>(url, {
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json"
}
});
const { data } = await makeRequestWithRetry(() =>
request.get<
{
envVar: {
key: string;
value: string;
};
cursor: string;
}[]
>(url, {
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json"
}
})
);
const secrets = data.map((item) => ({
key: item.envVar.key,
@@ -44,13 +66,20 @@ const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredential
allSecrets.push(...secrets);
cursor = data[data.length - 1]?.cursor;
if (data.length > 0 && data[data.length - 1]?.cursor) {
cursor = data[data.length - 1].cursor;
} else {
cursor = undefined;
}
} while (cursor);
return allSecrets;
};
const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap, key: string) => {
const batchUpdateEnvironmentSecrets = async (
secretSync: TRenderSyncWithCredentials,
envVars: Array<{ key: string; value: string }>
): Promise<void> => {
const {
destinationConfig,
connection: {
@@ -58,93 +87,69 @@ const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secr
}
} = secretSync;
await request.put(
`${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars/${key}`,
{
key,
value: secretMap[key].value
},
{
await makeRequestWithRetry(() =>
request.put(`${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars`, envVars, {
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json"
}
}
})
);
};
const deleteEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secret: Pick<TRenderSecret, "key">) => {
const {
destinationConfig,
connection: {
credentials: { apiKey }
}
} = secretSync;
try {
await request.delete(
`${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars/${secret.key}`,
{
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json"
}
}
);
} catch (error) {
if (isAxiosError(error) && error.response?.status === 404) {
// If the secret does not exist, we can ignore this error
return;
}
throw error;
}
};
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 500);
});
export const RenderSyncFns = {
syncSecrets: async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap) => {
const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
for await (const key of Object.keys(secretMap)) {
// If value is empty skip it as render does not allow empty variables
if (secretMap[key].value === "") {
const finalEnvVars: Array<{ key: string; value: string }> = [];
for (const renderSecret of renderSecrets) {
const shouldKeep =
secretMap[renderSecret.key] ||
(secretSync.syncOptions.disableSecretDeletion &&
!matchesSchema(renderSecret.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema));
if (shouldKeep && !secretMap[renderSecret.key]) {
finalEnvVars.push({
key: renderSecret.key,
value: renderSecret.value
});
}
}
for (const [key, secret] of Object.entries(secretMap)) {
// Skip empty values as render does not allow empty variables
if (secret.value === "") {
// eslint-disable-next-line no-continue
continue;
}
await putEnvironmentSecret(secretSync, secretMap, key);
await sleep();
finalEnvVars.push({
key,
value: secret.value
});
}
if (secretSync.syncOptions.disableSecretDeletion) return;
for await (const renderSecret of renderSecrets) {
if (!matchesSchema(renderSecret.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema))
// eslint-disable-next-line no-continue
continue;
if (!secretMap[renderSecret.key]) {
await deleteEnvironmentSecret(secretSync, renderSecret);
await sleep();
}
}
await batchUpdateEnvironmentSecrets(secretSync, finalEnvVars);
},
getSecrets: async (secretSync: TRenderSyncWithCredentials): Promise<TSecretMap> => {
const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
return Object.fromEntries(renderSecrets.map((secret) => [secret.key, { value: secret.value ?? "" }]));
},
removeSecrets: async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap) => {
const encryptedSecrets = await getRenderEnvironmentSecrets(secretSync);
const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
const finalEnvVars: Array<{ key: string; value: string }> = [];
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.key in secretMap) {
await deleteEnvironmentSecret(secretSync, encryptedSecret);
await sleep();
for (const renderSecret of renderSecrets) {
if (!(renderSecret.key in secretMap)) {
finalEnvVars.push({
key: renderSecret.key,
value: renderSecret.value
});
}
}
await batchUpdateEnvironmentSecrets(secretSync, finalEnvVars);
}
};

45
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
View File

@@ -875,6 +875,48 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
};
const findSecretsWithReminderRecipientsOld = async (ids: string[], limit: number, tx?: Knex) => {
try {
// Create a subquery to get limited secret IDs
const limitedSecretIds = (tx || db)(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, ids)
.limit(limit)
.select("id");
// Join with all recipients for the limited secrets
const docs = await (tx || db)(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
.leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`)
.leftJoin(
TableName.SecretReminderRecipients,
`${TableName.SecretV2}.id`,
`${TableName.SecretReminderRecipients}.secretId`
)
.select(selectAllTableCols(TableName.SecretV2))
.select(db.ref("userId").withSchema(TableName.SecretReminderRecipients).as("reminderRecipientUserId"));
const data = sqlNestRelationships({
data: docs,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...SecretsV2Schema.parse(el)
}),
childrenMapper: [
{
key: "reminderRecipientUserId",
label: "recipients" as const,
mapper: ({ reminderRecipientUserId }) => reminderRecipientUserId
}
]
});
return data;
} catch (error) {
throw new DatabaseError({ error, name: "findSecretsWithReminderRecipientsOld" });
}
};
return {
...secretOrm,
update,
@@ -893,6 +935,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
findOne,
find,
invalidateSecretCacheByProjectId,
findSecretsWithReminderRecipients
findSecretsWithReminderRecipients,
findSecretsWithReminderRecipientsOld
};
};