mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-19 11:42:52 +00:00
feat(external-migrations): vault migrations
This commit is contained in:
Daniel Hougaard
@@ -0,0 +1,495 @@
|
||||
import sjcl from "sjcl";
|
||||
import tweetnacl from "tweetnacl";
|
||||
import tweetnaclUtil from "tweetnacl-util";
|
||||
|
||||
import { crypto } from "@app/lib/crypto/cryptography";
|
||||
import { BadRequestError } from "@app/lib/errors";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service";
|
||||
import { TKmsServiceFactory } from "../../kms/kms-service";
|
||||
import { TProjectDALFactory } from "../../project/project-dal";
|
||||
import { TProjectServiceFactory } from "../../project/project-service";
|
||||
import { TProjectEnvDALFactory } from "../../project-env/project-env-dal";
|
||||
import { TProjectEnvServiceFactory } from "../../project-env/project-env-service";
|
||||
import { TResourceMetadataDALFactory } from "../../resource-metadata/resource-metadata-dal";
|
||||
import { TSecretFolderDALFactory } from "../../secret-folder/secret-folder-dal";
|
||||
import { TSecretFolderVersionDALFactory } from "../../secret-folder/secret-folder-version-dal";
|
||||
import { TSecretTagDALFactory } from "../../secret-tag/secret-tag-dal";
|
||||
import { TSecretV2BridgeDALFactory } from "../../secret-v2-bridge/secret-v2-bridge-dal";
|
||||
import type { TSecretV2BridgeServiceFactory } from "../../secret-v2-bridge/secret-v2-bridge-service";
|
||||
import { TSecretVersionV2DALFactory } from "../../secret-v2-bridge/secret-version-dal";
|
||||
import { TSecretVersionV2TagDALFactory } from "../../secret-v2-bridge/secret-version-tag-dal";
|
||||
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "../external-migration-types";
|
||||
|
||||
export type TImportDataIntoInfisicalDTO = {
|
||||
projectDAL: Pick<TProjectDALFactory, "transaction">;
|
||||
projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
|
||||
secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "find">;
|
||||
secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;
|
||||
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create" | "find">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
|
||||
|
||||
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany">;
|
||||
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath" | "findById">;
|
||||
projectService: Pick<TProjectServiceFactory, "createProject">;
|
||||
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
|
||||
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
|
||||
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
|
||||
folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "create">;
|
||||
|
||||
input: TImportInfisicalDataCreate;
|
||||
};
|
||||
|
||||
const { codec, hash } = sjcl;
|
||||
const { secretbox } = tweetnacl;
|
||||
|
||||
export const decryptEnvKeyDataFn = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => {
|
||||
const key = tweetnaclUtil.decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey)));
|
||||
const nonce = tweetnaclUtil.decodeBase64(encryptedJson.nonce);
|
||||
const encryptedData = tweetnaclUtil.decodeBase64(encryptedJson.data);
|
||||
|
||||
const decrypted = secretbox.open(encryptedData, nonce, key);
|
||||
|
||||
if (!decrypted) {
|
||||
throw new BadRequestError({ message: "Decryption failed, please check the entered encryption key" });
|
||||
}
|
||||
|
||||
const decryptedJson = tweetnaclUtil.encodeUTF8(decrypted);
|
||||
return decryptedJson;
|
||||
};
|
||||
|
||||
export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<InfisicalImportData> => {
|
||||
const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
|
||||
|
||||
const infisicalImportData: InfisicalImportData = {
|
||||
projects: [],
|
||||
environments: [],
|
||||
folders: [],
|
||||
secrets: []
|
||||
};
|
||||
|
||||
parsedJson.apps.forEach((app: { name: string; id: string }) => {
|
||||
infisicalImportData.projects.push({ name: app.name, id: app.id });
|
||||
});
|
||||
|
||||
// string to string map for env templates
|
||||
const envTemplates = new Map<string, string>();
|
||||
for (const env of parsedJson.defaultEnvironmentRoles) {
|
||||
envTemplates.set(env.id, env.defaultName);
|
||||
}
|
||||
|
||||
// custom base environments
|
||||
for (const env of parsedJson.nonDefaultEnvironmentRoles) {
|
||||
envTemplates.set(env.id, env.name);
|
||||
}
|
||||
|
||||
// environments
|
||||
for (const env of parsedJson.baseEnvironments) {
|
||||
const appId = parsedJson.apps.find((a) => a.id === env.envParentId)?.id;
|
||||
|
||||
// If we find the app from the envParentId, we know this is a root-level environment.
|
||||
if (appId) {
|
||||
infisicalImportData.environments.push({
|
||||
id: env.id,
|
||||
name: envTemplates.get(env.environmentRoleId)!,
|
||||
projectId: appId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const findRootInheritedSecret = (
|
||||
secret: { val?: string; inheritsEnvironmentId?: string },
|
||||
secretName: string,
|
||||
envs: typeof parsedJson.envs
|
||||
): { val?: string } => {
|
||||
if (!secret) {
|
||||
return {
|
||||
val: ""
|
||||
};
|
||||
}
|
||||
|
||||
// If we have a direct value, return it
|
||||
if (secret.val !== undefined) {
|
||||
return secret;
|
||||
}
|
||||
|
||||
// If there's no inheritance, return the secret as is
|
||||
if (!secret.inheritsEnvironmentId) {
|
||||
return secret;
|
||||
}
|
||||
|
||||
const inheritedEnv = envs[secret.inheritsEnvironmentId];
|
||||
if (!inheritedEnv) return secret;
|
||||
return findRootInheritedSecret(inheritedEnv.variables[secretName], secretName, envs);
|
||||
};
|
||||
|
||||
const targetIdToFolderIdsMap = new Map<string, string>();
|
||||
|
||||
const processBranches = () => {
|
||||
for (const subEnv of parsedJson.subEnvironments) {
|
||||
const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId);
|
||||
const block = parsedJson.blocks.find((b) => b.id === subEnv.envParentId);
|
||||
|
||||
if (app) {
|
||||
// Handle regular app branches
|
||||
const branchEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId);
|
||||
|
||||
// check if the folder already exists in the same parent environment with the same name
|
||||
|
||||
const folderExists = infisicalImportData.folders.some(
|
||||
(f) => f.name === subEnv.subName && f.parentFolderId === subEnv.parentEnvironmentId
|
||||
);
|
||||
|
||||
// No need to map to target ID's here, because we are not dealing with blocks
|
||||
if (!folderExists) {
|
||||
infisicalImportData.folders.push({
|
||||
name: subEnv.subName,
|
||||
parentFolderId: subEnv.parentEnvironmentId,
|
||||
environmentId: branchEnvironment!.id,
|
||||
id: subEnv.id
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (block) {
|
||||
// Handle block branches
|
||||
// 1. Find all apps that use this block
|
||||
const appsUsingBlock = parsedJson.appBlocks.filter((ab) => ab.blockId === block.id);
|
||||
|
||||
for (const { appId, orderIndex } of appsUsingBlock) {
|
||||
// 2. Find the matching environment in the app based on the environment role
|
||||
const blockBaseEnv = parsedJson.baseEnvironments.find((be) => be.id === subEnv.parentEnvironmentId);
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
if (!blockBaseEnv) continue;
|
||||
|
||||
const matchingAppEnv = parsedJson.baseEnvironments.find(
|
||||
(be) => be.envParentId === appId && be.environmentRoleId === blockBaseEnv.environmentRoleId
|
||||
);
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
if (!matchingAppEnv) continue;
|
||||
|
||||
const folderExists = infisicalImportData.folders.some(
|
||||
(f) => f.name === subEnv.subName && f.parentFolderId === matchingAppEnv.id
|
||||
);
|
||||
|
||||
if (!folderExists) {
|
||||
// 3. Create a folder in the matching app environment
|
||||
infisicalImportData.folders.push({
|
||||
name: subEnv.subName,
|
||||
parentFolderId: matchingAppEnv.id,
|
||||
environmentId: matchingAppEnv.id,
|
||||
id: `${subEnv.id}-${appId}` // Create unique ID for each app's copy of the branch
|
||||
});
|
||||
} else {
|
||||
// folder already exists, so lets map the old folder id to the new folder id
|
||||
targetIdToFolderIdsMap.set(subEnv.id, `${subEnv.id}-${appId}`);
|
||||
}
|
||||
|
||||
// 4. Process secrets in the block branch for this app
|
||||
const branchSecrets = parsedJson.envs[subEnv.id]?.variables || {};
|
||||
for (const [secretName, secretData] of Object.entries(branchSecrets)) {
|
||||
if (secretData.inheritsEnvironmentId) {
|
||||
const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs);
|
||||
|
||||
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
|
||||
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
|
||||
(s) => s.name === secretName && s.environmentId === matchingAppEnv.id
|
||||
);
|
||||
|
||||
if (preExistingSecretIndex !== -1) {
|
||||
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
|
||||
|
||||
if (
|
||||
preExistingSecret.appBlockOrderIndex !== undefined &&
|
||||
orderIndex > preExistingSecret.appBlockOrderIndex
|
||||
) {
|
||||
// if the existing secret has a lower orderIndex, we should replace it
|
||||
infisicalImportData.secrets[preExistingSecretIndex] = {
|
||||
...preExistingSecret,
|
||||
value: resolvedSecret.val || "",
|
||||
appBlockOrderIndex: orderIndex
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secretName,
|
||||
environmentId: matchingAppEnv.id,
|
||||
value: resolvedSecret.val || "",
|
||||
folderId: `${subEnv.id}-${appId}`,
|
||||
appBlockOrderIndex: orderIndex
|
||||
});
|
||||
} else {
|
||||
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
|
||||
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
|
||||
(s) => s.name === secretName && s.environmentId === matchingAppEnv.id
|
||||
);
|
||||
|
||||
if (preExistingSecretIndex !== -1) {
|
||||
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
|
||||
|
||||
if (
|
||||
preExistingSecret.appBlockOrderIndex !== undefined &&
|
||||
orderIndex > preExistingSecret.appBlockOrderIndex
|
||||
) {
|
||||
// if the existing secret has a lower orderIndex, we should replace it
|
||||
infisicalImportData.secrets[preExistingSecretIndex] = {
|
||||
...preExistingSecret,
|
||||
value: secretData.val || "",
|
||||
appBlockOrderIndex: orderIndex
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secretName,
|
||||
environmentId: matchingAppEnv.id,
|
||||
value: secretData.val || "",
|
||||
folderId: `${subEnv.id}-${appId}`,
|
||||
appBlockOrderIndex: orderIndex
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const processBlocksForApp = (appIds: string[]) => {
|
||||
for (const appId of appIds) {
|
||||
const blocksInApp = parsedJson.appBlocks.filter((ab) => ab.appId === appId);
|
||||
logger.info(
|
||||
{
|
||||
blocksInApp
|
||||
},
|
||||
"[processBlocksForApp]: Processing blocks for app"
|
||||
);
|
||||
|
||||
for (const appBlock of blocksInApp) {
|
||||
// 1. find all base environments for this block
|
||||
const blockBaseEnvironments = parsedJson.baseEnvironments.filter((env) => env.envParentId === appBlock.blockId);
|
||||
logger.info(
|
||||
{
|
||||
blockBaseEnvironments
|
||||
},
|
||||
"[processBlocksForApp]: Processing block base environments"
|
||||
);
|
||||
|
||||
for (const blockBaseEnvironment of blockBaseEnvironments) {
|
||||
// 2. find the corresponding environment that is not from the block
|
||||
const matchingEnv = parsedJson.baseEnvironments.find(
|
||||
(be) =>
|
||||
be.environmentRoleId === blockBaseEnvironment.environmentRoleId && be.envParentId !== appBlock.blockId
|
||||
);
|
||||
|
||||
if (!matchingEnv) {
|
||||
throw new Error(`Could not find environment for block ${appBlock.blockId}`);
|
||||
}
|
||||
|
||||
// 3. find all the secrets for this environment block
|
||||
const blockSecrets = parsedJson.envs[blockBaseEnvironment.id].variables;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
blockSecretsLength: Object.keys(blockSecrets).length
|
||||
},
|
||||
"[processBlocksForApp]: Processing block secrets"
|
||||
);
|
||||
|
||||
// 4. process each secret
|
||||
for (const secret of Object.keys(blockSecrets)) {
|
||||
const selectedSecret = blockSecrets[secret];
|
||||
|
||||
if (selectedSecret.inheritsEnvironmentId) {
|
||||
const resolvedSecret = findRootInheritedSecret(selectedSecret, secret, parsedJson.envs);
|
||||
|
||||
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
|
||||
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
|
||||
(s) => s.name === secret && s.environmentId === matchingEnv.id
|
||||
);
|
||||
|
||||
if (preExistingSecretIndex !== -1) {
|
||||
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
|
||||
|
||||
if (
|
||||
preExistingSecret.appBlockOrderIndex !== undefined &&
|
||||
appBlock.orderIndex > preExistingSecret.appBlockOrderIndex
|
||||
) {
|
||||
// if the existing secret has a lower orderIndex, we should replace it
|
||||
infisicalImportData.secrets[preExistingSecretIndex] = {
|
||||
...preExistingSecret,
|
||||
value: selectedSecret.val || "",
|
||||
appBlockOrderIndex: appBlock.orderIndex
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secret,
|
||||
environmentId: matchingEnv.id,
|
||||
value: resolvedSecret.val || "",
|
||||
appBlockOrderIndex: appBlock.orderIndex
|
||||
});
|
||||
} else {
|
||||
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
|
||||
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
|
||||
(s) => s.name === secret && s.environmentId === matchingEnv.id
|
||||
);
|
||||
|
||||
if (preExistingSecretIndex !== -1) {
|
||||
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
|
||||
|
||||
if (
|
||||
preExistingSecret.appBlockOrderIndex !== undefined &&
|
||||
appBlock.orderIndex > preExistingSecret.appBlockOrderIndex
|
||||
) {
|
||||
// if the existing secret has a lower orderIndex, we should replace it
|
||||
infisicalImportData.secrets[preExistingSecretIndex] = {
|
||||
...preExistingSecret,
|
||||
value: selectedSecret.val || "",
|
||||
appBlockOrderIndex: appBlock.orderIndex
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secret,
|
||||
environmentId: matchingEnv.id,
|
||||
value: selectedSecret.val || "",
|
||||
appBlockOrderIndex: appBlock.orderIndex
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
processBranches();
|
||||
processBlocksForApp(infisicalImportData.projects.map((app) => app.id));
|
||||
|
||||
for (const env of Object.keys(parsedJson.envs)) {
|
||||
// Skip user-specific environments
|
||||
// eslint-disable-next-line no-continue
|
||||
if (env.includes("|")) continue;
|
||||
|
||||
const envData = parsedJson.envs[env];
|
||||
const baseEnv = parsedJson.baseEnvironments.find((be) => be.id === env);
|
||||
const subEnv = parsedJson.subEnvironments.find((se) => se.id === env);
|
||||
|
||||
// Skip if we can't find either a base environment or sub-environment
|
||||
if (!baseEnv && !subEnv) {
|
||||
logger.info(
|
||||
{
|
||||
envId: env
|
||||
},
|
||||
"[parseEnvKeyDataFn]: Could not find base or sub environment for env, skipping"
|
||||
);
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
// If this is a base environment of a block, skip it (handled by processBlocksForApp)
|
||||
if (baseEnv) {
|
||||
const isBlock = parsedJson.appBlocks.some((block) => block.blockId === baseEnv.envParentId);
|
||||
if (isBlock) {
|
||||
logger.info(
|
||||
{
|
||||
envId: env,
|
||||
baseEnv
|
||||
},
|
||||
"[parseEnvKeyDataFn]: Skipping block environment (handled separately)"
|
||||
);
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Process each secret in this environment or branch
|
||||
for (const [secretName, secretData] of Object.entries(envData.variables)) {
|
||||
const indexOfExistingSecret = infisicalImportData.secrets.findIndex(
|
||||
(s) =>
|
||||
s.name === secretName &&
|
||||
(s.environmentId === subEnv?.parentEnvironmentId || s.environmentId === env) &&
|
||||
(s.folderId ? s.folderId === subEnv?.id : true) &&
|
||||
(secretData.val ? s.value === secretData.val : true)
|
||||
);
|
||||
|
||||
if (secretData.inheritsEnvironmentId) {
|
||||
const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs);
|
||||
// Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block.
|
||||
// Variables from the normal environment should take precedence over variables from the block.
|
||||
if (indexOfExistingSecret !== -1) {
|
||||
// if a existing secret is found, we should replace it directly
|
||||
const newSecret: (typeof infisicalImportData.secrets)[number] = {
|
||||
...infisicalImportData.secrets[indexOfExistingSecret],
|
||||
value: resolvedSecret.val || ""
|
||||
};
|
||||
|
||||
infisicalImportData.secrets[indexOfExistingSecret] = newSecret;
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secretName,
|
||||
environmentId: subEnv ? subEnv.parentEnvironmentId : env,
|
||||
value: resolvedSecret.val || "",
|
||||
...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret
|
||||
});
|
||||
} else {
|
||||
// Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block.
|
||||
// Variables from the normal environment should take precedence over variables from the block.
|
||||
|
||||
if (indexOfExistingSecret !== -1) {
|
||||
// if a existing secret is found, we should replace it directly
|
||||
const newSecret: (typeof infisicalImportData.secrets)[number] = {
|
||||
...infisicalImportData.secrets[indexOfExistingSecret],
|
||||
value: secretData.val || ""
|
||||
};
|
||||
|
||||
infisicalImportData.secrets[indexOfExistingSecret] = newSecret;
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
const folderId = targetIdToFolderIdsMap.get(subEnv?.id || "") || subEnv?.id;
|
||||
|
||||
infisicalImportData.secrets.push({
|
||||
id: crypto.nativeCrypto.randomUUID(),
|
||||
name: secretName,
|
||||
environmentId: subEnv ? subEnv.parentEnvironmentId : env,
|
||||
value: secretData.val || "",
|
||||
...(folderId && { folderId })
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return infisicalImportData;
|
||||
};
|
||||
@@ -0,0 +1,352 @@
|
||||
import slugify from "@sindresorhus/slugify";
|
||||
|
||||
import { SecretType, TSecretFolders } from "@app/db/schemas";
|
||||
import { BadRequestError, NotFoundError } from "@app/lib/errors";
|
||||
import { chunkArray } from "@app/lib/fn";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
import { CommitType } from "@app/services/folder-commit/folder-commit-service";
|
||||
import { KmsDataKey } from "@app/services/kms/kms-types";
|
||||
import { fnSecretBulkInsert, getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
|
||||
|
||||
import { TImportDataIntoInfisicalDTO } from "./envkey";
|
||||
|
||||
export const importDataIntoInfisicalFn = async ({
|
||||
projectService,
|
||||
projectEnvDAL,
|
||||
projectDAL,
|
||||
secretDAL,
|
||||
kmsService,
|
||||
secretVersionDAL,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
folderDAL,
|
||||
resourceMetadataDAL,
|
||||
folderVersionDAL,
|
||||
folderCommitService,
|
||||
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
|
||||
}: TImportDataIntoInfisicalDTO) => {
|
||||
// Import data to infisical
|
||||
if (!data || !data.projects) {
|
||||
throw new BadRequestError({ message: "No projects found in data" });
|
||||
}
|
||||
|
||||
const originalToNewProjectId = new Map<string, string>();
|
||||
const originalToNewEnvironmentId = new Map<
|
||||
string,
|
||||
{ envId: string; envSlug: string; rootFolderId?: string; projectId: string }
|
||||
>();
|
||||
const originalToNewFolderId = new Map<
|
||||
string,
|
||||
{
|
||||
envId: string;
|
||||
envSlug: string;
|
||||
folderId: string;
|
||||
projectId: string;
|
||||
}
|
||||
>();
|
||||
const projectsNotImported: string[] = [];
|
||||
|
||||
await projectDAL.transaction(async (tx) => {
|
||||
for await (const project of data.projects) {
|
||||
const newProject = await projectService
|
||||
.createProject({
|
||||
actor,
|
||||
actorId,
|
||||
actorOrgId,
|
||||
actorAuthMethod,
|
||||
workspaceName: project.name,
|
||||
createDefaultEnvs: false,
|
||||
tx
|
||||
})
|
||||
.catch((e) => {
|
||||
logger.error(e, `Failed to import to project [name:${project.name}]`);
|
||||
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
|
||||
});
|
||||
originalToNewProjectId.set(project.id, newProject.id);
|
||||
}
|
||||
|
||||
// Import environments
|
||||
if (data.environments) {
|
||||
for await (const environment of data.environments) {
|
||||
const projectId = originalToNewProjectId.get(environment.projectId);
|
||||
const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
|
||||
|
||||
if (!projectId) {
|
||||
projectsNotImported.push(environment.projectId);
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
|
||||
|
||||
if (existingEnv) {
|
||||
throw new BadRequestError({
|
||||
message: `Environment with slug '${slug}' already exist`,
|
||||
name: "CreateEnvironment"
|
||||
});
|
||||
}
|
||||
|
||||
const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
|
||||
const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
|
||||
const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
|
||||
|
||||
originalToNewEnvironmentId.set(environment.id, {
|
||||
envSlug: doc.slug,
|
||||
envId: doc.id,
|
||||
rootFolderId: folder.id,
|
||||
projectId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (data.folders) {
|
||||
for await (const folder of data.folders) {
|
||||
const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
|
||||
const parentFolder = originalToNewFolderId.get(folder.parentFolderId as string);
|
||||
|
||||
let newFolder: TSecretFolders;
|
||||
|
||||
if (parentEnv?.rootFolderId) {
|
||||
newFolder = await folderDAL.create(
|
||||
{
|
||||
name: folder.name,
|
||||
envId: parentEnv.envId,
|
||||
parentId: parentEnv.rootFolderId
|
||||
},
|
||||
tx
|
||||
);
|
||||
} else if (parentFolder) {
|
||||
newFolder = await folderDAL.create(
|
||||
{
|
||||
name: folder.name,
|
||||
envId: parentFolder.envId,
|
||||
parentId: parentFolder.folderId
|
||||
},
|
||||
tx
|
||||
);
|
||||
} else {
|
||||
logger.info({ folder }, "No parent environment found for folder");
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
const newFolderVersion = await folderVersionDAL.create(
|
||||
{
|
||||
name: newFolder.name,
|
||||
envId: newFolder.envId,
|
||||
version: newFolder.version,
|
||||
folderId: newFolder.id
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
await folderCommitService.createCommit(
|
||||
{
|
||||
actor: {
|
||||
type: actor,
|
||||
metadata: {
|
||||
id: actorId
|
||||
}
|
||||
},
|
||||
message: "Changed by external migration",
|
||||
folderId: parentEnv?.rootFolderId || parentFolder?.folderId || "",
|
||||
changes: [
|
||||
{
|
||||
type: CommitType.ADD,
|
||||
folderVersionId: newFolderVersion.id
|
||||
}
|
||||
]
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
originalToNewFolderId.set(folder.id, {
|
||||
folderId: newFolder.id,
|
||||
envId: parentEnv?.envId || parentFolder?.envId || "",
|
||||
envSlug: parentEnv?.envSlug || parentFolder?.envSlug || "",
|
||||
projectId: parentEnv?.projectId || parentFolder?.projectId || ""
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Useful for debugging:
|
||||
// console.log("data.secrets", data.secrets);
|
||||
// console.log("data.folders", data.folders);
|
||||
// console.log("data.environment", data.environments);
|
||||
|
||||
if (data.secrets && data.secrets.length > 0) {
|
||||
const mappedToEnvironmentId = new Map<
|
||||
string,
|
||||
{
|
||||
secretKey: string;
|
||||
secretValue: string;
|
||||
folderId?: string;
|
||||
isFromBlock?: boolean;
|
||||
}[]
|
||||
>();
|
||||
|
||||
for (const secret of data.secrets) {
|
||||
const targetId = secret.folderId || secret.environmentId;
|
||||
|
||||
// Skip if we can't find either an environment or folder mapping for this secret
|
||||
if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
|
||||
logger.info({ secret }, "[importDataIntoInfisicalFn]: Could not find environment or folder for secret");
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!mappedToEnvironmentId.has(targetId)) {
|
||||
mappedToEnvironmentId.set(targetId, []);
|
||||
}
|
||||
|
||||
const alreadyHasSecret = mappedToEnvironmentId
|
||||
.get(targetId)!
|
||||
.find((el) => el.secretKey === secret.name && el.folderId === secret.folderId);
|
||||
|
||||
if (alreadyHasSecret && alreadyHasSecret.isFromBlock) {
|
||||
// remove the existing secret if any
|
||||
mappedToEnvironmentId
|
||||
.get(targetId)!
|
||||
.splice(mappedToEnvironmentId.get(targetId)!.indexOf(alreadyHasSecret), 1);
|
||||
}
|
||||
mappedToEnvironmentId.get(targetId)!.push({
|
||||
secretKey: secret.name,
|
||||
secretValue: secret.value || "",
|
||||
folderId: secret.folderId,
|
||||
isFromBlock: secret.appBlockOrderIndex !== undefined
|
||||
});
|
||||
}
|
||||
|
||||
// for each of the mappedEnvironmentId
|
||||
for await (const [targetId, secrets] of mappedToEnvironmentId) {
|
||||
logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
|
||||
|
||||
let selectedFolder: TSecretFolders | undefined;
|
||||
let selectedProjectId: string | undefined;
|
||||
|
||||
// Case 1: Secret belongs to a folder / branch / branch of a block
|
||||
const foundFolder = originalToNewFolderId.get(targetId);
|
||||
if (foundFolder) {
|
||||
logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
|
||||
selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
|
||||
selectedProjectId = foundFolder.projectId;
|
||||
} else {
|
||||
logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
|
||||
const environment = data.environments.find((env) => env.id === targetId);
|
||||
if (!environment) {
|
||||
logger.info(
|
||||
{
|
||||
targetId
|
||||
},
|
||||
"[importDataIntoInfisicalFn]: Could not find environment for secret"
|
||||
);
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
const projectId = originalToNewProjectId.get(environment.projectId)!;
|
||||
|
||||
if (!projectId) {
|
||||
throw new BadRequestError({ message: `Failed to import secret, project not found` });
|
||||
}
|
||||
|
||||
const env = originalToNewEnvironmentId.get(targetId);
|
||||
if (!env) {
|
||||
logger.info(
|
||||
{
|
||||
targetId
|
||||
},
|
||||
"[importDataIntoInfisicalFn]: Could not find environment for secret"
|
||||
);
|
||||
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
|
||||
|
||||
if (!folder) {
|
||||
throw new NotFoundError({
|
||||
message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
|
||||
name: "Create secret"
|
||||
});
|
||||
}
|
||||
|
||||
selectedFolder = folder;
|
||||
selectedProjectId = projectId;
|
||||
}
|
||||
|
||||
if (!selectedFolder) {
|
||||
throw new NotFoundError({
|
||||
message: `Folder not found for the given environment slug & secret path`,
|
||||
name: "CreateSecret"
|
||||
});
|
||||
}
|
||||
|
||||
if (!selectedProjectId) {
|
||||
throw new NotFoundError({
|
||||
message: `Project not found for the given environment slug & secret path`,
|
||||
name: "CreateSecret"
|
||||
});
|
||||
}
|
||||
|
||||
const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
|
||||
{
|
||||
type: KmsDataKey.SecretManager,
|
||||
projectId: selectedProjectId
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
const secretBatches = chunkArray(secrets, 2500);
|
||||
for await (const secretBatch of secretBatches) {
|
||||
const secretsByKeys = await secretDAL.findBySecretKeys(
|
||||
selectedFolder.id,
|
||||
secretBatch.map((el) => ({
|
||||
key: el.secretKey,
|
||||
type: SecretType.Shared
|
||||
})),
|
||||
tx
|
||||
);
|
||||
if (secretsByKeys.length) {
|
||||
throw new BadRequestError({
|
||||
message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
|
||||
});
|
||||
}
|
||||
await fnSecretBulkInsert({
|
||||
inputSecrets: secretBatch.map((el) => {
|
||||
const references = getAllSecretReferences(el.secretValue).nestedReferences;
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
encryptedValue: el.secretValue
|
||||
? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
|
||||
: undefined,
|
||||
key: el.secretKey,
|
||||
references,
|
||||
type: SecretType.Shared
|
||||
};
|
||||
}),
|
||||
folderId: selectedFolder.id,
|
||||
orgId: actorOrgId,
|
||||
resourceMetadataDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
folderCommitService,
|
||||
actor: {
|
||||
type: actor,
|
||||
actorId
|
||||
},
|
||||
tx
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { projectsNotImported };
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./envkey";
|
||||
export * from "./import";
|
||||
export * from "./vault";
|
||||
@@ -0,0 +1,312 @@
|
||||
import axios, { AxiosInstance } from "axios";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
|
||||
|
||||
type VaultData = {
|
||||
namespace: string;
|
||||
mount: string;
|
||||
path: string;
|
||||
secretData: Record<string, string>;
|
||||
};
|
||||
|
||||
const vaultFactory = () => {
|
||||
const getMounts = async (request: AxiosInstance) => {
|
||||
const response = await request.get<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
accessor: string;
|
||||
options: {
|
||||
version?: string;
|
||||
} | null;
|
||||
type: string;
|
||||
}
|
||||
>
|
||||
>("/v1/sys/mounts");
|
||||
return response.data;
|
||||
};
|
||||
|
||||
const getPaths = async (
|
||||
request: AxiosInstance,
|
||||
{ mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
|
||||
) => {
|
||||
try {
|
||||
// For KV v2: /v1/{mount}/metadata/{path}?list=true
|
||||
const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
|
||||
const response = await request.get<{
|
||||
data: {
|
||||
keys: string[];
|
||||
};
|
||||
}>(`/v1/${path}?list=true`);
|
||||
|
||||
return response.data.data.keys;
|
||||
} catch (err) {
|
||||
if (axios.isAxiosError(err) && err.response?.status === 404) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const getSecrets = async (
|
||||
request: AxiosInstance,
|
||||
{ mountPath, secretPath }: { mountPath: string; secretPath: string }
|
||||
) => {
|
||||
// For KV v2: /v1/{mount}/data/{path}
|
||||
const response = await request.get<{
|
||||
data: {
|
||||
data: Record<string, string>; // KV v2 has nested data structure
|
||||
metadata: {
|
||||
created_time: string;
|
||||
deletion_time: string;
|
||||
destroyed: boolean;
|
||||
version: number;
|
||||
};
|
||||
};
|
||||
}>(`/v1/${mountPath}/data/${secretPath}`);
|
||||
|
||||
return response.data.data.data;
|
||||
};
|
||||
|
||||
// helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
|
||||
// const isKvV2Mount = (mountInfo: { type: string; options?: { version?: string } | null }) => {
|
||||
// return mountInfo.type === "kv" && mountInfo.options?.version === "2";
|
||||
// };
|
||||
|
||||
const recursivelyGetAllPaths = async (
|
||||
request: AxiosInstance,
|
||||
mountPath: string,
|
||||
currentPath: string = ""
|
||||
): Promise<string[]> => {
|
||||
const paths = await getPaths(request, { mountPath, secretPath: currentPath });
|
||||
|
||||
if (paths === null || paths.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const allSecrets: string[] = [];
|
||||
|
||||
for await (const path of paths) {
|
||||
const cleanPath = path.endsWith("/") ? path.slice(0, -1) : path;
|
||||
const fullItemPath = currentPath ? `${currentPath}/${cleanPath}` : cleanPath;
|
||||
|
||||
if (path.endsWith("/")) {
|
||||
// it's a folder so we recurse into it
|
||||
const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
|
||||
allSecrets.push(...subSecrets);
|
||||
} else {
|
||||
// it's a secret so we add it to our results
|
||||
allSecrets.push(`${mountPath}/${fullItemPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
return allSecrets;
|
||||
};
|
||||
|
||||
async function collectVaultData({
|
||||
baseUrl,
|
||||
namespace,
|
||||
accessToken
|
||||
}: {
|
||||
baseUrl: string;
|
||||
namespace: string;
|
||||
accessToken: string;
|
||||
}): Promise<VaultData[]> {
|
||||
const request = axios.create({
|
||||
baseURL: baseUrl,
|
||||
headers: {
|
||||
"X-Vault-Namespace": namespace,
|
||||
"X-Vault-Token": accessToken
|
||||
}
|
||||
});
|
||||
|
||||
const allData: VaultData[] = [];
|
||||
|
||||
// Get all mounts in this namespace
|
||||
const mounts = await getMounts(request);
|
||||
|
||||
for (const mount of Object.keys(mounts)) {
|
||||
if (!mount.endsWith("/")) {
|
||||
delete mounts[mount];
|
||||
}
|
||||
}
|
||||
|
||||
for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
|
||||
// skip non-KV mounts
|
||||
if (!mountInfo.type.startsWith("kv")) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
// get all paths in this mount
|
||||
const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
|
||||
|
||||
const cleanMountPath = mountPath.replace(/\/$/, "");
|
||||
|
||||
for await (const secretPath of paths) {
|
||||
// get the actual secret data
|
||||
const secretData = await getSecrets(request, {
|
||||
mountPath: cleanMountPath,
|
||||
secretPath: secretPath.replace(`${cleanMountPath}/`, "")
|
||||
});
|
||||
|
||||
allData.push({
|
||||
namespace,
|
||||
mount: mountPath.replace(/\/$/, ""),
|
||||
path: secretPath.replace(`${cleanMountPath}/`, ""),
|
||||
secretData
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return allData;
|
||||
}
|
||||
|
||||
return {
|
||||
collectVaultData,
|
||||
getMounts,
|
||||
getPaths,
|
||||
getSecrets,
|
||||
recursivelyGetAllPaths
|
||||
};
|
||||
};
|
||||
|
||||
export const transformToInfisicalFormatNamespaceToProjects = (
|
||||
vaultData: VaultData[],
|
||||
mappingType: VaultMappingType
|
||||
): InfisicalImportData => {
|
||||
const projects: Array<{ name: string; id: string }> = [];
|
||||
const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = [];
|
||||
const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = [];
|
||||
const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = [];
|
||||
|
||||
// track created entities to avoid duplicates
|
||||
const projectMap = new Map<string, string>(); // namespace -> projectId
|
||||
const environmentMap = new Map<string, string>(); // namespace:mount -> environmentId
|
||||
const folderMap = new Map<string, string>(); // namespace:mount:folderPath -> folderId
|
||||
|
||||
let environmentId: string = "";
|
||||
for (const data of vaultData) {
|
||||
const { namespace, mount, path, secretData } = data;
|
||||
|
||||
if (mappingType === "namespace") {
|
||||
// create project (namespace)
|
||||
if (!projectMap.has(namespace)) {
|
||||
const projectId = uuidv4();
|
||||
projectMap.set(namespace, projectId);
|
||||
projects.push({
|
||||
name: namespace,
|
||||
id: projectId
|
||||
});
|
||||
}
|
||||
const projectId = projectMap.get(namespace)!;
|
||||
|
||||
// create environment (mount)
|
||||
const envKey = `${namespace}:${mount}`;
|
||||
if (!environmentMap.has(envKey)) {
|
||||
environmentId = uuidv4();
|
||||
environmentMap.set(envKey, environmentId);
|
||||
environments.push({
|
||||
name: mount,
|
||||
id: environmentId,
|
||||
projectId
|
||||
});
|
||||
}
|
||||
environmentId = environmentMap.get(envKey)!;
|
||||
} else if (mappingType === "key-vault") {
|
||||
if (!projectMap.has(mount)) {
|
||||
const projectId = uuidv4();
|
||||
projectMap.set(mount, projectId);
|
||||
projects.push({
|
||||
name: mount,
|
||||
id: projectId
|
||||
});
|
||||
}
|
||||
const projectId = projectMap.get(mount)!;
|
||||
|
||||
// create single "Production" environment per project, because we have no good way of determining environments from vault
|
||||
if (!environmentMap.has(mount)) {
|
||||
environmentId = uuidv4();
|
||||
environmentMap.set(mount, environmentId);
|
||||
environments.push({
|
||||
name: "Production",
|
||||
id: environmentId,
|
||||
projectId
|
||||
});
|
||||
}
|
||||
environmentId = environmentMap.get(mount)!;
|
||||
}
|
||||
|
||||
// create folder structure
|
||||
let currentFolderId: string | undefined;
|
||||
let currentPath = "";
|
||||
|
||||
if (path.includes("/")) {
|
||||
const pathParts = path.split("/").filter(Boolean);
|
||||
|
||||
const folderParts = pathParts;
|
||||
|
||||
// create nested folder structure for the entire path
|
||||
for (const folderName of folderParts) {
|
||||
currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
|
||||
const folderKey = `${namespace}:${mount}:${currentPath}`;
|
||||
|
||||
if (!folderMap.has(folderKey)) {
|
||||
const folderId = uuidv4();
|
||||
folderMap.set(folderKey, folderId);
|
||||
folders.push({
|
||||
id: folderId,
|
||||
name: folderName,
|
||||
environmentId,
|
||||
parentFolderId: currentFolderId || environmentId
|
||||
});
|
||||
currentFolderId = folderId;
|
||||
} else {
|
||||
currentFolderId = folderMap.get(folderKey)!;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(secretData)) {
|
||||
secrets.push({
|
||||
id: uuidv4(),
|
||||
name: key,
|
||||
environmentId,
|
||||
value: String(value),
|
||||
folderId: currentFolderId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
projects,
|
||||
environments,
|
||||
folders,
|
||||
secrets
|
||||
};
|
||||
};
|
||||
|
||||
export const importVaultDataFn = async ({
|
||||
vaultAccessToken,
|
||||
vaultNamespace,
|
||||
vaultUrl,
|
||||
mappingType
|
||||
}: {
|
||||
vaultAccessToken: string;
|
||||
vaultNamespace: string;
|
||||
vaultUrl: string;
|
||||
mappingType: VaultMappingType;
|
||||
}) => {
|
||||
const vaultApi = vaultFactory();
|
||||
|
||||
const vaultData = await vaultApi.collectVaultData({
|
||||
accessToken: vaultAccessToken,
|
||||
baseUrl: vaultUrl,
|
||||
namespace: vaultNamespace
|
||||
});
|
||||
|
||||
const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);
|
||||
|
||||
return infisicalData;
|
||||
};
|
||||
Reference in New Issue
Block a user