feat: redis-based external imports

This commit is contained in:
Daniel Hougaard
2024-10-11 11:11:34 +04:00
parent 09d28156f8
commit 162005d72f
14 changed files with 338 additions and 110 deletions

View File

@@ -1,9 +1,15 @@
import { ForbiddenError, subject } from "@casl/ability";
import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas";
import {
TableName,
TSecretSnapshotFolders,
TSecretSnapshotSecretsV2,
TSecretTagJunctionInsert,
TSecretV2TagJunctionInsert
} from "@app/db/schemas";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { InternalServerError, NotFoundError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { chunkArray, groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
@@ -240,22 +246,37 @@ export const secretSnapshotServiceFactory = ({
},
tx
);
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
secretVersions.map(({ id }) => ({
secretVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions.map(({ id }) => ({
folderVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500);
const chunkedSnapshotFolders = chunkArray(folderVersions, 2500);
const snapshotSecrets: TSecretSnapshotSecretsV2[] = [];
const snapshotFolders: TSecretSnapshotFolders[] = [];
for await (const chunk of chunkedSnapshotSecrets) {
const result = await snapshotSecretV2BridgeDAL.insertMany(
chunk.map(({ id }) => ({
secretVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
snapshotSecrets.push(...result);
}
for await (const chunk of chunkedSnapshotFolders) {
const result = await snapshotFolderDAL.insertMany(
chunk.map(({ id }) => ({
folderVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
snapshotFolders.push(...result);
}
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
});

View File

@@ -70,3 +70,14 @@ export const objectify = <T, Key extends string | number | symbol, Value = T>(
{} as Record<Key, Value>
);
};
/**
* Chunks an array into smaller arrays of the given size.
*/
export const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
const chunks: T[][] = [];
for (let i = 0; i < array.length; i += chunkSize) {
chunks.push(array.slice(i, i + chunkSize));
}
return chunks;
};

View File

@@ -1,7 +1,7 @@
import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
import Redis from "ioredis";
import { SecretKeyEncoding } from "@app/db/schemas";
import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
import {
TScanFullRepoEventPayload,
@@ -32,7 +32,8 @@ export enum QueueName {
SecretReplication = "secret-replication",
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
ProjectV3Migration = "project-v3-migration",
AccessTokenStatusUpdate = "access-token-status-update"
AccessTokenStatusUpdate = "access-token-status-update",
ImportSecretsFromExternalSource = "import-secrets-from-external-source"
}
export enum QueueJobs {
@@ -56,7 +57,8 @@ export enum QueueJobs {
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
ProjectV3Migration = "project-v3-migration",
IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
ServiceTokenStatusUpdate = "service-token-status-update"
ServiceTokenStatusUpdate = "service-token-status-update",
ImportSecretsFromExternalSource = "import-secrets-from-external-source"
}
export type TQueueJobTypes = {
@@ -166,6 +168,19 @@ export type TQueueJobTypes = {
name: QueueJobs.ProjectV3Migration;
payload: { projectId: string };
};
[QueueName.ImportSecretsFromExternalSource]: {
name: QueueJobs.ImportSecretsFromExternalSource;
payload: {
actorEmail: string;
data: {
iv: string;
tag: string;
ciphertext: string;
algorithm: SecretEncryptionAlgo;
encoding: SecretKeyEncoding;
};
};
};
};
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

View File

@@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat
import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue";
import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
@@ -1202,12 +1203,19 @@ export const registerRoutes = async (
permissionService
});
const migrationService = externalMigrationServiceFactory({
projectService,
const externalMigrationQueue = externalMigrationQueueFactory({
orgService,
projectEnvService,
permissionService,
secretService
projectService,
smtpService,
queueService,
secretV2BridgeService
});
const migrationService = externalMigrationServiceFactory({
externalMigrationQueue,
userDAL,
permissionService
});
await superAdminService.initServerCfg();

View File

@@ -4,22 +4,23 @@ import sjcl from "sjcl";
import tweetnacl from "tweetnacl";
import tweetnaclUtil from "tweetnacl-util";
import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas";
import { BadRequestError } from "@app/lib/errors";
import { chunkArray } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TOrgServiceFactory } from "../org/org-service";
import { TProjectServiceFactory } from "../project/project-service";
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
import { TSecretServiceFactory } from "../secret/secret-service";
import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
export type TImportDataIntoInfisicalDTO = {
projectService: TProjectServiceFactory;
orgService: TOrgServiceFactory;
projectEnvService: TProjectEnvServiceFactory;
secretService: TSecretServiceFactory;
projectService: Pick<TProjectServiceFactory, "createProject">;
orgService: Pick<TOrgServiceFactory, "inviteUserToOrganization">;
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
input: TImportInfisicalDataCreate;
};
@@ -46,13 +47,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
const infisicalImportData: InfisicalImportData = {
projects: new Map<string, { name: string; id: string }>(),
environments: new Map<string, { name: string; id: string; projectId: string }>(),
secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
projects: [],
environments: [],
secrets: []
};
parsedJson.apps.forEach((app: { name: string; id: string }) => {
infisicalImportData.projects.set(app.id, { name: app.name, id: app.id });
infisicalImportData.projects.push({ name: app.name, id: app.id });
});
// string to string map for env templates
@@ -63,7 +64,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
// environments
for (const env of parsedJson.baseEnvironments) {
infisicalImportData.environments?.set(env.id, {
infisicalImportData.environments.push({
id: env.id,
name: envTemplates.get(env.environmentRoleId)!,
projectId: env.envParentId
@@ -75,9 +76,8 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
if (!env.includes("|")) {
const envData = parsedJson.envs[env];
for (const secret of Object.keys(envData.variables)) {
const id = randomUUID();
infisicalImportData.secrets?.set(id, {
id,
infisicalImportData.secrets.push({
id: randomUUID(),
name: secret,
environmentId: env,
value: envData.variables[secret].val
@@ -93,7 +93,7 @@ export const importDataIntoInfisicalFn = async ({
projectService,
orgService,
projectEnvService,
secretService,
secretV2BridgeService,
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
}: TImportDataIntoInfisicalDTO) => {
// Import data to infisical
@@ -104,7 +104,7 @@ export const importDataIntoInfisicalFn = async ({
const originalToNewProjectId = new Map<string, string>();
const originalToNewEnvironmentId = new Map<string, string>();
for await (const [id, project] of data.projects) {
for await (const project of data.projects) {
const newProject = await projectService
.createProject({
actor,
@@ -115,7 +115,7 @@ export const importDataIntoInfisicalFn = async ({
createDefaultEnvs: false
})
.catch(() => {
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` });
});
originalToNewProjectId.set(project.id, newProject.id);
@@ -141,7 +141,7 @@ export const importDataIntoInfisicalFn = async ({
// Import environments
if (data.environments) {
for await (const [id, environment] of data.environments) {
for await (const environment of data.environments) {
try {
const newEnvironment = await projectEnvService.createEnvironment({
actor,
@@ -154,12 +154,12 @@ export const importDataIntoInfisicalFn = async ({
});
if (!newEnvironment) {
logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
logger.error(`Failed to import environment: [name:${environment.name}]`);
throw new BadRequestError({
message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
message: `Failed to import environment: [name:${environment.name}]`
});
}
originalToNewEnvironmentId.set(id, newEnvironment.slug);
originalToNewEnvironmentId.set(environment.id, newEnvironment.slug);
} catch (error) {
throw new BadRequestError({
message: `Failed to import environment: ${environment.name}]`,
@@ -169,28 +169,47 @@ export const importDataIntoInfisicalFn = async ({
}
}
// Import secrets
if (data.secrets) {
for await (const [id, secret] of data.secrets) {
const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
if (!dataProjectId) {
throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
if (data.secrets && data.secrets.length > 0) {
const mappedToEnvironmentId = new Map<
string,
{
secretKey: string;
secretValue: string;
}[]
>();
for (const secret of data.secrets) {
if (!mappedToEnvironmentId.has(secret.environmentId)) {
mappedToEnvironmentId.set(secret.environmentId, []);
}
const projectId = originalToNewProjectId.get(dataProjectId);
const newSecret = await secretService.createSecretRaw({
actorId,
actor,
actorOrgId,
environment: originalToNewEnvironmentId.get(secret.environmentId)!,
actorAuthMethod,
projectId: projectId!,
secretPath: "/",
secretName: secret.name,
type: SecretType.Shared,
mappedToEnvironmentId.get(secret.environmentId)!.push({
secretKey: secret.name,
secretValue: secret.value || ""
});
if (!newSecret) {
throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
}
// for each of the mappedEnvironmentId
for await (const [envId, secrets] of mappedToEnvironmentId) {
const environment = data.environments.find((env) => env.id === envId);
const projectId = environment?.projectId;
if (!projectId) {
throw new BadRequestError({ message: `Failed to import secret, project not found` });
}
const secretBatches = chunkArray(secrets, 2500);
for await (const secretBatch of secretBatches) {
await secretV2BridgeService.createManySecret({
actorId,
actor,
actorOrgId,
environment: originalToNewEnvironmentId.get(envId)!,
actorAuthMethod,
projectId: originalToNewProjectId.get(projectId)!,
secretPath: "/",
secrets: secretBatch
});
}
}
}

View File

@@ -0,0 +1,111 @@
import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TOrgServiceFactory } from "../org/org-service";
import { TProjectServiceFactory } from "../project/project-service";
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { importDataIntoInfisicalFn } from "./external-migration-fns";
import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
export type TExternalMigrationQueueFactoryDep = {
projectService: Pick<TProjectServiceFactory, "createProject">;
orgService: Pick<TOrgServiceFactory, "inviteUserToOrganization">;
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
smtpService: TSmtpService;
queueService: TQueueServiceFactory;
};
export type TExternalMigrationQueueFactory = ReturnType<typeof externalMigrationQueueFactory>;
export const externalMigrationQueueFactory = ({
queueService,
projectService,
orgService,
smtpService,
projectEnvService,
secretV2BridgeService
}: TExternalMigrationQueueFactoryDep) => {
const startImport = async (dto: {
actorEmail: string;
data: {
iv: string;
tag: string;
ciphertext: string;
algorithm: SecretEncryptionAlgo;
encoding: SecretKeyEncoding;
};
}) => {
await queueService.queue(
QueueName.ImportSecretsFromExternalSource,
QueueJobs.ImportSecretsFromExternalSource,
dto,
{
removeOnComplete: true,
removeOnFail: true
}
);
};
queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => {
try {
const { data, actorEmail } = job.data;
await smtpService.sendMail({
recipients: [actorEmail],
subjectLine: "Infisical import started",
substitutions: {
provider: ExternalPlatforms.EnvKey
},
template: SmtpTemplates.ExternalImportStarted
});
const decrypted = infisicalSymmetricDecrypt({
ciphertext: data.ciphertext,
iv: data.iv,
keyEncoding: data.encoding,
tag: data.tag
});
const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate;
await importDataIntoInfisicalFn({
input: decryptedJson,
projectService,
orgService,
projectEnvService,
secretV2BridgeService
});
await smtpService.sendMail({
recipients: [actorEmail],
subjectLine: "Infisical import successful",
substitutions: {
provider: ExternalPlatforms.EnvKey
},
template: SmtpTemplates.ExternalImportSuccessful
});
} catch (err) {
await smtpService.sendMail({
recipients: [job.data.actorEmail],
subjectLine: "Infisical import failed",
substitutions: {
provider: ExternalPlatforms.EnvKey,
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
error: (err as any)?.message || "Unknown error"
},
template: SmtpTemplates.ExternalImportFailed
});
logger.error(err, "Failed to import data from external source");
}
});
return {
startImport
};
};

View File

@@ -1,30 +1,25 @@
import { OrgMembershipRole } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { ForbiddenRequestError } from "@app/lib/errors";
import { TOrgServiceFactory } from "../org/org-service";
import { TProjectServiceFactory } from "../project/project-service";
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
import { TSecretServiceFactory } from "../secret/secret-service";
import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns";
import { TUserDALFactory } from "../user/user-dal";
import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
import { TExternalMigrationQueueFactory } from "./external-migration-queue";
import { TImportEnvKeyDataCreate } from "./external-migration-types";
type TExternalMigrationServiceFactoryDep = {
projectService: TProjectServiceFactory;
orgService: TOrgServiceFactory;
projectEnvService: TProjectEnvServiceFactory;
secretService: TSecretServiceFactory;
permissionService: TPermissionServiceFactory;
externalMigrationQueue: TExternalMigrationQueueFactory;
userDAL: Pick<TUserDALFactory, "findById">;
};
export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
export const externalMigrationServiceFactory = ({
projectService,
orgService,
projectEnvService,
permissionService,
secretService
externalMigrationQueue,
userDAL
}: TExternalMigrationServiceFactoryDep) => {
const importEnvKeyData = async ({
decryptionKey,
@@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({
actorAuthMethod,
actorOrgId
);
if (membership.role !== OrgMembershipRole.Admin) {
throw new ForbiddenRequestError({ message: "Only admins can import data" });
}
const user = await userDAL.findById(actorId);
const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
const envKeyData = await parseEnvKeyDataFn(json);
const response = await importDataIntoInfisicalFn({
input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod },
projectService,
orgService,
projectEnvService,
secretService
const stringifiedJson = JSON.stringify({
data: envKeyData,
actor,
actorId,
actorOrgId,
actorAuthMethod
});
const encrypted = infisicalSymmetricEncypt(stringifiedJson);
await externalMigrationQueue.startImport({
actorEmail: user.email!,
data: encrypted
});
return response;
};
return {

View File

@@ -1,26 +1,9 @@
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export type InfisicalImportData = {
projects: Map<string, { name: string; id: string }>;
environments?: Map<
string,
{
name: string;
id: string;
projectId: string;
}
>;
secrets?: Map<
string,
{
name: string;
id: string;
environmentId: string;
value?: string;
}
>;
projects: Array<{ name: string; id: string }>;
environments: Array<{ name: string; id: string; projectId: string }>;
secrets: Array<{ name: string; id: string; environmentId: string; value: string }>;
};
export type TImportEnvKeyDataCreate = {
@@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = {
}
>;
};
export enum ExternalPlatforms {
EnvKey = "EnvKey"
}

View File

@@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = {
secretTagDAL: TSecretTagDALFactory;
userDAL: Pick<TUserDALFactory, "findById">;
secretVersionTagDAL: TSecretVersionTagDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
kmsService: TKmsServiceFactory;
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;

View File

@@ -34,7 +34,10 @@ export enum SmtpTemplates {
WorkspaceInvite = "workspaceInvitation.handlebars",
ScimUserProvisioned = "scimUserProvisioned.handlebars",
PkiExpirationAlert = "pkiExpirationAlert.handlebars",
IntegrationSyncFailed = "integrationSyncFailed.handlebars"
IntegrationSyncFailed = "integrationSyncFailed.handlebars",
ExternalImportSuccessful = "externalImportSuccessful.handlebars",
ExternalImportFailed = "externalImportFailed.handlebars",
ExternalImportStarted = "externalImportStarted.handlebars"
}
export enum SmtpHost {

View File

@@ -0,0 +1,21 @@
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="x-ua-compatible" content="ie=edge" />
<title>Import failed</title>
</head>
<body>
<h2>An import from {{provider}} to Infisical has failed</h2>
<p>An import from
{{provider}}
to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you
can contact the Infisical team at team@infisical.com.
</p>
<p>Error: {{error}}</p>
</body>
</html>

View File

@@ -0,0 +1,17 @@
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="x-ua-compatible" content="ie=edge" />
<title>Import in progress</title>
</head>
<body>
<h2>An import from {{provider}} to Infisical is in progress</h2>
<p>An import from
{{provider}}
to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import
has finished or if it fails.</p>
</body>
</html>

View File

@@ -0,0 +1,14 @@
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="x-ua-compatible" content="ie=edge" />
<title>Import successful</title>
</head>
<body>
<h2>An import from {{provider}} to Infisical was successful</h2>
<p>An import from {{provider}} was successful. Your data is now available in Infisical.</p>
</body>
</html>

View File

@@ -18,11 +18,10 @@ export const useImportEnvKey = () => {
};
decryptionKey: string;
}) => {
const { data } = await apiRequest.post("/api/v3/migrate/env-key/", {
await apiRequest.post("/api/v3/migrate/env-key/", {
encryptedJson,
decryptionKey
});
return data;
},
onSuccess: () => {
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);