mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-07 20:27:43 +00:00
feat: redis-based external imports
This commit is contained in:
@@ -1,9 +1,15 @@
|
|||||||
import { ForbiddenError, subject } from "@casl/ability";
|
import { ForbiddenError, subject } from "@casl/ability";
|
||||||
|
|
||||||
import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas";
|
import {
|
||||||
|
TableName,
|
||||||
|
TSecretSnapshotFolders,
|
||||||
|
TSecretSnapshotSecretsV2,
|
||||||
|
TSecretTagJunctionInsert,
|
||||||
|
TSecretV2TagJunctionInsert
|
||||||
|
} from "@app/db/schemas";
|
||||||
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
|
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
|
||||||
import { InternalServerError, NotFoundError } from "@app/lib/errors";
|
import { InternalServerError, NotFoundError } from "@app/lib/errors";
|
||||||
import { groupBy } from "@app/lib/fn";
|
import { chunkArray, groupBy } from "@app/lib/fn";
|
||||||
import { logger } from "@app/lib/logger";
|
import { logger } from "@app/lib/logger";
|
||||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||||
import { KmsDataKey } from "@app/services/kms/kms-types";
|
import { KmsDataKey } from "@app/services/kms/kms-types";
|
||||||
@@ -240,22 +246,37 @@ export const secretSnapshotServiceFactory = ({
|
|||||||
},
|
},
|
||||||
tx
|
tx
|
||||||
);
|
);
|
||||||
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
|
|
||||||
secretVersions.map(({ id }) => ({
|
const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500);
|
||||||
secretVersionId: id,
|
const chunkedSnapshotFolders = chunkArray(folderVersions, 2500);
|
||||||
envId: folder.environment.envId,
|
const snapshotSecrets: TSecretSnapshotSecretsV2[] = [];
|
||||||
snapshotId: newSnapshot.id
|
const snapshotFolders: TSecretSnapshotFolders[] = [];
|
||||||
})),
|
|
||||||
tx
|
for await (const chunk of chunkedSnapshotSecrets) {
|
||||||
);
|
const result = await snapshotSecretV2BridgeDAL.insertMany(
|
||||||
const snapshotFolders = await snapshotFolderDAL.insertMany(
|
chunk.map(({ id }) => ({
|
||||||
folderVersions.map(({ id }) => ({
|
secretVersionId: id,
|
||||||
folderVersionId: id,
|
envId: folder.environment.envId,
|
||||||
envId: folder.environment.envId,
|
snapshotId: newSnapshot.id
|
||||||
snapshotId: newSnapshot.id
|
})),
|
||||||
})),
|
tx
|
||||||
tx
|
);
|
||||||
);
|
|
||||||
|
snapshotSecrets.push(...result);
|
||||||
|
}
|
||||||
|
|
||||||
|
for await (const chunk of chunkedSnapshotFolders) {
|
||||||
|
const result = await snapshotFolderDAL.insertMany(
|
||||||
|
chunk.map(({ id }) => ({
|
||||||
|
folderVersionId: id,
|
||||||
|
envId: folder.environment.envId,
|
||||||
|
snapshotId: newSnapshot.id
|
||||||
|
})),
|
||||||
|
tx
|
||||||
|
);
|
||||||
|
|
||||||
|
snapshotFolders.push(...result);
|
||||||
|
}
|
||||||
|
|
||||||
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
|
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -70,3 +70,14 @@ export const objectify = <T, Key extends string | number | symbol, Value = T>(
|
|||||||
{} as Record<Key, Value>
|
{} as Record<Key, Value>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chunks an array into smaller arrays of the given size.
|
||||||
|
*/
|
||||||
|
export const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
|
||||||
|
const chunks: T[][] = [];
|
||||||
|
for (let i = 0; i < array.length; i += chunkSize) {
|
||||||
|
chunks.push(array.slice(i, i + chunkSize));
|
||||||
|
}
|
||||||
|
return chunks;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
|
import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
|
||||||
import Redis from "ioredis";
|
import Redis from "ioredis";
|
||||||
|
|
||||||
import { SecretKeyEncoding } from "@app/db/schemas";
|
import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
|
||||||
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
|
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
|
||||||
import {
|
import {
|
||||||
TScanFullRepoEventPayload,
|
TScanFullRepoEventPayload,
|
||||||
@@ -32,7 +32,8 @@ export enum QueueName {
|
|||||||
SecretReplication = "secret-replication",
|
SecretReplication = "secret-replication",
|
||||||
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
|
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
|
||||||
ProjectV3Migration = "project-v3-migration",
|
ProjectV3Migration = "project-v3-migration",
|
||||||
AccessTokenStatusUpdate = "access-token-status-update"
|
AccessTokenStatusUpdate = "access-token-status-update",
|
||||||
|
ImportSecretsFromExternalSource = "import-secrets-from-external-source"
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum QueueJobs {
|
export enum QueueJobs {
|
||||||
@@ -56,7 +57,8 @@ export enum QueueJobs {
|
|||||||
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
|
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
|
||||||
ProjectV3Migration = "project-v3-migration",
|
ProjectV3Migration = "project-v3-migration",
|
||||||
IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
|
IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
|
||||||
ServiceTokenStatusUpdate = "service-token-status-update"
|
ServiceTokenStatusUpdate = "service-token-status-update",
|
||||||
|
ImportSecretsFromExternalSource = "import-secrets-from-external-source"
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TQueueJobTypes = {
|
export type TQueueJobTypes = {
|
||||||
@@ -166,6 +168,19 @@ export type TQueueJobTypes = {
|
|||||||
name: QueueJobs.ProjectV3Migration;
|
name: QueueJobs.ProjectV3Migration;
|
||||||
payload: { projectId: string };
|
payload: { projectId: string };
|
||||||
};
|
};
|
||||||
|
[QueueName.ImportSecretsFromExternalSource]: {
|
||||||
|
name: QueueJobs.ImportSecretsFromExternalSource;
|
||||||
|
payload: {
|
||||||
|
actorEmail: string;
|
||||||
|
data: {
|
||||||
|
iv: string;
|
||||||
|
tag: string;
|
||||||
|
ciphertext: string;
|
||||||
|
algorithm: SecretEncryptionAlgo;
|
||||||
|
encoding: SecretKeyEncoding;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
|
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat
|
|||||||
import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
|
import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
|
||||||
import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
|
import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
|
||||||
import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
|
import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
|
||||||
|
import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue";
|
||||||
import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
|
import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
|
||||||
import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
|
import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
|
||||||
import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
|
import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
|
||||||
@@ -1202,12 +1203,19 @@ export const registerRoutes = async (
|
|||||||
permissionService
|
permissionService
|
||||||
});
|
});
|
||||||
|
|
||||||
const migrationService = externalMigrationServiceFactory({
|
const externalMigrationQueue = externalMigrationQueueFactory({
|
||||||
projectService,
|
|
||||||
orgService,
|
orgService,
|
||||||
projectEnvService,
|
projectEnvService,
|
||||||
permissionService,
|
projectService,
|
||||||
secretService
|
smtpService,
|
||||||
|
queueService,
|
||||||
|
secretV2BridgeService
|
||||||
|
});
|
||||||
|
|
||||||
|
const migrationService = externalMigrationServiceFactory({
|
||||||
|
externalMigrationQueue,
|
||||||
|
userDAL,
|
||||||
|
permissionService
|
||||||
});
|
});
|
||||||
|
|
||||||
await superAdminService.initServerCfg();
|
await superAdminService.initServerCfg();
|
||||||
|
|||||||
@@ -4,22 +4,23 @@ import sjcl from "sjcl";
|
|||||||
import tweetnacl from "tweetnacl";
|
import tweetnacl from "tweetnacl";
|
||||||
import tweetnaclUtil from "tweetnacl-util";
|
import tweetnaclUtil from "tweetnacl-util";
|
||||||
|
|
||||||
import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
|
import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas";
|
||||||
import { BadRequestError } from "@app/lib/errors";
|
import { BadRequestError } from "@app/lib/errors";
|
||||||
|
import { chunkArray } from "@app/lib/fn";
|
||||||
import { logger } from "@app/lib/logger";
|
import { logger } from "@app/lib/logger";
|
||||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||||
|
|
||||||
import { TOrgServiceFactory } from "../org/org-service";
|
import { TOrgServiceFactory } from "../org/org-service";
|
||||||
import { TProjectServiceFactory } from "../project/project-service";
|
import { TProjectServiceFactory } from "../project/project-service";
|
||||||
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
|
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
|
||||||
import { TSecretServiceFactory } from "../secret/secret-service";
|
import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
|
||||||
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
|
import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
|
||||||
|
|
||||||
export type TImportDataIntoInfisicalDTO = {
|
export type TImportDataIntoInfisicalDTO = {
|
||||||
projectService: TProjectServiceFactory;
|
projectService: Pick<TProjectServiceFactory, "createProject">;
|
||||||
orgService: TOrgServiceFactory;
|
orgService: Pick<TOrgServiceFactory, "inviteUserToOrganization">;
|
||||||
projectEnvService: TProjectEnvServiceFactory;
|
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
|
||||||
secretService: TSecretServiceFactory;
|
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
|
||||||
|
|
||||||
input: TImportInfisicalDataCreate;
|
input: TImportInfisicalDataCreate;
|
||||||
};
|
};
|
||||||
@@ -46,13 +47,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
|
|||||||
const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
|
const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
|
||||||
|
|
||||||
const infisicalImportData: InfisicalImportData = {
|
const infisicalImportData: InfisicalImportData = {
|
||||||
projects: new Map<string, { name: string; id: string }>(),
|
projects: [],
|
||||||
environments: new Map<string, { name: string; id: string; projectId: string }>(),
|
environments: [],
|
||||||
secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
|
secrets: []
|
||||||
};
|
};
|
||||||
|
|
||||||
parsedJson.apps.forEach((app: { name: string; id: string }) => {
|
parsedJson.apps.forEach((app: { name: string; id: string }) => {
|
||||||
infisicalImportData.projects.set(app.id, { name: app.name, id: app.id });
|
infisicalImportData.projects.push({ name: app.name, id: app.id });
|
||||||
});
|
});
|
||||||
|
|
||||||
// string to string map for env templates
|
// string to string map for env templates
|
||||||
@@ -63,7 +64,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
|
|||||||
|
|
||||||
// environments
|
// environments
|
||||||
for (const env of parsedJson.baseEnvironments) {
|
for (const env of parsedJson.baseEnvironments) {
|
||||||
infisicalImportData.environments?.set(env.id, {
|
infisicalImportData.environments.push({
|
||||||
id: env.id,
|
id: env.id,
|
||||||
name: envTemplates.get(env.environmentRoleId)!,
|
name: envTemplates.get(env.environmentRoleId)!,
|
||||||
projectId: env.envParentId
|
projectId: env.envParentId
|
||||||
@@ -75,9 +76,8 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
|
|||||||
if (!env.includes("|")) {
|
if (!env.includes("|")) {
|
||||||
const envData = parsedJson.envs[env];
|
const envData = parsedJson.envs[env];
|
||||||
for (const secret of Object.keys(envData.variables)) {
|
for (const secret of Object.keys(envData.variables)) {
|
||||||
const id = randomUUID();
|
infisicalImportData.secrets.push({
|
||||||
infisicalImportData.secrets?.set(id, {
|
id: randomUUID(),
|
||||||
id,
|
|
||||||
name: secret,
|
name: secret,
|
||||||
environmentId: env,
|
environmentId: env,
|
||||||
value: envData.variables[secret].val
|
value: envData.variables[secret].val
|
||||||
@@ -93,7 +93,7 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
projectService,
|
projectService,
|
||||||
orgService,
|
orgService,
|
||||||
projectEnvService,
|
projectEnvService,
|
||||||
secretService,
|
secretV2BridgeService,
|
||||||
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
|
input: { data, actor, actorId, actorOrgId, actorAuthMethod }
|
||||||
}: TImportDataIntoInfisicalDTO) => {
|
}: TImportDataIntoInfisicalDTO) => {
|
||||||
// Import data to infisical
|
// Import data to infisical
|
||||||
@@ -104,7 +104,7 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
const originalToNewProjectId = new Map<string, string>();
|
const originalToNewProjectId = new Map<string, string>();
|
||||||
const originalToNewEnvironmentId = new Map<string, string>();
|
const originalToNewEnvironmentId = new Map<string, string>();
|
||||||
|
|
||||||
for await (const [id, project] of data.projects) {
|
for await (const project of data.projects) {
|
||||||
const newProject = await projectService
|
const newProject = await projectService
|
||||||
.createProject({
|
.createProject({
|
||||||
actor,
|
actor,
|
||||||
@@ -115,7 +115,7 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
createDefaultEnvs: false
|
createDefaultEnvs: false
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
|
throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` });
|
||||||
});
|
});
|
||||||
|
|
||||||
originalToNewProjectId.set(project.id, newProject.id);
|
originalToNewProjectId.set(project.id, newProject.id);
|
||||||
@@ -141,7 +141,7 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
|
|
||||||
// Import environments
|
// Import environments
|
||||||
if (data.environments) {
|
if (data.environments) {
|
||||||
for await (const [id, environment] of data.environments) {
|
for await (const environment of data.environments) {
|
||||||
try {
|
try {
|
||||||
const newEnvironment = await projectEnvService.createEnvironment({
|
const newEnvironment = await projectEnvService.createEnvironment({
|
||||||
actor,
|
actor,
|
||||||
@@ -154,12 +154,12 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!newEnvironment) {
|
if (!newEnvironment) {
|
||||||
logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
|
logger.error(`Failed to import environment: [name:${environment.name}]`);
|
||||||
throw new BadRequestError({
|
throw new BadRequestError({
|
||||||
message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
|
message: `Failed to import environment: [name:${environment.name}]`
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
originalToNewEnvironmentId.set(id, newEnvironment.slug);
|
originalToNewEnvironmentId.set(environment.id, newEnvironment.slug);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new BadRequestError({
|
throw new BadRequestError({
|
||||||
message: `Failed to import environment: ${environment.name}]`,
|
message: `Failed to import environment: ${environment.name}]`,
|
||||||
@@ -169,28 +169,47 @@ export const importDataIntoInfisicalFn = async ({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Import secrets
|
if (data.secrets && data.secrets.length > 0) {
|
||||||
if (data.secrets) {
|
const mappedToEnvironmentId = new Map<
|
||||||
for await (const [id, secret] of data.secrets) {
|
string,
|
||||||
const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
|
{
|
||||||
if (!dataProjectId) {
|
secretKey: string;
|
||||||
throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
|
secretValue: string;
|
||||||
|
}[]
|
||||||
|
>();
|
||||||
|
|
||||||
|
for (const secret of data.secrets) {
|
||||||
|
if (!mappedToEnvironmentId.has(secret.environmentId)) {
|
||||||
|
mappedToEnvironmentId.set(secret.environmentId, []);
|
||||||
}
|
}
|
||||||
const projectId = originalToNewProjectId.get(dataProjectId);
|
mappedToEnvironmentId.get(secret.environmentId)!.push({
|
||||||
const newSecret = await secretService.createSecretRaw({
|
secretKey: secret.name,
|
||||||
actorId,
|
|
||||||
actor,
|
|
||||||
actorOrgId,
|
|
||||||
environment: originalToNewEnvironmentId.get(secret.environmentId)!,
|
|
||||||
actorAuthMethod,
|
|
||||||
projectId: projectId!,
|
|
||||||
secretPath: "/",
|
|
||||||
secretName: secret.name,
|
|
||||||
type: SecretType.Shared,
|
|
||||||
secretValue: secret.value || ""
|
secretValue: secret.value || ""
|
||||||
});
|
});
|
||||||
if (!newSecret) {
|
}
|
||||||
throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
|
|
||||||
|
// for each of the mappedEnvironmentId
|
||||||
|
for await (const [envId, secrets] of mappedToEnvironmentId) {
|
||||||
|
const environment = data.environments.find((env) => env.id === envId);
|
||||||
|
const projectId = environment?.projectId;
|
||||||
|
|
||||||
|
if (!projectId) {
|
||||||
|
throw new BadRequestError({ message: `Failed to import secret, project not found` });
|
||||||
|
}
|
||||||
|
|
||||||
|
const secretBatches = chunkArray(secrets, 2500);
|
||||||
|
|
||||||
|
for await (const secretBatch of secretBatches) {
|
||||||
|
await secretV2BridgeService.createManySecret({
|
||||||
|
actorId,
|
||||||
|
actor,
|
||||||
|
actorOrgId,
|
||||||
|
environment: originalToNewEnvironmentId.get(envId)!,
|
||||||
|
actorAuthMethod,
|
||||||
|
projectId: originalToNewProjectId.get(projectId)!,
|
||||||
|
secretPath: "/",
|
||||||
|
secrets: secretBatch
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
|
||||||
|
import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
|
||||||
|
import { logger } from "@app/lib/logger";
|
||||||
|
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||||
|
|
||||||
|
import { TOrgServiceFactory } from "../org/org-service";
|
||||||
|
import { TProjectServiceFactory } from "../project/project-service";
|
||||||
|
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
|
||||||
|
import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
|
||||||
|
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
|
||||||
|
import { importDataIntoInfisicalFn } from "./external-migration-fns";
|
||||||
|
import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
|
||||||
|
|
||||||
|
export type TExternalMigrationQueueFactoryDep = {
|
||||||
|
projectService: Pick<TProjectServiceFactory, "createProject">;
|
||||||
|
orgService: Pick<TOrgServiceFactory, "inviteUserToOrganization">;
|
||||||
|
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
|
||||||
|
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
|
||||||
|
|
||||||
|
smtpService: TSmtpService;
|
||||||
|
queueService: TQueueServiceFactory;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type TExternalMigrationQueueFactory = ReturnType<typeof externalMigrationQueueFactory>;
|
||||||
|
|
||||||
|
export const externalMigrationQueueFactory = ({
|
||||||
|
queueService,
|
||||||
|
projectService,
|
||||||
|
orgService,
|
||||||
|
smtpService,
|
||||||
|
projectEnvService,
|
||||||
|
secretV2BridgeService
|
||||||
|
}: TExternalMigrationQueueFactoryDep) => {
|
||||||
|
const startImport = async (dto: {
|
||||||
|
actorEmail: string;
|
||||||
|
data: {
|
||||||
|
iv: string;
|
||||||
|
tag: string;
|
||||||
|
ciphertext: string;
|
||||||
|
algorithm: SecretEncryptionAlgo;
|
||||||
|
encoding: SecretKeyEncoding;
|
||||||
|
};
|
||||||
|
}) => {
|
||||||
|
await queueService.queue(
|
||||||
|
QueueName.ImportSecretsFromExternalSource,
|
||||||
|
QueueJobs.ImportSecretsFromExternalSource,
|
||||||
|
dto,
|
||||||
|
{
|
||||||
|
removeOnComplete: true,
|
||||||
|
removeOnFail: true
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => {
|
||||||
|
try {
|
||||||
|
const { data, actorEmail } = job.data;
|
||||||
|
|
||||||
|
await smtpService.sendMail({
|
||||||
|
recipients: [actorEmail],
|
||||||
|
subjectLine: "Infisical import started",
|
||||||
|
substitutions: {
|
||||||
|
provider: ExternalPlatforms.EnvKey
|
||||||
|
},
|
||||||
|
template: SmtpTemplates.ExternalImportStarted
|
||||||
|
});
|
||||||
|
|
||||||
|
const decrypted = infisicalSymmetricDecrypt({
|
||||||
|
ciphertext: data.ciphertext,
|
||||||
|
iv: data.iv,
|
||||||
|
keyEncoding: data.encoding,
|
||||||
|
tag: data.tag
|
||||||
|
});
|
||||||
|
|
||||||
|
const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate;
|
||||||
|
|
||||||
|
await importDataIntoInfisicalFn({
|
||||||
|
input: decryptedJson,
|
||||||
|
projectService,
|
||||||
|
orgService,
|
||||||
|
projectEnvService,
|
||||||
|
secretV2BridgeService
|
||||||
|
});
|
||||||
|
|
||||||
|
await smtpService.sendMail({
|
||||||
|
recipients: [actorEmail],
|
||||||
|
subjectLine: "Infisical import successful",
|
||||||
|
substitutions: {
|
||||||
|
provider: ExternalPlatforms.EnvKey
|
||||||
|
},
|
||||||
|
template: SmtpTemplates.ExternalImportSuccessful
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
await smtpService.sendMail({
|
||||||
|
recipients: [job.data.actorEmail],
|
||||||
|
subjectLine: "Infisical import failed",
|
||||||
|
substitutions: {
|
||||||
|
provider: ExternalPlatforms.EnvKey,
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
|
||||||
|
error: (err as any)?.message || "Unknown error"
|
||||||
|
},
|
||||||
|
template: SmtpTemplates.ExternalImportFailed
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.error(err, "Failed to import data from external source");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
startImport
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -1,30 +1,25 @@
|
|||||||
import { OrgMembershipRole } from "@app/db/schemas";
|
import { OrgMembershipRole } from "@app/db/schemas";
|
||||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||||
|
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
|
||||||
import { ForbiddenRequestError } from "@app/lib/errors";
|
import { ForbiddenRequestError } from "@app/lib/errors";
|
||||||
|
|
||||||
import { TOrgServiceFactory } from "../org/org-service";
|
import { TUserDALFactory } from "../user/user-dal";
|
||||||
import { TProjectServiceFactory } from "../project/project-service";
|
import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
|
||||||
import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
|
import { TExternalMigrationQueueFactory } from "./external-migration-queue";
|
||||||
import { TSecretServiceFactory } from "../secret/secret-service";
|
|
||||||
import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns";
|
|
||||||
import { TImportEnvKeyDataCreate } from "./external-migration-types";
|
import { TImportEnvKeyDataCreate } from "./external-migration-types";
|
||||||
|
|
||||||
type TExternalMigrationServiceFactoryDep = {
|
type TExternalMigrationServiceFactoryDep = {
|
||||||
projectService: TProjectServiceFactory;
|
|
||||||
orgService: TOrgServiceFactory;
|
|
||||||
projectEnvService: TProjectEnvServiceFactory;
|
|
||||||
secretService: TSecretServiceFactory;
|
|
||||||
permissionService: TPermissionServiceFactory;
|
permissionService: TPermissionServiceFactory;
|
||||||
|
externalMigrationQueue: TExternalMigrationQueueFactory;
|
||||||
|
userDAL: Pick<TUserDALFactory, "findById">;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
|
export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
|
||||||
|
|
||||||
export const externalMigrationServiceFactory = ({
|
export const externalMigrationServiceFactory = ({
|
||||||
projectService,
|
|
||||||
orgService,
|
|
||||||
projectEnvService,
|
|
||||||
permissionService,
|
permissionService,
|
||||||
secretService
|
externalMigrationQueue,
|
||||||
|
userDAL
|
||||||
}: TExternalMigrationServiceFactoryDep) => {
|
}: TExternalMigrationServiceFactoryDep) => {
|
||||||
const importEnvKeyData = async ({
|
const importEnvKeyData = async ({
|
||||||
decryptionKey,
|
decryptionKey,
|
||||||
@@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({
|
|||||||
actorAuthMethod,
|
actorAuthMethod,
|
||||||
actorOrgId
|
actorOrgId
|
||||||
);
|
);
|
||||||
|
|
||||||
if (membership.role !== OrgMembershipRole.Admin) {
|
if (membership.role !== OrgMembershipRole.Admin) {
|
||||||
throw new ForbiddenRequestError({ message: "Only admins can import data" });
|
throw new ForbiddenRequestError({ message: "Only admins can import data" });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const user = await userDAL.findById(actorId);
|
||||||
const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
|
const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
|
||||||
const envKeyData = await parseEnvKeyDataFn(json);
|
const envKeyData = await parseEnvKeyDataFn(json);
|
||||||
const response = await importDataIntoInfisicalFn({
|
|
||||||
input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod },
|
const stringifiedJson = JSON.stringify({
|
||||||
projectService,
|
data: envKeyData,
|
||||||
orgService,
|
actor,
|
||||||
projectEnvService,
|
actorId,
|
||||||
secretService
|
actorOrgId,
|
||||||
|
actorAuthMethod
|
||||||
|
});
|
||||||
|
|
||||||
|
const encrypted = infisicalSymmetricEncypt(stringifiedJson);
|
||||||
|
|
||||||
|
await externalMigrationQueue.startImport({
|
||||||
|
actorEmail: user.email!,
|
||||||
|
data: encrypted
|
||||||
});
|
});
|
||||||
return response;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,26 +1,9 @@
|
|||||||
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
|
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
|
||||||
|
|
||||||
export type InfisicalImportData = {
|
export type InfisicalImportData = {
|
||||||
projects: Map<string, { name: string; id: string }>;
|
projects: Array<{ name: string; id: string }>;
|
||||||
|
environments: Array<{ name: string; id: string; projectId: string }>;
|
||||||
environments?: Map<
|
secrets: Array<{ name: string; id: string; environmentId: string; value: string }>;
|
||||||
string,
|
|
||||||
{
|
|
||||||
name: string;
|
|
||||||
id: string;
|
|
||||||
projectId: string;
|
|
||||||
}
|
|
||||||
>;
|
|
||||||
|
|
||||||
secrets?: Map<
|
|
||||||
string,
|
|
||||||
{
|
|
||||||
name: string;
|
|
||||||
id: string;
|
|
||||||
environmentId: string;
|
|
||||||
value?: string;
|
|
||||||
}
|
|
||||||
>;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type TImportEnvKeyDataCreate = {
|
export type TImportEnvKeyDataCreate = {
|
||||||
@@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = {
|
|||||||
}
|
}
|
||||||
>;
|
>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export enum ExternalPlatforms {
|
||||||
|
EnvKey = "EnvKey"
|
||||||
|
}
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = {
|
|||||||
secretTagDAL: TSecretTagDALFactory;
|
secretTagDAL: TSecretTagDALFactory;
|
||||||
userDAL: Pick<TUserDALFactory, "findById">;
|
userDAL: Pick<TUserDALFactory, "findById">;
|
||||||
secretVersionTagDAL: TSecretVersionTagDALFactory;
|
secretVersionTagDAL: TSecretVersionTagDALFactory;
|
||||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
kmsService: TKmsServiceFactory;
|
||||||
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
|
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
|
||||||
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
|
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
|
||||||
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
|
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
|
||||||
|
|||||||
@@ -34,7 +34,10 @@ export enum SmtpTemplates {
|
|||||||
WorkspaceInvite = "workspaceInvitation.handlebars",
|
WorkspaceInvite = "workspaceInvitation.handlebars",
|
||||||
ScimUserProvisioned = "scimUserProvisioned.handlebars",
|
ScimUserProvisioned = "scimUserProvisioned.handlebars",
|
||||||
PkiExpirationAlert = "pkiExpirationAlert.handlebars",
|
PkiExpirationAlert = "pkiExpirationAlert.handlebars",
|
||||||
IntegrationSyncFailed = "integrationSyncFailed.handlebars"
|
IntegrationSyncFailed = "integrationSyncFailed.handlebars",
|
||||||
|
ExternalImportSuccessful = "externalImportSuccessful.handlebars",
|
||||||
|
ExternalImportFailed = "externalImportFailed.handlebars",
|
||||||
|
ExternalImportStarted = "externalImportStarted.handlebars"
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum SmtpHost {
|
export enum SmtpHost {
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta http-equiv="x-ua-compatible" content="ie=edge" />
|
||||||
|
<title>Import failed</title>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<h2>An import from {{provider}} to Infisical has failed</h2>
|
||||||
|
<p>An import from
|
||||||
|
{{provider}}
|
||||||
|
to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you
|
||||||
|
can contact the Infisical team at [email protected].
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<p>Error: {{error}}</p>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta http-equiv="x-ua-compatible" content="ie=edge" />
|
||||||
|
<title>Import in progress</title>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<h2>An import from {{provider}} to Infisical is in progress</h2>
|
||||||
|
<p>An import from
|
||||||
|
{{provider}}
|
||||||
|
to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import
|
||||||
|
has finished or if it fails.</p>
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta http-equiv="x-ua-compatible" content="ie=edge" />
|
||||||
|
<title>Import successful</title>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<h2>An import from {{provider}} to Infisical was successful</h2>
|
||||||
|
<p>An import from {{provider}} was successful. Your data is now available in Infisical.</p>
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
@@ -18,11 +18,10 @@ export const useImportEnvKey = () => {
|
|||||||
};
|
};
|
||||||
decryptionKey: string;
|
decryptionKey: string;
|
||||||
}) => {
|
}) => {
|
||||||
const { data } = await apiRequest.post("/api/v3/migrate/env-key/", {
|
await apiRequest.post("/api/v3/migrate/env-key/", {
|
||||||
encryptedJson,
|
encryptedJson,
|
||||||
decryptionKey
|
decryptionKey
|
||||||
});
|
});
|
||||||
return data;
|
|
||||||
},
|
},
|
||||||
onSuccess: () => {
|
onSuccess: () => {
|
||||||
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);
|
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);
|
||||||
|
|||||||
Reference in New Issue
Block a user