awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-07 10:27:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

patches for migration

Browse Source
This commit is contained in:
Maidul Islam
2024-01-27 12:39:54 +05:30
committed by Akhil Mohan
parent 32a1a0a9e5
commit bfe5ee672b
3 changed files with 120 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pg-migrator/src/index.ts
+116 -38
View File
@@ -66,6 +66,9 @@ enum SecretKeyEncoding {
const kdb = new Level<string, any>("./db", { valueEncoding: "json" }); const kdb = new Level<string, any>("./db", { valueEncoding: "json" });
const getFolderVersionKey = (folderId: string, version: number) => const getFolderVersionKey = (folderId: string, version: number) =>
`${folderId}:${version}`; `${folderId}:${version}`;
const projectKv = kdb.sublevel(TableName.Project);
/** /**
* Sorts an array of items into groups. The return value is a map where the keys are * Sorts an array of items into groups. The return value is a map where the keys are
* the group ids the given getGroupId function produced and the value is an array of * the group ids the given getGroupId function produced and the value is an array of
@@ -124,9 +127,7 @@ const migrateCollection = async <
console.log("Total batches", Math.ceil(totalMongoCount / 1000)); console.log("Total batches", Math.ceil(totalMongoCount / 1000));
let batch = 1; let batch = 1;
for await (const doc of mongooseCollection for await (const doc of mongooseCollection.find(filter || {}).cursor({ batchSize: 100 })) {
.find(filter || {})
.cursor({ batchSize: 100 })) {
mongooseDoc.push(doc); mongooseDoc.push(doc);
const preProcessedData = await preProcessing( const preProcessedData = await preProcessing(
doc.toObject({ virtuals: true }), doc.toObject({ virtuals: true }),
@@ -156,8 +157,8 @@ const migrateCollection = async <
await postPgProcessing?.(mongooseDoc, newUserIds); await postPgProcessing?.(mongooseDoc, newUserIds);
} }
batch += 1; batch += 1;
mongooseDoc.slice(0, mongooseDoc.length); mongooseDoc.splice(0, mongooseDoc.length);
pgDoc.slice(0, pgDoc.length); pgDoc.splice(0, pgDoc.length);
} }
} }
if (mongooseDoc.length) { if (mongooseDoc.length) {
@@ -176,8 +177,8 @@ const migrateCollection = async <
await postPgProcessing?.(mongooseDoc, newUserIds); await postPgProcessing?.(mongooseDoc, newUserIds);
} }
batch += 1; batch += 1;
mongooseDoc.slice(0, mongooseDoc.length); mongooseDoc.splice(0, mongooseDoc.length);
pgDoc.slice(0, pgDoc.length); pgDoc.splice(0, pgDoc.length);
} }
console.log("Finished migration of ", mongooseCollection.modelName); console.log("Finished migration of ", mongooseCollection.modelName);
@@ -216,7 +217,7 @@ const main = async () => {
console.log("Starting rolling back to latest, comment this out later"); console.log("Starting rolling back to latest, comment this out later");
await db.migrate.rollback({}, true); await db.migrate.rollback({}, true);
kdb.clear(); await kdb.clear();
console.log("Rolling back completed"); console.log("Rolling back completed");
console.log("Executing migration"); console.log("Executing migration");
@@ -229,9 +230,13 @@ const main = async () => {
mongooseCollection: User, mongooseCollection: User,
postgresTableName: TableName.Users, postgresTableName: TableName.Users,
returnKeys: ["id", "email"], returnKeys: ["id", "email"],
preProcessing: (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userKeyExists = await userKv.get(id).catch(()=> null)
if (!userKeyExists) return
userKv.put(doc.id, id); userKv.put(doc.id, id);
return { return {
id, id,
firstName: doc.firstName, firstName: doc.firstName,
@@ -256,7 +261,8 @@ const main = async () => {
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
if (!doc.publicKey || !doc.encryptedPrivateKey || !doc.iv) return; if (!doc.publicKey || !doc.encryptedPrivateKey || !doc.iv) return;
const userId = await userKv.get(doc.id.toString()); const userId = await userKv.get(doc.id.toString()).catch(() => null);
if (!userId) return;
return { return {
id, id,
@@ -287,7 +293,9 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userId = await userKv.get(doc.user.toString());
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return
return { return {
id, id,
@@ -333,7 +341,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
await orgRoleKv.put(doc._id.toString(), id); await orgRoleKv.put(doc._id.toString(), id);
return { return {
id, id,
@@ -357,8 +368,13 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const userId = doc?.user ? await userKv.get(doc.user.toString()) : null; const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
const userId = doc?.user ? await userKv.get(doc.user.toString()).catch(() => null) : null;
if (!userId) return;
const roleId = doc.customRole const roleId = doc.customRole
? await orgRoleKv.get(doc.customRole.toString()) ? await orgRoleKv.get(doc.customRole.toString())
: null; : null;
@@ -384,7 +400,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
return { return {
id, id,
email: doc.email, email: doc.email,
@@ -402,7 +421,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userId = await userKv.get(doc.user.toString());
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return
return { return {
id, id,
userId, userId,
@@ -437,7 +459,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userId = await userKv.get(doc.user.toString());
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return
// expired tokens can be removed // expired tokens can be removed
if (new Date(doc.expiresAt) < new Date()) return; if (new Date(doc.expiresAt) < new Date()) return;
return { return {
@@ -459,12 +484,17 @@ const main = async () => {
postgresTableName: TableName.Project, postgresTableName: TableName.Project,
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null ) : null;
if (!orgId) return;
await projectKv.put(doc._id.toString(),doc._id.toString());
// expired tokens can be removed // expired tokens can be removed
// cannot use this uuid for the org id // cannot use this uuid for the org id
return { return {
id: doc._id.toString(), id: doc._id.toString(),
name: doc.name, name: doc.name.slice(0, 60),
orgId, orgId,
autoCapitalization: doc.autoCapitalization, autoCapitalization: doc.autoCapitalization,
createdAt: new Date(), createdAt: new Date(),
@@ -493,8 +523,15 @@ const main = async () => {
postgresTableName: TableName.Environment, postgresTableName: TableName.Environment,
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null ) : null;
if (!orgId) return;
const projectKvRes = await projectKv.get(doc._id.toString()).catch(() => null);
if (!projectKvRes) return
// to we scope environments into each project then map each slug with respective id // to we scope environments into each project then map each slug with respective id
const envKv = envPKv.sublevel(doc._id.toString()); const envKv = envPKv.sublevel(doc._id.toString());
// expired tokens can be removed // expired tokens can be removed
// cannot use this uuid for the org id // cannot use this uuid for the org id
return doc.environments.map((env, index) => { return doc.environments.map((env, index) => {
@@ -519,11 +556,16 @@ const main = async () => {
mongooseCollection: Workspace, mongooseCollection: Workspace,
postgresTableName: TableName.SecretFolder, postgresTableName: TableName.SecretFolder,
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => preProcessing: async (doc) => {
Promise.all( const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null ) : null;
if (!orgId) return;
return Promise.all(
doc.environments.map(async (env) => { doc.environments.map(async (env) => {
const id = uuidV4(); const id = uuidV4();
const envId = await getEnvId(doc._id.toString(), env.slug); const envId = await getEnvId(doc._id.toString(), env.slug);
const folderKv = getFolderKv(doc._id.toString(), env.slug); const folderKv = getFolderKv(doc._id.toString(), env.slug);
await folderKv.put("root", id); await folderKv.put("root", id);
return { return {
@@ -535,7 +577,8 @@ const main = async () => {
updatedAt: new Date(), updatedAt: new Date(),
}; };
}), }),
), )
}
}); });
await migrateCollection({ await migrateCollection({
@@ -547,8 +590,11 @@ const main = async () => {
// expired tokens can be removed // expired tokens can be removed
// cannot use this uuid for the org id // cannot use this uuid for the org id
const id = uuidV4(); const id = uuidV4();
const senderId = (await userKv.get(doc.sender.toString())) || null;
const receiverId = await userKv.get(doc.receiver.toString()); const senderId = (await userKv.get(doc.sender.toString()).catch(() => null)) || null;
if (!senderId) return;
const receiverId = await userKv.get(doc.receiver.toString()).catch(() => null);
if (!receiverId) return; if (!receiverId) return;
return { return {
@@ -598,7 +644,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userId = await userKv.get(doc.user.toString());
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return
const roleId = doc.customRole const roleId = doc.customRole
? await projectRoleKv.get(doc.customRole.toString()) ? await projectRoleKv.get(doc.customRole.toString())
: null; : null;
@@ -729,7 +778,9 @@ const main = async () => {
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
await tagKv.put(doc._id.toString(), id); await tagKv.put(doc._id.toString(), id);
const createdBy = await userKv.get(doc.user.toString());
const createdBy = await userKv.get(doc.user.toString()).catch(() => null);
if (!createdBy) return;
return { return {
id, id,
@@ -776,7 +827,9 @@ const main = async () => {
const folderKv = getFolderKv(doc.workspace.toString(), doc.environment); const folderKv = getFolderKv(doc.workspace.toString(), doc.environment);
const folderId = await folderKv.get(doc.folder || "root"); const folderId = await folderKv.get(doc.folder || "root");
const userId = doc.user ? await userKv.get(doc.user.toString()) : null; const userId = doc.user ? await userKv.get(doc.user.toString()).catch(() => null) : null;
if (!userId) return
const id = uuidV4(); const id = uuidV4();
await secKv.put(doc._id.toString(), id); await secKv.put(doc._id.toString(), id);
@@ -843,7 +896,9 @@ const main = async () => {
const envId = await getEnvId(doc.workspace.toString(), doc.environment); const envId = await getEnvId(doc.workspace.toString(), doc.environment);
const folderId = await folderKv.get(doc.folder || "root"); const folderId = await folderKv.get(doc.folder || "root");
const userId = doc.user ? await userKv.get(doc.user.toString()) : null;
const userId = doc.user ? await userKv.get(doc.user.toString()).catch(() => null) : null;
if (!userId) return
const id = uuidV4(); const id = uuidV4();
await secVerKv.put(doc._id.toString(), id); await secVerKv.put(doc._id.toString(), id);
@@ -884,9 +939,13 @@ const main = async () => {
const botKey = await BotKey.findOne({ const botKey = await BotKey.findOne({
workspace: doc.workspace, workspace: doc.workspace,
}).lean(); }).lean();
const senderId = botKey?.sender const senderId = botKey?.sender
? await userKv.get(botKey.sender.toString()) ? await userKv.get(botKey.sender.toString()).catch(() => null)
: null; : null;
if (!senderId) return
return { return {
id, id,
algorithm: doc.algorithm, algorithm: doc.algorithm,
@@ -987,7 +1046,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const userId = await userKv.get(doc.user.toString());
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return;
return { return {
id, id,
projectId: doc.workspace.toString(), projectId: doc.workspace.toString(),
@@ -1150,7 +1212,10 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
const identityId = await identityKv.get(doc.identity.toString()); const identityId = await identityKv.get(doc.identity.toString());
const roleId = doc.customRole const roleId = doc.customRole
? await orgRoleKv.get(doc.customRole.toString()) ? await orgRoleKv.get(doc.customRole.toString())
@@ -1310,7 +1375,9 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
return { return {
id, id,
@@ -1339,7 +1406,8 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString()); const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
return { return {
id, id,
@@ -1369,8 +1437,12 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const userId = await userKv.get(doc.user.toString()); const orgId = doc?.organization ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return;
return { return {
id, id,
@@ -1390,8 +1462,12 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organizationId);
const userId = await userKv.get(doc.user.toString()); const orgId = doc?.organizationId.toString() ? await orgKv.get(doc.organizationId.toString()).catch(() => null) : null;
if (!orgId) return;
const userId = await userKv.get(doc.user.toString()).catch(() => null);
if (!userId) return;
return { return {
id, id,
@@ -1411,7 +1487,9 @@ const main = async () => {
returnKeys: ["id"], returnKeys: ["id"],
preProcessing: async (doc) => { preProcessing: async (doc) => {
const id = uuidV4(); const id = uuidV4();
const orgId = await orgKv.get(doc.organization.toString());
const orgId = doc?.organization.toString() ? await orgKv.get(doc.organization.toString()).catch(() => null) : null;
if (!orgId) return;
return { return {
id, id,
pg-migrator/src/migrations/20231128092347_user-encryption-key.ts
+3 -3
View File
@@ -10,9 +10,9 @@ export async function up(knex: Knex): Promise<void> {
t.text("clientPublicKey"); t.text("clientPublicKey");
t.text("serverPrivateKey"); t.text("serverPrivateKey");
t.integer("encryptionVersion").defaultTo(2); t.integer("encryptionVersion").defaultTo(2);
t.text("protectedKey").notNullable(); t.text("protectedKey");
t.text("protectedKeyIV").notNullable(); t.text("protectedKeyIV");
t.text("protectedKeyTag").notNullable(); t.text("protectedKeyTag");
t.text("publicKey").notNullable(); t.text("publicKey").notNullable();
t.text("encryptedPrivateKey").notNullable(); t.text("encryptedPrivateKey").notNullable();
t.text("iv").notNullable(); t.text("iv").notNullable();
pg-migrator/src/migrations/20231212110939_project.ts
+1 -1
View File
@@ -7,7 +7,7 @@ export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasTable(TableName.Project))) { if (!(await knex.schema.hasTable(TableName.Project))) {
await knex.schema.createTable(TableName.Project, (t) => { await knex.schema.createTable(TableName.Project, (t) => {
t.string("id", 36).primary().defaultTo(knex.fn.uuid()); t.string("id", 36).primary().defaultTo(knex.fn.uuid());
t.string("name").notNullable(); t.string("name", 60).notNullable();
t.boolean("autoCapitalization").defaultTo(true); t.boolean("autoCapitalization").defaultTo(true);
t.uuid("orgId").notNullable(); t.uuid("orgId").notNullable();
t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");