From 36b7911bcc4783639192de9da63589351c44a55c Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Tue, 30 Apr 2024 22:41:44 +0530 Subject: [PATCH 01/27] feat: poc for secret replication completed --- backend/src/@types/fastify.d.ts | 2 + .../20240430024805_secret-replication.ts | 43 ++ backend/src/db/schemas/secret-imports.ts | 3 +- backend/src/db/schemas/secret-versions.ts | 3 +- backend/src/db/schemas/secrets.ts | 3 +- backend/src/keystore/keystore.ts | 30 +- backend/src/lib/red-lock/index.ts | 682 ++++++++++++++++++ backend/src/queue/queue-service.ts | 15 +- backend/src/server/routes/index.ts | 16 + .../server/routes/v1/secret-import-router.ts | 11 +- .../secret-import/secret-import-service.ts | 4 +- .../secret-import/secret-import-types.ts | 1 + .../secret-replication-dal.ts | 58 ++ .../secret-replication-service.ts | 247 +++++++ .../secret-replication-types.ts | 17 + backend/src/services/secret/secret-dal.ts | 2 +- backend/src/services/secret/secret-fns.ts | 2 +- backend/src/services/secret/secret-service.ts | 68 +- .../src/hooks/api/secretImports/mutation.tsx | 5 +- frontend/src/hooks/api/secretImports/types.ts | 1 + .../ActionBar/CreateSecretImportForm.tsx | 29 +- 21 files changed, 1213 insertions(+), 29 deletions(-) create mode 100644 backend/src/db/migrations/20240430024805_secret-replication.ts create mode 100644 backend/src/lib/red-lock/index.ts create mode 100644 backend/src/services/secret-replication/secret-replication-dal.ts create mode 100644 backend/src/services/secret-replication/secret-replication-service.ts create mode 100644 backend/src/services/secret-replication/secret-replication-types.ts diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 2bc1d9f26..3f1ca94e9 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -52,6 +52,7 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service"; import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; +import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service"; @@ -108,6 +109,7 @@ declare module "fastify" { projectKey: TProjectKeyServiceFactory; projectRole: TProjectRoleServiceFactory; secret: TSecretServiceFactory; + secretReplication: TSecretReplicationServiceFactory; secretTag: TSecretTagServiceFactory; secretImport: TSecretImportServiceFactory; projectBot: TProjectBotServiceFactory; diff --git a/backend/src/db/migrations/20240430024805_secret-replication.ts b/backend/src/db/migrations/20240430024805_secret-replication.ts new file mode 100644 index 000000000..2cb6b4c83 --- /dev/null +++ b/backend/src/db/migrations/20240430024805_secret-replication.ts @@ -0,0 +1,43 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + t.boolean("isReplication").defaultTo(false); + }); + } + + if (await knex.schema.hasTable(TableName.Secret)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.boolean("isReplicated"); + }); + } + + if (await knex.schema.hasTable(TableName.SecretVersion)) { + await knex.schema.alterTable(TableName.SecretVersion, (t) => { + t.boolean("isReplicated"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + t.dropColumns("isReplication"); + }); + } + + if (await knex.schema.hasTable(TableName.Secret)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.dropColumns("isReplicated"); + }); + } + + if (await knex.schema.hasTable(TableName.SecretVersion)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.dropColumns("isReplicated"); + }); + } +} diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 9d42d8da5..56148cdd6 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -15,7 +15,8 @@ export const SecretImportsSchema = z.object({ position: z.number(), createdAt: z.date(), updatedAt: z.date(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + isReplication: z.boolean().default(false).nullable().optional() }); export type TSecretImports = z.infer; diff --git a/backend/src/db/schemas/secret-versions.ts b/backend/src/db/schemas/secret-versions.ts index d60db9b75..08cfc49cd 100644 --- a/backend/src/db/schemas/secret-versions.ts +++ b/backend/src/db/schemas/secret-versions.ts @@ -32,7 +32,8 @@ export const SecretVersionsSchema = z.object({ folderId: z.string().uuid(), userId: z.string().uuid().nullable().optional(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecretVersions = z.infer; diff --git a/backend/src/db/schemas/secrets.ts b/backend/src/db/schemas/secrets.ts index f261c40bb..8174f5171 100644 --- a/backend/src/db/schemas/secrets.ts +++ b/backend/src/db/schemas/secrets.ts @@ -30,7 +30,8 @@ export const SecretsSchema = z.object({ userId: z.string().uuid().nullable().optional(), folderId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecrets = z.infer; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 5e2c3aab3..4ad568e11 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,20 +1,42 @@ import { Redis } from "ioredis"; +import { Redlock, Settings } from "@app/lib/red-lock"; + export type TKeyStoreFactory = ReturnType; +// all the key prefixes used must be set here to avoid conflict +export enum KeyStorePrefixes { + SecretReplication = "secret-replication" +} + export const keyStoreFactory = (redisUrl: string) => { const redis = new Redis(redisUrl); + const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 }); - const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value); + const setItem = async (key: string, value: string | number | Buffer, prefix?: string) => + redis.set(prefix ? `${prefix}:${key}` : key, value); const getItem = async (key: string) => redis.get(key); - const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) => - redis.setex(key, exp, value); + const setItemWithExpiry = async ( + key: string, + exp: number | string, + value: string | number | Buffer, + prefix?: string + ) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value); const deleteItem = async (key: string) => redis.del(key); const incrementBy = async (key: string, value: number) => redis.incrby(key, value); - return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; + return { + setItem, + getItem, + setItemWithExpiry, + deleteItem, + incrementBy, + acquireLock(resources: string[], duration: number, settings?: Partial) { + return redisLock.acquire(resources, duration, settings); + } + }; }; diff --git a/backend/src/lib/red-lock/index.ts b/backend/src/lib/red-lock/index.ts new file mode 100644 index 000000000..e1cc4f587 --- /dev/null +++ b/backend/src/lib/red-lock/index.ts @@ -0,0 +1,682 @@ +/* eslint-disable */ +// Source code credits: https://github.com/mike-marcacci/node-redlock +// Taken to avoid external dependency +import { randomBytes, createHash } from "crypto"; +import { EventEmitter } from "events"; + +// AbortController became available as a global in node version 16. Once version +// 14 reaches its end-of-life, this can be removed. + +import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; + +type Client = IORedisClient | IORedisCluster; + +// Define script constants. +const ACQUIRE_SCRIPT = ` + -- Return 0 if an entry already exists. + for i, key in ipairs(KEYS) do + if redis.call("exists", key) == 1 then + return 0 + end + end + + -- Create an entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries added. + return #KEYS +`; + +const EXTEND_SCRIPT = ` + -- Return 0 if an entry exists with a *different* lock value. + for i, key in ipairs(KEYS) do + if redis.call("get", key) ~= ARGV[1] then + return 0 + end + end + + -- Update the entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries updated. + return #KEYS +`; + +const RELEASE_SCRIPT = ` + local count = 0 + for i, key in ipairs(KEYS) do + -- Only remove entries for *this* lock value. + if redis.call("get", key) == ARGV[1] then + redis.pcall("del", key) + count = count + 1 + end + end + + -- Return the number of entries removed. + return count +`; + +export type ClientExecutionResult = + | { + client: Client; + vote: "for"; + value: number; + } + | { + client: Client; + vote: "against"; + error: Error; + }; + +/* + * This object contains a summary of results. + */ +export type ExecutionStats = { + readonly membershipSize: number; + readonly quorumSize: number; + readonly votesFor: Set; + readonly votesAgainst: Map; +}; + +/* + * This object contains a summary of results. Because the result of an attempt + * can sometimes be determined before all requests are finished, each attempt + * contains a Promise that will resolve ExecutionStats once all requests are + * finished. A rejection of these promises should be considered undefined + * behavior and should cause a crash. + */ +export type ExecutionResult = { + attempts: ReadonlyArray>; + start: number; +}; + +/** + * + */ +export interface Settings { + readonly driftFactor: number; + readonly retryCount: number; + readonly retryDelay: number; + readonly retryJitter: number; + readonly automaticExtensionThreshold: number; +} + +// Define default settings. +const defaultSettings: Readonly = { + driftFactor: 0.01, + retryCount: 10, + retryDelay: 200, + retryJitter: 100, + automaticExtensionThreshold: 500 +}; + +// Modifyng this object is forbidden. +Object.freeze(defaultSettings); + +/* + * This error indicates a failure due to the existence of another lock for one + * or more of the requested resources. + */ +export class ResourceLockedError extends Error { + constructor(public readonly message: string) { + super(); + this.name = "ResourceLockedError"; + } +} + +/* + * This error indicates a failure of an operation to pass with a quorum. + */ +export class ExecutionError extends Error { + constructor( + public readonly message: string, + public readonly attempts: ReadonlyArray> + ) { + super(); + this.name = "ExecutionError"; + } +} + +/* + * An object of this type is returned when a resource is successfully locked. It + * contains convenience methods `release` and `extend` which perform the + * associated Redlock method on itself. + */ +export class Lock { + constructor( + public readonly redlock: Redlock, + public readonly resources: string[], + public readonly value: string, + public readonly attempts: ReadonlyArray>, + public expiration: number + ) {} + + async release(): Promise { + return this.redlock.release(this); + } + + async extend(duration: number): Promise { + return this.redlock.extend(this, duration); + } +} + +export type RedlockAbortSignal = AbortSignal & { error?: Error }; + +/** + * A redlock object is instantiated with an array of at least one redis client + * and an optional `options` object. Properties of the Redlock object should NOT + * be changed after it is first used, as doing so could have unintended + * consequences for live locks. + */ +export class Redlock extends EventEmitter { + public readonly clients: Set; + public readonly settings: Settings; + public readonly scripts: { + readonly acquireScript: { value: string; hash: string }; + readonly extendScript: { value: string; hash: string }; + readonly releaseScript: { value: string; hash: string }; + }; + + public constructor( + clients: Iterable, + settings: Partial = {}, + scripts: { + readonly acquireScript?: string | ((script: string) => string); + readonly extendScript?: string | ((script: string) => string); + readonly releaseScript?: string | ((script: string) => string); + } = {} + ) { + super(); + + // Prevent crashes on error events. + this.on("error", () => { + // Because redlock is designed for high availability, it does not care if + // a minority of redis instances/clusters fail at an operation. + // + // However, it can be helpful to monitor and log such cases. Redlock emits + // an "error" event whenever it encounters an error, even if the error is + // ignored in its normal operation. + // + // This function serves to prevent node's default behavior of crashing + // when an "error" event is emitted in the absence of listeners. + }); + + // Create a new array of client, to ensure no accidental mutation. + this.clients = new Set(clients); + if (this.clients.size === 0) { + throw new Error("Redlock must be instantiated with at least one redis client."); + } + + // Customize the settings for this instance. + this.settings = { + driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor, + retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount, + retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay, + retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter, + automaticExtensionThreshold: + typeof settings.automaticExtensionThreshold === "number" + ? settings.automaticExtensionThreshold + : defaultSettings.automaticExtensionThreshold + }; + + // Use custom scripts and script modifiers. + const acquireScript = + typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT; + const extendScript = + typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT; + const releaseScript = + typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT; + + this.scripts = { + acquireScript: { + value: acquireScript, + hash: this._hash(acquireScript) + }, + extendScript: { + value: extendScript, + hash: this._hash(extendScript) + }, + releaseScript: { + value: releaseScript, + hash: this._hash(releaseScript) + } + }; + } + + /** + * Generate a sha1 hash compatible with redis evalsha. + */ + private _hash(value: string): string { + return createHash("sha1").update(value).digest("hex"); + } + + /** + * Generate a cryptographically random string. + */ + private _random(): string { + return randomBytes(16).toString("hex"); + } + + /** + * This method runs `.quit()` on all client connections. + */ + public async quit(): Promise { + const results = []; + for (const client of this.clients) { + results.push(client.quit()); + } + + await Promise.all(results); + } + + /** + * This method acquires a locks on the resources for the duration specified by + * the `duration`. + */ + public async acquire(resources: string[], duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const value = this._random(); + + try { + const { attempts, start } = await this._execute( + this.scripts.acquireScript, + resources, + [value, duration], + settings + ); + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + return new Lock(this, resources, value, attempts, start + duration - drift); + } catch (error) { + // If there was an error acquiring the lock, release any partial lock + // state that may exist on a minority of clients. + await this._execute(this.scripts.releaseScript, resources, [value], { + retryCount: 0 + }).catch(() => { + // Any error here will be ignored. + }); + + throw error; + } + } + + /** + * This method unlocks the provided lock from all servers still persisting it. + * It will fail with an error if it is unable to release the lock on a quorum + * of nodes, but will make no attempt to restore the lock in the case of a + * failure to release. It is safe to re-attempt a release or to ignore the + * error, as the lock will automatically expire after its timeout. + */ + public async release(lock: Lock, settings?: Partial): Promise { + // Immediately invalidate the lock. + lock.expiration = 0; + + // Attempt to release the lock. + return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings); + } + + /** + * This method extends a valid lock by the provided `duration`. + */ + public async extend(existing: Lock, duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + // The lock has already expired. + if (existing.expiration < Date.now()) { + throw new ExecutionError("Cannot extend an already-expired lock.", []); + } + + const { attempts, start } = await this._execute( + this.scripts.extendScript, + existing.resources, + [existing.value, duration], + settings + ); + + // Invalidate the existing lock. + existing.expiration = 0; + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift); + + return replacement; + } + + /** + * Execute a script on all clients. The resulting promise is resolved or + * rejected as soon as this quorum is reached; the resolution or rejection + * will contains a `stats` property that is resolved once all votes are in. + */ + private async _execute( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[], + _settings?: Partial + ): Promise { + const settings = _settings + ? { + ...this.settings, + ..._settings + } + : this.settings; + + // For the purpose of easy config serialization, we treat a retryCount of + // -1 a equivalent to Infinity. + const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1; + + const attempts: Promise[] = []; + + while (true) { + const { vote, stats, start } = await this._attemptOperation(script, keys, args); + + attempts.push(stats); + + // The operation achieved a quorum in favor. + if (vote === "for") { + return { attempts, start }; + } + + // Wait before reattempting. + if (attempts.length < maxAttempts) { + await new Promise((resolve) => { + setTimeout( + resolve, + Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)), + undefined + ); + }); + } else { + throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts); + } + } + } + + private async _attemptOperation( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise< + | { vote: "for"; stats: Promise; start: number } + | { vote: "against"; stats: Promise; start: number } + > { + const start = Date.now(); + + return await new Promise((resolve) => { + const clientResults = []; + for (const client of this.clients) { + clientResults.push(this._attemptOperationOnClient(client, script, keys, args)); + } + + const stats: ExecutionStats = { + membershipSize: clientResults.length, + quorumSize: Math.floor(clientResults.length / 2) + 1, + votesFor: new Set(), + votesAgainst: new Map() + }; + + let done: () => void; + const statsPromise = new Promise((resolve) => { + done = () => resolve(stats); + }); + + // This is the expected flow for all successful and unsuccessful requests. + const onResultResolve = (clientResult: ClientExecutionResult): void => { + switch (clientResult.vote) { + case "for": + stats.votesFor.add(clientResult.client); + break; + case "against": + stats.votesAgainst.set(clientResult.client, clientResult.error); + break; + } + + // A quorum has determined a success. + if (stats.votesFor.size === stats.quorumSize) { + resolve({ + vote: "for", + stats: statsPromise, + start + }); + } + + // A quorum has determined a failure. + if (stats.votesAgainst.size === stats.quorumSize) { + resolve({ + vote: "against", + stats: statsPromise, + start + }); + } + + // All votes are in. + if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) { + done(); + } + }; + + // This is unexpected and should crash to prevent undefined behavior. + const onResultReject = (error: Error): void => { + throw error; + }; + + for (const result of clientResults) { + result.then(onResultResolve, onResultReject); + } + }); + } + + private async _attemptOperationOnClient( + client: Client, + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise { + try { + let result: number; + try { + // Attempt to evaluate the script by its hash. + // @ts-expect-error + const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown; + + if (typeof shaResult !== "number") { + throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`); + } + + result = shaResult; + } catch (error) { + // If the redis server does not already have the script cached, + // reattempt the request with the script's raw text. + if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) { + throw error; + } + // @ts-expect-error + const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown; + + if (typeof rawResult !== "number") { + throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`); + } + + result = rawResult; + } + + // One or more of the resources was already locked. + if (result !== keys.length) { + throw new ResourceLockedError( + `The operation was applied to: ${result} of the ${keys.length} requested resources.` + ); + } + + return { + vote: "for", + client, + value: result + }; + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`); + } + + // Emit the error on the redlock instance for observability. + this.emit("error", error); + + return { + vote: "against", + client, + error + }; + } + } + + /** + * Wrap and execute a routine in the context of an auto-extending lock, + * returning a promise of the routine's value. In the case that auto-extension + * fails, an AbortSignal will be updated to indicate that abortion of the + * routine is in order, and to pass along the encountered error. + * + * @example + * ```ts + * await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => { + * const senderBalance = await getBalance(senderId); + * const recipientBalance = await getBalance(recipientId); + * + * if (senderBalance < amountToSend) { + * throw new Error("Insufficient balance."); + * } + * + * // The abort signal will be true if: + * // 1. the above took long enough that the lock needed to be extended + * // 2. redlock was unable to extend the lock + * // + * // In such a case, exclusivity can no longer be guaranteed for further + * // operations, and should be handled as an exceptional case. + * if (signal.aborted) { + * throw signal.error; + * } + * + * await setBalances([ + * {id: senderId, balance: senderBalance - amountToSend}, + * {id: recipientId, balance: recipientBalance + amountToSend}, + * ]); + * }); + * ``` + */ + + public async using( + resources: string[], + duration: number, + settings: Partial, + routine?: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + settingsOrRoutine: undefined | Partial | ((signal: RedlockAbortSignal) => Promise), + optionalRoutine?: (signal: RedlockAbortSignal) => Promise + ): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const settings = + settingsOrRoutine && typeof settingsOrRoutine !== "function" + ? { + ...this.settings, + ...settingsOrRoutine + } + : this.settings; + + const routine = optionalRoutine ?? settingsOrRoutine; + if (typeof routine !== "function") { + throw new Error("INVARIANT: routine is not a function."); + } + + if (settings.automaticExtensionThreshold > duration - 100) { + throw new Error( + "A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting." + ); + } + + // The AbortController/AbortSignal pattern allows the routine to be notified + // of a failure to extend the lock, and subsequent expiration. In the event + // of an abort, the error object will be made available at `signal.error`. + const controller = new AbortController(); + + const signal = controller.signal as RedlockAbortSignal; + + function queue(): void { + timeout = setTimeout( + () => (extension = extend()), + lock.expiration - Date.now() - settings.automaticExtensionThreshold + ); + } + + async function extend(): Promise { + timeout = undefined; + + try { + lock = await lock.extend(duration); + queue(); + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected thrown ${typeof error}: ${error}.`); + } + + if (lock.expiration > Date.now()) { + return (extension = extend()); + } + + signal.error = error instanceof Error ? error : new Error(`${error}`); + controller.abort(); + } + } + + let timeout: undefined | NodeJS.Timeout; + let extension: undefined | Promise; + let lock = await this.acquire(resources, duration, settings); + queue(); + + try { + return await routine(signal); + } finally { + // Clean up the timer. + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + + // Wait for an in-flight extension to finish. + if (extension) { + await extension.catch(() => { + // An error here doesn't matter at all, because the routine has + // already completed, and a release will be attempted regardless. The + // only reason for waiting here is to prevent possible contention + // between the extension and release. + }); + } + + await lock.release(); + } + } +} diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 9d85b6015..32af011ec 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -7,6 +7,7 @@ import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types"; export enum QueueName { SecretRotation = "secret-rotation", @@ -21,7 +22,8 @@ export enum QueueName { SecretFullRepoScan = "secret-full-repo-scan", SecretPushEventScan = "secret-push-event-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost", - DynamicSecretRevocation = "dynamic-secret-revocation" + DynamicSecretRevocation = "dynamic-secret-revocation", + SecretReplication = "secret-replication" } export enum QueueJobs { @@ -37,7 +39,8 @@ export enum QueueJobs { SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job", DynamicSecretRevocation = "dynamic-secret-revocation", - DynamicSecretPruning = "dynamic-secret-pruning" + DynamicSecretPruning = "dynamic-secret-pruning", + SecretReplication = "secret-replication" } export type TQueueJobTypes = { @@ -116,6 +119,10 @@ export type TQueueJobTypes = { dynamicSecretCfgId: string; }; }; + [QueueName.SecretReplication]: { + name: QueueJobs.SecretReplication; + payload: TSyncSecretReplicationDTO; + }; }; export type TQueueServiceFactory = ReturnType; @@ -132,7 +139,7 @@ export const queueServiceFactory = (redisUrl: string) => { const start = ( name: T, - jobFn: (job: Job) => Promise, + jobFn: (job: Job, token?: string) => Promise, queueSettings: Omit = {} ) => { if (queueContainer[name]) { @@ -166,7 +173,7 @@ export const queueServiceFactory = (redisUrl: string) => { name: T, job: TQueueJobTypes[T]["name"], data: TQueueJobTypes[T]["payload"], - opts: JobsOptions & { jobId?: string } + opts?: JobsOptions & { jobId?: string } ) => { const q = queueContainer[name]; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1593515ec..c3fdb87cf 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -130,6 +130,8 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; +import { secretReplicationDALFactory } from "@app/services/secret-replication/secret-replication-dal"; +import { secretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal"; import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; @@ -193,6 +195,7 @@ export const registerRoutes = async ( const projectBotDAL = projectBotDALFactory(db); const secretDAL = secretDALFactory(db); + const secretReplicationDAL = secretReplicationDALFactory(db); const secretTagDAL = secretTagDALFactory(db); const folderDAL = secretFolderDALFactory(db); const folderVersionDAL = secretFolderVersionDALFactory(db); @@ -600,6 +603,17 @@ export const registerRoutes = async ( secretDAL, secretBlindIndexDAL }); + const secretReplicationService = secretReplicationServiceFactory({ + secretTagDAL, + secretVersionTagDAL, + secretDAL, + secretVersionDAL, + secretImportDAL, + keyStore, + queueService, + secretReplicationDAL, + folderDAL + }); const secretService = secretServiceFactory({ folderDAL, secretVersionDAL, @@ -611,6 +625,7 @@ export const registerRoutes = async ( secretTagDAL, snapshotService, secretQueueService, + secretReplicationService, secretImportDAL, projectEnvDAL, projectBotService @@ -826,6 +841,7 @@ export const registerRoutes = async ( projectEnv: projectEnvService, projectRole: projectRoleService, secret: secretService, + secretReplication: secretReplicationService, secretTag: secretTagService, folder: folderService, secretImport: secretImportService, diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index d036fdbdd..d540344ca 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => import: z.object({ environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment), path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path) - }) + }), + isReplication: z.boolean().default(false) }), response: { 200: z.object({ @@ -232,11 +233,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => 200: z.object({ message: z.string(), secretImports: SecretImportsSchema.omit({ importEnv: true }) - .merge( - z.object({ - importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) - }) - ) + .extend({ + importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) + }) .array() }) } diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 43676ba04..2aba9d89c 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -50,6 +50,7 @@ export const secretImportServiceFactory = ({ actorOrgId, actorAuthMethod, projectId, + isReplication, path }: TCreateSecretImportDTO) => { const { permission } = await permissionService.getProjectPermission( @@ -100,7 +101,8 @@ export const secretImportServiceFactory = ({ folderId: folder.id, position: lastPos + 1, importEnv: importEnv.id, - importPath: data.path + importPath: data.path, + isReplication }, tx ); diff --git a/backend/src/services/secret-import/secret-import-types.ts b/backend/src/services/secret-import/secret-import-types.ts index d123f28da..0dca0c306 100644 --- a/backend/src/services/secret-import/secret-import-types.ts +++ b/backend/src/services/secret-import/secret-import-types.ts @@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; } & TProjectPermission; export type TUpdateSecretImportDTO = { diff --git a/backend/src/services/secret-replication/secret-replication-dal.ts b/backend/src/services/secret-replication/secret-replication-dal.ts new file mode 100644 index 000000000..d977f3fa2 --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-dal.ts @@ -0,0 +1,58 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TSecretVersions } from "@app/db/schemas"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TSecretReplicationDALFactory = ReturnType; + +export const secretReplicationDALFactory = (db: TDbClient) => { + const orm = ormify(db, TableName.SecretVersion); + + const findSecrets = async (filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex) => { + if (!filter.secrets) return []; + + const sqlRawDocs = await (tx || db)(TableName.SecretVersion) + .where({ folderId: filter.folderId }) + .andWhere((bd) => { + filter.secrets.forEach((el) => { + void bd.orWhere({ + [`${TableName.SecretVersion}.secretId` as "secretId"]: el.id, + [`${TableName.SecretVersion}.version` as "version"]: el.version + }); + }); + }) + .leftJoin( + (tx || db)(TableName.SecretVersion) + .where("isReplicated", true) + .groupBy(["secretId", "version"]) + .max("version") + .select("version", "secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersion}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersion}.version`, + "latestVersion.max" + ); + } + ) + // .leftJoin( + // (tx || db)(TableName.SecretVersion).select("isReplicated", "version", "secretId").as("previousVersion"), + // (bd) => { + // bd.on(`${TableName.SecretVersion}.secretId`, "previousVersion.secretId").andOn( + // "previousVersion.version", + // (tx || db).raw(`${TableName.SecretVersion}.version - 1`) + // ); + // } + // ) + .select(db.ref("version").withSchema("latestVersion").as("latestReplicatedVersion")) + .select(selectAllTableCols(TableName.SecretVersion)); + + return sqlRawDocs; + }; + + return { + findSecrets, + ...orm + }; +}; diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts new file mode 100644 index 000000000..2a98193ea --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -0,0 +1,247 @@ +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { groupBy } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretReplicationDALFactory } from "./secret-replication-dal"; +import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types"; + +type TSecretReplicationServiceFactoryDep = { + secretReplicationDAL: TSecretReplicationDALFactory; + secretDAL: Pick; + secretImportDAL: Pick; + folderDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + queueService: Pick; + keyStore: Pick; +}; + +export type TSecretReplicationServiceFactory = ReturnType; + +// function getRandomError(): number { +// const minCeiled: number = Math.ceil(0); +// const maxFloored: number = Math.floor(20); +// const val = Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled); // The maximum is exclusive and the minimum is inclusive +// if (val >= 10) throw new Error("Random error point"); +// return val; +// } + +export const secretReplicationServiceFactory = ({ + secretReplicationDAL, + secretDAL, + queueService, + secretVersionDAL, + secretImportDAL, + keyStore, + secretVersionTagDAL, + secretTagDAL, + folderDAL +}: TSecretReplicationServiceFactoryDep) => { + queueService.start(QueueName.SecretReplication, async (job) => { + logger.info(job.data, "Replication started"); + const { secrets, folderId, secretPath, environmentId, projectId } = job.data; + const secretImports = await secretImportDAL.find({ + importPath: secretPath, + importEnv: environmentId, + isReplication: true + }); + console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId); + if (!secretImports.length) return; + + // unfiltered secrets to be replicated + console.log(secrets.length); + const toBeReplicatedSecrets = await secretReplicationDAL.findSecrets({ folderId, secrets }); + const replicatedSecrets = toBeReplicatedSecrets.filter( + ({ version, latestReplicatedVersion, secretBlindIndex }) => + secretBlindIndex && (version === 1 || latestReplicatedVersion <= version) + ); + + const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId); + console.log("replicated ", replicatedSecretsGroupBySecretId); + const lock = await keyStore.acquireLock( + replicatedSecrets.map(({ id }) => id), + 5000 + ); + + try { + /* eslint-disable no-await-in-loop */ + for (const secretImport of secretImports) { + const importFolderId = secretImport.folderId; + + const localSecrets = await secretDAL.find({ + $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, + folderId: importFolderId, + isReplicated: true + }); + const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); + + const locallyCreatedSecrets = secrets.filter(({ operation, id }) => { + return ( + (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + }); + + const locallyUpdatedSecrets = secrets.filter( + ({ operation, id }) => + (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + + console.log(replicatedSecretsGroupBySecretId); + console.log(locallyCreatedSecrets); + console.log("update", locallyUpdatedSecrets); + console.log("local board", localSecrets); + + const locallyDeletedSecrets = secrets + .filter( + ({ operation, id }) => + operation === SecretReplicationOperations.Delete && + Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ) + .map( + ({ id }) => + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0] + ); + + let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = []; + await secretReplicationDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + const newSecrets = await fnSecretBulkInsert({ + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + folderId, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id })) + ); + } + if (locallyUpdatedSecrets.length) { + const newSecrets = await fnSecretBulkUpdate({ + projectId, + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + filter: { + folderId: importFolderId, + id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id })) + ); + } + if (locallyDeletedSecrets.length) { + const newSecrets = await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + isReplicated: true, + folderId: importFolderId + }, + tx + ); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id })) + ); + } + }); + const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); + console.log("Environment ID -> slug", folder.envId, folder.environmentSlug); + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, { + folderId: folder.id, + projectId, + secrets: nestedImportSecrets, + secretPath: folder.path, + environmentId: folder.envId + }); + } + await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true }); + /* eslint-enable no-await-in-loop */ + } finally { + await lock.release(); + } + }); + + queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => { + logger.error(err, "Failed to replicate secret", job?.data); + }); + + const replicate = async (data: TSyncSecretReplicationDTO) => { + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, { + attempts: 3, + backoff: { + type: "exponential", + delay: 1000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + return { + replicate + }; +}; diff --git a/backend/src/services/secret-replication/secret-replication-types.ts b/backend/src/services/secret-replication/secret-replication-types.ts new file mode 100644 index 000000000..509bf5b02 --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-types.ts @@ -0,0 +1,17 @@ +export enum SecretReplicationOperations { + Create = "create", + Update = "update", + Delete = "delete" +} + +export type TSyncSecretReplicationDTO = { + secretPath: string; + projectId: string; + environmentId: string; + folderId: string; + secrets: { + operation: SecretReplicationOperations; + id: string; + version: number; + }[]; +}; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 1a2e414dd..9c72f9e38 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => { }; const deleteMany = async ( - data: Array<{ blindIndex: string; type: SecretType }>, + data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>, folderId: string, userId: string, tx?: Knex diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 51ad7a6aa..e59804bec 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -509,7 +509,7 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, references, ...el }) => ({ + inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({ ...el, folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 39e47a28e..da8c48b5b 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -30,6 +30,8 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind- import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; +import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service"; +import { SecretReplicationOperations } from "../secret-replication/secret-replication-types"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; import { @@ -85,6 +87,7 @@ type TSecretServiceFactoryDep = { projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; + secretReplicationService: Pick; }; export type TSecretServiceFactory = ReturnType; @@ -101,7 +104,8 @@ export const secretServiceFactory = ({ projectDAL, projectBotService, secretImportDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretReplicationService }: TSecretServiceFactoryDep) => { const getSecretReference = async (projectId: string) => { // if bot key missing means e2e still exist @@ -285,6 +289,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Create, + id: secret[0].id, + version: 1 + } + ] + }); return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; @@ -414,7 +431,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Update, + id: updatedSecret[0].id, + version: updatedSecret[0].version + } + ] + }); return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; @@ -482,7 +511,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Delete, + id: deletedSecret[0].id, + version: deletedSecret[0].version + } + ] + }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; }; @@ -768,6 +809,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create })) + }); return newSecrets; }; @@ -868,6 +916,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update })) + }); return secrets; }; @@ -930,6 +985,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete })) + }); return secretsDeleted; }; diff --git a/frontend/src/hooks/api/secretImports/mutation.tsx b/frontend/src/hooks/api/secretImports/mutation.tsx index 928322a3c..0bee77ae8 100644 --- a/frontend/src/hooks/api/secretImports/mutation.tsx +++ b/frontend/src/hooks/api/secretImports/mutation.tsx @@ -9,12 +9,13 @@ export const useCreateSecretImport = () => { const queryClient = useQueryClient(); return useMutation<{}, {}, TCreateSecretImportDTO>({ - mutationFn: async ({ import: secretImport, environment, projectId, path }) => { + mutationFn: async ({ import: secretImport, environment, isReplication, projectId, path }) => { const { data } = await apiRequest.post("/api/v1/secret-imports", { import: secretImport, environment, workspaceId: projectId, - path + path, + isReplication }); return data; }, diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 950fc20c4..58d38047b 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -60,6 +60,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; }; export type TUpdateSecretImportDTO = { diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index a6dfe7c0e..8c2739c2f 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -4,7 +4,15 @@ import { AxiosError } from "axios"; import { z } from "zod"; import { createNotification } from "@app/components/notifications"; -import { Button, FormControl, Modal, ModalContent, Select, SelectItem } from "@app/components/v2"; +import { + Button, + Checkbox, + FormControl, + Modal, + ModalContent, + Select, + SelectItem +} from "@app/components/v2"; import { SecretPathInput } from "@app/components/v2/SecretPathInput"; import { useWorkspace } from "@app/context"; import { useCreateSecretImport } from "@app/hooks/api"; @@ -16,7 +24,8 @@ const typeSchema = z.object({ .trim() .transform((val) => typeof val === "string" && val.at(-1) === "/" && val.length > 1 ? val.slice(0, -1) : val - ) + ), + isReplication: z.boolean().default(false) }); type TFormSchema = z.infer; @@ -54,13 +63,15 @@ export const CreateSecretImportForm = ({ const handleFormSubmit = async ({ environment: importedEnv, - secretPath: importedSecPath + secretPath: importedSecPath, + isReplication }: TFormSchema) => { try { await createSecretImport({ environment, projectId: workspaceId, path: secretPath, + isReplication, import: { environment: importedEnv, path: importedSecPath @@ -127,7 +138,17 @@ export const CreateSecretImportForm = ({ )} /> - + ( + + The replication mode retrieves secrets when changes occur in the specified + environment and secret path. + + )} + />
); diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 80dbe9f73..449d0b774 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -185,6 +185,7 @@ export const SecretApprovalRequestChanges = ({
{generateCommitText(secretApprovalRequestDetails.commits)} + {secretApprovalRequestDetails.isReplication && (replication)}
{committer?.user?.firstName} diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index dd0f5fbec..744a4be54 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -11,7 +11,7 @@ import { import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { ProjectPermissionCan } from "@app/components/permissions"; -import { EmptyState, IconButton, SecretInput, TableContainer } from "@app/components/v2"; +import { EmptyState, IconButton, SecretInput, TableContainer, Tag } from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; import { useToggle } from "@app/hooks"; @@ -19,6 +19,7 @@ type Props = { onDelete: () => void; environment: string; secretPath?: string; + isReplication?: boolean; importEnvName: string; importEnvPath: string; importedSecrets: { key: string; value: string; overriden: { env: string; secretPath: string } }[]; @@ -27,11 +28,20 @@ type Props = { }; // to show the environment and folder icon -export const EnvFolderIcon = ({ env, secretPath }: { env: string; secretPath: string }) => ( +export const EnvFolderIcon = ({ + env, + secretPath, + isReplication +}: { + env: string; + secretPath: string; + isReplication?: boolean; +}) => (
{env || "-"}
{secretPath && (
+ {isReplication && Replication Mode} {secretPath}
@@ -44,6 +54,7 @@ export const SecretImportItem = ({ id, importEnvName, importEnvPath, + isReplication, importedSecrets = [], searchTerm = "", secretPath, @@ -92,7 +103,11 @@ export const SecretImportItem = ({
- +
{items?.map((item) => { - const { importPath, importEnv, id } = item; + const { importPath, importEnv, id, isReplication } = item; return ( handlePopUpToggle("deleteSecretImport", isOpen)} onDeleteApproved={handleSecretImportDelete} /> diff --git a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx index 8b7077417..e8901b8cc 100644 --- a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx @@ -420,9 +420,8 @@ export const SecretItem = memo( 0 - ? `Every ${secretReminderRepeatDays} day${ - Number(secretReminderRepeatDays) > 1 ? "s" : "" - } + ? `Every ${secretReminderRepeatDays} day${Number(secretReminderRepeatDays) > 1 ? "s" : "" + } ` : "Reminder" } From a11f120a83a14ef044f81fe27a4f9f5cdb804cbf Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Thu, 2 May 2024 20:23:23 +0530 Subject: [PATCH 06/27] feat: updated ui replication text and approval replication flag --- .../20240430024805_secret-replication.ts | 12 ++++++++ .../db/schemas/secret-approval-requests.ts | 3 +- .../v1/secret-approval-request-router.ts | 30 +++++++++---------- .../secret-replication-service.ts | 3 +- .../hooks/api/secretApprovalRequest/types.ts | 2 +- .../SecretApprovalRequest.tsx | 4 +-- .../SecretApprovalRequestChanges.tsx | 4 ++- .../ActionBar/CreateSecretImportForm.tsx | 3 +- .../SecretImportListView/SecretImportItem.tsx | 2 +- 9 files changed, 38 insertions(+), 25 deletions(-) diff --git a/backend/src/db/migrations/20240430024805_secret-replication.ts b/backend/src/db/migrations/20240430024805_secret-replication.ts index d3ec4d81f..d71bf465f 100644 --- a/backend/src/db/migrations/20240430024805_secret-replication.ts +++ b/backend/src/db/migrations/20240430024805_secret-replication.ts @@ -26,6 +26,12 @@ export async function up(knex: Knex): Promise { t.boolean("isReplicated"); }); } + + if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { + await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { + t.boolean("isReplicated"); + }); + } } export async function down(knex: Knex): Promise { @@ -52,4 +58,10 @@ export async function down(knex: Knex): Promise { t.dropColumns("isReplicated"); }); } + + if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { + await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { + t.dropColumn("isReplicated"); + }); + } } diff --git a/backend/src/db/schemas/secret-approval-requests.ts b/backend/src/db/schemas/secret-approval-requests.ts index 6ee97fbb6..77ad370b7 100644 --- a/backend/src/db/schemas/secret-approval-requests.ts +++ b/backend/src/db/schemas/secret-approval-requests.ts @@ -18,7 +18,8 @@ export const SecretApprovalRequestsSchema = z.object({ statusChangeBy: z.string().uuid().nullable().optional(), committerId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecretApprovalRequests = z.infer; diff --git a/backend/src/ee/routes/v1/secret-approval-request-router.ts b/backend/src/ee/routes/v1/secret-approval-request-router.ts index 2a9cc405d..b7204f72e 100644 --- a/backend/src/ee/routes/v1/secret-approval-request-router.ts +++ b/backend/src/ee/routes/v1/secret-approval-request-router.ts @@ -32,22 +32,20 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv }), response: { 200: z.object({ - approvals: SecretApprovalRequestsSchema.merge( - z.object({ - // secretPath: z.string(), - policy: z.object({ - id: z.string(), - name: z.string(), - approvals: z.number(), - approvers: z.string().array(), - secretPath: z.string().optional().nullable() - }), - commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(), - environment: z.string(), - reviewers: z.object({ member: z.string(), status: z.string() }).array(), - approvers: z.string().array() - }) - ).array() + approvals: SecretApprovalRequestsSchema.extend({ + // secretPath: z.string(), + policy: z.object({ + id: z.string(), + name: z.string(), + approvals: z.number(), + approvers: z.string().array(), + secretPath: z.string().optional().nullable() + }), + commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(), + environment: z.string(), + reviewers: z.object({ member: z.string(), status: z.string() }).array(), + approvers: z.string().array() + }).array() }) } }, diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index 0d126db2f..1b1aed5a9 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -154,7 +154,8 @@ export const secretReplicationServiceFactory = ({ policyId: policy.id, status: "open", hasMerged: false, - committerId: membershipId + committerId: membershipId, + isReplicated: true }, tx ); diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index b7f8784a6..8c2ba6963 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -44,7 +44,7 @@ export type TSecretApprovalSecChange = { export type TSecretApprovalRequest = { id: string; - isReplication?: boolean; + isReplicated?: boolean; slug: string; createdAt: string; committerId: string; diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx index 8ab051217..ca4d3b897 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx @@ -213,7 +213,7 @@ export const SecretApprovalRequest = () => { policy, reviewers, status, - isReplication + isReplicated: isReplication } = secretApproval; const isApprover = policy?.approvers?.indexOf(myMembershipId || "") !== -1; const isReviewed = @@ -242,7 +242,7 @@ export const SecretApprovalRequest = () => { {membersGroupById?.[committerId]?.user?.firstName}{" "} {membersGroupById?.[committerId]?.user?.lastName} ( {membersGroupById?.[committerId]?.user?.email}) - {isReplication && "via replication"} + {isReplication && " via replication"} {isApprover && !isReviewed && status === "open" && " - Review required"}
diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 449d0b774..4815c7535 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -185,7 +185,9 @@ export const SecretApprovalRequestChanges = ({
{generateCommitText(secretApprovalRequestDetails.commits)} - {secretApprovalRequestDetails.isReplication && (replication)} + {secretApprovalRequestDetails.isReplicated && ( + (replication) + )}
{committer?.user?.firstName} diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index 8c2739c2f..9afc73b06 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -144,8 +144,7 @@ export const CreateSecretImportForm = ({ defaultValue={false} render={({ field }) => ( - The replication mode retrieves secrets when changes occur in the specified - environment and secret path. + Enable replication mode to synchronize changes across boards. )} /> diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index 744a4be54..fa3897e4d 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -156,7 +156,7 @@ export const SecretImportItem = ({
- {isExpanded && !isDragging && ( + {!isReplication && isExpanded && !isDragging && ( Date: Thu, 2 May 2024 21:11:15 +0530 Subject: [PATCH 07/27] feat: added resync replication feature --- .../server/routes/v1/secret-import-router.ts | 43 ++++++++++ .../secret-import/secret-import-service.ts | 79 ++++++++++++++++++- .../secret-import/secret-import-types.ts | 6 ++ frontend/src/hooks/api/secretImports/index.ts | 7 +- .../src/hooks/api/secretImports/mutation.tsx | 20 ++++- frontend/src/hooks/api/secretImports/types.ts | 7 ++ .../ActionBar/CreateSecretImportForm.tsx | 3 +- .../SecretImportListView/SecretImportItem.tsx | 55 ++++++++++++- 8 files changed, 213 insertions(+), 7 deletions(-) diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index d540344ca..50311273c 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -211,6 +211,49 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => } }); + server.route({ + method: "POST", + url: "/:secretImportId/replication-resync", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Resync secret replication of secret imports", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + secretImportId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.secretImportId) + }), + body: z.object({ + workspaceId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.workspaceId), + environment: z.string().trim().describe(SECRET_IMPORTS.UPDATE.environment), + path: z.string().trim().default("/").transform(removeTrailingSlash).describe(SECRET_IMPORTS.UPDATE.path) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { message } = await server.services.secretImport.resyncSecretImportReplication({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + id: req.params.secretImportId, + ...req.body, + projectId: req.body.workspaceId + }); + + return { message }; + } + }); + server.route({ method: "GET", url: "/", diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index f002aefbd..0ff82370a 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -1,5 +1,6 @@ import { ForbiddenError, subject } from "@casl/ability"; +import { TableName } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { BadRequestError } from "@app/lib/errors"; @@ -17,6 +18,7 @@ import { TDeleteSecretImportDTO, TGetSecretImportsDTO, TGetSecretsFromImportDTO, + TResyncSecretImportReplicationDTO, TUpdateSecretImportDTO } from "./secret-import-types"; @@ -109,17 +111,17 @@ export const secretImportServiceFactory = ({ ); }); - if (secImport.isReplication) { + if (secImport.isReplication && sourceFolder && membership) { const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); await secretQueueService.replicateSecrets({ secretPath: secImport.importPath, projectId, environmentSlug: importEnv.slug, pickOnlyImportIds: [secImport.id], - folderId: sourceFolder?.id as string, + folderId: sourceFolder.id, secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), // TODO(akhilmhdh): approval based replication this will fail for identity - membershipId: membership?.id as string, + membershipId: membership.id, environmentId: importEnv.id }); } else { @@ -247,6 +249,76 @@ export const secretImportServiceFactory = ({ return secImport; }; + const resyncSecretImportReplication = async ({ + environment, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + path, + id: secretImportDocId + }: TResyncSecretImportReplicationDTO) => { + const { permission, membership } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + // check if user has permission to import into destination path + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); + + const [secretImportDoc] = await secretImportDAL.find({ + folderId: folder.id, + [`${TableName.SecretImport}.id` as "id"]: secretImportDocId + }); + if (!secretImportDoc) throw new BadRequestError({ message: "Failed to find secret import" }); + + if (!secretImportDoc.isReplication) throw new BadRequestError({ message: "Import is not in replication mode" }); + + // check if user has permission to import from target path + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { + environment: secretImportDoc.importEnv.slug, + secretPath: secretImportDoc.importPath + }) + ); + + await projectDAL.checkProjectUpgradeStatus(projectId); + + const sourceFolder = await folderDAL.findBySecretPath( + projectId, + secretImportDoc.importEnv.slug, + secretImportDoc.importPath + ); + + const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); + if (membership && sourceFolder) { + await secretQueueService.replicateSecrets({ + secretPath: secretImportDoc.importPath, + projectId, + environmentSlug: secretImportDoc.importEnv.slug, + pickOnlyImportIds: [secretImportDoc.id], + folderId: sourceFolder.id, + secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), + // TODO(akhilmhdh): approval based replication this will fail for identity + membershipId: membership.id, + environmentId: secretImportDoc.importEnv.id + }); + } + + return { message: "replication started" }; + }; + const getImports = async ({ path, environment, @@ -319,6 +391,7 @@ export const secretImportServiceFactory = ({ deleteImport, getImports, getSecretsFromImports, + resyncSecretImportReplication, fnSecretsFromImports }; }; diff --git a/backend/src/services/secret-import/secret-import-types.ts b/backend/src/services/secret-import/secret-import-types.ts index 0dca0c306..01847738b 100644 --- a/backend/src/services/secret-import/secret-import-types.ts +++ b/backend/src/services/secret-import/secret-import-types.ts @@ -17,6 +17,12 @@ export type TUpdateSecretImportDTO = { data: Partial<{ environment: string; path: string; position: number }>; } & TProjectPermission; +export type TResyncSecretImportReplicationDTO = { + environment: string; + path: string; + id: string; +} & TProjectPermission; + export type TDeleteSecretImportDTO = { environment: string; path: string; diff --git a/frontend/src/hooks/api/secretImports/index.ts b/frontend/src/hooks/api/secretImports/index.ts index f30506b6b..fed0f13d4 100644 --- a/frontend/src/hooks/api/secretImports/index.ts +++ b/frontend/src/hooks/api/secretImports/index.ts @@ -1,4 +1,9 @@ -export { useCreateSecretImport, useDeleteSecretImport, useUpdateSecretImport } from "./mutation"; +export { + useCreateSecretImport, + useDeleteSecretImport, + useResyncSecretReplication, + useUpdateSecretImport +} from "./mutation"; export { useGetImportedFoldersByEnv, useGetImportedSecretsAllEnvs, diff --git a/frontend/src/hooks/api/secretImports/mutation.tsx b/frontend/src/hooks/api/secretImports/mutation.tsx index 0bee77ae8..04f1f01e6 100644 --- a/frontend/src/hooks/api/secretImports/mutation.tsx +++ b/frontend/src/hooks/api/secretImports/mutation.tsx @@ -3,7 +3,12 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { apiRequest } from "@app/config/request"; import { secretImportKeys } from "./queries"; -import { TCreateSecretImportDTO, TDeleteSecretImportDTO, TUpdateSecretImportDTO } from "./types"; +import { + TCreateSecretImportDTO, + TDeleteSecretImportDTO, + TResyncSecretReplicationDTO, + TUpdateSecretImportDTO +} from "./types"; export const useCreateSecretImport = () => { const queryClient = useQueryClient(); @@ -54,6 +59,19 @@ export const useUpdateSecretImport = () => { }); }; +export const useResyncSecretReplication = () => { + return useMutation<{}, {}, TResyncSecretReplicationDTO>({ + mutationFn: async ({ environment, projectId, path, id }) => { + const { data } = await apiRequest.post(`/api/v1/secret-imports/${id}/replication-resync`, { + environment, + path, + workspaceId: projectId + }); + return data; + } + }); +}; + export const useDeleteSecretImport = () => { const queryClient = useQueryClient(); diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 8aafb0e88..b9c0ddf57 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -76,6 +76,13 @@ export type TUpdateSecretImportDTO = { }>; }; +export type TResyncSecretReplicationDTO = { + id: string; + projectId: string; + environment: string; + path?: string; +}; + export type TDeleteSecretImportDTO = { id: string; projectId: string; diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index 9afc73b06..0d6aee8c2 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -81,7 +81,8 @@ export const CreateSecretImportForm = ({ reset(); createNotification({ type: "success", - text: "Successfully linked" + text: `Successfully linked.${isReplication ? " Kindly refresh the board to see changes." : "" + }` }); } catch (err) { console.error(err); diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index fa3897e4d..a88214a4e 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -6,14 +6,18 @@ import { faFileImport, faFolder, faKey, + faRotate, faUpDown } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { twMerge } from "tailwind-merge"; +import { createNotification } from "@app/components/notifications"; import { ProjectPermissionCan } from "@app/components/permissions"; import { EmptyState, IconButton, SecretInput, TableContainer, Tag } from "@app/components/v2"; -import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; +import { ProjectPermissionActions, ProjectPermissionSub, useWorkspace } from "@app/context"; import { useToggle } from "@app/hooks"; +import { useResyncSecretReplication } from "@app/hooks/api"; type Props = { onDelete: () => void; @@ -60,10 +64,12 @@ export const SecretImportItem = ({ secretPath, environment }: Props) => { + const { currentWorkspace } = useWorkspace(); const [isExpanded, setIsExpanded] = useToggle(); const { attributes, listeners, transform, transition, setNodeRef, isDragging } = useSortable({ id }); + const resyncSecretReplication = useResyncSecretReplication(); useEffect(() => { const filteredSecrets = importedSecrets.filter((secret) => @@ -88,6 +94,28 @@ export const SecretImportItem = ({ transition }; + const handleResyncSecretReplication = async () => { + if (resyncSecretReplication.isLoading) return; + try { + await resyncSecretReplication.mutateAsync({ + id, + environment, + path: secretPath, + projectId: currentWorkspace?.id || "" + }); + createNotification({ + text: "Kindly refresh the board to see changes.", + type: "success" + }); + } catch (error) { + console.error(error); + createNotification({ + text: "Failed to resync replication", + type: "error" + }); + } + }; + return ( <>
+
+ + {(isAllowed) => ( + + + + )} + +
Date: Tue, 14 May 2024 16:31:53 +0530 Subject: [PATCH 08/27] feat: switched to actor and actorId for replication --- .../secret-approval-request-service.ts | 5 ++- backend/src/server/routes/index.ts | 3 +- .../secret-import/secret-import-service.ts | 12 +++--- .../secret-replication-service.ts | 21 ++++++--- backend/src/services/secret/secret-queue.ts | 6 ++- backend/src/services/secret/secret-service.ts | 43 ++++++++----------- backend/src/services/secret/secret-types.ts | 5 ++- 7 files changed, 54 insertions(+), 41 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index c9cbf3b15..67a477141 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -484,13 +484,14 @@ export const secretApprovalRequestServiceFactory = ({ }); await snapshotService.performSnapshot(folderId); const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); - // TODO(akhilmhdh-pg): change query to do secret path from folder + if (!folder) throw new BadRequestError({ message: "Folder not found" }); await secretQueueService.syncSecrets({ projectId, secretPath: folder.path, environmentSlug: folder.environmentSlug, folderId: folder.id, - membershipId: membership.id, + actorId, + actor, environmentId: folder.envId, secrets: mergeStatus.secrets.created .map(({ id, version }) => ({ diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index c018cfc26..34eb5fa28 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -679,7 +679,8 @@ export const registerRoutes = async ( secretApprovalRequestDAL, secretApprovalRequestSecretDAL, secretQueueService, - snapshotService + snapshotService, + projectMembershipDAL }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 0ff82370a..2e9c682a2 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -56,7 +56,7 @@ export const secretImportServiceFactory = ({ isReplication, path }: TCreateSecretImportDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -111,7 +111,7 @@ export const secretImportServiceFactory = ({ ); }); - if (secImport.isReplication && sourceFolder && membership) { + if (secImport.isReplication && sourceFolder) { const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); await secretQueueService.replicateSecrets({ secretPath: secImport.importPath, @@ -120,8 +120,8 @@ export const secretImportServiceFactory = ({ pickOnlyImportIds: [secImport.id], folderId: sourceFolder.id, secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), - // TODO(akhilmhdh): approval based replication this will fail for identity - membershipId: membership.id, + actorId, + actor, environmentId: importEnv.id }); } else { @@ -310,8 +310,8 @@ export const secretImportServiceFactory = ({ pickOnlyImportIds: [secretImportDoc.id], folderId: sourceFolder.id, secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), - // TODO(akhilmhdh): approval based replication this will fail for identity - membershipId: membership.id, + actorId, + actor, environmentId: secretImportDoc.importEnv.id }); } diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index 1b1aed5a9..f5a585930 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -9,6 +9,8 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { ActorType } from "../auth/auth-type"; +import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns"; import { TSecretQueueFactory } from "../secret/secret-queue"; @@ -36,6 +38,7 @@ type TSecretReplicationServiceFactoryDep = { secretBlindIndexDAL: Pick; secretTagDAL: Pick; secretApprovalRequestDAL: Pick; + projectMembershipDAL: Pick; secretApprovalRequestSecretDAL: Pick< TSecretApprovalRequestSecretDALFactory, "insertMany" | "insertApprovalSecretTags" @@ -60,11 +63,12 @@ export const secretReplicationServiceFactory = ({ secretApprovalRequestSecretDAL, secretApprovalRequestDAL, secretQueueService, - snapshotService + snapshotService, + projectMembershipDAL }: TSecretReplicationServiceFactoryDep) => { queueService.start(QueueName.SecretReplication, async (job) => { logger.info(job.data, "Replication started"); - const { secrets, folderId, secretPath, environmentId, projectId, membershipId, pickOnlyImportIds } = job.data; + const { secrets, folderId, secretPath, environmentId, projectId, actorId, actor, pickOnlyImportIds } = job.data; let secretImports = await secretImportDAL.find({ importPath: secretPath, importEnv: environmentId, @@ -140,7 +144,13 @@ export const secretReplicationServiceFactory = ({ importedFolder.path ); // this means it should be a approval request rather than direct replication - if (policy) { + if (policy && actor === ActorType.USER) { + const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId }); + if (!membership) { + logger.error("Project membership not found in %s for user %s", projectId, actorId); + return; + } + const localSecretsLatestVersions = localSecrets.map(({ id }) => id); const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( importFolderId, @@ -154,7 +164,7 @@ export const secretReplicationServiceFactory = ({ policyId: policy.id, status: "open", hasMerged: false, - committerId: membershipId, + committerId: membership.id, isReplicated: true }, tx @@ -294,7 +304,8 @@ export const secretReplicationServiceFactory = ({ secrets: nestedImportSecrets, secretPath: importedFolder.path, environmentId: importedFolder.envId, - membershipId + actorId, + actor }); const folderLock = await keyStore .acquireLock([`secret-replication-${importFolderId}`], 5000) diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index a4533ebc3..3055c48c3 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -354,7 +354,8 @@ export const secretQueueFactory = ({ secrets, folderId, excludeReplication, - membershipId + actorId, + actor } = job.data; await queueService.queue( QueueName.SecretWebhook, @@ -380,7 +381,8 @@ export const secretQueueFactory = ({ secretPath, folderId, secrets, - membershipId, + actorId, + actor, excludeReplication, environmentSlug: environment }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 207a3871d..35bff3772 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -149,7 +149,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TCreateSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -238,9 +238,9 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + folderId: folder.id, + actorId, + actor, projectId, environmentSlug: folder.environment.slug, environmentId: folder.envId, @@ -265,7 +265,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TUpdateSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -381,9 +381,8 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + actor, + actorId, secretPath: path, folderId: folder.id, projectId, @@ -410,7 +409,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TDeleteSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -466,9 +465,8 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + actor, + actorId, secretPath: path, folderId: folder.id, projectId, @@ -703,7 +701,7 @@ export const secretServiceFactory = ({ projectId, secrets: inputSecrets }: TCreateBulkSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -768,9 +766,8 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + actor, + actorId, secretPath: path, folderId: folder.id, projectId, @@ -792,7 +789,7 @@ export const secretServiceFactory = ({ projectId, secrets: inputSecrets }: TUpdateBulkSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -878,9 +875,8 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + actor, + actorId, secretPath: path, folderId: folder.id, projectId, @@ -902,7 +898,7 @@ export const secretServiceFactory = ({ actorAuthMethod, actorOrgId }: TDeleteBulkSecretDTO) => { - const { permission, membership } = await permissionService.getProjectPermission( + const { permission } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -952,9 +948,8 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ - // if secret service reached means there was no secret policy - // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. - membershipId: membership?.id as string, + actor, + actorId, secretPath: path, folderId: folder.id, projectId, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 723aedaf4..fb62ec052 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -11,6 +11,8 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { ActorType } from "../auth/auth-type"; + type TPartialSecret = Pick; type TPartialInputSecret = Pick; @@ -388,7 +390,8 @@ export type TSyncSecretsDTO = { : { environmentId: string; folderId: string; - membershipId: string; + actor: ActorType; + actorId: string; // used for import creation to trigger replication pickOnlyImportIds?: string[]; secrets: { From cec884ce343979e174f232a2cd6baca8c6f2d0b0 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 14 May 2024 16:39:45 +0530 Subject: [PATCH 09/27] fix: merge conflicts --- backend/src/server/routes/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 34eb5fa28..0d7a7e907 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -291,7 +291,7 @@ export const registerRoutes = async ( permissionService, auditLogStreamDAL }); - const sapService = secretApprovalPolicyServiceFactory({ + const secretApprovalPolicyService = secretApprovalPolicyServiceFactory({ projectMembershipDAL, projectEnvDAL, secretApprovalPolicyApproverDAL: sapApproverDAL, From 2d0d90785fbef8c3f3274e774c42e9eac3e98331 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 14:46:54 +0530 Subject: [PATCH 10/27] feat: added icon for replicated secret --- .../components/SecretListView/SecretItem.tsx | 13 +++++++++---- .../SecretListView/SecretListView.utils.ts | 7 +++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx index e8901b8cc..077018017 100644 --- a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx @@ -224,7 +224,11 @@ export const SecretItem = memo( "ml-3 block h-3.5 w-3.5 group-hover:hidden", isSelected && "hidden" )} - symbolName={FontAwesomeSpriteName.SecretKey} + symbolName={ + secret.isReplicated + ? FontAwesomeSpriteName.ReplicatedSecretKey + : FontAwesomeSpriteName.SecretKey + } />
@@ -420,8 +424,9 @@ export const SecretItem = memo( 0 - ? `Every ${secretReminderRepeatDays} day${Number(secretReminderRepeatDays) > 1 ? "s" : "" - } + ? `Every ${secretReminderRepeatDays} day${ + Number(secretReminderRepeatDays) > 1 ? "s" : "" + } ` : "Reminder" } @@ -491,7 +496,7 @@ export const SecretItem = memo( ariaLabel="more" variant="plain" size="md" - className="p-0 opacity-0 group-hover:opacity-100 h-5 w-4" + className="h-5 w-4 p-0 opacity-0 group-hover:opacity-100" onClick={() => onDetailViewSecret(secret)} > Date: Wed, 29 May 2024 14:47:16 +0530 Subject: [PATCH 11/27] feat: resolved personal secret breaking secret replication --- .../secret-replication/secret-replication-dal.ts | 12 ++++++++---- .../secret-replication-service.ts | 14 +++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/backend/src/services/secret-replication/secret-replication-dal.ts b/backend/src/services/secret-replication/secret-replication-dal.ts index 2a8b035ea..e1013df99 100644 --- a/backend/src/services/secret-replication/secret-replication-dal.ts +++ b/backend/src/services/secret-replication/secret-replication-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TSecretVersions } from "@app/db/schemas"; +import { SecretType, TableName, TSecretVersions } from "@app/db/schemas"; import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretReplicationDALFactory = ReturnType; @@ -9,7 +9,10 @@ export type TSecretReplicationDALFactory = ReturnType { const orm = ormify(db, TableName.SecretVersion); - const findSecrets = async (filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex) => { + const findSecretVersions = async ( + filter: { folderId: string; secrets: { id: string; version: number }[] }, + tx?: Knex + ) => { if (!filter.secrets) return []; const sqlRawDocs = await (tx || db)(TableName.SecretVersion) @@ -18,7 +21,8 @@ export const secretReplicationDALFactory = (db: TDbClient) => { filter.secrets.forEach((el) => { void bd.orWhere({ [`${TableName.SecretVersion}.secretId` as "secretId"]: el.id, - [`${TableName.SecretVersion}.version` as "version"]: el.version + [`${TableName.SecretVersion}.version` as "version"]: el.version, + [`${TableName.SecretVersion}.type` as "type"]: SecretType.Shared }); }); }) @@ -39,7 +43,7 @@ export const secretReplicationDALFactory = (db: TDbClient) => { }; return { - findSecrets, + findSecretVersions, ...orm }; }; diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index f5a585930..ab1cf1c40 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -80,13 +80,16 @@ export const secretReplicationServiceFactory = ({ if (!secretImports.length || !secrets.length) return; // unfiltered secrets to be replicated - const toBeReplicatedSecrets = await secretReplicationDAL.findSecrets({ folderId, secrets }); + const toBeReplicatedSecrets = await secretReplicationDAL.findSecretVersions({ folderId, secrets }); const replicatedSecrets = toBeReplicatedSecrets.filter( ({ version, latestReplicatedVersion, secretBlindIndex }) => secretBlindIndex && (version === 1 || latestReplicatedVersion <= version) ); - const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId); + // this is to filter out personal secrets + const sanitizedSecrets = secrets.filter(({ id }) => Object.hasOwn(replicatedSecretsGroupBySecretId, id)); + if (!sanitizedSecrets.length) return; + const lock = await keyStore.acquireLock( replicatedSecrets.map(({ id }) => id), 5000 @@ -118,20 +121,20 @@ export const secretReplicationServiceFactory = ({ }); const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); - const locallyCreatedSecrets = secrets.filter(({ operation, id }) => { + const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => { return ( (operation === SecretOperations.Create || operation === SecretOperations.Update) && !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] ); }); - const locallyUpdatedSecrets = secrets.filter( + const locallyUpdatedSecrets = sanitizedSecrets.filter( ({ operation, id }) => (operation === SecretOperations.Create || operation === SecretOperations.Update) && localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] ); - const locallyDeletedSecrets = secrets.filter( + const locallyDeletedSecrets = sanitizedSecrets.filter( ({ operation, id }) => operation === SecretOperations.Delete && Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && @@ -333,6 +336,7 @@ export const secretReplicationServiceFactory = ({ /* eslint-enable no-await-in-loop */ } finally { await lock.release(); + logger.info(job.data, "Replication finished"); } }); From 4502d394a3a2b5eec593122ba42bafdd844ff448 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 16:29:29 +0530 Subject: [PATCH 12/27] feat: added back dedupe queue for both replication and syncing ops --- .../secret-approval-request-service.ts | 7 +- .../secret-replication-service.ts | 53 ++++-- backend/src/services/secret/secret-queue.ts | 173 ++++++++++-------- backend/src/services/secret/secret-service.ts | 15 +- backend/src/services/secret/secret-types.ts | 2 +- 5 files changed, 143 insertions(+), 107 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 67a477141..175435e0a 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -15,13 +15,13 @@ import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; -import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; import { fnSecretBlindIndexCheck, fnSecretBlindIndexCheckV2, fnSecretBulkDelete, fnSecretBulkInsert, - fnSecretBulkUpdate + fnSecretBulkUpdate, + getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; import { SecretOperations } from "@app/services/secret/secret-types"; @@ -51,6 +51,7 @@ import { type TSecretApprovalRequestServiceFactoryDep = { permissionService: Pick; + projectBotService: Pick; secretApprovalRequestDAL: TSecretApprovalRequestDALFactory; secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory; secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory; @@ -356,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({ } const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); - + const botKey = await projectBotService.getBotKey(projectId).catch(() => null); const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { const newSecrets = secretCreationCommits.length ? await fnSecretBulkInsert({ diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index ab1cf1c40..a289e4600 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -7,7 +7,7 @@ import { BadRequestError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "../auth/auth-type"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; @@ -25,7 +25,10 @@ import { TSecretReplicationDALFactory } from "./secret-replication-dal"; type TSecretReplicationServiceFactoryDep = { secretReplicationDAL: TSecretReplicationDALFactory; - secretDAL: Pick; + secretDAL: Pick< + TSecretDALFactory, + "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" + >; secretVersionDAL: Pick; secretImportDAL: Pick; folderDAL: Pick; @@ -48,6 +51,7 @@ type TSecretReplicationServiceFactoryDep = { export type TSecretReplicationServiceFactory = ReturnType; const SECRET_IMPORT_SUCCESS_LOCK = 10; const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`; +const getReplicationKeyLockPrefix = (keyName: string) => `REPLICATION_SECRET_${keyName}`; export const secretReplicationServiceFactory = ({ secretReplicationDAL, @@ -68,7 +72,20 @@ export const secretReplicationServiceFactory = ({ }: TSecretReplicationServiceFactoryDep) => { queueService.start(QueueName.SecretReplication, async (job) => { logger.info(job.data, "Replication started"); - const { secrets, folderId, secretPath, environmentId, projectId, actorId, actor, pickOnlyImportIds } = job.data; + const { + secrets, + folderId, + secretPath, + environmentId, + projectId, + actorId, + actor, + pickOnlyImportIds, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue + } = job.data; + + // filter for initial filling let secretImports = await secretImportDAL.find({ importPath: secretPath, importEnv: environmentId, @@ -91,7 +108,7 @@ export const secretReplicationServiceFactory = ({ if (!sanitizedSecrets.length) return; const lock = await keyStore.acquireLock( - replicatedSecrets.map(({ id }) => id), + replicatedSecrets.map(({ id }) => getReplicationKeyLockPrefix(id)), 5000 ); @@ -301,28 +318,26 @@ export const secretReplicationServiceFactory = ({ } }); - await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, { - folderId: importedFolder.id, - projectId, - secrets: nestedImportSecrets, - secretPath: importedFolder.path, - environmentId: importedFolder.envId, - actorId, - actor - }); const folderLock = await keyStore .acquireLock([`secret-replication-${importFolderId}`], 5000) .catch(() => null); if (folderLock) { await snapshotService.performSnapshot(importFolderId); await folderLock.release(); - await secretQueueService.syncSecrets({ - excludeReplication: true, - projectId, - secretPath: importedFolder.path, - environmentSlug: importedFolder.environmentSlug - }); } + + await secretQueueService.syncSecrets({ + projectId, + secretPath: importedFolder.path, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue, + environmentSlug: importedFolder.environmentSlug, + actorId, + actor, + secrets: nestedImportSecrets, + folderId: importedFolder.id, + environmentId: importedFolder.envId + }); } await keyStore.setItemWithExpiry( diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 3055c48c3..32be44bc1 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -64,8 +64,10 @@ export type TGetSecrets = { }; const MAX_SYNC_SECRET_DEPTH = 5; -const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`; +const uniqueSecretQueueKey = (environment: string, secretPath: string) => + `secret-queue-dedupe-${environment}-${secretPath}`; +type TIntegrationSecret = Record; export const secretQueueFactory = ({ queueService, integrationDAL, @@ -86,75 +88,6 @@ export const secretQueueFactory = ({ secretTagDAL, secretVersionTagDAL }: TSecretQueueFactoryDep) => { - const createManySecretsRawFn = createManySecretsRawFnFactory({ - projectDAL, - projectBotDAL, - secretDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - folderDAL - }); - - const updateManySecretsRawFn = updateManySecretsRawFnFactory({ - projectDAL, - projectBotDAL, - secretDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - folderDAL - }); - - const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { - await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { - attempts: 3, - delay: 1000, - backoff: { - type: "exponential", - delay: 3000 - }, - removeOnComplete: true, - removeOnFail: true - }); - }; - - const syncSecrets = async ({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO) => { - logger.info( - `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]` - ); - const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath); - if (deDupeQueue?.[deDuplicationKey]) { - return; - } - // eslint-disable-next-line - deDupeQueue[deDuplicationKey] = true; - await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, { - removeOnFail: true, - removeOnComplete: true, - delay: 1000, - attempts: 5, - backoff: { - type: "exponential", - delay: 3000 - } - }); - }; - - const replicateSecrets = async (dto: TSyncSecretsDTO) => { - await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, { - attempts: 3, - backoff: { - type: "exponential", - delay: 2000 - }, - removeOnComplete: true, - removeOnFail: true - }); - }; - const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); await queueService.stopRepeatableJob( @@ -249,8 +182,27 @@ export const secretQueueFactory = ({ } } }; + const createManySecretsRawFn = createManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL + }); - type Content = Record; + const updateManySecretsRawFn = updateManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL + }); /** * Return the secrets in a given [folderId] including secrets from @@ -263,7 +215,7 @@ export const secretQueueFactory = ({ key: string; depth: number; }) => { - let content: Content = {}; + let content: TIntegrationSecret = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) { logger.info( `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]` @@ -345,8 +297,69 @@ export const secretQueueFactory = ({ return content; }; + const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { + await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { + attempts: 3, + delay: 1000, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + const replicateSecrets = async (dto: Omit) => { + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, { + attempts: 3, + backoff: { + type: "exponential", + delay: 2000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + const syncSecrets = async ({ + // seperate de-dupe queue for integration sync and replication sync + _deDupeQueue: deDupeQueue = {}, + _deDupeReplicationQueue: deDupeReplicationQueue = {}, + ...dto + }: TSyncSecretsDTO) => { + logger.info( + `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]` + ); + const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath); + if (!dto.excludeReplication ? deDupeReplicationQueue?.[deDuplicationKey] : deDupeQueue?.[deDuplicationKey]) { + return; + } + // eslint-disable-next-line + deDupeQueue[deDuplicationKey] = true; + // eslint-disable-next-line + deDupeReplicationQueue[deDuplicationKey] = true; + await queueService.queue( + QueueName.SecretSync, + QueueJobs.SecretSync, + { ...dto, _deDupeQueue: deDupeQueue, _deDupeReplicationQueue: deDupeReplicationQueue } as TSyncSecretsDTO, + { + removeOnFail: true, + removeOnComplete: true, + delay: 1000, + attempts: 5, + backoff: { + type: "exponential", + delay: 3000 + } + } + ); + }; + queueService.start(QueueName.SecretSync, async (job) => { const { + _deDupeQueue: deDupeQueue, + _deDupeReplicationQueue: deDupeReplicationQueue, secretPath, environmentId, projectId, @@ -373,9 +386,10 @@ export const secretQueueFactory = ({ } } ); - await syncIntegrations({ secretPath, projectId, environment }); - if (!excludeReplication) + await syncIntegrations({ secretPath, projectId, environment, deDupeQueue }); + if (!excludeReplication) { await replicateSecrets({ + _deDupeReplicationQueue: deDupeReplicationQueue, environmentId, projectId, secretPath, @@ -386,6 +400,7 @@ export const secretQueueFactory = ({ excludeReplication, environmentSlug: environment }); + } }); queueService.start(QueueName.IntegrationSync, async (job) => { @@ -409,7 +424,7 @@ export const secretQueueFactory = ({ const imports = await secretImportDAL.find(linkSourceDto); if (imports.length) { - // keep calling sync secret for all the imports made + // keep calling sync secret for all the imports made const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); @@ -423,15 +438,14 @@ export const secretQueueFactory = ({ .filter( ({ folderId }) => !deDupeQueue[ - uniqueIntegrationKey( + uniqueSecretQueueKey( foldersGroupedById[folderId][0]?.environmentSlug as string, - foldersGroupedById[folderId][0]?.path + foldersGroupedById[folderId][0]?.path as string ) ] ) .map(({ folderId }) => syncSecrets({ - _depth: depth + 1, projectId, secretPath: foldersGroupedById[folderId][0]?.path as string, environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, @@ -461,7 +475,7 @@ export const secretQueueFactory = ({ .filter( ({ folderId }) => !deDupeQueue[ - uniqueIntegrationKey( + uniqueSecretQueueKey( referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, referencedFoldersGroupedById[folderId][0]?.path as string ) @@ -469,7 +483,6 @@ export const secretQueueFactory = ({ ) .map(({ folderId }) => syncSecrets({ - _depth: depth + 1, projectId, secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 35bff3772..d43b48268 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1251,7 +1251,9 @@ export const secretServiceFactory = ({ }) }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const updateManySecretsRaw = async ({ @@ -1300,7 +1302,9 @@ export const secretServiceFactory = ({ }) }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const deleteManySecretsRaw = async ({ @@ -1331,7 +1335,9 @@ export const secretServiceFactory = ({ secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + return secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ); }; const getSecretVersions = async ({ @@ -1637,6 +1643,7 @@ export const secretServiceFactory = ({ createManySecretsRaw, updateManySecretsRaw, deleteManySecretsRaw, - getSecretVersions + getSecretVersions, + backfillSecretReferences }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index fb62ec052..8c6d5db06 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -378,8 +378,8 @@ export enum SecretOperations { } export type TSyncSecretsDTO = { - _depth?: number; _deDupeQueue?: Record; + _deDupeReplicationQueue?: Record; secretPath: string; projectId: string; environmentSlug: string; From 25a615cbb312edce4cc5fe0186636efa5a9a6e1a Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 16:34:27 +0530 Subject: [PATCH 13/27] feat: made sure secret updates make replicated into normal ones --- backend/src/services/secret/secret-service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index d43b48268..63a6aa928 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -362,6 +362,7 @@ export const secretServiceFactory = ({ "secretReminderRepeatDays", "tags" ]), + isReplicated: false, secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName], references: references({ ciphertext: inputSecret.secretValueCiphertext, @@ -850,6 +851,7 @@ export const secretServiceFactory = ({ ...el, folderId, type: SecretType.Shared, + isReplicated: false, secretBlindIndex: newSecretName && newKeyName2BlindIndex[newSecretName] ? newKeyName2BlindIndex[newSecretName] From b0c472b5e1c6a471dba738ebf0630a863030ef41 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 17:29:47 +0530 Subject: [PATCH 14/27] feat: added user status signs for replication failure etc --- ...s => 20240529111503_secret-replication.ts} | 6 + backend/src/db/schemas/secret-imports.ts | 5 +- .../secret-replication-service.ts | 396 +++++++++--------- frontend/src/hooks/api/secretImports/types.ts | 3 + .../SecretImportListView/SecretImportItem.tsx | 63 ++- .../SecretImportListView.tsx | 13 +- 6 files changed, 292 insertions(+), 194 deletions(-) rename backend/src/db/migrations/{20240430024805_secret-replication.ts => 20240529111503_secret-replication.ts} (88%) diff --git a/backend/src/db/migrations/20240430024805_secret-replication.ts b/backend/src/db/migrations/20240529111503_secret-replication.ts similarity index 88% rename from backend/src/db/migrations/20240430024805_secret-replication.ts rename to backend/src/db/migrations/20240529111503_secret-replication.ts index d71bf465f..3cf555007 100644 --- a/backend/src/db/migrations/20240430024805_secret-replication.ts +++ b/backend/src/db/migrations/20240529111503_secret-replication.ts @@ -6,6 +6,9 @@ export async function up(knex: Knex): Promise { if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { t.boolean("isReplication").defaultTo(false); + t.boolean("isReplicationSuccess").nullable(); + t.text("replicationStatus").nullable(); + t.datetime("lastReplicated").nullable(); }); } @@ -38,6 +41,9 @@ export async function down(knex: Knex): Promise { if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { t.dropColumns("isReplication"); + t.dropColumns("isReplicationSuccess"); + t.dropColumns("replicationStatus"); + t.dropColumns("lastReplicated"); }); } diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 56148cdd6..c8795098e 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -16,7 +16,10 @@ export const SecretImportsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), folderId: z.string().uuid(), - isReplication: z.boolean().default(false).nullable().optional() + isReplication: z.boolean().default(false).nullable().optional(), + isReplicationSuccess: z.boolean().nullable().optional(), + replicationStatus: z.string().nullable().optional(), + lastReplicated: z.date().nullable().optional() }); export type TSecretImports = z.infer; diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index a289e4600..da004426e 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -30,7 +30,7 @@ type TSecretReplicationServiceFactoryDep = { "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" >; secretVersionDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; folderDAL: Pick; secretVersionTagDAL: Pick; secretQueueService: Pick; @@ -115,131 +115,92 @@ export const secretReplicationServiceFactory = ({ try { /* eslint-disable no-await-in-loop */ for (const secretImport of secretImports) { - const hasJobCompleted = await keyStore.getItem( - keystoreReplicationSuccessKey(job.id as string, secretImport.id), - KeyStorePrefixes.SecretReplication - ); - if (hasJobCompleted) { - logger.info( - { jobId: job.id, importId: secretImport.id }, - "Skipping this job as this has been successfully replicated." + try { + const hasJobCompleted = await keyStore.getItem( + keystoreReplicationSuccessKey(job.id as string, secretImport.id), + KeyStorePrefixes.SecretReplication ); - // eslint-disable-next-line - continue; - } - - const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); - if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" }); - const importFolderId = importedFolder.id; - - const localSecrets = await secretDAL.find({ - $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, - folderId: importFolderId - }); - const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); - - const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => { - return ( - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); - }); - - const locallyUpdatedSecrets = sanitizedSecrets.filter( - ({ operation, id }) => - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); - - const locallyDeletedSecrets = sanitizedSecrets.filter( - ({ operation, id }) => - operation === SecretOperations.Delete && - Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); - - const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( - projectId, - importedFolder.environmentSlug, - importedFolder.path - ); - // this means it should be a approval request rather than direct replication - if (policy && actor === ActorType.USER) { - const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId }); - if (!membership) { - logger.error("Project membership not found in %s for user %s", projectId, actorId); - return; + if (hasJobCompleted) { + logger.info( + { jobId: job.id, importId: secretImport.id }, + "Skipping this job as this has been successfully replicated." + ); + // eslint-disable-next-line + continue; } - const localSecretsLatestVersions = localSecrets.map(({ id }) => id); - const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( - importFolderId, - localSecretsLatestVersions - ); - await secretApprovalRequestDAL.transaction(async (tx) => { - const approvalRequestDoc = await secretApprovalRequestDAL.create( - { - folderId: importFolderId, - slug: alphaNumericNanoId(), - policyId: policy.id, - status: "open", - hasMerged: false, - committerId: membership.id, - isReplicated: true - }, - tx - ); - const commits = locallyCreatedSecrets - .concat(locallyUpdatedSecrets) - .concat(locallyDeletedSecrets) - .map(({ id, operation }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; - const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; - return { - op: operation, - keyEncoding: doc.keyEncoding, - algorithm: doc.algorithm, - requestId: approvalRequestDoc.id, - metadata: doc.metadata, - secretKeyIV: doc.secretKeyIV, - secretKeyTag: doc.secretKeyTag, - secretKeyCiphertext: doc.secretKeyCiphertext, - secretValueIV: doc.secretValueIV, - secretValueTag: doc.secretValueTag, - secretValueCiphertext: doc.secretValueCiphertext, - secretBlindIndex: doc.secretBlindIndex, - secretCommentIV: doc.secretCommentIV, - secretCommentTag: doc.secretCommentTag, - secretCommentCiphertext: doc.secretCommentCiphertext, - isReplicated: true, - skipMultilineEncoding: doc.skipMultilineEncoding, - // except create operation other two needs the secret id and version id - ...(operation !== SecretOperations.Create - ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } - : {}) - }; - }); - const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); + if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" }); + const importFolderId = importedFolder.id; - return { ...approvalRequestDoc, commits: approvalCommits }; + const localSecrets = await secretDAL.find({ + $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, + folderId: importFolderId }); - } else { - let nestedImportSecrets: TSyncSecretsDTO["secrets"] = []; - await secretReplicationDAL.transaction(async (tx) => { - if (locallyCreatedSecrets.length) { - const newSecrets = await fnSecretBulkInsert({ - folderId: importFolderId, - secretVersionDAL, - secretDAL, - tx, - secretTagDAL, - secretVersionTagDAL, - inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); + + const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => { + return ( + (operation === SecretOperations.Create || operation === SecretOperations.Update) && + !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + }); + + const locallyUpdatedSecrets = sanitizedSecrets.filter( + ({ operation, id }) => + (operation === SecretOperations.Create || operation === SecretOperations.Update) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + + const locallyDeletedSecrets = sanitizedSecrets.filter( + ({ operation, id }) => + operation === SecretOperations.Delete && + Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + + const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + importedFolder.environmentSlug, + importedFolder.path + ); + // this means it should be a approval request rather than direct replication + if (policy && actor === ActorType.USER) { + const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId }); + if (!membership) { + logger.error("Project membership not found in %s for user %s", projectId, actorId); + return; + } + + const localSecretsLatestVersions = localSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( + importFolderId, + localSecretsLatestVersions + ); + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: importFolderId, + slug: alphaNumericNanoId(), + policyId: policy.id, + status: "open", + hasMerged: false, + committerId: membership.id, + isReplicated: true + }, + tx + ); + const commits = locallyCreatedSecrets + .concat(locallyUpdatedSecrets) + .concat(locallyDeletedSecrets) + .map(({ id, operation }) => { const doc = replicatedSecretsGroupBySecretId[id][0]; + const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; return { + op: operation, keyEncoding: doc.keyEncoding, algorithm: doc.algorithm, - type: doc.type, + requestId: approvalRequestDoc.id, metadata: doc.metadata, secretKeyIV: doc.secretKeyIV, secretKeyTag: doc.secretKeyTag, @@ -252,31 +213,31 @@ export const secretReplicationServiceFactory = ({ secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, isReplicated: true, - skipMultilineEncoding: doc.skipMultilineEncoding + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) }; - }) - }); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })) - ); - } - if (locallyUpdatedSecrets.length) { - const newSecrets = await fnSecretBulkUpdate({ - projectId, - folderId: importFolderId, - secretVersionDAL, - secretDAL, - tx, - secretTagDAL, - secretVersionTagDAL, - inputSecrets: locallyUpdatedSecrets.map(({ id }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; - return { - filter: { - folderId: importFolderId, - id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id - }, - data: { + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + let nestedImportSecrets: TSyncSecretsDTO["secrets"] = []; + await secretReplicationDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + const newSecrets = await fnSecretBulkInsert({ + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { keyEncoding: doc.keyEncoding, algorithm: doc.algorithm, type: doc.type, @@ -293,60 +254,119 @@ export const secretReplicationServiceFactory = ({ secretCommentCiphertext: doc.secretCommentCiphertext, isReplicated: true, skipMultilineEncoding: doc.skipMultilineEncoding - } - }; - }) - }); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id })) - ); - } - if (locallyDeletedSecrets.length) { - const newSecrets = await secretDAL.delete( - { - $in: { - id: locallyDeletedSecrets.map(({ id }) => id) + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })) + ); + } + if (locallyUpdatedSecrets.length) { + const newSecrets = await fnSecretBulkUpdate({ + projectId, + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + filter: { + folderId: importFolderId, + id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id })) + ); + } + if (locallyDeletedSecrets.length) { + const newSecrets = await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + isReplicated: true, + folderId: importFolderId }, - isReplicated: true, - folderId: importFolderId - }, - tx - ); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id })) - ); - } - }); + tx + ); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id })) + ); + } + }); - const folderLock = await keyStore - .acquireLock([`secret-replication-${importFolderId}`], 5000) - .catch(() => null); - if (folderLock) { - await snapshotService.performSnapshot(importFolderId); - await folderLock.release(); + const folderLock = await keyStore + .acquireLock([`secret-replication-${importFolderId}`], 5000) + .catch(() => null); + if (folderLock) { + await snapshotService.performSnapshot(importFolderId); + await folderLock.release(); + } + + await secretQueueService.syncSecrets({ + projectId, + secretPath: importedFolder.path, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue, + environmentSlug: importedFolder.environmentSlug, + actorId, + actor, + secrets: nestedImportSecrets, + folderId: importedFolder.id, + environmentId: importedFolder.envId + }); } - await secretQueueService.syncSecrets({ - projectId, - secretPath: importedFolder.path, - _deDupeReplicationQueue: deDupeReplicationQueue, - _deDupeQueue: deDupeQueue, - environmentSlug: importedFolder.environmentSlug, - actorId, - actor, - secrets: nestedImportSecrets, - folderId: importedFolder.id, - environmentId: importedFolder.envId + // this is used to avoid multiple times generating secret approval by failed one + await keyStore.setItemWithExpiry( + keystoreReplicationSuccessKey(job.id as string, secretImport.id), + SECRET_IMPORT_SUCCESS_LOCK, + 1, + KeyStorePrefixes.SecretReplication + ); + + await secretImportDAL.updateById(secretImport.id, { + lastReplicated: new Date(), + replicationStatus: null, + isReplicationSuccess: true + }); + } catch (err) { + logger.error( + err, + `Failed to replicate secret with import id=[${secretImport.id}] env=[${secretImport.importEnv.slug}] path=[${secretImport.importPath}]` + ); + await secretImportDAL.updateById(secretImport.id, { + lastReplicated: new Date(), + replicationStatus: (err as Error)?.message.slice(0, 500), + isReplicationSuccess: false }); } - - await keyStore.setItemWithExpiry( - keystoreReplicationSuccessKey(job.id as string, secretImport.id), - SECRET_IMPORT_SUCCESS_LOCK, - 1, - KeyStorePrefixes.SecretReplication - ); } + await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true }); /* eslint-enable no-await-in-loop */ } finally { diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index b9c0ddf57..64b55425d 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -11,6 +11,9 @@ export type TSecretImport = { createdAt: string; updatedAt: string; isReplication?: boolean; + isReplicationSuccess?: boolean; + replicationStatus?: string; + lastReplicated?: string; }; export type TGetImportedFoldersByEnvDTO = { diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index a88214a4e..46c8134be 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -2,19 +2,31 @@ import { useEffect } from "react"; import { subject } from "@casl/ability"; import { useSortable } from "@dnd-kit/sortable"; import { + faCalendarCheck, faClose, faFileImport, faFolder, + faInfoCircle, faKey, faRotate, - faUpDown + faUpDown, + faWarning, + faXmark } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { format } from "date-fns"; import { twMerge } from "tailwind-merge"; import { createNotification } from "@app/components/notifications"; import { ProjectPermissionCan } from "@app/components/permissions"; -import { EmptyState, IconButton, SecretInput, TableContainer, Tag } from "@app/components/v2"; +import { + EmptyState, + IconButton, + SecretInput, + TableContainer, + Tag, + Tooltip +} from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub, useWorkspace } from "@app/context"; import { useToggle } from "@app/hooks"; import { useResyncSecretReplication } from "@app/hooks/api"; @@ -24,6 +36,9 @@ type Props = { environment: string; secretPath?: string; isReplication?: boolean; + isReplicationSuccess?: boolean; + replicationStatus?: string; + lastReplicated?: string; importEnvName: string; importEnvPath: string; importedSecrets: { key: string; value: string; overriden: { env: string; secretPath: string } }[]; @@ -62,7 +77,10 @@ export const SecretImportItem = ({ importedSecrets = [], searchTerm = "", secretPath, - environment + environment, + isReplicationSuccess, + replicationStatus, + lastReplicated }: Props) => { const { currentWorkspace } = useWorkspace(); const [isExpanded, setIsExpanded] = useToggle(); @@ -137,7 +155,44 @@ export const SecretImportItem = ({ isReplication={isReplication} />
-
+
+ {lastReplicated && ( + +
+ +
Last Replication
+
+
+ {lastReplicated + ? format(new Date(lastReplicated), "yyyy-MM-dd, hh:mm aaa") + : "-"} +
+ {!isReplicationSuccess && ( + <> +
+ +
Fail reason
+
+
{replicationStatus}
+ + )} +
+ } + > +
+ +
+ + )} {items?.map((item) => { - const { importPath, importEnv, id, isReplication } = item; + const { + importPath, + importEnv, + id, + isReplication, + replicationStatus, + lastReplicated, + isReplicationSuccess + } = item; return ( Date: Wed, 29 May 2024 18:46:11 +0530 Subject: [PATCH 15/27] feat: made migration script idempotent --- .../20240529111503_secret-replication.ts | 73 +++++++++++++++---- 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/backend/src/db/migrations/20240529111503_secret-replication.ts b/backend/src/db/migrations/20240529111503_secret-replication.ts index 3cf555007..a51d557a7 100644 --- a/backend/src/db/migrations/20240529111503_secret-replication.ts +++ b/backend/src/db/migrations/20240529111503_secret-replication.ts @@ -3,71 +3,112 @@ import { Knex } from "knex"; import { TableName } from "../schemas"; export async function up(knex: Knex): Promise { + const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication"); + const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn( + TableName.SecretImport, + "isReplicationSuccess" + ); + const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn( + TableName.SecretImport, + "replicationStatus" + ); + const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); + if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { - t.boolean("isReplication").defaultTo(false); - t.boolean("isReplicationSuccess").nullable(); - t.text("replicationStatus").nullable(); - t.datetime("lastReplicated").nullable(); + if (!doesSecretImportIsReplicationExist) t.boolean("isReplication").defaultTo(false); + if (!doesSecretImportIsReplicationSuccessExist) t.boolean("isReplicationSuccess").nullable(); + if (!doesSecretImportReplicationStatusExist) t.text("replicationStatus").nullable(); + if (!doesSecretImportLastReplicatedExist) t.datetime("lastReplicated").nullable(); }); } + const doesSecretIsReplicatedExist = await knex.schema.hasColumn(TableName.Secret, "isReplicated"); if (await knex.schema.hasTable(TableName.Secret)) { await knex.schema.alterTable(TableName.Secret, (t) => { - t.boolean("isReplicated"); + if (!doesSecretIsReplicatedExist) t.boolean("isReplicated"); }); } + const doesSecretVersionIsReplicatedExist = await knex.schema.hasColumn(TableName.SecretVersion, "isReplicated"); if (await knex.schema.hasTable(TableName.SecretVersion)) { await knex.schema.alterTable(TableName.SecretVersion, (t) => { - t.boolean("isReplicated"); + if (!doesSecretVersionIsReplicatedExist) t.boolean("isReplicated"); }); } + const doesSecretApprovalRequestSecretIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequestSecret, + "isReplicated" + ); if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { - t.boolean("isReplicated"); + if (!doesSecretApprovalRequestSecretIsReplicatedExist) t.boolean("isReplicated"); }); } + const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequest, + "isReplicated" + ); if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { - t.boolean("isReplicated"); + if (!doesSecretApprovalRequestIsReplicatedExist) t.boolean("isReplicated"); }); } } export async function down(knex: Knex): Promise { + const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication"); + const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn( + TableName.SecretImport, + "isReplicationSuccess" + ); + const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn( + TableName.SecretImport, + "replicationStatus" + ); + const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { - t.dropColumns("isReplication"); - t.dropColumns("isReplicationSuccess"); - t.dropColumns("replicationStatus"); - t.dropColumns("lastReplicated"); + if (doesSecretImportIsReplicationExist) t.dropColumn("isReplication"); + if (doesSecretImportIsReplicationSuccessExist) t.dropColumn("isReplicationSuccess"); + if (doesSecretImportReplicationStatusExist) t.dropColumn("replicationStatus"); + if (doesSecretImportLastReplicatedExist) t.dropColumn("lastReplicated"); }); } + const doesSecretIsReplicatedExist = await knex.schema.hasColumn(TableName.Secret, "isReplicated"); if (await knex.schema.hasTable(TableName.Secret)) { await knex.schema.alterTable(TableName.Secret, (t) => { - t.dropColumns("isReplicated"); + if (doesSecretIsReplicatedExist) t.dropColumns("isReplicated"); }); } + const doesSecretVersionIsReplicatedExist = await knex.schema.hasColumn(TableName.SecretVersion, "isReplicated"); if (await knex.schema.hasTable(TableName.SecretVersion)) { await knex.schema.alterTable(TableName.SecretVersion, (t) => { - t.dropColumns("isReplicated"); + if (doesSecretVersionIsReplicatedExist) t.dropColumns("isReplicated"); }); } + const doesSecretApprovalRequestSecretIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequestSecret, + "isReplicated" + ); if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { - t.dropColumns("isReplicated"); + if (doesSecretApprovalRequestSecretIsReplicatedExist) t.dropColumns("isReplicated"); }); } + const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequest, + "isReplicated" + ); if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { - t.dropColumn("isReplicated"); + if (doesSecretApprovalRequestIsReplicatedExist) t.dropColumn("isReplicated"); }); } } From 7311cf8f6cbc331144140dc0d0c718248799c2d0 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 18:48:12 +0530 Subject: [PATCH 16/27] feat: updated enable replication text --- .../components/ActionBar/CreateSecretImportForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index 0d6aee8c2..bbc5cc978 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -145,7 +145,8 @@ export const CreateSecretImportForm = ({ defaultValue={false} render={({ field }) => ( - Enable replication mode to synchronize changes across boards. + Enable replication to synchronize changes. Warning: this will overwrite any existing + secrets with the same name. )} /> From 316259f218d4e6917f4ac34607827e77f159edb1 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 21:03:52 +0530 Subject: [PATCH 17/27] feat: added subscription plan to secret replication --- .../src/ee/services/license/licence-fns.ts | 1 + .../src/ee/services/license/license-types.ts | 1 + backend/src/server/routes/index.ts | 1 + .../secret-import/secret-import-service.ts | 20 ++++++++++++++++++- frontend/src/hooks/api/subscriptions/types.ts | 1 + .../components/ActionBar/ActionBar.tsx | 1 + .../ActionBar/CreateSecretImportForm.tsx | 12 +++++++++-- 7 files changed, 34 insertions(+), 3 deletions(-) diff --git a/backend/src/ee/services/license/licence-fns.ts b/backend/src/ee/services/license/licence-fns.ts index 189a3c4e0..411882b25 100644 --- a/backend/src/ee/services/license/licence-fns.ts +++ b/backend/src/ee/services/license/licence-fns.ts @@ -16,6 +16,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentLimit: null, environmentsUsed: 0, dynamicSecret: false, + secretReplication: false, secretVersioning: true, pitRecovery: false, ipAllowlisting: false, diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index 0c8fdc197..f1ee6ab13 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -52,6 +52,7 @@ export type TFeatureSet = { has_used_trial: true; secretApproval: false; secretRotation: true; + secretReplication: false; }; export type TOrgPlansTableDTO = { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 0d7a7e907..9bbb49d64 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -590,6 +590,7 @@ export const registerRoutes = async ( secretVersionTagDAL }); const secretImportService = secretImportServiceFactory({ + licenseService, projectEnvDAL, folderDAL, permissionService, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 2e9c682a2..0d351d3e9 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -1,6 +1,7 @@ import { ForbiddenError, subject } from "@casl/ability"; import { TableName } from "@app/db/schemas"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { BadRequestError } from "@app/lib/errors"; @@ -30,6 +31,7 @@ type TSecretImportServiceFactoryDep = { projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; secretQueueService: Pick; + licenseService: Pick; }; const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" }); @@ -43,7 +45,8 @@ export const secretImportServiceFactory = ({ folderDAL, projectDAL, secretDAL, - secretQueueService + secretQueueService, + licenseService }: TSecretImportServiceFactoryDep) => { const createImport = async ({ environment, @@ -78,6 +81,14 @@ export const secretImportServiceFactory = ({ secretPath: data.path }) ); + if (isReplication) { + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.secretReplication) { + throw new BadRequestError({ + message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." + }); + } + } await projectDAL.checkProjectUpgradeStatus(projectId); @@ -273,6 +284,13 @@ export const secretImportServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.secretReplication) { + throw new BadRequestError({ + message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." + }); + } + const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); diff --git a/frontend/src/hooks/api/subscriptions/types.ts b/frontend/src/hooks/api/subscriptions/types.ts index 45414292d..4eb54cc11 100644 --- a/frontend/src/hooks/api/subscriptions/types.ts +++ b/frontend/src/hooks/api/subscriptions/types.ts @@ -24,6 +24,7 @@ export type SubscriptionPlan = { scim: boolean; ldap: boolean; groups: boolean; + secretReplication: boolean; status: | "incomplete" | "incomplete_expired" diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx index 9b235867b..bed5d6a84 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx @@ -482,6 +482,7 @@ export const ActionBar = ({ environment={environment} workspaceId={workspaceId} secretPath={secretPath} + onUpgradePlan={() => handlePopUpOpen("upgradePlan")} isOpen={popUp.addSecretImport.isOpen} onClose={() => handlePopUpClose("addSecretImport")} onTogglePopUp={(isOpen) => handlePopUpToggle("addSecretImport", isOpen)} diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index bbc5cc978..b9ddb7bfe 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -14,7 +14,7 @@ import { SelectItem } from "@app/components/v2"; import { SecretPathInput } from "@app/components/v2/SecretPathInput"; -import { useWorkspace } from "@app/context"; +import { useSubscription, useWorkspace } from "@app/context"; import { useCreateSecretImport } from "@app/hooks/api"; const typeSchema = z.object({ @@ -38,6 +38,7 @@ type Props = { isOpen?: boolean; onClose: () => void; onTogglePopUp: (isOpen: boolean) => void; + onUpgradePlan: () => void; }; export const CreateSecretImportForm = ({ @@ -46,7 +47,8 @@ export const CreateSecretImportForm = ({ secretPath = "/", isOpen, onClose, - onTogglePopUp + onTogglePopUp, + onUpgradePlan }: Props) => { const { handleSubmit, @@ -58,6 +60,7 @@ export const CreateSecretImportForm = ({ const { currentWorkspace } = useWorkspace(); const environments = currentWorkspace?.environments || []; const selectedEnvironment = watch("environment"); + const { subscription } = useSubscription(); const { mutateAsync: createSecretImport } = useCreateSecretImport(); @@ -67,6 +70,11 @@ export const CreateSecretImportForm = ({ isReplication }: TFormSchema) => { try { + if (isReplication && !subscription?.secretReplication) { + onUpgradePlan(); + return; + } + await createSecretImport({ environment, projectId: workspaceId, From 51d4fcf9ee04e8d2a35ae744c45624e8a3eedb0d Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 May 2024 21:18:31 +0530 Subject: [PATCH 18/27] feat: moved secret replication to ee folder --- .../secret-replication-dal.ts | 0 .../secret-replication-service.ts | 24 +++++++++---------- .../secret-replication-types.ts | 0 backend/src/server/routes/index.ts | 4 ++-- .../ActionBar/CreateSecretImportForm.tsx | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) rename backend/src/{ => ee}/services/secret-replication/secret-replication-dal.ts (100%) rename backend/src/{ => ee}/services/secret-replication/secret-replication-service.ts (94%) rename backend/src/{ => ee}/services/secret-replication/secret-replication-types.ts (100%) diff --git a/backend/src/services/secret-replication/secret-replication-dal.ts b/backend/src/ee/services/secret-replication/secret-replication-dal.ts similarity index 100% rename from backend/src/services/secret-replication/secret-replication-dal.ts rename to backend/src/ee/services/secret-replication/secret-replication-dal.ts diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts similarity index 94% rename from backend/src/services/secret-replication/secret-replication-service.ts rename to backend/src/ee/services/secret-replication/secret-replication-service.ts index da004426e..a8296536a 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -8,19 +8,19 @@ import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; +import { ActorType } from "@app/services/auth/auth-type"; +import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal"; +import { TSecretDALFactory } from "@app/services/secret/secret-dal"; +import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns"; +import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; +import { SecretOperations, TSyncSecretsDTO } from "@app/services/secret/secret-types"; +import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { ActorType } from "../auth/auth-type"; -import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; -import { TSecretDALFactory } from "../secret/secret-dal"; -import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns"; -import { TSecretQueueFactory } from "../secret/secret-queue"; -import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types"; -import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; -import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal"; -import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; -import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; -import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; -import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretReplicationDALFactory } from "./secret-replication-dal"; type TSecretReplicationServiceFactoryDep = { diff --git a/backend/src/services/secret-replication/secret-replication-types.ts b/backend/src/ee/services/secret-replication/secret-replication-types.ts similarity index 100% rename from backend/src/services/secret-replication/secret-replication-types.ts rename to backend/src/ee/services/secret-replication/secret-replication-types.ts diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 9bbb49d64..05b9982bb 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -44,6 +44,8 @@ import { secretApprovalRequestDALFactory } from "@app/ee/services/secret-approva import { secretApprovalRequestReviewerDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-reviewer-dal"; import { secretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service"; +import { secretReplicationDALFactory } from "@app/ee/services/secret-replication/secret-replication-dal"; +import { secretReplicationServiceFactory } from "@app/ee/services/secret-replication/secret-replication-service"; import { secretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue"; import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service"; @@ -130,8 +132,6 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; -import { secretReplicationDALFactory } from "@app/services/secret-replication/secret-replication-dal"; -import { secretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal"; import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index b9ddb7bfe..13aa8ee42 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -153,8 +153,8 @@ export const CreateSecretImportForm = ({ defaultValue={false} render={({ field }) => ( - Enable replication to synchronize changes. Warning: this will overwrite any existing - secrets with the same name. + Enable replication to synchronize changes.
Warning: This will overwrite any + existing secrets with the same name. )} /> From d6fcba91697ace4fe9d34ad35c539a6498d4f1e1 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Wed, 29 May 2024 20:52:33 -0400 Subject: [PATCH 19/27] update texts for secret replication --- .../components/ActionBar/CreateSecretImportForm.tsx | 2 +- .../components/SecretImportListView/SecretImportItem.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index 13aa8ee42..a2a0b4354 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -89,7 +89,7 @@ export const CreateSecretImportForm = ({ reset(); createNotification({ type: "success", - text: `Successfully linked.${isReplication ? " Kindly refresh the board to see changes." : "" + text: `Successfully linked. ${isReplication ? "Please refresh the dashboard to view changes" : "" }` }); } catch (err) { diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index 46c8134be..c78af8ffd 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -122,7 +122,7 @@ export const SecretImportItem = ({ projectId: currentWorkspace?.id || "" }); createNotification({ - text: "Kindly refresh the board to see changes.", + text: "Please refresh the dashboard to view changes", type: "success" }); } catch (error) { From f426025fd51e5644ee3be1880f59cbf553305cf3 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 30 May 2024 11:25:33 +0530 Subject: [PATCH 20/27] fix: resolved approval failing when secret was missing in board --- .../secret-replication-service.ts | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index a8296536a..83a846233 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -139,18 +139,25 @@ export const secretReplicationServiceFactory = ({ }); const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); - const locallyCreatedSecrets = sanitizedSecrets.filter(({ operation, id }) => { - return ( - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); - }); + const locallyCreatedSecrets = sanitizedSecrets + .filter( + ({ operation, id }) => + // upsert: irrespective of create or update its a create if not found in dashboard + (operation === SecretOperations.Create || operation === SecretOperations.Update) && + !localSecretsGroupedByBlindIndex[ + replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string + ]?.[0] + ) + .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create - const locallyUpdatedSecrets = sanitizedSecrets.filter( - ({ operation, id }) => - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); + const locallyUpdatedSecrets = sanitizedSecrets + .filter( + ({ operation, id }) => + // upsert: irrespective of create or update its an update if not found in dashboard + (operation === SecretOperations.Create || operation === SecretOperations.Update) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ) + .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite create ops to update const locallyDeletedSecrets = sanitizedSecrets.filter( ({ operation, id }) => @@ -196,6 +203,7 @@ export const secretReplicationServiceFactory = ({ .map(({ id, operation }) => { const doc = replicatedSecretsGroupBySecretId[id][0]; const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; + return { op: operation, keyEncoding: doc.keyEncoding, From 5db6ac711cc546bd9efac779d1455da36745bf01 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 30 May 2024 20:40:36 +0530 Subject: [PATCH 21/27] feat: implemented replication to a folder strategy --- .../20240529111503_secret-replication.ts | 19 ++++ backend/src/db/schemas/secret-folders.ts | 3 +- backend/src/db/schemas/secret-imports.ts | 3 +- .../secret-approval-request-service.ts | 4 +- .../secret-replication-service.ts | 43 +++++--- .../secret-snapshot-service.ts | 2 +- .../secret-folder/secret-folder-service.ts | 8 +- .../secret-folder/secret-folder-types.ts | 4 + .../secret-folder-version-dal.ts | 2 +- .../secret-import/secret-import-dal.ts | 9 +- .../secret-import/secret-import-service.ts | 103 +++++++++++++----- backend/src/services/secret/secret-fns.ts | 3 +- 12 files changed, 147 insertions(+), 56 deletions(-) diff --git a/backend/src/db/migrations/20240529111503_secret-replication.ts b/backend/src/db/migrations/20240529111503_secret-replication.ts index a51d557a7..00bf85c38 100644 --- a/backend/src/db/migrations/20240529111503_secret-replication.ts +++ b/backend/src/db/migrations/20240529111503_secret-replication.ts @@ -13,6 +13,7 @@ export async function up(knex: Knex): Promise { "replicationStatus" ); const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); + const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved"); if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { @@ -20,6 +21,14 @@ export async function up(knex: Knex): Promise { if (!doesSecretImportIsReplicationSuccessExist) t.boolean("isReplicationSuccess").nullable(); if (!doesSecretImportReplicationStatusExist) t.text("replicationStatus").nullable(); if (!doesSecretImportLastReplicatedExist) t.datetime("lastReplicated").nullable(); + if (!doesSecretImportIsReservedExist) t.boolean("isReserved").defaultTo(false); + }); + } + + const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved"); + if (await knex.schema.hasTable(TableName.SecretFolder)) { + await knex.schema.alterTable(TableName.SecretFolder, (t) => { + if (!doesSecretFolderReservedExist) t.boolean("isReserved").defaultTo(false); }); } @@ -69,12 +78,22 @@ export async function down(knex: Knex): Promise { "replicationStatus" ); const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); + const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved"); + if (await knex.schema.hasTable(TableName.SecretImport)) { await knex.schema.alterTable(TableName.SecretImport, (t) => { if (doesSecretImportIsReplicationExist) t.dropColumn("isReplication"); if (doesSecretImportIsReplicationSuccessExist) t.dropColumn("isReplicationSuccess"); if (doesSecretImportReplicationStatusExist) t.dropColumn("replicationStatus"); if (doesSecretImportLastReplicatedExist) t.dropColumn("lastReplicated"); + if (doesSecretImportIsReservedExist) t.dropColumn("isReserved"); + }); + } + + const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved"); + if (await knex.schema.hasTable(TableName.SecretFolder)) { + await knex.schema.alterTable(TableName.SecretFolder, (t) => { + if (doesSecretFolderReservedExist) t.dropColumn("isReserved"); }); } diff --git a/backend/src/db/schemas/secret-folders.ts b/backend/src/db/schemas/secret-folders.ts index 0f9684d0e..ad43ed1ad 100644 --- a/backend/src/db/schemas/secret-folders.ts +++ b/backend/src/db/schemas/secret-folders.ts @@ -14,7 +14,8 @@ export const SecretFoldersSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - parentId: z.string().uuid().nullable().optional() + parentId: z.string().uuid().nullable().optional(), + isReserved: z.boolean().default(false).nullable().optional() }); export type TSecretFolders = z.infer; diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index c8795098e..4bb1e669d 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -19,7 +19,8 @@ export const SecretImportsSchema = z.object({ isReplication: z.boolean().default(false).nullable().optional(), isReplicationSuccess: z.boolean().nullable().optional(), replicationStatus: z.string().nullable().optional(), - lastReplicated: z.date().nullable().optional() + lastReplicated: z.date().nullable().optional(), + isReserved: z.boolean().default(false).nullable().optional() }); export type TSecretImports = z.infer; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 175435e0a..86d8920a8 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -601,7 +601,7 @@ export const secretApprovalRequestServiceFactory = ({ // same process as above const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({ - inputSecrets: nameUpdatedSecrets, + inputSecrets: nameUpdatedSecrets.map(({ newSecretName }) => ({ secretName: newSecretName as string })), folderId, isNew: true, blindIndexCfg, @@ -618,7 +618,7 @@ export const secretApprovalRequestServiceFactory = ({ const secretId = secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0].id; const secretBlindIndex = newSecretName && newKeyName2BlindIndex[newSecretName] - ? newKeyName2BlindIndex?.[secretName] + ? newKeyName2BlindIndex?.[newSecretName] : keyName2BlindIndex[secretName]; // add tags if (tagIds?.length) commitTagIds[keyName2BlindIndex[secretName]] = tagIds; diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 83a846233..d1b970b68 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -18,6 +18,7 @@ import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-da import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types"; import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; @@ -31,7 +32,7 @@ type TSecretReplicationServiceFactoryDep = { >; secretVersionDAL: Pick; secretImportDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; secretVersionTagDAL: Pick; secretQueueService: Pick; snapshotService: Pick; @@ -52,6 +53,7 @@ export type TSecretReplicationServiceFactory = ReturnType `${jobId}-${secretImportId}`; const getReplicationKeyLockPrefix = (keyName: string) => `REPLICATION_SECRET_${keyName}`; +export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`; export const secretReplicationServiceFactory = ({ secretReplicationDAL, @@ -67,7 +69,6 @@ export const secretReplicationServiceFactory = ({ secretApprovalRequestSecretDAL, secretApprovalRequestDAL, secretQueueService, - snapshotService, projectMembershipDAL }: TSecretReplicationServiceFactoryDep) => { queueService.start(QueueName.SecretReplication, async (job) => { @@ -131,11 +132,25 @@ export const secretReplicationServiceFactory = ({ const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" }); - const importFolderId = importedFolder.id; + + let replicationFolder = await folderDAL.findOne({ + parentId: importedFolder.id, + name: getReplicationFolderName(secretImport.id), + isReserved: true + }); + if (!replicationFolder) { + replicationFolder = await folderDAL.create({ + parentId: importedFolder.id, + name: getReplicationFolderName(secretImport.id), + envId: importedFolder.envId, + isReserved: true + }); + } + const replicationFolderId = replicationFolder.id; const localSecrets = await secretDAL.find({ $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, - folderId: importFolderId + folderId: replicationFolderId }); const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); @@ -181,13 +196,13 @@ export const secretReplicationServiceFactory = ({ const localSecretsLatestVersions = localSecrets.map(({ id }) => id); const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( - importFolderId, + replicationFolderId, localSecretsLatestVersions ); await secretApprovalRequestDAL.transaction(async (tx) => { const approvalRequestDoc = await secretApprovalRequestDAL.create( { - folderId: importFolderId, + folderId: replicationFolderId, slug: alphaNumericNanoId(), policyId: policy.id, status: "open", @@ -237,7 +252,7 @@ export const secretReplicationServiceFactory = ({ await secretReplicationDAL.transaction(async (tx) => { if (locallyCreatedSecrets.length) { const newSecrets = await fnSecretBulkInsert({ - folderId: importFolderId, + folderId: replicationFolderId, secretVersionDAL, secretDAL, tx, @@ -272,7 +287,7 @@ export const secretReplicationServiceFactory = ({ if (locallyUpdatedSecrets.length) { const newSecrets = await fnSecretBulkUpdate({ projectId, - folderId: importFolderId, + folderId: replicationFolderId, secretVersionDAL, secretDAL, tx, @@ -282,7 +297,7 @@ export const secretReplicationServiceFactory = ({ const doc = replicatedSecretsGroupBySecretId[id][0]; return { filter: { - folderId: importFolderId, + folderId: replicationFolderId, id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id }, data: { @@ -317,7 +332,7 @@ export const secretReplicationServiceFactory = ({ id: locallyDeletedSecrets.map(({ id }) => id) }, isReplicated: true, - folderId: importFolderId + folderId: replicationFolderId }, tx ); @@ -327,14 +342,6 @@ export const secretReplicationServiceFactory = ({ } }); - const folderLock = await keyStore - .acquireLock([`secret-replication-${importFolderId}`], 5000) - .catch(() => null); - if (folderLock) { - await snapshotService.performSnapshot(importFolderId); - await folderLock.release(); - } - await secretQueueService.syncSecrets({ projectId, secretPath: importedFolder.path, diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 0e71ad126..bd8750577 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -220,7 +220,7 @@ export const secretSnapshotServiceFactory = ({ const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); // this will remove all secrets and folders on child // due to sql foreign key and link list connection removing the folders removes everything below too - const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx); + const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); const deletedTopLevelFolders = groupBy( deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId), (item) => item.id diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index da429d88a..ee537577e 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -276,7 +276,11 @@ export const secretFolderServiceFactory = ({ } const newFolder = await folderDAL.transaction(async (tx) => { - const [doc] = await folderDAL.update({ envId: env.id, id: folder.id, parentId: parentFolder.id }, { name }, tx); + const [doc] = await folderDAL.update( + { envId: env.id, id: folder.id, parentId: parentFolder.id, isReserved: false }, + { name }, + tx + ); await folderVersionDAL.create( { name: doc.name, @@ -354,7 +358,7 @@ export const secretFolderServiceFactory = ({ const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!parentFolder) return []; - const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id }); + const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false }); return folders; }; diff --git a/backend/src/services/secret-folder/secret-folder-types.ts b/backend/src/services/secret-folder/secret-folder-types.ts index 1405f8bd7..c01d5f7b8 100644 --- a/backend/src/services/secret-folder/secret-folder-types.ts +++ b/backend/src/services/secret-folder/secret-folder-types.ts @@ -1,5 +1,9 @@ import { TProjectPermission } from "@app/lib/types"; +export enum ReservedFolders { + SecretReplication = "__reserve_replication_" +} + export type TCreateFolderDTO = { environment: string; path: string; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index f133308cf..73b536b48 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -15,7 +15,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { try { const docs = await (tx || db)(TableName.SecretFolderVersion) .join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) - .where({ parentId: folderId }) + .where({ parentId: folderId, isReserved: false }) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("envId", "folderId") diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index aa45d410d..0e73a8c23 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -20,14 +20,14 @@ export const secretImportDALFactory = (db: TDbClient) => { return lastPos?.position || 0; }; - const updateAllPosition = async (folderId: string, pos: number, targetPos: number, tx?: Knex) => { + const updateAllPosition = async (folderId: string, pos: number, targetPos: number, positionInc = 1, tx?: Knex) => { try { if (targetPos === -1) { // this means delete await (tx || db)(TableName.SecretImport) .where({ folderId }) .andWhere("position", ">", pos) - .decrement("position", 1); + .decrement("position", positionInc); return; } @@ -36,13 +36,13 @@ export const secretImportDALFactory = (db: TDbClient) => { .where({ folderId }) .where("position", "<=", targetPos) .andWhere("position", ">", pos) - .decrement("position", 1); + .decrement("position", positionInc); } else { await (tx || db)(TableName.SecretImport) .where({ folderId }) .where("position", ">=", targetPos) .andWhere("position", "<", pos) - .increment("position", 1); + .increment("position", positionInc); } } catch (error) { throw new DatabaseError({ error, name: "Update position" }); @@ -74,6 +74,7 @@ export const secretImportDALFactory = (db: TDbClient) => { try { const docs = await (tx || db)(TableName.SecretImport) .whereIn("folderId", folderIds) + .where("isReplication", false) .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) .select( db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 0d351d3e9..83eaf2c90 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -1,9 +1,12 @@ +import path from "node:path"; + import { ForbiddenError, subject } from "@casl/ability"; import { TableName } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service"; import { BadRequestError } from "@app/lib/errors"; import { TProjectDALFactory } from "../project/project-dal"; @@ -57,7 +60,7 @@ export const secretImportServiceFactory = ({ actorAuthMethod, projectId, isReplication, - path + path: secretPath }: TCreateSecretImportDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -70,7 +73,7 @@ export const secretImportServiceFactory = ({ // check if user has permission to import into destination path ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); // check if user has permission to import from target path @@ -92,7 +95,7 @@ export const secretImportServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]); @@ -103,14 +106,14 @@ export const secretImportServiceFactory = ({ const existingImport = await secretImportDAL.findOne({ folderId: sourceFolder.id, importEnv: folder.environment.id, - importPath: path + importPath: secretPath }); if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); } const secImport = await secretImportDAL.transaction(async (tx) => { const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx); - return secretImportDAL.create( + const doc = await secretImportDAL.create( { folderId: folder.id, position: lastPos + 1, @@ -120,6 +123,19 @@ export const secretImportServiceFactory = ({ }, tx ); + if (doc.isReplication) { + await secretImportDAL.create( + { + folderId: folder.id, + position: lastPos + 2, + isReserved: true, + importEnv: folder.environment.id, + importPath: path.join(secretPath, getReplicationFolderName(doc.id)) + }, + tx + ); + } + return doc; }); if (secImport.isReplication && sourceFolder) { @@ -148,7 +164,7 @@ export const secretImportServiceFactory = ({ }; const updateImport = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -167,10 +183,10 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Edit, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); const secImpDoc = await secretImportDAL.findOne({ folderId: folder.id, id }); @@ -190,7 +206,7 @@ export const secretImportServiceFactory = ({ const existingImport = await secretImportDAL.findOne({ folderId: sourceFolder.id, importEnv: folder.environment.id, - importPath: path + importPath: secretPath }); if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); } @@ -199,12 +215,31 @@ export const secretImportServiceFactory = ({ const secImp = await secretImportDAL.findOne({ folderId: folder.id, id }); if (!secImp) throw ERR_SEC_IMP_NOT_FOUND; if (data.position) { - await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, tx); + if (secImp.isReplication) { + await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 2, tx); + } else { + await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 1, tx); + } + } + if (secImp.isReplication) { + const replicationFolderPath = path.join(secretPath, getReplicationFolderName(secImp.id)); + await secretImportDAL.update( + { + folderId: folder.id, + importEnv: folder.environment.id, + importPath: replicationFolderPath, + isReserved: true + }, + { position: data?.position ? data.position + 1 : undefined }, + tx + ); } const [doc] = await secretImportDAL.update( { id, folderId: folder.id }, { - position: data?.position, + // when moving replicated import, the position is meant for reserved import + // replicated one should always be behind the reserved import + position: data.position, importEnv: data?.environment ? importedEnv.id : undefined, importPath: data?.path }, @@ -216,7 +251,7 @@ export const secretImportServiceFactory = ({ }; const deleteImport = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -234,16 +269,34 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Delete, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Delete import" }); const secImport = await secretImportDAL.transaction(async (tx) => { const [doc] = await secretImportDAL.delete({ folderId: folder.id, id }, tx); if (!doc) throw new BadRequestError({ name: "Sec imp del", message: "Secret import doc not found" }); - await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, tx); + if (doc.isReplication) { + const replicationFolderPath = path.join(secretPath, getReplicationFolderName(doc.id)); + const replicatedFolder = await folderDAL.findBySecretPath(projectId, environment, replicationFolderPath, tx); + if (replicatedFolder) { + await secretImportDAL.delete( + { + folderId: folder.id, + importEnv: folder.environment.id, + importPath: replicationFolderPath, + isReserved: true + }, + tx + ); + await folderDAL.deleteById(replicatedFolder.id, tx); + } + await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 2, tx); + } else { + await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 1, tx); + } const importEnv = await projectEnvDAL.findById(doc.importEnv); if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); @@ -251,7 +304,7 @@ export const secretImportServiceFactory = ({ }); await secretQueueService.syncSecrets({ - secretPath: path, + secretPath, projectId, environmentSlug: environment, excludeReplication: true @@ -267,7 +320,7 @@ export const secretImportServiceFactory = ({ actorOrgId, actorAuthMethod, projectId, - path, + path: secretPath, id: secretImportDocId }: TResyncSecretImportReplicationDTO) => { const { permission, membership } = await permissionService.getProjectPermission( @@ -281,7 +334,7 @@ export const secretImportServiceFactory = ({ // check if user has permission to import into destination path ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); const plan = await licenseService.getPlan(actorOrgId); @@ -291,7 +344,7 @@ export const secretImportServiceFactory = ({ }); } - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); const [secretImportDoc] = await secretImportDAL.find({ @@ -338,7 +391,7 @@ export const secretImportServiceFactory = ({ }; const getImports = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -355,10 +408,10 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" }); const secImports = await secretImportDAL.find({ folderId: folder.id }); @@ -366,7 +419,7 @@ export const secretImportServiceFactory = ({ }; const getSecretsFromImports = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -383,9 +436,9 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) return []; // this will already order by position // so anything based on this order will also be in right position diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index f21144cfc..b3fee7f7a 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -151,7 +151,8 @@ export const recursivelyGetSecretPaths = ({ // Fetch all folders in env once with a single query const folders = await folderDAL.find({ - envId: env.id + envId: env.id, + isReserved: false }); // Build the folder hierarchy map From a13d4a49701efae157b58b2a4e503a212dcfced9 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 30 May 2024 20:40:51 +0530 Subject: [PATCH 22/27] feat: implemented replication to a folder strategy ui --- frontend/src/hooks/api/secretFolders/types.ts | 4 + frontend/src/hooks/api/secretImports/types.ts | 1 + frontend/src/lib/fn/string.ts | 9 ++ .../SecretApprovalRequestChanges.tsx | 5 +- .../ActionBar/CreateSecretImportForm.tsx | 25 ++++- .../SecretImportListView/SecretImportItem.tsx | 106 ++++++++++-------- .../SecretImportListView.tsx | 48 ++++---- 7 files changed, 127 insertions(+), 71 deletions(-) create mode 100644 frontend/src/lib/fn/string.ts diff --git a/frontend/src/hooks/api/secretFolders/types.ts b/frontend/src/hooks/api/secretFolders/types.ts index 8fde9c63d..412f2686d 100644 --- a/frontend/src/hooks/api/secretFolders/types.ts +++ b/frontend/src/hooks/api/secretFolders/types.ts @@ -1,3 +1,7 @@ +export enum ReservedFolders { + SecretReplication = "__reserve_replication_" +} + export type TSecretFolder = { id: string; name: string; diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 64b55425d..1a6c06dd3 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -10,6 +10,7 @@ export type TSecretImport = { position: string; createdAt: string; updatedAt: string; + isReserved?: boolean; isReplication?: boolean; isReplicationSuccess?: boolean; replicationStatus?: string; diff --git a/frontend/src/lib/fn/string.ts b/frontend/src/lib/fn/string.ts new file mode 100644 index 000000000..a7de20262 --- /dev/null +++ b/frontend/src/lib/fn/string.ts @@ -0,0 +1,9 @@ +import { ReservedFolders } from "@app/hooks/api/secretFolders/types"; + +export const formatReservedPaths = (secretPath: string) => { + const i = secretPath.indexOf(ReservedFolders.SecretReplication); + if (i !== -1) { + return secretPath.slice(0, i); + } + return secretPath; +}; diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 4815c7535..33025342f 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -20,6 +20,7 @@ import { useUpdateSecretApprovalReviewStatus } from "@app/hooks/api"; import { ApprovalStatus, CommitType, TWorkspaceUser } from "@app/hooks/api/types"; +import { formatReservedPaths } from "@app/lib/fn/string"; import { SecretApprovalRequestAction } from "./SecretApprovalRequestAction"; import { SecretApprovalRequestChangeItem } from "./SecretApprovalRequestChangeItem"; @@ -200,7 +201,9 @@ export const SecretApprovalRequestChanges = ({
-
{secretApprovalRequestDetails.secretPath}
+
+ {formatReservedPaths(secretApprovalRequestDetails.secretPath)} +
diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index a2a0b4354..5832779e3 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -6,7 +6,6 @@ import { z } from "zod"; import { createNotification } from "@app/components/notifications"; import { Button, - Checkbox, FormControl, Modal, ModalContent, @@ -151,11 +150,25 @@ export const CreateSecretImportForm = ({ name="isReplication" control={control} defaultValue={false} - render={({ field }) => ( - - Enable replication to synchronize changes.
Warning: This will overwrite any - existing secrets with the same name. - + render={({ field: { value, onChange }, fieldState: { error } }) => ( + + + )} />
diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index c78af8ffd..07c66f17c 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -30,20 +30,17 @@ import { import { ProjectPermissionActions, ProjectPermissionSub, useWorkspace } from "@app/context"; import { useToggle } from "@app/hooks"; import { useResyncSecretReplication } from "@app/hooks/api"; +import { TSecretImport } from "@app/hooks/api/types"; type Props = { onDelete: () => void; environment: string; secretPath?: string; - isReplication?: boolean; - isReplicationSuccess?: boolean; - replicationStatus?: string; - lastReplicated?: string; - importEnvName: string; - importEnvPath: string; + secretImport?: TSecretImport; + isReplicationExpand?: boolean; importedSecrets: { key: string; value: string; overriden: { env: string; secretPath: string } }[]; searchTerm: string; - id: string; + onExpandReplicateSecrets: (id: string) => void; }; // to show the environment and folder icon @@ -70,25 +67,29 @@ export const EnvFolderIcon = ({ export const SecretImportItem = ({ onDelete, - id, - importEnvName, - importEnvPath, - isReplication, + isReplicationExpand, importedSecrets = [], searchTerm = "", secretPath, environment, - isReplicationSuccess, - replicationStatus, - lastReplicated + secretImport, + onExpandReplicateSecrets: onExpandReplicate }: Props) => { + const { + isReserved, + id, + isReplication, + isReplicationSuccess, + replicationStatus, + lastReplicated, + importEnv + } = secretImport as TSecretImport; const { currentWorkspace } = useWorkspace(); const [isExpanded, setIsExpanded] = useToggle(); const { attributes, listeners, transform, transition, setNodeRef, isDragging } = useSortable({ id }); const resyncSecretReplication = useResyncSecretReplication(); - useEffect(() => { const filteredSecrets = importedSecrets.filter((secret) => secret.key.toUpperCase().includes(searchTerm.toUpperCase()) @@ -134,24 +135,39 @@ export const SecretImportItem = ({ } }; + const handleRowClick = () => { + if (isReplication) { + onExpandReplicate(id); + } else { + setIsExpanded.toggle(); + } + }; + return ( <>
setIsExpanded.toggle()} - onKeyDown={() => setIsExpanded.toggle()} + onClick={handleRowClick} + onKeyDown={(e) => { + if (e.key === "Enter") { + handleRowClick(); + } + }} >
@@ -193,29 +209,31 @@ export const SecretImportItem = ({
)} - - {(isAllowed) => ( - - - - )} - + {isReplication && ( + + {(isAllowed) => ( + + + + )} + + )}
- {!isReplication && isExpanded && !isDragging && ( + {!isReplication && (isReplicationExpand || isExpanded) && !isDragging && ( { overridenSec[el.key] = { env: importSecrets[i].environmentInfo.name, - secretPath: importSecrets[i].secretPath + secretPath: formatReservedPaths(importSecrets[i].secretPath) }; }); } @@ -91,6 +93,8 @@ export const SecretImportListView = ({ "deleteSecretImport" ] as const); + const [replicationSecrets, setReplicationSecrets] = useState>({}); + const sensors = useSensors( useSensor(MouseSensor, {}), useSensor(TouchSensor, {}), @@ -149,6 +153,22 @@ export const SecretImportListView = ({ } }; + const handleOpenReplicationSecrets = (replicationImportId: string) => { + console.log(secretImports); + const reservedImport = secretImports.find( + ({ isReserved, importPath, importEnv }) => + importEnv.slug === environment && + isReserved && + importPath === `/${ReservedFolders.SecretReplication}${replicationImportId}` + ); + if (reservedImport) { + setReplicationSecrets((state) => ({ + ...state, + [reservedImport.id]: !state?.[reservedImport.id] + })); + } + }; + return ( <> {items?.map((item) => { - const { - importPath, - importEnv, - id, - isReplication, - replicationStatus, - lastReplicated, - isReplicationSuccess - } = item; + // TODO(akhilmhdh): change this and pass this whole object instead of one by one return ( Date: Fri, 31 May 2024 00:03:27 +0530 Subject: [PATCH 23/27] feat: switched to secretapproval check for license --- backend/src/ee/services/license/licence-fns.ts | 1 - backend/src/ee/services/license/license-types.ts | 1 - backend/src/services/secret-import/secret-import-service.ts | 4 ++-- frontend/src/hooks/api/subscriptions/types.ts | 1 - .../components/ActionBar/CreateSecretImportForm.tsx | 2 +- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/license/licence-fns.ts b/backend/src/ee/services/license/licence-fns.ts index 411882b25..189a3c4e0 100644 --- a/backend/src/ee/services/license/licence-fns.ts +++ b/backend/src/ee/services/license/licence-fns.ts @@ -16,7 +16,6 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentLimit: null, environmentsUsed: 0, dynamicSecret: false, - secretReplication: false, secretVersioning: true, pitRecovery: false, ipAllowlisting: false, diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index f1ee6ab13..0c8fdc197 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -52,7 +52,6 @@ export type TFeatureSet = { has_used_trial: true; secretApproval: false; secretRotation: true; - secretReplication: false; }; export type TOrgPlansTableDTO = { diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 83eaf2c90..31c057d99 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -86,7 +86,7 @@ export const secretImportServiceFactory = ({ ); if (isReplication) { const plan = await licenseService.getPlan(actorOrgId); - if (!plan.secretReplication) { + if (!plan.secretApproval) { throw new BadRequestError({ message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." }); @@ -338,7 +338,7 @@ export const secretImportServiceFactory = ({ ); const plan = await licenseService.getPlan(actorOrgId); - if (!plan.secretReplication) { + if (!plan.secretApproval) { throw new BadRequestError({ message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." }); diff --git a/frontend/src/hooks/api/subscriptions/types.ts b/frontend/src/hooks/api/subscriptions/types.ts index 4eb54cc11..45414292d 100644 --- a/frontend/src/hooks/api/subscriptions/types.ts +++ b/frontend/src/hooks/api/subscriptions/types.ts @@ -24,7 +24,6 @@ export type SubscriptionPlan = { scim: boolean; ldap: boolean; groups: boolean; - secretReplication: boolean; status: | "incomplete" | "incomplete_expired" diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index 5832779e3..f579852a2 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -69,7 +69,7 @@ export const CreateSecretImportForm = ({ isReplication }: TFormSchema) => { try { - if (isReplication && !subscription?.secretReplication) { + if (isReplication && !subscription?.secretApproval) { onUpgradePlan(); return; } From c67642786fa063aa8caec8074f82802c2a15fea7 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Thu, 30 May 2024 16:38:01 -0400 Subject: [PATCH 24/27] make replicated secrets more intuitive --- .../secret-replication/secret-replication-dal.ts | 10 ++++++++++ .../secret-replication/secret-replication-service.ts | 6 ++++-- .../services/secret-folder/secret-folder-service.ts | 9 +++++++-- .../components/ActionBar/CreateSecretImportForm.tsx | 8 ++++---- .../SecretImportListView/SecretImportItem.tsx | 9 ++++----- 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/backend/src/ee/services/secret-replication/secret-replication-dal.ts b/backend/src/ee/services/secret-replication/secret-replication-dal.ts index e1013df99..5426a51cf 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-dal.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-dal.ts @@ -9,6 +9,16 @@ export type TSecretReplicationDALFactory = ReturnType { const orm = ormify(db, TableName.SecretVersion); + /** + * Retrieves secret versions based on the specified filter criteria. + * + * @param {Object} filter - The filter criteria for querying secret versions. + * @param {string} filter.folderId - The ID of the folder containing the secrets. + * @param {Array} filter.secrets - An array of secret objects containing the ID and version of each secret. + * @param {Knex} [tx] - An optional Knex transaction object. If provided, the query will be executed within this transaction. + * + * @returns {Promise>} A promise that resolves to an array of secret version documents that match the filter criteria. + */ const findSecretVersions = async ( filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index d1b970b68..1f8b46058 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -84,7 +84,7 @@ export const secretReplicationServiceFactory = ({ pickOnlyImportIds, _deDupeReplicationQueue: deDupeReplicationQueue, _deDupeQueue: deDupeQueue - } = job.data; + } = job.data; // source import details (this is where the secrets are to be synced from) // filter for initial filling let secretImports = await secretImportDAL.find({ @@ -97,8 +97,10 @@ export const secretReplicationServiceFactory = ({ : secretImports; if (!secretImports.length || !secrets.length) return; - // unfiltered secrets to be replicated + // unfiltered secrets to be replicated (will fetch the latest versions in case another queue already processed this request) const toBeReplicatedSecrets = await secretReplicationDAL.findSecretVersions({ folderId, secrets }); + + // case: https://www.notion.so/infisical/Secret-Replication-6907fbe3130c4124976f7cba1b9fc4c7 const replicatedSecrets = toBeReplicatedSecrets.filter( ({ version, latestReplicatedVersion, secretBlindIndex }) => secretBlindIndex && (version === 1 || latestReplicatedVersion <= version) diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index ee537577e..97258c006 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -253,7 +253,7 @@ export const secretFolderServiceFactory = ({ const env = await projectEnvDAL.findOne({ projectId, slug: environment }); if (!env) throw new BadRequestError({ message: "Environment not found", name: "Update folder" }); const folder = await folderDAL - .findOne({ envId: env.id, id, parentId: parentFolder.id }) + .findOne({ envId: env.id, id, parentId: parentFolder.id, isReserved: false }) // now folder api accepts id based change // this is for cli backward compatiability and when cli removes this, we will remove this logic .catch(() => folderDAL.findOne({ envId: env.id, name: id, parentId: parentFolder.id })); @@ -328,7 +328,12 @@ export const secretFolderServiceFactory = ({ if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); const [doc] = await folderDAL.delete( - { envId: env.id, [uuidValidate(idOrName) ? "id" : "name"]: idOrName, parentId: parentFolder.id }, + { + envId: env.id, + [uuidValidate(idOrName) ? "id" : "name"]: idOrName, + parentId: parentFolder.id, + isReserved: false + }, tx ); if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Delete folder" }); diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index f579852a2..7f9f2a4fe 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -156,8 +156,8 @@ export const CreateSecretImportForm = ({ errorText={error?.message} helperText={ value - ? "Manual control over when updates propagate in approval mode, giving you the flexibility to push changes as needed." - : "Instantaneous updates from the linked source on approval mode, ensuring real-time synchronization." + ? "Secrets from the source will be automatically sent to the destination. If approval policies exist at the destination, the secrets will be sent as approval requests instead of being applied immediately." + : "Secrets from the source location will be imported to the selected destination immediately, ignoring any approval policies at the destination." } > )} diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx index 07c66f17c..fee86bc14 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportItem.tsx @@ -24,7 +24,6 @@ import { IconButton, SecretInput, TableContainer, - Tag, Tooltip } from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub, useWorkspace } from "@app/context"; @@ -47,17 +46,17 @@ type Props = { export const EnvFolderIcon = ({ env, secretPath, - isReplication + // isReplication }: { env: string; secretPath: string; - isReplication?: boolean; + // isReplication?: boolean; }) => (
{env || "-"}
{secretPath && (
- {isReplication && Replication Mode} + {/* {isReplication && Replication Mode} */} {secretPath}
@@ -168,7 +167,7 @@ export const SecretImportItem = ({
From cf690e2e1616b093ffabab68fefa5752e56b5d8a Mon Sep 17 00:00:00 2001 From: = Date: Fri, 31 May 2024 17:36:01 +0530 Subject: [PATCH 25/27] feat: switched to pull all and selectively replicated strategy and simplified logic --- .../20240529111503_secret-replication.ts | 48 --- .../secret-approval-requests-secrets.ts | 3 +- backend/src/db/schemas/secret-versions.ts | 3 +- backend/src/db/schemas/secrets.ts | 3 +- .../secret-approval-request-service.ts | 26 +- .../secret-replication-constants.ts | 1 + .../secret-replication-dal.ts | 55 +-- .../secret-replication-service.ts | 333 +++++++++++------- backend/src/queue/queue-service.ts | 2 +- backend/src/server/routes/index.ts | 7 +- .../secret-import/secret-import-service.ts | 23 +- backend/src/services/secret/secret-dal.ts | 2 +- backend/src/services/secret/secret-fns.ts | 2 +- backend/src/services/secret/secret-queue.ts | 30 +- backend/src/services/secret/secret-service.ts | 51 +-- backend/src/services/secret/secret-types.ts | 8 +- 16 files changed, 259 insertions(+), 338 deletions(-) create mode 100644 backend/src/ee/services/secret-replication/secret-replication-constants.ts diff --git a/backend/src/db/migrations/20240529111503_secret-replication.ts b/backend/src/db/migrations/20240529111503_secret-replication.ts index 00bf85c38..ddb965df4 100644 --- a/backend/src/db/migrations/20240529111503_secret-replication.ts +++ b/backend/src/db/migrations/20240529111503_secret-replication.ts @@ -32,30 +32,6 @@ export async function up(knex: Knex): Promise { }); } - const doesSecretIsReplicatedExist = await knex.schema.hasColumn(TableName.Secret, "isReplicated"); - if (await knex.schema.hasTable(TableName.Secret)) { - await knex.schema.alterTable(TableName.Secret, (t) => { - if (!doesSecretIsReplicatedExist) t.boolean("isReplicated"); - }); - } - - const doesSecretVersionIsReplicatedExist = await knex.schema.hasColumn(TableName.SecretVersion, "isReplicated"); - if (await knex.schema.hasTable(TableName.SecretVersion)) { - await knex.schema.alterTable(TableName.SecretVersion, (t) => { - if (!doesSecretVersionIsReplicatedExist) t.boolean("isReplicated"); - }); - } - - const doesSecretApprovalRequestSecretIsReplicatedExist = await knex.schema.hasColumn( - TableName.SecretApprovalRequestSecret, - "isReplicated" - ); - if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { - await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { - if (!doesSecretApprovalRequestSecretIsReplicatedExist) t.boolean("isReplicated"); - }); - } - const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( TableName.SecretApprovalRequest, "isReplicated" @@ -97,30 +73,6 @@ export async function down(knex: Knex): Promise { }); } - const doesSecretIsReplicatedExist = await knex.schema.hasColumn(TableName.Secret, "isReplicated"); - if (await knex.schema.hasTable(TableName.Secret)) { - await knex.schema.alterTable(TableName.Secret, (t) => { - if (doesSecretIsReplicatedExist) t.dropColumns("isReplicated"); - }); - } - - const doesSecretVersionIsReplicatedExist = await knex.schema.hasColumn(TableName.SecretVersion, "isReplicated"); - if (await knex.schema.hasTable(TableName.SecretVersion)) { - await knex.schema.alterTable(TableName.SecretVersion, (t) => { - if (doesSecretVersionIsReplicatedExist) t.dropColumns("isReplicated"); - }); - } - - const doesSecretApprovalRequestSecretIsReplicatedExist = await knex.schema.hasColumn( - TableName.SecretApprovalRequestSecret, - "isReplicated" - ); - if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { - await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { - if (doesSecretApprovalRequestSecretIsReplicatedExist) t.dropColumns("isReplicated"); - }); - } - const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( TableName.SecretApprovalRequest, "isReplicated" diff --git a/backend/src/db/schemas/secret-approval-requests-secrets.ts b/backend/src/db/schemas/secret-approval-requests-secrets.ts index 12b97e006..b795b47b4 100644 --- a/backend/src/db/schemas/secret-approval-requests-secrets.ts +++ b/backend/src/db/schemas/secret-approval-requests-secrets.ts @@ -31,8 +31,7 @@ export const SecretApprovalRequestsSecretsSchema = z.object({ requestId: z.string().uuid(), op: z.string(), secretId: z.string().uuid().nullable().optional(), - secretVersion: z.string().uuid().nullable().optional(), - isReplicated: z.boolean().nullable().optional() + secretVersion: z.string().uuid().nullable().optional() }); export type TSecretApprovalRequestsSecrets = z.infer; diff --git a/backend/src/db/schemas/secret-versions.ts b/backend/src/db/schemas/secret-versions.ts index 08cfc49cd..d60db9b75 100644 --- a/backend/src/db/schemas/secret-versions.ts +++ b/backend/src/db/schemas/secret-versions.ts @@ -32,8 +32,7 @@ export const SecretVersionsSchema = z.object({ folderId: z.string().uuid(), userId: z.string().uuid().nullable().optional(), createdAt: z.date(), - updatedAt: z.date(), - isReplicated: z.boolean().nullable().optional() + updatedAt: z.date() }); export type TSecretVersions = z.infer; diff --git a/backend/src/db/schemas/secrets.ts b/backend/src/db/schemas/secrets.ts index 8174f5171..f261c40bb 100644 --- a/backend/src/db/schemas/secrets.ts +++ b/backend/src/db/schemas/secrets.ts @@ -30,8 +30,7 @@ export const SecretsSchema = z.object({ userId: z.string().uuid().nullable().optional(), folderId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date(), - isReplicated: z.boolean().nullable().optional() + updatedAt: z.date() }); export type TSecrets = z.infer; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 86d8920a8..5d0977134 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -380,8 +380,7 @@ export const secretApprovalRequestServiceFactory = ({ "secretReminderRepeatDays", "algorithm", "keyEncoding", - "secretBlindIndex", - "isReplicated" + "secretBlindIndex" ]), tags: el?.tags.map(({ id }) => id), version: 1, @@ -426,7 +425,6 @@ export const secretApprovalRequestServiceFactory = ({ "secretKeyTag", "secretKeyIV", "metadata", - "isReplicated", "skipMultilineEncoding", "secretReminderNote", "secretReminderRepeatDays", @@ -490,28 +488,8 @@ export const secretApprovalRequestServiceFactory = ({ projectId, secretPath: folder.path, environmentSlug: folder.environmentSlug, - folderId: folder.id, actorId, - actor, - environmentId: folder.envId, - secrets: mergeStatus.secrets.created - .map(({ id, version }) => ({ - operation: SecretOperations.Create, - version, - id - })) - .concat( - mergeStatus.secrets.updated.map(({ id, version }) => ({ - operation: SecretOperations.Update, - version, - id - })), - mergeStatus.secrets.deleted.map(({ id, version }) => ({ - operation: SecretOperations.Delete, - version, - id - })) - ) + actor }); return mergeStatus; }; diff --git a/backend/src/ee/services/secret-replication/secret-replication-constants.ts b/backend/src/ee/services/secret-replication/secret-replication-constants.ts new file mode 100644 index 000000000..88c9ee166 --- /dev/null +++ b/backend/src/ee/services/secret-replication/secret-replication-constants.ts @@ -0,0 +1 @@ +export const MAX_REPLICATION_DEPTH = 5; diff --git a/backend/src/ee/services/secret-replication/secret-replication-dal.ts b/backend/src/ee/services/secret-replication/secret-replication-dal.ts index 5426a51cf..3c4c021fd 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-dal.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-dal.ts @@ -1,59 +1,10 @@ -import { Knex } from "knex"; - import { TDbClient } from "@app/db"; -import { SecretType, TableName, TSecretVersions } from "@app/db/schemas"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; export type TSecretReplicationDALFactory = ReturnType; export const secretReplicationDALFactory = (db: TDbClient) => { const orm = ormify(db, TableName.SecretVersion); - - /** - * Retrieves secret versions based on the specified filter criteria. - * - * @param {Object} filter - The filter criteria for querying secret versions. - * @param {string} filter.folderId - The ID of the folder containing the secrets. - * @param {Array} filter.secrets - An array of secret objects containing the ID and version of each secret. - * @param {Knex} [tx] - An optional Knex transaction object. If provided, the query will be executed within this transaction. - * - * @returns {Promise>} A promise that resolves to an array of secret version documents that match the filter criteria. - */ - const findSecretVersions = async ( - filter: { folderId: string; secrets: { id: string; version: number }[] }, - tx?: Knex - ) => { - if (!filter.secrets) return []; - - const sqlRawDocs = await (tx || db)(TableName.SecretVersion) - .where({ folderId: filter.folderId }) - .andWhere((bd) => { - filter.secrets.forEach((el) => { - void bd.orWhere({ - [`${TableName.SecretVersion}.secretId` as "secretId"]: el.id, - [`${TableName.SecretVersion}.version` as "version"]: el.version, - [`${TableName.SecretVersion}.type` as "type"]: SecretType.Shared - }); - }); - }) - .leftJoin( - (tx || db)(TableName.SecretVersion) - .where("isReplicated", true) - .groupBy("secretId") - .max("version") - .select("secretId") - .as("latestVersion"), - `${TableName.SecretVersion}.secretId`, - "latestVersion.secretId" - ) - .select(db.ref("max").withSchema("latestVersion").as("latestReplicatedVersion")) - .select(selectAllTableCols(TableName.SecretVersion)); - - return sqlRawDocs; - }; - - return { - findSecretVersions, - ...orm - }; + return orm; }; diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 1f8b46058..fd2f7cc1a 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -1,41 +1,45 @@ +import { SecretType, TSecrets } from "@app/db/schemas"; import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; -import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; -import { groupBy } from "@app/lib/fn"; +import { groupBy, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns"; -import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; -import { SecretOperations, TSyncSecretsDTO } from "@app/services/secret/secret-types"; +import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue"; +import { SecretOperations } from "@app/services/secret/secret-types"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types"; import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; +import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { TSecretReplicationDALFactory } from "./secret-replication-dal"; +import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants"; type TSecretReplicationServiceFactoryDep = { - secretReplicationDAL: TSecretReplicationDALFactory; secretDAL: Pick< TSecretDALFactory, - "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" + "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction" >; secretVersionDAL: Pick; - secretImportDAL: Pick; - folderDAL: Pick; + secretImportDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" + >; secretVersionTagDAL: Pick; - secretQueueService: Pick; - snapshotService: Pick; + secretQueueService: Pick; queueService: Pick; secretApprovalPolicyService: Pick; keyStore: Pick; @@ -47,16 +51,35 @@ type TSecretReplicationServiceFactoryDep = { TSecretApprovalRequestSecretDALFactory, "insertMany" | "insertApprovalSecretTags" >; + projectBotService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; const SECRET_IMPORT_SUCCESS_LOCK = 10; + const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`; -const getReplicationKeyLockPrefix = (keyName: string) => `REPLICATION_SECRET_${keyName}`; +const getReplicationKeyLockPrefix = (projectId: string, environmentSlug: string, secretPath: string) => + `REPLICATION_SECRET_${projectId}-${environmentSlug}-${secretPath}`; export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`; +const getDecryptedKeyValue = (key: string, secret: TSecrets) => { + const secretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key + }); + + const secretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key + }); + return { key: secretKey, value: secretValue }; +}; + export const secretReplicationServiceFactory = ({ - secretReplicationDAL, secretDAL, queueService, secretVersionDAL, @@ -69,124 +92,208 @@ export const secretReplicationServiceFactory = ({ secretApprovalRequestSecretDAL, secretApprovalRequestDAL, secretQueueService, - projectMembershipDAL + projectMembershipDAL, + projectBotService }: TSecretReplicationServiceFactoryDep) => { + const getReplicatedSecrets = ( + botKey: string, + localSecrets: TSecrets[], + importedSecrets: { secrets: TSecrets[] }[] + ) => { + const deDupe = new Set(); + const secrets = localSecrets + .filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)) + .map((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + deDupe.add(decryptedSecret.key); + return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }; + }); + + for (let i = importedSecrets.length - 1; i >= 0; i = -1) { + importedSecrets[i].secrets.forEach((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + if (deDupe.has(decryptedSecret.key) || !el.secretBlindIndex) { + return; + } + deDupe.add(decryptedSecret.key); + secrets.push({ ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }); + }); + } + return secrets; + }; + + // IMPORTANT NOTE BEFORE READING THE FUNCTION + // SOURCE - Where secrets are copied from + // DESTINATION - Where the replicated imports that points to SOURCE from Destination queueService.start(QueueName.SecretReplication, async (job) => { logger.info(job.data, "Replication started"); const { - secrets, - folderId, secretPath, - environmentId, + environmentSlug, projectId, actorId, actor, pickOnlyImportIds, _deDupeReplicationQueue: deDupeReplicationQueue, - _deDupeQueue: deDupeQueue - } = job.data; // source import details (this is where the secrets are to be synced from) + _deDupeQueue: deDupeQueue, + _depth: depth = 0 + } = job.data; + if (depth > MAX_REPLICATION_DEPTH) return; - // filter for initial filling - let secretImports = await secretImportDAL.find({ + const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath); + if (!folder) return; + + // the the replicated imports made to the source. These are the destinations + const destinationSecretImports = await secretImportDAL.find({ importPath: secretPath, - importEnv: environmentId, - isReplication: true + importEnv: folder.envId }); - secretImports = pickOnlyImportIds - ? secretImports.filter(({ id }) => pickOnlyImportIds?.includes(id)) - : secretImports; - if (!secretImports.length || !secrets.length) return; - // unfiltered secrets to be replicated (will fetch the latest versions in case another queue already processed this request) - const toBeReplicatedSecrets = await secretReplicationDAL.findSecretVersions({ folderId, secrets }); + // CASE: normal mode <- link import <- replicated import + const nonReplicatedDestinationImports = destinationSecretImports.filter(({ isReplication }) => !isReplication); + if (nonReplicatedDestinationImports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(nonReplicatedDestinationImports, (i) => i.folderId).map( + ({ folderId }) => folderId + ); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); + await Promise.all( + nonReplicatedDestinationImports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue?.[ + uniqueSecretQueueKey( + foldersGroupedById[folderId][0]?.environmentSlug as string, + foldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + secretQueueService.replicateSecrets({ + projectId, + secretPath: foldersGroupedById[folderId][0]?.path as string, + environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, + actorId, + actor, + _depth: depth + 1, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue + }) + ) + ); + } - // case: https://www.notion.so/infisical/Secret-Replication-6907fbe3130c4124976f7cba1b9fc4c7 - const replicatedSecrets = toBeReplicatedSecrets.filter( - ({ version, latestReplicatedVersion, secretBlindIndex }) => - secretBlindIndex && (version === 1 || latestReplicatedVersion <= version) + let destinationReplicatedSecretImports = destinationSecretImports.filter(({ isReplication }) => + Boolean(isReplication) ); - const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId); - // this is to filter out personal secrets - const sanitizedSecrets = secrets.filter(({ id }) => Object.hasOwn(replicatedSecretsGroupBySecretId, id)); - if (!sanitizedSecrets.length) return; + destinationReplicatedSecretImports = pickOnlyImportIds + ? destinationReplicatedSecretImports.filter(({ id }) => pickOnlyImportIds?.includes(id)) + : destinationReplicatedSecretImports; + if (!destinationReplicatedSecretImports.length) return; + + const botKey = await projectBotService.getBotKey(projectId); + + // these are the secrets to be added in replicated folders + const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared }); + const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); + const sourceImportedSecrets = await fnSecretsFromImports({ + allowedImports: sourceSecretImports, + secretDAL, + folderDAL, + secretImportDAL + }); + // secrets that gets replicated across imports + const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets); + const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string); const lock = await keyStore.acquireLock( - replicatedSecrets.map(({ id }) => getReplicationKeyLockPrefix(id)), + [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)], 5000 ); try { /* eslint-disable no-await-in-loop */ - for (const secretImport of secretImports) { + for (const destinationSecretImport of destinationReplicatedSecretImports) { try { const hasJobCompleted = await keyStore.getItem( - keystoreReplicationSuccessKey(job.id as string, secretImport.id), + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), KeyStorePrefixes.SecretReplication ); if (hasJobCompleted) { logger.info( - { jobId: job.id, importId: secretImport.id }, + { jobId: job.id, importId: destinationSecretImport.id }, "Skipping this job as this has been successfully replicated." ); // eslint-disable-next-line continue; } - const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); - if (!importedFolder) throw new BadRequestError({ message: "Imported folder not found" }); + const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [ + destinationSecretImport.folderId + ]); + if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" }); - let replicationFolder = await folderDAL.findOne({ - parentId: importedFolder.id, - name: getReplicationFolderName(secretImport.id), + let destinationReplicationFolder = await folderDAL.findOne({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), isReserved: true }); - if (!replicationFolder) { - replicationFolder = await folderDAL.create({ - parentId: importedFolder.id, - name: getReplicationFolderName(secretImport.id), - envId: importedFolder.envId, + if (!destinationReplicationFolder) { + destinationReplicationFolder = await folderDAL.create({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), + envId: destinationFolder.envId, isReserved: true }); } - const replicationFolderId = replicationFolder.id; + const destinationReplicationFolderId = destinationReplicationFolder.id; - const localSecrets = await secretDAL.find({ - $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, - folderId: replicationFolderId + const destinationLocalSecretsFromDB = await secretDAL.find({ + folderId: destinationReplicationFolderId + }); + const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }; }); - const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); - const locallyCreatedSecrets = sanitizedSecrets + const destinationLocalSecretsGroupedByBlindIndex = groupBy( + destinationLocalSecrets.filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)), + (i) => i.secretBlindIndex as string + ); + + const locallyCreatedSecrets = sourceSecrets .filter( - ({ operation, id }) => - // upsert: irrespective of create or update its a create if not found in dashboard - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - !localSecretsGroupedByBlindIndex[ - replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string - ]?.[0] + ({ secretBlindIndex }) => !destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] ) .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create - const locallyUpdatedSecrets = sanitizedSecrets + const locallyUpdatedSecrets = sourceSecrets .filter( - ({ operation, id }) => - // upsert: irrespective of create or update its an update if not found in dashboard - (operation === SecretOperations.Create || operation === SecretOperations.Update) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ({ secretBlindIndex, secretKey, secretValue }) => + destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] && + // if key or value changed + (destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretKey !== secretKey || + destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretValue !== + secretValue) ) - .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite create ops to update + .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create - const locallyDeletedSecrets = sanitizedSecrets.filter( - ({ operation, id }) => - operation === SecretOperations.Delete && - Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ); + const locallyDeletedSecrets = destinationLocalSecrets + .filter(({ secretBlindIndex }) => !sourceSecretsGroupByBlindIndex[secretBlindIndex as string]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Delete })); + + const isEmtpy = + locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0; + // eslint-disable-next-line + if (isEmtpy) continue; const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( projectId, - importedFolder.environmentSlug, - importedFolder.path + destinationFolder.environmentSlug, + destinationFolder.path ); // this means it should be a approval request rather than direct replication if (policy && actor === ActorType.USER) { @@ -196,15 +303,15 @@ export const secretReplicationServiceFactory = ({ return; } - const localSecretsLatestVersions = localSecrets.map(({ id }) => id); + const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id); const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( - replicationFolderId, + destinationReplicationFolderId, localSecretsLatestVersions ); await secretApprovalRequestDAL.transaction(async (tx) => { const approvalRequestDoc = await secretApprovalRequestDAL.create( { - folderId: replicationFolderId, + folderId: destinationReplicationFolderId, slug: alphaNumericNanoId(), policyId: policy.id, status: "open", @@ -217,9 +324,9 @@ export const secretReplicationServiceFactory = ({ const commits = locallyCreatedSecrets .concat(locallyUpdatedSecrets) .concat(locallyDeletedSecrets) - .map(({ id, operation }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; - const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; + .map((doc) => { + const { operation } = doc; + const localSecret = destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; return { op: operation, @@ -237,7 +344,6 @@ export const secretReplicationServiceFactory = ({ secretCommentIV: doc.secretCommentIV, secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, - isReplicated: true, skipMultilineEncoding: doc.skipMultilineEncoding, // except create operation other two needs the secret id and version id ...(operation !== SecretOperations.Create @@ -250,18 +356,16 @@ export const secretReplicationServiceFactory = ({ return { ...approvalRequestDoc, commits: approvalCommits }; }); } else { - let nestedImportSecrets: TSyncSecretsDTO["secrets"] = []; - await secretReplicationDAL.transaction(async (tx) => { + await secretDAL.transaction(async (tx) => { if (locallyCreatedSecrets.length) { - const newSecrets = await fnSecretBulkInsert({ - folderId: replicationFolderId, + await fnSecretBulkInsert({ + folderId: destinationReplicationFolderId, secretVersionDAL, secretDAL, tx, secretTagDAL, secretVersionTagDAL, - inputSecrets: locallyCreatedSecrets.map(({ id }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; + inputSecrets: locallyCreatedSecrets.map((doc) => { return { keyEncoding: doc.keyEncoding, algorithm: doc.algorithm, @@ -277,30 +381,25 @@ export const secretReplicationServiceFactory = ({ secretCommentIV: doc.secretCommentIV, secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, - isReplicated: true, skipMultilineEncoding: doc.skipMultilineEncoding }; }) }); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })) - ); } if (locallyUpdatedSecrets.length) { - const newSecrets = await fnSecretBulkUpdate({ + await fnSecretBulkUpdate({ projectId, - folderId: replicationFolderId, + folderId: destinationReplicationFolderId, secretVersionDAL, secretDAL, tx, secretTagDAL, secretVersionTagDAL, - inputSecrets: locallyUpdatedSecrets.map(({ id }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; + inputSecrets: locallyUpdatedSecrets.map((doc) => { return { filter: { - folderId: replicationFolderId, - id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + folderId: destinationReplicationFolderId, + id: destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id }, data: { keyEncoding: doc.keyEncoding, @@ -317,56 +416,46 @@ export const secretReplicationServiceFactory = ({ secretCommentIV: doc.secretCommentIV, secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, - isReplicated: true, skipMultilineEncoding: doc.skipMultilineEncoding } }; }) }); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id })) - ); } if (locallyDeletedSecrets.length) { - const newSecrets = await secretDAL.delete( + await secretDAL.delete( { $in: { id: locallyDeletedSecrets.map(({ id }) => id) }, - isReplicated: true, - folderId: replicationFolderId + folderId: destinationReplicationFolderId }, tx ); - nestedImportSecrets = nestedImportSecrets.concat( - newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id })) - ); } }); await secretQueueService.syncSecrets({ projectId, - secretPath: importedFolder.path, - _deDupeReplicationQueue: deDupeReplicationQueue, - _deDupeQueue: deDupeQueue, - environmentSlug: importedFolder.environmentSlug, + secretPath: destinationFolder.path, + environmentSlug: destinationFolder.environmentSlug, actorId, actor, - secrets: nestedImportSecrets, - folderId: importedFolder.id, - environmentId: importedFolder.envId + _depth: depth + 1, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue }); } // this is used to avoid multiple times generating secret approval by failed one await keyStore.setItemWithExpiry( - keystoreReplicationSuccessKey(job.id as string, secretImport.id), + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), SECRET_IMPORT_SUCCESS_LOCK, 1, KeyStorePrefixes.SecretReplication ); - await secretImportDAL.updateById(secretImport.id, { + await secretImportDAL.updateById(destinationSecretImport.id, { lastReplicated: new Date(), replicationStatus: null, isReplicationSuccess: true @@ -374,17 +463,15 @@ export const secretReplicationServiceFactory = ({ } catch (err) { logger.error( err, - `Failed to replicate secret with import id=[${secretImport.id}] env=[${secretImport.importEnv.slug}] path=[${secretImport.importPath}]` + `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]` ); - await secretImportDAL.updateById(secretImport.id, { + await secretImportDAL.updateById(destinationSecretImport.id, { lastReplicated: new Date(), replicationStatus: (err as Error)?.message.slice(0, 500), isReplicationSuccess: false }); } } - - await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true }); /* eslint-enable no-await-in-loop */ } finally { await lock.release(); diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index f0ce62318..7046058b7 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -123,7 +123,7 @@ export type TQueueJobTypes = { }; [QueueName.SecretReplication]: { name: QueueJobs.SecretReplication; - payload: Omit; + payload: TSyncSecretsDTO; }; [QueueName.SecretSync]: { name: QueueJobs.SecretSync; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 05b9982bb..ed2d31254 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -44,7 +44,6 @@ import { secretApprovalRequestDALFactory } from "@app/ee/services/secret-approva import { secretApprovalRequestReviewerDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-reviewer-dal"; import { secretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service"; -import { secretReplicationDALFactory } from "@app/ee/services/secret-replication/secret-replication-dal"; import { secretReplicationServiceFactory } from "@app/ee/services/secret-replication/secret-replication-service"; import { secretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue"; @@ -195,7 +194,6 @@ export const registerRoutes = async ( const projectBotDAL = projectBotDALFactory(db); const secretDAL = secretDALFactory(db); - const secretReplicationDAL = secretReplicationDALFactory(db); const secretTagDAL = secretTagDALFactory(db); const folderDAL = secretFolderDALFactory(db); const folderVersionDAL = secretFolderVersionDALFactory(db); @@ -673,15 +671,14 @@ export const registerRoutes = async ( secretImportDAL, keyStore, queueService, - secretReplicationDAL, folderDAL, secretApprovalPolicyService, secretBlindIndexDAL, secretApprovalRequestDAL, secretApprovalRequestSecretDAL, secretQueueService, - snapshotService, - projectMembershipDAL + projectMembershipDAL, + projectBotService }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 31c057d99..237c7cfe4 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -13,7 +13,6 @@ import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretQueueFactory } from "../secret/secret-queue"; -import { SecretOperations } from "../secret/secret-types"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "./secret-import-dal"; import { fnSecretsFromImports } from "./secret-import-fns"; @@ -139,24 +138,21 @@ export const secretImportServiceFactory = ({ }); if (secImport.isReplication && sourceFolder) { - const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); await secretQueueService.replicateSecrets({ secretPath: secImport.importPath, projectId, environmentSlug: importEnv.slug, pickOnlyImportIds: [secImport.id], - folderId: sourceFolder.id, - secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), actorId, - actor, - environmentId: importEnv.id + actor }); } else { await secretQueueService.syncSecrets({ - secretPath: secImport.importPath, + secretPath, projectId, - environmentSlug: importEnv.slug, - excludeReplication: true + environmentSlug: environment, + actorId, + actor }); } @@ -307,7 +303,8 @@ export const secretImportServiceFactory = ({ secretPath, projectId, environmentSlug: environment, - excludeReplication: true + actor, + actorId }); return secImport; @@ -372,18 +369,14 @@ export const secretImportServiceFactory = ({ secretImportDoc.importPath ); - const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id }); if (membership && sourceFolder) { await secretQueueService.replicateSecrets({ secretPath: secretImportDoc.importPath, projectId, environmentSlug: secretImportDoc.importEnv.slug, pickOnlyImportIds: [secretImportDoc.id], - folderId: sourceFolder.id, - secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })), actorId, - actor, - environmentId: secretImportDoc.importEnv.id + actor }); } diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 9c72f9e38..1a2e414dd 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => { }; const deleteMany = async ( - data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>, + data: Array<{ blindIndex: string; type: SecretType }>, folderId: string, userId: string, tx?: Knex diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index b3fee7f7a..3cd6c4e6e 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -536,7 +536,7 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({ + inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 32be44bc1..d40a18e5e 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -64,7 +64,7 @@ export type TGetSecrets = { }; const MAX_SYNC_SECRET_DEPTH = 5; -const uniqueSecretQueueKey = (environment: string, secretPath: string) => +export const uniqueSecretQueueKey = (environment: string, secretPath: string) => `secret-queue-dedupe-${environment}-${secretPath}`; type TIntegrationSecret = Record; @@ -325,6 +325,7 @@ export const secretQueueFactory = ({ const syncSecrets = async ({ // seperate de-dupe queue for integration sync and replication sync _deDupeQueue: deDupeQueue = {}, + _depth: depth = 0, _deDupeReplicationQueue: deDupeReplicationQueue = {}, ...dto }: TSyncSecretsDTO) => { @@ -332,7 +333,11 @@ export const secretQueueFactory = ({ `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]` ); const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath); - if (!dto.excludeReplication ? deDupeReplicationQueue?.[deDuplicationKey] : deDupeQueue?.[deDuplicationKey]) { + if ( + !dto.excludeReplication + ? deDupeReplicationQueue?.[deDuplicationKey] + : deDupeQueue?.[deDuplicationKey] || depth > MAX_SYNC_SECRET_DEPTH + ) { return; } // eslint-disable-next-line @@ -342,7 +347,12 @@ export const secretQueueFactory = ({ await queueService.queue( QueueName.SecretSync, QueueJobs.SecretSync, - { ...dto, _deDupeQueue: deDupeQueue, _deDupeReplicationQueue: deDupeReplicationQueue } as TSyncSecretsDTO, + { + ...dto, + _deDupeQueue: deDupeQueue, + _deDupeReplicationQueue: deDupeReplicationQueue, + _depth: depth + } as TSyncSecretsDTO, { removeOnFail: true, removeOnComplete: true, @@ -360,22 +370,21 @@ export const secretQueueFactory = ({ const { _deDupeQueue: deDupeQueue, _deDupeReplicationQueue: deDupeReplicationQueue, + _depth: depth, secretPath, - environmentId, projectId, environmentSlug: environment, - secrets, - folderId, excludeReplication, actorId, actor } = job.data; + await queueService.queue( QueueName.SecretWebhook, QueueJobs.SecWebhook, { environment, projectId, secretPath }, { - jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`, + jobId: `secret-webhook-${environment}-${projectId}-${secretPath}`, removeOnFail: { count: 5 }, removeOnComplete: true, delay: 1000, @@ -390,11 +399,9 @@ export const secretQueueFactory = ({ if (!excludeReplication) { await replicateSecrets({ _deDupeReplicationQueue: deDupeReplicationQueue, - environmentId, + _depth: depth, projectId, secretPath, - folderId, - secrets, actorId, actor, excludeReplication, @@ -405,6 +412,7 @@ export const secretQueueFactory = ({ queueService.start(QueueName.IntegrationSync, async (job) => { const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data; + if (depth > MAX_SYNC_SECRET_DEPTH) return; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { @@ -450,6 +458,7 @@ export const secretQueueFactory = ({ secretPath: foldersGroupedById[folderId][0]?.path as string, environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, _deDupeQueue: deDupeQueue, + _depth: depth + 1, excludeReplication: true }) ) @@ -487,6 +496,7 @@ export const secretQueueFactory = ({ secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, _deDupeQueue: deDupeQueue, + _depth: depth + 1, excludeReplication: true }) ) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 63a6aa928..5688f7f15 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -44,7 +44,6 @@ import { } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { - SecretOperations, TAttachSecretTagsDTO, TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, @@ -238,19 +237,10 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, - folderId: folder.id, actorId, actor, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: [ - { - operation: SecretOperations.Create, - id: secret[0].id, - version: 1 - } - ] + environmentSlug: folder.environment.slug }); return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; @@ -362,7 +352,6 @@ export const secretServiceFactory = ({ "secretReminderRepeatDays", "tags" ]), - isReplicated: false, secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName], references: references({ ciphertext: inputSecret.secretValueCiphertext, @@ -385,17 +374,8 @@ export const secretServiceFactory = ({ actor, actorId, secretPath: path, - folderId: folder.id, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: [ - { - operation: SecretOperations.Update, - id: updatedSecret[0].id, - version: updatedSecret[0].version - } - ] + environmentSlug: folder.environment.slug }); return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; @@ -469,17 +449,8 @@ export const secretServiceFactory = ({ actor, actorId, secretPath: path, - folderId: folder.id, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: [ - { - operation: SecretOperations.Delete, - id: deletedSecret[0].id, - version: deletedSecret[0].version - } - ] + environmentSlug: folder.environment.slug }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; @@ -770,11 +741,8 @@ export const secretServiceFactory = ({ actor, actorId, secretPath: path, - folderId: folder.id, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create })) + environmentSlug: folder.environment.slug }); return newSecrets; @@ -851,7 +819,6 @@ export const secretServiceFactory = ({ ...el, folderId, type: SecretType.Shared, - isReplicated: false, secretBlindIndex: newSecretName && newKeyName2BlindIndex[newSecretName] ? newKeyName2BlindIndex[newSecretName] @@ -880,11 +847,8 @@ export const secretServiceFactory = ({ actor, actorId, secretPath: path, - folderId: folder.id, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update })) + environmentSlug: folder.environment.slug }); return secrets; @@ -953,11 +917,8 @@ export const secretServiceFactory = ({ actor, actorId, secretPath: path, - folderId: folder.id, projectId, - environmentSlug: folder.environment.slug, - environmentId: folder.envId, - secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete })) + environmentSlug: folder.environment.slug }); return secretsDeleted; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 8c6d5db06..18a0077fe 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -380,6 +380,7 @@ export enum SecretOperations { export type TSyncSecretsDTO = { _deDupeQueue?: Record; _deDupeReplicationQueue?: Record; + _depth?: number; secretPath: string; projectId: string; environmentSlug: string; @@ -388,15 +389,8 @@ export type TSyncSecretsDTO = { } & (T extends true ? object : { - environmentId: string; - folderId: string; actor: ActorType; actorId: string; // used for import creation to trigger replication pickOnlyImportIds?: string[]; - secrets: { - operation: SecretOperations; - id: string; - version: number; - }[]; }); From c6043568cf58086872d481b552dab9585e8e899d Mon Sep 17 00:00:00 2001 From: = Date: Fri, 31 May 2024 17:36:32 +0530 Subject: [PATCH 26/27] feat: removed isReplicated fields from secret as no longer needed corresponding its changes in ui --- frontend/src/hooks/api/secrets/queries.tsx | 1 - frontend/src/hooks/api/secrets/types.ts | 2 -- frontend/src/lib/fn/string.ts | 2 +- .../components/SecretApprovalRequestChanges.tsx | 8 +++++--- .../SecretImportListView/SecretImportListView.tsx | 1 - .../components/SecretListView/SecretItem.tsx | 11 +++-------- 6 files changed, 9 insertions(+), 16 deletions(-) diff --git a/frontend/src/hooks/api/secrets/queries.tsx b/frontend/src/hooks/api/secrets/queries.tsx index 485f9638e..28999389e 100644 --- a/frontend/src/hooks/api/secrets/queries.tsx +++ b/frontend/src/hooks/api/secrets/queries.tsx @@ -67,7 +67,6 @@ export const decryptSecrets = ( env: encSecret.environment, key: secretKey, value: secretValue, - isReplicated: encSecret.isReplicated, tags: encSecret.tags, comment: secretComment, reminderRepeatDays: encSecret.secretReminderRepeatDays, diff --git a/frontend/src/hooks/api/secrets/types.ts b/frontend/src/hooks/api/secrets/types.ts index d818afe6c..f36872e43 100644 --- a/frontend/src/hooks/api/secrets/types.ts +++ b/frontend/src/hooks/api/secrets/types.ts @@ -14,7 +14,6 @@ export type EncryptedSecret = { secretValueIV: string; secretValueTag: string; __v: number; - isReplicated?: boolean; createdAt: string; updatedAt: string; skipMultilineEncoding?: boolean; @@ -32,7 +31,6 @@ export type DecryptedSecret = { key: string; value: string; comment: string; - isReplicated?: boolean; reminderRepeatDays?: number | null; reminderNote?: string | null; tags: WsTag[]; diff --git a/frontend/src/lib/fn/string.ts b/frontend/src/lib/fn/string.ts index a7de20262..9d3d01cc5 100644 --- a/frontend/src/lib/fn/string.ts +++ b/frontend/src/lib/fn/string.ts @@ -3,7 +3,7 @@ import { ReservedFolders } from "@app/hooks/api/secretFolders/types"; export const formatReservedPaths = (secretPath: string) => { const i = secretPath.indexOf(ReservedFolders.SecretReplication); if (i !== -1) { - return secretPath.slice(0, i); + return `${secretPath.slice(0, i)} - (replication)`; } return secretPath; }; diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 33025342f..85d970d4b 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -201,9 +201,11 @@ export const SecretApprovalRequestChanges = ({
-
- {formatReservedPaths(secretApprovalRequestDetails.secretPath)} -
+ +
+ {formatReservedPaths(secretApprovalRequestDetails.secretPath)} +
+ diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx index d130bcb65..e7537e46f 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx @@ -154,7 +154,6 @@ export const SecretImportListView = ({ }; const handleOpenReplicationSecrets = (replicationImportId: string) => { - console.log(secretImports); const reservedImport = secretImports.find( ({ isReserved, importPath, importEnv }) => importEnv.slug === environment && diff --git a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx index 077018017..98a803020 100644 --- a/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretListView/SecretItem.tsx @@ -224,11 +224,7 @@ export const SecretItem = memo( "ml-3 block h-3.5 w-3.5 group-hover:hidden", isSelected && "hidden" )} - symbolName={ - secret.isReplicated - ? FontAwesomeSpriteName.ReplicatedSecretKey - : FontAwesomeSpriteName.SecretKey - } + symbolName={FontAwesomeSpriteName.SecretKey} />
@@ -424,9 +420,8 @@ export const SecretItem = memo( 0 - ? `Every ${secretReminderRepeatDays} day${ - Number(secretReminderRepeatDays) > 1 ? "s" : "" - } + ? `Every ${secretReminderRepeatDays} day${Number(secretReminderRepeatDays) > 1 ? "s" : "" + } ` : "Reminder" } From 1dfad876cff5bf6447993fd6c232da42a69beb96 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 1 Jun 2024 00:39:09 +0530 Subject: [PATCH 27/27] fix: replicated import not expanding inside folder --- .../components/SecretImportListView/SecretImportListView.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx index e7537e46f..1a41b0d35 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx @@ -158,7 +158,9 @@ export const SecretImportListView = ({ ({ isReserved, importPath, importEnv }) => importEnv.slug === environment && isReserved && - importPath === `/${ReservedFolders.SecretReplication}${replicationImportId}` + importPath === + `${secretPath === "/" ? "" : secretPath}/${ReservedFolders.SecretReplication + }${replicationImportId}` ); if (reservedImport) { setReplicationSecrets((state) => ({