diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 2bc1d9f26..3f1ca94e9 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -52,6 +52,7 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service"; import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; +import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service"; @@ -108,6 +109,7 @@ declare module "fastify" { projectKey: TProjectKeyServiceFactory; projectRole: TProjectRoleServiceFactory; secret: TSecretServiceFactory; + secretReplication: TSecretReplicationServiceFactory; secretTag: TSecretTagServiceFactory; secretImport: TSecretImportServiceFactory; projectBot: TProjectBotServiceFactory; diff --git a/backend/src/db/migrations/20240430024805_secret-replication.ts b/backend/src/db/migrations/20240430024805_secret-replication.ts new file mode 100644 index 000000000..2cb6b4c83 --- /dev/null +++ b/backend/src/db/migrations/20240430024805_secret-replication.ts @@ -0,0 +1,43 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + t.boolean("isReplication").defaultTo(false); + }); + } + + if (await knex.schema.hasTable(TableName.Secret)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.boolean("isReplicated"); + }); + } + + if (await knex.schema.hasTable(TableName.SecretVersion)) { + await knex.schema.alterTable(TableName.SecretVersion, (t) => { + t.boolean("isReplicated"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + t.dropColumns("isReplication"); + }); + } + + if (await knex.schema.hasTable(TableName.Secret)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.dropColumns("isReplicated"); + }); + } + + if (await knex.schema.hasTable(TableName.SecretVersion)) { + await knex.schema.alterTable(TableName.Secret, (t) => { + t.dropColumns("isReplicated"); + }); + } +} diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 9d42d8da5..56148cdd6 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -15,7 +15,8 @@ export const SecretImportsSchema = z.object({ position: z.number(), createdAt: z.date(), updatedAt: z.date(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + isReplication: z.boolean().default(false).nullable().optional() }); export type TSecretImports = z.infer; diff --git a/backend/src/db/schemas/secret-versions.ts b/backend/src/db/schemas/secret-versions.ts index d60db9b75..08cfc49cd 100644 --- a/backend/src/db/schemas/secret-versions.ts +++ b/backend/src/db/schemas/secret-versions.ts @@ -32,7 +32,8 @@ export const SecretVersionsSchema = z.object({ folderId: z.string().uuid(), userId: z.string().uuid().nullable().optional(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecretVersions = z.infer; diff --git a/backend/src/db/schemas/secrets.ts b/backend/src/db/schemas/secrets.ts index f261c40bb..8174f5171 100644 --- a/backend/src/db/schemas/secrets.ts +++ b/backend/src/db/schemas/secrets.ts @@ -30,7 +30,8 @@ export const SecretsSchema = z.object({ userId: z.string().uuid().nullable().optional(), folderId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecrets = z.infer; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 5e2c3aab3..4ad568e11 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,20 +1,42 @@ import { Redis } from "ioredis"; +import { Redlock, Settings } from "@app/lib/red-lock"; + export type TKeyStoreFactory = ReturnType; +// all the key prefixes used must be set here to avoid conflict +export enum KeyStorePrefixes { + SecretReplication = "secret-replication" +} + export const keyStoreFactory = (redisUrl: string) => { const redis = new Redis(redisUrl); + const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 }); - const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value); + const setItem = async (key: string, value: string | number | Buffer, prefix?: string) => + redis.set(prefix ? `${prefix}:${key}` : key, value); const getItem = async (key: string) => redis.get(key); - const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) => - redis.setex(key, exp, value); + const setItemWithExpiry = async ( + key: string, + exp: number | string, + value: string | number | Buffer, + prefix?: string + ) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value); const deleteItem = async (key: string) => redis.del(key); const incrementBy = async (key: string, value: number) => redis.incrby(key, value); - return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; + return { + setItem, + getItem, + setItemWithExpiry, + deleteItem, + incrementBy, + acquireLock(resources: string[], duration: number, settings?: Partial) { + return redisLock.acquire(resources, duration, settings); + } + }; }; diff --git a/backend/src/lib/red-lock/index.ts b/backend/src/lib/red-lock/index.ts new file mode 100644 index 000000000..e1cc4f587 --- /dev/null +++ b/backend/src/lib/red-lock/index.ts @@ -0,0 +1,682 @@ +/* eslint-disable */ +// Source code credits: https://github.com/mike-marcacci/node-redlock +// Taken to avoid external dependency +import { randomBytes, createHash } from "crypto"; +import { EventEmitter } from "events"; + +// AbortController became available as a global in node version 16. Once version +// 14 reaches its end-of-life, this can be removed. + +import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; + +type Client = IORedisClient | IORedisCluster; + +// Define script constants. +const ACQUIRE_SCRIPT = ` + -- Return 0 if an entry already exists. + for i, key in ipairs(KEYS) do + if redis.call("exists", key) == 1 then + return 0 + end + end + + -- Create an entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries added. + return #KEYS +`; + +const EXTEND_SCRIPT = ` + -- Return 0 if an entry exists with a *different* lock value. + for i, key in ipairs(KEYS) do + if redis.call("get", key) ~= ARGV[1] then + return 0 + end + end + + -- Update the entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries updated. + return #KEYS +`; + +const RELEASE_SCRIPT = ` + local count = 0 + for i, key in ipairs(KEYS) do + -- Only remove entries for *this* lock value. + if redis.call("get", key) == ARGV[1] then + redis.pcall("del", key) + count = count + 1 + end + end + + -- Return the number of entries removed. + return count +`; + +export type ClientExecutionResult = + | { + client: Client; + vote: "for"; + value: number; + } + | { + client: Client; + vote: "against"; + error: Error; + }; + +/* + * This object contains a summary of results. + */ +export type ExecutionStats = { + readonly membershipSize: number; + readonly quorumSize: number; + readonly votesFor: Set; + readonly votesAgainst: Map; +}; + +/* + * This object contains a summary of results. Because the result of an attempt + * can sometimes be determined before all requests are finished, each attempt + * contains a Promise that will resolve ExecutionStats once all requests are + * finished. A rejection of these promises should be considered undefined + * behavior and should cause a crash. + */ +export type ExecutionResult = { + attempts: ReadonlyArray>; + start: number; +}; + +/** + * + */ +export interface Settings { + readonly driftFactor: number; + readonly retryCount: number; + readonly retryDelay: number; + readonly retryJitter: number; + readonly automaticExtensionThreshold: number; +} + +// Define default settings. +const defaultSettings: Readonly = { + driftFactor: 0.01, + retryCount: 10, + retryDelay: 200, + retryJitter: 100, + automaticExtensionThreshold: 500 +}; + +// Modifyng this object is forbidden. +Object.freeze(defaultSettings); + +/* + * This error indicates a failure due to the existence of another lock for one + * or more of the requested resources. + */ +export class ResourceLockedError extends Error { + constructor(public readonly message: string) { + super(); + this.name = "ResourceLockedError"; + } +} + +/* + * This error indicates a failure of an operation to pass with a quorum. + */ +export class ExecutionError extends Error { + constructor( + public readonly message: string, + public readonly attempts: ReadonlyArray> + ) { + super(); + this.name = "ExecutionError"; + } +} + +/* + * An object of this type is returned when a resource is successfully locked. It + * contains convenience methods `release` and `extend` which perform the + * associated Redlock method on itself. + */ +export class Lock { + constructor( + public readonly redlock: Redlock, + public readonly resources: string[], + public readonly value: string, + public readonly attempts: ReadonlyArray>, + public expiration: number + ) {} + + async release(): Promise { + return this.redlock.release(this); + } + + async extend(duration: number): Promise { + return this.redlock.extend(this, duration); + } +} + +export type RedlockAbortSignal = AbortSignal & { error?: Error }; + +/** + * A redlock object is instantiated with an array of at least one redis client + * and an optional `options` object. Properties of the Redlock object should NOT + * be changed after it is first used, as doing so could have unintended + * consequences for live locks. + */ +export class Redlock extends EventEmitter { + public readonly clients: Set; + public readonly settings: Settings; + public readonly scripts: { + readonly acquireScript: { value: string; hash: string }; + readonly extendScript: { value: string; hash: string }; + readonly releaseScript: { value: string; hash: string }; + }; + + public constructor( + clients: Iterable, + settings: Partial = {}, + scripts: { + readonly acquireScript?: string | ((script: string) => string); + readonly extendScript?: string | ((script: string) => string); + readonly releaseScript?: string | ((script: string) => string); + } = {} + ) { + super(); + + // Prevent crashes on error events. + this.on("error", () => { + // Because redlock is designed for high availability, it does not care if + // a minority of redis instances/clusters fail at an operation. + // + // However, it can be helpful to monitor and log such cases. Redlock emits + // an "error" event whenever it encounters an error, even if the error is + // ignored in its normal operation. + // + // This function serves to prevent node's default behavior of crashing + // when an "error" event is emitted in the absence of listeners. + }); + + // Create a new array of client, to ensure no accidental mutation. + this.clients = new Set(clients); + if (this.clients.size === 0) { + throw new Error("Redlock must be instantiated with at least one redis client."); + } + + // Customize the settings for this instance. + this.settings = { + driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor, + retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount, + retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay, + retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter, + automaticExtensionThreshold: + typeof settings.automaticExtensionThreshold === "number" + ? settings.automaticExtensionThreshold + : defaultSettings.automaticExtensionThreshold + }; + + // Use custom scripts and script modifiers. + const acquireScript = + typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT; + const extendScript = + typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT; + const releaseScript = + typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT; + + this.scripts = { + acquireScript: { + value: acquireScript, + hash: this._hash(acquireScript) + }, + extendScript: { + value: extendScript, + hash: this._hash(extendScript) + }, + releaseScript: { + value: releaseScript, + hash: this._hash(releaseScript) + } + }; + } + + /** + * Generate a sha1 hash compatible with redis evalsha. + */ + private _hash(value: string): string { + return createHash("sha1").update(value).digest("hex"); + } + + /** + * Generate a cryptographically random string. + */ + private _random(): string { + return randomBytes(16).toString("hex"); + } + + /** + * This method runs `.quit()` on all client connections. + */ + public async quit(): Promise { + const results = []; + for (const client of this.clients) { + results.push(client.quit()); + } + + await Promise.all(results); + } + + /** + * This method acquires a locks on the resources for the duration specified by + * the `duration`. + */ + public async acquire(resources: string[], duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const value = this._random(); + + try { + const { attempts, start } = await this._execute( + this.scripts.acquireScript, + resources, + [value, duration], + settings + ); + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + return new Lock(this, resources, value, attempts, start + duration - drift); + } catch (error) { + // If there was an error acquiring the lock, release any partial lock + // state that may exist on a minority of clients. + await this._execute(this.scripts.releaseScript, resources, [value], { + retryCount: 0 + }).catch(() => { + // Any error here will be ignored. + }); + + throw error; + } + } + + /** + * This method unlocks the provided lock from all servers still persisting it. + * It will fail with an error if it is unable to release the lock on a quorum + * of nodes, but will make no attempt to restore the lock in the case of a + * failure to release. It is safe to re-attempt a release or to ignore the + * error, as the lock will automatically expire after its timeout. + */ + public async release(lock: Lock, settings?: Partial): Promise { + // Immediately invalidate the lock. + lock.expiration = 0; + + // Attempt to release the lock. + return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings); + } + + /** + * This method extends a valid lock by the provided `duration`. + */ + public async extend(existing: Lock, duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + // The lock has already expired. + if (existing.expiration < Date.now()) { + throw new ExecutionError("Cannot extend an already-expired lock.", []); + } + + const { attempts, start } = await this._execute( + this.scripts.extendScript, + existing.resources, + [existing.value, duration], + settings + ); + + // Invalidate the existing lock. + existing.expiration = 0; + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift); + + return replacement; + } + + /** + * Execute a script on all clients. The resulting promise is resolved or + * rejected as soon as this quorum is reached; the resolution or rejection + * will contains a `stats` property that is resolved once all votes are in. + */ + private async _execute( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[], + _settings?: Partial + ): Promise { + const settings = _settings + ? { + ...this.settings, + ..._settings + } + : this.settings; + + // For the purpose of easy config serialization, we treat a retryCount of + // -1 a equivalent to Infinity. + const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1; + + const attempts: Promise[] = []; + + while (true) { + const { vote, stats, start } = await this._attemptOperation(script, keys, args); + + attempts.push(stats); + + // The operation achieved a quorum in favor. + if (vote === "for") { + return { attempts, start }; + } + + // Wait before reattempting. + if (attempts.length < maxAttempts) { + await new Promise((resolve) => { + setTimeout( + resolve, + Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)), + undefined + ); + }); + } else { + throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts); + } + } + } + + private async _attemptOperation( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise< + | { vote: "for"; stats: Promise; start: number } + | { vote: "against"; stats: Promise; start: number } + > { + const start = Date.now(); + + return await new Promise((resolve) => { + const clientResults = []; + for (const client of this.clients) { + clientResults.push(this._attemptOperationOnClient(client, script, keys, args)); + } + + const stats: ExecutionStats = { + membershipSize: clientResults.length, + quorumSize: Math.floor(clientResults.length / 2) + 1, + votesFor: new Set(), + votesAgainst: new Map() + }; + + let done: () => void; + const statsPromise = new Promise((resolve) => { + done = () => resolve(stats); + }); + + // This is the expected flow for all successful and unsuccessful requests. + const onResultResolve = (clientResult: ClientExecutionResult): void => { + switch (clientResult.vote) { + case "for": + stats.votesFor.add(clientResult.client); + break; + case "against": + stats.votesAgainst.set(clientResult.client, clientResult.error); + break; + } + + // A quorum has determined a success. + if (stats.votesFor.size === stats.quorumSize) { + resolve({ + vote: "for", + stats: statsPromise, + start + }); + } + + // A quorum has determined a failure. + if (stats.votesAgainst.size === stats.quorumSize) { + resolve({ + vote: "against", + stats: statsPromise, + start + }); + } + + // All votes are in. + if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) { + done(); + } + }; + + // This is unexpected and should crash to prevent undefined behavior. + const onResultReject = (error: Error): void => { + throw error; + }; + + for (const result of clientResults) { + result.then(onResultResolve, onResultReject); + } + }); + } + + private async _attemptOperationOnClient( + client: Client, + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise { + try { + let result: number; + try { + // Attempt to evaluate the script by its hash. + // @ts-expect-error + const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown; + + if (typeof shaResult !== "number") { + throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`); + } + + result = shaResult; + } catch (error) { + // If the redis server does not already have the script cached, + // reattempt the request with the script's raw text. + if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) { + throw error; + } + // @ts-expect-error + const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown; + + if (typeof rawResult !== "number") { + throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`); + } + + result = rawResult; + } + + // One or more of the resources was already locked. + if (result !== keys.length) { + throw new ResourceLockedError( + `The operation was applied to: ${result} of the ${keys.length} requested resources.` + ); + } + + return { + vote: "for", + client, + value: result + }; + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`); + } + + // Emit the error on the redlock instance for observability. + this.emit("error", error); + + return { + vote: "against", + client, + error + }; + } + } + + /** + * Wrap and execute a routine in the context of an auto-extending lock, + * returning a promise of the routine's value. In the case that auto-extension + * fails, an AbortSignal will be updated to indicate that abortion of the + * routine is in order, and to pass along the encountered error. + * + * @example + * ```ts + * await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => { + * const senderBalance = await getBalance(senderId); + * const recipientBalance = await getBalance(recipientId); + * + * if (senderBalance < amountToSend) { + * throw new Error("Insufficient balance."); + * } + * + * // The abort signal will be true if: + * // 1. the above took long enough that the lock needed to be extended + * // 2. redlock was unable to extend the lock + * // + * // In such a case, exclusivity can no longer be guaranteed for further + * // operations, and should be handled as an exceptional case. + * if (signal.aborted) { + * throw signal.error; + * } + * + * await setBalances([ + * {id: senderId, balance: senderBalance - amountToSend}, + * {id: recipientId, balance: recipientBalance + amountToSend}, + * ]); + * }); + * ``` + */ + + public async using( + resources: string[], + duration: number, + settings: Partial, + routine?: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + settingsOrRoutine: undefined | Partial | ((signal: RedlockAbortSignal) => Promise), + optionalRoutine?: (signal: RedlockAbortSignal) => Promise + ): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const settings = + settingsOrRoutine && typeof settingsOrRoutine !== "function" + ? { + ...this.settings, + ...settingsOrRoutine + } + : this.settings; + + const routine = optionalRoutine ?? settingsOrRoutine; + if (typeof routine !== "function") { + throw new Error("INVARIANT: routine is not a function."); + } + + if (settings.automaticExtensionThreshold > duration - 100) { + throw new Error( + "A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting." + ); + } + + // The AbortController/AbortSignal pattern allows the routine to be notified + // of a failure to extend the lock, and subsequent expiration. In the event + // of an abort, the error object will be made available at `signal.error`. + const controller = new AbortController(); + + const signal = controller.signal as RedlockAbortSignal; + + function queue(): void { + timeout = setTimeout( + () => (extension = extend()), + lock.expiration - Date.now() - settings.automaticExtensionThreshold + ); + } + + async function extend(): Promise { + timeout = undefined; + + try { + lock = await lock.extend(duration); + queue(); + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected thrown ${typeof error}: ${error}.`); + } + + if (lock.expiration > Date.now()) { + return (extension = extend()); + } + + signal.error = error instanceof Error ? error : new Error(`${error}`); + controller.abort(); + } + } + + let timeout: undefined | NodeJS.Timeout; + let extension: undefined | Promise; + let lock = await this.acquire(resources, duration, settings); + queue(); + + try { + return await routine(signal); + } finally { + // Clean up the timer. + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + + // Wait for an in-flight extension to finish. + if (extension) { + await extension.catch(() => { + // An error here doesn't matter at all, because the routine has + // already completed, and a release will be attempted regardless. The + // only reason for waiting here is to prevent possible contention + // between the extension and release. + }); + } + + await lock.release(); + } + } +} diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 9d85b6015..32af011ec 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -7,6 +7,7 @@ import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types"; export enum QueueName { SecretRotation = "secret-rotation", @@ -21,7 +22,8 @@ export enum QueueName { SecretFullRepoScan = "secret-full-repo-scan", SecretPushEventScan = "secret-push-event-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost", - DynamicSecretRevocation = "dynamic-secret-revocation" + DynamicSecretRevocation = "dynamic-secret-revocation", + SecretReplication = "secret-replication" } export enum QueueJobs { @@ -37,7 +39,8 @@ export enum QueueJobs { SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job", DynamicSecretRevocation = "dynamic-secret-revocation", - DynamicSecretPruning = "dynamic-secret-pruning" + DynamicSecretPruning = "dynamic-secret-pruning", + SecretReplication = "secret-replication" } export type TQueueJobTypes = { @@ -116,6 +119,10 @@ export type TQueueJobTypes = { dynamicSecretCfgId: string; }; }; + [QueueName.SecretReplication]: { + name: QueueJobs.SecretReplication; + payload: TSyncSecretReplicationDTO; + }; }; export type TQueueServiceFactory = ReturnType; @@ -132,7 +139,7 @@ export const queueServiceFactory = (redisUrl: string) => { const start = ( name: T, - jobFn: (job: Job) => Promise, + jobFn: (job: Job, token?: string) => Promise, queueSettings: Omit = {} ) => { if (queueContainer[name]) { @@ -166,7 +173,7 @@ export const queueServiceFactory = (redisUrl: string) => { name: T, job: TQueueJobTypes[T]["name"], data: TQueueJobTypes[T]["payload"], - opts: JobsOptions & { jobId?: string } + opts?: JobsOptions & { jobId?: string } ) => { const q = queueContainer[name]; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1593515ec..c3fdb87cf 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -130,6 +130,8 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; +import { secretReplicationDALFactory } from "@app/services/secret-replication/secret-replication-dal"; +import { secretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal"; import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; @@ -193,6 +195,7 @@ export const registerRoutes = async ( const projectBotDAL = projectBotDALFactory(db); const secretDAL = secretDALFactory(db); + const secretReplicationDAL = secretReplicationDALFactory(db); const secretTagDAL = secretTagDALFactory(db); const folderDAL = secretFolderDALFactory(db); const folderVersionDAL = secretFolderVersionDALFactory(db); @@ -600,6 +603,17 @@ export const registerRoutes = async ( secretDAL, secretBlindIndexDAL }); + const secretReplicationService = secretReplicationServiceFactory({ + secretTagDAL, + secretVersionTagDAL, + secretDAL, + secretVersionDAL, + secretImportDAL, + keyStore, + queueService, + secretReplicationDAL, + folderDAL + }); const secretService = secretServiceFactory({ folderDAL, secretVersionDAL, @@ -611,6 +625,7 @@ export const registerRoutes = async ( secretTagDAL, snapshotService, secretQueueService, + secretReplicationService, secretImportDAL, projectEnvDAL, projectBotService @@ -826,6 +841,7 @@ export const registerRoutes = async ( projectEnv: projectEnvService, projectRole: projectRoleService, secret: secretService, + secretReplication: secretReplicationService, secretTag: secretTagService, folder: folderService, secretImport: secretImportService, diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index d036fdbdd..d540344ca 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => import: z.object({ environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment), path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path) - }) + }), + isReplication: z.boolean().default(false) }), response: { 200: z.object({ @@ -232,11 +233,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => 200: z.object({ message: z.string(), secretImports: SecretImportsSchema.omit({ importEnv: true }) - .merge( - z.object({ - importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) - }) - ) + .extend({ + importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) + }) .array() }) } diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 43676ba04..2aba9d89c 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -50,6 +50,7 @@ export const secretImportServiceFactory = ({ actorOrgId, actorAuthMethod, projectId, + isReplication, path }: TCreateSecretImportDTO) => { const { permission } = await permissionService.getProjectPermission( @@ -100,7 +101,8 @@ export const secretImportServiceFactory = ({ folderId: folder.id, position: lastPos + 1, importEnv: importEnv.id, - importPath: data.path + importPath: data.path, + isReplication }, tx ); diff --git a/backend/src/services/secret-import/secret-import-types.ts b/backend/src/services/secret-import/secret-import-types.ts index d123f28da..0dca0c306 100644 --- a/backend/src/services/secret-import/secret-import-types.ts +++ b/backend/src/services/secret-import/secret-import-types.ts @@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; } & TProjectPermission; export type TUpdateSecretImportDTO = { diff --git a/backend/src/services/secret-replication/secret-replication-dal.ts b/backend/src/services/secret-replication/secret-replication-dal.ts new file mode 100644 index 000000000..d977f3fa2 --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-dal.ts @@ -0,0 +1,58 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TSecretVersions } from "@app/db/schemas"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TSecretReplicationDALFactory = ReturnType; + +export const secretReplicationDALFactory = (db: TDbClient) => { + const orm = ormify(db, TableName.SecretVersion); + + const findSecrets = async (filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex) => { + if (!filter.secrets) return []; + + const sqlRawDocs = await (tx || db)(TableName.SecretVersion) + .where({ folderId: filter.folderId }) + .andWhere((bd) => { + filter.secrets.forEach((el) => { + void bd.orWhere({ + [`${TableName.SecretVersion}.secretId` as "secretId"]: el.id, + [`${TableName.SecretVersion}.version` as "version"]: el.version + }); + }); + }) + .leftJoin( + (tx || db)(TableName.SecretVersion) + .where("isReplicated", true) + .groupBy(["secretId", "version"]) + .max("version") + .select("version", "secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersion}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersion}.version`, + "latestVersion.max" + ); + } + ) + // .leftJoin( + // (tx || db)(TableName.SecretVersion).select("isReplicated", "version", "secretId").as("previousVersion"), + // (bd) => { + // bd.on(`${TableName.SecretVersion}.secretId`, "previousVersion.secretId").andOn( + // "previousVersion.version", + // (tx || db).raw(`${TableName.SecretVersion}.version - 1`) + // ); + // } + // ) + .select(db.ref("version").withSchema("latestVersion").as("latestReplicatedVersion")) + .select(selectAllTableCols(TableName.SecretVersion)); + + return sqlRawDocs; + }; + + return { + findSecrets, + ...orm + }; +}; diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts new file mode 100644 index 000000000..2a98193ea --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -0,0 +1,247 @@ +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { groupBy } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretReplicationDALFactory } from "./secret-replication-dal"; +import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types"; + +type TSecretReplicationServiceFactoryDep = { + secretReplicationDAL: TSecretReplicationDALFactory; + secretDAL: Pick; + secretImportDAL: Pick; + folderDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + queueService: Pick; + keyStore: Pick; +}; + +export type TSecretReplicationServiceFactory = ReturnType; + +// function getRandomError(): number { +// const minCeiled: number = Math.ceil(0); +// const maxFloored: number = Math.floor(20); +// const val = Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled); // The maximum is exclusive and the minimum is inclusive +// if (val >= 10) throw new Error("Random error point"); +// return val; +// } + +export const secretReplicationServiceFactory = ({ + secretReplicationDAL, + secretDAL, + queueService, + secretVersionDAL, + secretImportDAL, + keyStore, + secretVersionTagDAL, + secretTagDAL, + folderDAL +}: TSecretReplicationServiceFactoryDep) => { + queueService.start(QueueName.SecretReplication, async (job) => { + logger.info(job.data, "Replication started"); + const { secrets, folderId, secretPath, environmentId, projectId } = job.data; + const secretImports = await secretImportDAL.find({ + importPath: secretPath, + importEnv: environmentId, + isReplication: true + }); + console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId); + if (!secretImports.length) return; + + // unfiltered secrets to be replicated + console.log(secrets.length); + const toBeReplicatedSecrets = await secretReplicationDAL.findSecrets({ folderId, secrets }); + const replicatedSecrets = toBeReplicatedSecrets.filter( + ({ version, latestReplicatedVersion, secretBlindIndex }) => + secretBlindIndex && (version === 1 || latestReplicatedVersion <= version) + ); + + const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId); + console.log("replicated ", replicatedSecretsGroupBySecretId); + const lock = await keyStore.acquireLock( + replicatedSecrets.map(({ id }) => id), + 5000 + ); + + try { + /* eslint-disable no-await-in-loop */ + for (const secretImport of secretImports) { + const importFolderId = secretImport.folderId; + + const localSecrets = await secretDAL.find({ + $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, + folderId: importFolderId, + isReplicated: true + }); + const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string); + + const locallyCreatedSecrets = secrets.filter(({ operation, id }) => { + return ( + (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + }); + + const locallyUpdatedSecrets = secrets.filter( + ({ operation, id }) => + (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); + + console.log(replicatedSecretsGroupBySecretId); + console.log(locallyCreatedSecrets); + console.log("update", locallyUpdatedSecrets); + console.log("local board", localSecrets); + + const locallyDeletedSecrets = secrets + .filter( + ({ operation, id }) => + operation === SecretReplicationOperations.Delete && + Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ) + .map( + ({ id }) => + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0] + ); + + let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = []; + await secretReplicationDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + const newSecrets = await fnSecretBulkInsert({ + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + folderId, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id })) + ); + } + if (locallyUpdatedSecrets.length) { + const newSecrets = await fnSecretBulkUpdate({ + projectId, + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + filter: { + folderId: importFolderId, + id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id })) + ); + } + if (locallyDeletedSecrets.length) { + const newSecrets = await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + isReplicated: true, + folderId: importFolderId + }, + tx + ); + nestedImportSecrets = nestedImportSecrets.concat( + ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id })) + ); + } + }); + const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); + console.log("Environment ID -> slug", folder.envId, folder.environmentSlug); + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, { + folderId: folder.id, + projectId, + secrets: nestedImportSecrets, + secretPath: folder.path, + environmentId: folder.envId + }); + } + await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true }); + /* eslint-enable no-await-in-loop */ + } finally { + await lock.release(); + } + }); + + queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => { + logger.error(err, "Failed to replicate secret", job?.data); + }); + + const replicate = async (data: TSyncSecretReplicationDTO) => { + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, { + attempts: 3, + backoff: { + type: "exponential", + delay: 1000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + return { + replicate + }; +}; diff --git a/backend/src/services/secret-replication/secret-replication-types.ts b/backend/src/services/secret-replication/secret-replication-types.ts new file mode 100644 index 000000000..509bf5b02 --- /dev/null +++ b/backend/src/services/secret-replication/secret-replication-types.ts @@ -0,0 +1,17 @@ +export enum SecretReplicationOperations { + Create = "create", + Update = "update", + Delete = "delete" +} + +export type TSyncSecretReplicationDTO = { + secretPath: string; + projectId: string; + environmentId: string; + folderId: string; + secrets: { + operation: SecretReplicationOperations; + id: string; + version: number; + }[]; +}; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 1a2e414dd..9c72f9e38 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => { }; const deleteMany = async ( - data: Array<{ blindIndex: string; type: SecretType }>, + data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>, folderId: string, userId: string, tx?: Knex diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 51ad7a6aa..e59804bec 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -509,7 +509,7 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, references, ...el }) => ({ + inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({ ...el, folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 39e47a28e..da8c48b5b 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -30,6 +30,8 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind- import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; +import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service"; +import { SecretReplicationOperations } from "../secret-replication/secret-replication-types"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; import { @@ -85,6 +87,7 @@ type TSecretServiceFactoryDep = { projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; + secretReplicationService: Pick; }; export type TSecretServiceFactory = ReturnType; @@ -101,7 +104,8 @@ export const secretServiceFactory = ({ projectDAL, projectBotService, secretImportDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretReplicationService }: TSecretServiceFactoryDep) => { const getSecretReference = async (projectId: string) => { // if bot key missing means e2e still exist @@ -285,6 +289,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Create, + id: secret[0].id, + version: 1 + } + ] + }); return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; @@ -414,7 +431,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Update, + id: updatedSecret[0].id, + version: updatedSecret[0].version + } + ] + }); return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; @@ -482,7 +511,19 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: [ + { + operation: SecretReplicationOperations.Delete, + id: deletedSecret[0].id, + version: deletedSecret[0].version + } + ] + }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; }; @@ -768,6 +809,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create })) + }); return newSecrets; }; @@ -868,6 +916,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update })) + }); return secrets; }; @@ -930,6 +985,13 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretReplicationService.replicate({ + folderId, + projectId, + environmentId: folder.envId, + secretPath: path, + secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete })) + }); return secretsDeleted; }; diff --git a/frontend/src/hooks/api/secretImports/mutation.tsx b/frontend/src/hooks/api/secretImports/mutation.tsx index 928322a3c..0bee77ae8 100644 --- a/frontend/src/hooks/api/secretImports/mutation.tsx +++ b/frontend/src/hooks/api/secretImports/mutation.tsx @@ -9,12 +9,13 @@ export const useCreateSecretImport = () => { const queryClient = useQueryClient(); return useMutation<{}, {}, TCreateSecretImportDTO>({ - mutationFn: async ({ import: secretImport, environment, projectId, path }) => { + mutationFn: async ({ import: secretImport, environment, isReplication, projectId, path }) => { const { data } = await apiRequest.post("/api/v1/secret-imports", { import: secretImport, environment, workspaceId: projectId, - path + path, + isReplication }); return data; }, diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 950fc20c4..58d38047b 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -60,6 +60,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; }; export type TUpdateSecretImportDTO = { diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index a6dfe7c0e..8c2739c2f 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -4,7 +4,15 @@ import { AxiosError } from "axios"; import { z } from "zod"; import { createNotification } from "@app/components/notifications"; -import { Button, FormControl, Modal, ModalContent, Select, SelectItem } from "@app/components/v2"; +import { + Button, + Checkbox, + FormControl, + Modal, + ModalContent, + Select, + SelectItem +} from "@app/components/v2"; import { SecretPathInput } from "@app/components/v2/SecretPathInput"; import { useWorkspace } from "@app/context"; import { useCreateSecretImport } from "@app/hooks/api"; @@ -16,7 +24,8 @@ const typeSchema = z.object({ .trim() .transform((val) => typeof val === "string" && val.at(-1) === "/" && val.length > 1 ? val.slice(0, -1) : val - ) + ), + isReplication: z.boolean().default(false) }); type TFormSchema = z.infer; @@ -54,13 +63,15 @@ export const CreateSecretImportForm = ({ const handleFormSubmit = async ({ environment: importedEnv, - secretPath: importedSecPath + secretPath: importedSecPath, + isReplication }: TFormSchema) => { try { await createSecretImport({ environment, projectId: workspaceId, path: secretPath, + isReplication, import: { environment: importedEnv, path: importedSecPath @@ -127,7 +138,17 @@ export const CreateSecretImportForm = ({ )} /> - + ( + + The replication mode retrieves secrets when changes occur in the specified + environment and secret path. + + )} + />