feat: poc for secret replication completed

This commit is contained in:
Akhil Mohan
2024-04-30 22:41:44 +05:30
committed by =
parent 520167a8ff
commit 36b7911bcc
21 changed files with 1213 additions and 29 deletions

View File

@@ -52,6 +52,7 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service";
import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service";
import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
@@ -108,6 +109,7 @@ declare module "fastify" {
projectKey: TProjectKeyServiceFactory;
projectRole: TProjectRoleServiceFactory;
secret: TSecretServiceFactory;
secretReplication: TSecretReplicationServiceFactory;
secretTag: TSecretTagServiceFactory;
secretImport: TSecretImportServiceFactory;
projectBot: TProjectBotServiceFactory;

View File

@@ -0,0 +1,43 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
t.boolean("isReplication").defaultTo(false);
});
}
if (await knex.schema.hasTable(TableName.Secret)) {
await knex.schema.alterTable(TableName.Secret, (t) => {
t.boolean("isReplicated");
});
}
if (await knex.schema.hasTable(TableName.SecretVersion)) {
await knex.schema.alterTable(TableName.SecretVersion, (t) => {
t.boolean("isReplicated");
});
}
}
export async function down(knex: Knex): Promise<void> {
if (await knex.schema.hasTable(TableName.SecretImport)) {
await knex.schema.alterTable(TableName.SecretImport, (t) => {
t.dropColumns("isReplication");
});
}
if (await knex.schema.hasTable(TableName.Secret)) {
await knex.schema.alterTable(TableName.Secret, (t) => {
t.dropColumns("isReplicated");
});
}
if (await knex.schema.hasTable(TableName.SecretVersion)) {
await knex.schema.alterTable(TableName.Secret, (t) => {
t.dropColumns("isReplicated");
});
}
}

View File

@@ -15,7 +15,8 @@ export const SecretImportsSchema = z.object({
position: z.number(),
createdAt: z.date(),
updatedAt: z.date(),
folderId: z.string().uuid()
folderId: z.string().uuid(),
isReplication: z.boolean().default(false).nullable().optional()
});
export type TSecretImports = z.infer<typeof SecretImportsSchema>;

View File

@@ -32,7 +32,8 @@ export const SecretVersionsSchema = z.object({
folderId: z.string().uuid(),
userId: z.string().uuid().nullable().optional(),
createdAt: z.date(),
updatedAt: z.date()
updatedAt: z.date(),
isReplicated: z.boolean().nullable().optional()
});
export type TSecretVersions = z.infer<typeof SecretVersionsSchema>;

View File

@@ -30,7 +30,8 @@ export const SecretsSchema = z.object({
userId: z.string().uuid().nullable().optional(),
folderId: z.string().uuid(),
createdAt: z.date(),
updatedAt: z.date()
updatedAt: z.date(),
isReplicated: z.boolean().nullable().optional()
});
export type TSecrets = z.infer<typeof SecretsSchema>;

View File

@@ -1,20 +1,42 @@
import { Redis } from "ioredis";
import { Redlock, Settings } from "@app/lib/red-lock";
export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
// all the key prefixes used must be set here to avoid conflict
export enum KeyStorePrefixes {
SecretReplication = "secret-replication"
}
export const keyStoreFactory = (redisUrl: string) => {
const redis = new Redis(redisUrl);
const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value);
const setItem = async (key: string, value: string | number | Buffer, prefix?: string) =>
redis.set(prefix ? `${prefix}:${key}` : key, value);
const getItem = async (key: string) => redis.get(key);
const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) =>
redis.setex(key, exp, value);
const setItemWithExpiry = async (
key: string,
exp: number | string,
value: string | number | Buffer,
prefix?: string
) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value);
const deleteItem = async (key: string) => redis.del(key);
const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy };
return {
setItem,
getItem,
setItemWithExpiry,
deleteItem,
incrementBy,
acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
return redisLock.acquire(resources, duration, settings);
}
};
};

View File

@@ -0,0 +1,682 @@
/* eslint-disable */
// Source code credits: https://github.com/mike-marcacci/node-redlock
// Taken to avoid external dependency
import { randomBytes, createHash } from "crypto";
import { EventEmitter } from "events";
// AbortController became available as a global in node version 16. Once version
// 14 reaches its end-of-life, this can be removed.
import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis";
type Client = IORedisClient | IORedisCluster;
// Define script constants.
const ACQUIRE_SCRIPT = `
-- Return 0 if an entry already exists.
for i, key in ipairs(KEYS) do
if redis.call("exists", key) == 1 then
return 0
end
end
-- Create an entry for each provided key.
for i, key in ipairs(KEYS) do
redis.call("set", key, ARGV[1], "PX", ARGV[2])
end
-- Return the number of entries added.
return #KEYS
`;
const EXTEND_SCRIPT = `
-- Return 0 if an entry exists with a *different* lock value.
for i, key in ipairs(KEYS) do
if redis.call("get", key) ~= ARGV[1] then
return 0
end
end
-- Update the entry for each provided key.
for i, key in ipairs(KEYS) do
redis.call("set", key, ARGV[1], "PX", ARGV[2])
end
-- Return the number of entries updated.
return #KEYS
`;
const RELEASE_SCRIPT = `
local count = 0
for i, key in ipairs(KEYS) do
-- Only remove entries for *this* lock value.
if redis.call("get", key) == ARGV[1] then
redis.pcall("del", key)
count = count + 1
end
end
-- Return the number of entries removed.
return count
`;
export type ClientExecutionResult =
| {
client: Client;
vote: "for";
value: number;
}
| {
client: Client;
vote: "against";
error: Error;
};
/*
* This object contains a summary of results.
*/
export type ExecutionStats = {
readonly membershipSize: number;
readonly quorumSize: number;
readonly votesFor: Set<Client>;
readonly votesAgainst: Map<Client, Error>;
};
/*
* This object contains a summary of results. Because the result of an attempt
* can sometimes be determined before all requests are finished, each attempt
* contains a Promise that will resolve ExecutionStats once all requests are
* finished. A rejection of these promises should be considered undefined
* behavior and should cause a crash.
*/
export type ExecutionResult = {
attempts: ReadonlyArray<Promise<ExecutionStats>>;
start: number;
};
/**
*
*/
export interface Settings {
readonly driftFactor: number;
readonly retryCount: number;
readonly retryDelay: number;
readonly retryJitter: number;
readonly automaticExtensionThreshold: number;
}
// Define default settings.
const defaultSettings: Readonly<Settings> = {
driftFactor: 0.01,
retryCount: 10,
retryDelay: 200,
retryJitter: 100,
automaticExtensionThreshold: 500
};
// Modifyng this object is forbidden.
Object.freeze(defaultSettings);
/*
* This error indicates a failure due to the existence of another lock for one
* or more of the requested resources.
*/
export class ResourceLockedError extends Error {
constructor(public readonly message: string) {
super();
this.name = "ResourceLockedError";
}
}
/*
* This error indicates a failure of an operation to pass with a quorum.
*/
export class ExecutionError extends Error {
constructor(
public readonly message: string,
public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>
) {
super();
this.name = "ExecutionError";
}
}
/*
* An object of this type is returned when a resource is successfully locked. It
* contains convenience methods `release` and `extend` which perform the
* associated Redlock method on itself.
*/
export class Lock {
constructor(
public readonly redlock: Redlock,
public readonly resources: string[],
public readonly value: string,
public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>,
public expiration: number
) {}
async release(): Promise<ExecutionResult> {
return this.redlock.release(this);
}
async extend(duration: number): Promise<Lock> {
return this.redlock.extend(this, duration);
}
}
export type RedlockAbortSignal = AbortSignal & { error?: Error };
/**
* A redlock object is instantiated with an array of at least one redis client
* and an optional `options` object. Properties of the Redlock object should NOT
* be changed after it is first used, as doing so could have unintended
* consequences for live locks.
*/
export class Redlock extends EventEmitter {
public readonly clients: Set<Client>;
public readonly settings: Settings;
public readonly scripts: {
readonly acquireScript: { value: string; hash: string };
readonly extendScript: { value: string; hash: string };
readonly releaseScript: { value: string; hash: string };
};
public constructor(
clients: Iterable<Client>,
settings: Partial<Settings> = {},
scripts: {
readonly acquireScript?: string | ((script: string) => string);
readonly extendScript?: string | ((script: string) => string);
readonly releaseScript?: string | ((script: string) => string);
} = {}
) {
super();
// Prevent crashes on error events.
this.on("error", () => {
// Because redlock is designed for high availability, it does not care if
// a minority of redis instances/clusters fail at an operation.
//
// However, it can be helpful to monitor and log such cases. Redlock emits
// an "error" event whenever it encounters an error, even if the error is
// ignored in its normal operation.
//
// This function serves to prevent node's default behavior of crashing
// when an "error" event is emitted in the absence of listeners.
});
// Create a new array of client, to ensure no accidental mutation.
this.clients = new Set(clients);
if (this.clients.size === 0) {
throw new Error("Redlock must be instantiated with at least one redis client.");
}
// Customize the settings for this instance.
this.settings = {
driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor,
retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount,
retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay,
retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter,
automaticExtensionThreshold:
typeof settings.automaticExtensionThreshold === "number"
? settings.automaticExtensionThreshold
: defaultSettings.automaticExtensionThreshold
};
// Use custom scripts and script modifiers.
const acquireScript =
typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT;
const extendScript =
typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT;
const releaseScript =
typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT;
this.scripts = {
acquireScript: {
value: acquireScript,
hash: this._hash(acquireScript)
},
extendScript: {
value: extendScript,
hash: this._hash(extendScript)
},
releaseScript: {
value: releaseScript,
hash: this._hash(releaseScript)
}
};
}
/**
* Generate a sha1 hash compatible with redis evalsha.
*/
private _hash(value: string): string {
return createHash("sha1").update(value).digest("hex");
}
/**
* Generate a cryptographically random string.
*/
private _random(): string {
return randomBytes(16).toString("hex");
}
/**
* This method runs `.quit()` on all client connections.
*/
public async quit(): Promise<void> {
const results = [];
for (const client of this.clients) {
results.push(client.quit());
}
await Promise.all(results);
}
/**
* This method acquires a locks on the resources for the duration specified by
* the `duration`.
*/
public async acquire(resources: string[], duration: number, settings?: Partial<Settings>): Promise<Lock> {
if (Math.floor(duration) !== duration) {
throw new Error("Duration must be an integer value in milliseconds.");
}
const value = this._random();
try {
const { attempts, start } = await this._execute(
this.scripts.acquireScript,
resources,
[value, duration],
settings
);
// Add 2 milliseconds to the drift to account for Redis expires precision,
// which is 1 ms, plus the configured allowable drift factor.
const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
return new Lock(this, resources, value, attempts, start + duration - drift);
} catch (error) {
// If there was an error acquiring the lock, release any partial lock
// state that may exist on a minority of clients.
await this._execute(this.scripts.releaseScript, resources, [value], {
retryCount: 0
}).catch(() => {
// Any error here will be ignored.
});
throw error;
}
}
/**
* This method unlocks the provided lock from all servers still persisting it.
* It will fail with an error if it is unable to release the lock on a quorum
* of nodes, but will make no attempt to restore the lock in the case of a
* failure to release. It is safe to re-attempt a release or to ignore the
* error, as the lock will automatically expire after its timeout.
*/
public async release(lock: Lock, settings?: Partial<Settings>): Promise<ExecutionResult> {
// Immediately invalidate the lock.
lock.expiration = 0;
// Attempt to release the lock.
return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings);
}
/**
* This method extends a valid lock by the provided `duration`.
*/
public async extend(existing: Lock, duration: number, settings?: Partial<Settings>): Promise<Lock> {
if (Math.floor(duration) !== duration) {
throw new Error("Duration must be an integer value in milliseconds.");
}
// The lock has already expired.
if (existing.expiration < Date.now()) {
throw new ExecutionError("Cannot extend an already-expired lock.", []);
}
const { attempts, start } = await this._execute(
this.scripts.extendScript,
existing.resources,
[existing.value, duration],
settings
);
// Invalidate the existing lock.
existing.expiration = 0;
// Add 2 milliseconds to the drift to account for Redis expires precision,
// which is 1 ms, plus the configured allowable drift factor.
const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift);
return replacement;
}
/**
* Execute a script on all clients. The resulting promise is resolved or
* rejected as soon as this quorum is reached; the resolution or rejection
* will contains a `stats` property that is resolved once all votes are in.
*/
private async _execute(
script: { value: string; hash: string },
keys: string[],
args: (string | number)[],
_settings?: Partial<Settings>
): Promise<ExecutionResult> {
const settings = _settings
? {
...this.settings,
..._settings
}
: this.settings;
// For the purpose of easy config serialization, we treat a retryCount of
// -1 a equivalent to Infinity.
const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1;
const attempts: Promise<ExecutionStats>[] = [];
while (true) {
const { vote, stats, start } = await this._attemptOperation(script, keys, args);
attempts.push(stats);
// The operation achieved a quorum in favor.
if (vote === "for") {
return { attempts, start };
}
// Wait before reattempting.
if (attempts.length < maxAttempts) {
await new Promise((resolve) => {
setTimeout(
resolve,
Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)),
undefined
);
});
} else {
throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts);
}
}
}
private async _attemptOperation(
script: { value: string; hash: string },
keys: string[],
args: (string | number)[]
): Promise<
| { vote: "for"; stats: Promise<ExecutionStats>; start: number }
| { vote: "against"; stats: Promise<ExecutionStats>; start: number }
> {
const start = Date.now();
return await new Promise((resolve) => {
const clientResults = [];
for (const client of this.clients) {
clientResults.push(this._attemptOperationOnClient(client, script, keys, args));
}
const stats: ExecutionStats = {
membershipSize: clientResults.length,
quorumSize: Math.floor(clientResults.length / 2) + 1,
votesFor: new Set<Client>(),
votesAgainst: new Map<Client, Error>()
};
let done: () => void;
const statsPromise = new Promise<typeof stats>((resolve) => {
done = () => resolve(stats);
});
// This is the expected flow for all successful and unsuccessful requests.
const onResultResolve = (clientResult: ClientExecutionResult): void => {
switch (clientResult.vote) {
case "for":
stats.votesFor.add(clientResult.client);
break;
case "against":
stats.votesAgainst.set(clientResult.client, clientResult.error);
break;
}
// A quorum has determined a success.
if (stats.votesFor.size === stats.quorumSize) {
resolve({
vote: "for",
stats: statsPromise,
start
});
}
// A quorum has determined a failure.
if (stats.votesAgainst.size === stats.quorumSize) {
resolve({
vote: "against",
stats: statsPromise,
start
});
}
// All votes are in.
if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) {
done();
}
};
// This is unexpected and should crash to prevent undefined behavior.
const onResultReject = (error: Error): void => {
throw error;
};
for (const result of clientResults) {
result.then(onResultResolve, onResultReject);
}
});
}
private async _attemptOperationOnClient(
client: Client,
script: { value: string; hash: string },
keys: string[],
args: (string | number)[]
): Promise<ClientExecutionResult> {
try {
let result: number;
try {
// Attempt to evaluate the script by its hash.
// @ts-expect-error
const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown;
if (typeof shaResult !== "number") {
throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`);
}
result = shaResult;
} catch (error) {
// If the redis server does not already have the script cached,
// reattempt the request with the script's raw text.
if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) {
throw error;
}
// @ts-expect-error
const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown;
if (typeof rawResult !== "number") {
throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`);
}
result = rawResult;
}
// One or more of the resources was already locked.
if (result !== keys.length) {
throw new ResourceLockedError(
`The operation was applied to: ${result} of the ${keys.length} requested resources.`
);
}
return {
vote: "for",
client,
value: result
};
} catch (error) {
if (!(error instanceof Error)) {
throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`);
}
// Emit the error on the redlock instance for observability.
this.emit("error", error);
return {
vote: "against",
client,
error
};
}
}
/**
* Wrap and execute a routine in the context of an auto-extending lock,
* returning a promise of the routine's value. In the case that auto-extension
* fails, an AbortSignal will be updated to indicate that abortion of the
* routine is in order, and to pass along the encountered error.
*
* @example
* ```ts
* await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => {
* const senderBalance = await getBalance(senderId);
* const recipientBalance = await getBalance(recipientId);
*
* if (senderBalance < amountToSend) {
* throw new Error("Insufficient balance.");
* }
*
* // The abort signal will be true if:
* // 1. the above took long enough that the lock needed to be extended
* // 2. redlock was unable to extend the lock
* //
* // In such a case, exclusivity can no longer be guaranteed for further
* // operations, and should be handled as an exceptional case.
* if (signal.aborted) {
* throw signal.error;
* }
*
* await setBalances([
* {id: senderId, balance: senderBalance - amountToSend},
* {id: recipientId, balance: recipientBalance + amountToSend},
* ]);
* });
* ```
*/
public async using<T>(
resources: string[],
duration: number,
settings: Partial<Settings>,
routine?: (signal: RedlockAbortSignal) => Promise<T>
): Promise<T>;
public async using<T>(
resources: string[],
duration: number,
routine: (signal: RedlockAbortSignal) => Promise<T>
): Promise<T>;
public async using<T>(
resources: string[],
duration: number,
settingsOrRoutine: undefined | Partial<Settings> | ((signal: RedlockAbortSignal) => Promise<T>),
optionalRoutine?: (signal: RedlockAbortSignal) => Promise<T>
): Promise<T> {
if (Math.floor(duration) !== duration) {
throw new Error("Duration must be an integer value in milliseconds.");
}
const settings =
settingsOrRoutine && typeof settingsOrRoutine !== "function"
? {
...this.settings,
...settingsOrRoutine
}
: this.settings;
const routine = optionalRoutine ?? settingsOrRoutine;
if (typeof routine !== "function") {
throw new Error("INVARIANT: routine is not a function.");
}
if (settings.automaticExtensionThreshold > duration - 100) {
throw new Error(
"A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting."
);
}
// The AbortController/AbortSignal pattern allows the routine to be notified
// of a failure to extend the lock, and subsequent expiration. In the event
// of an abort, the error object will be made available at `signal.error`.
const controller = new AbortController();
const signal = controller.signal as RedlockAbortSignal;
function queue(): void {
timeout = setTimeout(
() => (extension = extend()),
lock.expiration - Date.now() - settings.automaticExtensionThreshold
);
}
async function extend(): Promise<void> {
timeout = undefined;
try {
lock = await lock.extend(duration);
queue();
} catch (error) {
if (!(error instanceof Error)) {
throw new Error(`Unexpected thrown ${typeof error}: ${error}.`);
}
if (lock.expiration > Date.now()) {
return (extension = extend());
}
signal.error = error instanceof Error ? error : new Error(`${error}`);
controller.abort();
}
}
let timeout: undefined | NodeJS.Timeout;
let extension: undefined | Promise<void>;
let lock = await this.acquire(resources, duration, settings);
queue();
try {
return await routine(signal);
} finally {
// Clean up the timer.
if (timeout) {
clearTimeout(timeout);
timeout = undefined;
}
// Wait for an in-flight extension to finish.
if (extension) {
await extension.catch(() => {
// An error here doesn't matter at all, because the routine has
// already completed, and a release will be attempted regardless. The
// only reason for waiting here is to prevent possible contention
// between the extension and release.
});
}
await lock.release();
}
}
}

View File

@@ -7,6 +7,7 @@ import {
TScanFullRepoEventPayload,
TScanPushEventPayload
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types";
export enum QueueName {
SecretRotation = "secret-rotation",
@@ -21,7 +22,8 @@ export enum QueueName {
SecretFullRepoScan = "secret-full-repo-scan",
SecretPushEventScan = "secret-push-event-scan",
UpgradeProjectToGhost = "upgrade-project-to-ghost",
DynamicSecretRevocation = "dynamic-secret-revocation"
DynamicSecretRevocation = "dynamic-secret-revocation",
SecretReplication = "secret-replication"
}
export enum QueueJobs {
@@ -37,7 +39,8 @@ export enum QueueJobs {
SecretScan = "secret-scan",
UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
DynamicSecretRevocation = "dynamic-secret-revocation",
DynamicSecretPruning = "dynamic-secret-pruning"
DynamicSecretPruning = "dynamic-secret-pruning",
SecretReplication = "secret-replication"
}
export type TQueueJobTypes = {
@@ -116,6 +119,10 @@ export type TQueueJobTypes = {
dynamicSecretCfgId: string;
};
};
[QueueName.SecretReplication]: {
name: QueueJobs.SecretReplication;
payload: TSyncSecretReplicationDTO;
};
};
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
@@ -132,7 +139,7 @@ export const queueServiceFactory = (redisUrl: string) => {
const start = <T extends QueueName>(
name: T,
jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>) => Promise<void>,
jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>, token?: string) => Promise<void>,
queueSettings: Omit<QueueOptions, "connection"> = {}
) => {
if (queueContainer[name]) {
@@ -166,7 +173,7 @@ export const queueServiceFactory = (redisUrl: string) => {
name: T,
job: TQueueJobTypes[T]["name"],
data: TQueueJobTypes[T]["payload"],
opts: JobsOptions & { jobId?: string }
opts?: JobsOptions & { jobId?: string }
) => {
const q = queueContainer[name];

View File

@@ -130,6 +130,8 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f
import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
import { secretReplicationDALFactory } from "@app/services/secret-replication/secret-replication-dal";
import { secretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service";
import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal";
import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
@@ -193,6 +195,7 @@ export const registerRoutes = async (
const projectBotDAL = projectBotDALFactory(db);
const secretDAL = secretDALFactory(db);
const secretReplicationDAL = secretReplicationDALFactory(db);
const secretTagDAL = secretTagDALFactory(db);
const folderDAL = secretFolderDALFactory(db);
const folderVersionDAL = secretFolderVersionDALFactory(db);
@@ -600,6 +603,17 @@ export const registerRoutes = async (
secretDAL,
secretBlindIndexDAL
});
const secretReplicationService = secretReplicationServiceFactory({
secretTagDAL,
secretVersionTagDAL,
secretDAL,
secretVersionDAL,
secretImportDAL,
keyStore,
queueService,
secretReplicationDAL,
folderDAL
});
const secretService = secretServiceFactory({
folderDAL,
secretVersionDAL,
@@ -611,6 +625,7 @@ export const registerRoutes = async (
secretTagDAL,
snapshotService,
secretQueueService,
secretReplicationService,
secretImportDAL,
projectEnvDAL,
projectBotService
@@ -826,6 +841,7 @@ export const registerRoutes = async (
projectEnv: projectEnvService,
projectRole: projectRoleService,
secret: secretService,
secretReplication: secretReplicationService,
secretTag: secretTagService,
folder: folderService,
secretImport: secretImportService,

View File

@@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
import: z.object({
environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment),
path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path)
})
}),
isReplication: z.boolean().default(false)
}),
response: {
200: z.object({
@@ -232,11 +233,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
200: z.object({
message: z.string(),
secretImports: SecretImportsSchema.omit({ importEnv: true })
.merge(
z.object({
importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
})
)
.extend({
importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
})
.array()
})
}

View File

@@ -50,6 +50,7 @@ export const secretImportServiceFactory = ({
actorOrgId,
actorAuthMethod,
projectId,
isReplication,
path
}: TCreateSecretImportDTO) => {
const { permission } = await permissionService.getProjectPermission(
@@ -100,7 +101,8 @@ export const secretImportServiceFactory = ({
folderId: folder.id,
position: lastPos + 1,
importEnv: importEnv.id,
importPath: data.path
importPath: data.path,
isReplication
},
tx
);

View File

@@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = {
environment: string;
path: string;
};
isReplication?: boolean;
} & TProjectPermission;
export type TUpdateSecretImportDTO = {

View File

@@ -0,0 +1,58 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName, TSecretVersions } from "@app/db/schemas";
import { ormify, selectAllTableCols } from "@app/lib/knex";
export type TSecretReplicationDALFactory = ReturnType<typeof secretReplicationDALFactory>;
export const secretReplicationDALFactory = (db: TDbClient) => {
const orm = ormify(db, TableName.SecretVersion);
const findSecrets = async (filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex) => {
if (!filter.secrets) return [];
const sqlRawDocs = await (tx || db)(TableName.SecretVersion)
.where({ folderId: filter.folderId })
.andWhere((bd) => {
filter.secrets.forEach((el) => {
void bd.orWhere({
[`${TableName.SecretVersion}.secretId` as "secretId"]: el.id,
[`${TableName.SecretVersion}.version` as "version"]: el.version
});
});
})
.leftJoin<TSecretVersions>(
(tx || db)(TableName.SecretVersion)
.where("isReplicated", true)
.groupBy(["secretId", "version"])
.max("version")
.select("version", "secretId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.SecretVersion}.secretId`, "latestVersion.secretId").andOn(
`${TableName.SecretVersion}.version`,
"latestVersion.max"
);
}
)
// .leftJoin<TSecretVersions>(
// (tx || db)(TableName.SecretVersion).select("isReplicated", "version", "secretId").as("previousVersion"),
// (bd) => {
// bd.on(`${TableName.SecretVersion}.secretId`, "previousVersion.secretId").andOn(
// "previousVersion.version",
// (tx || db).raw(`${TableName.SecretVersion}.version - 1`)
// );
// }
// )
.select(db.ref("version").withSchema("latestVersion").as("latestReplicatedVersion"))
.select(selectAllTableCols(TableName.SecretVersion));
return sqlRawDocs;
};
return {
findSecrets,
...orm
};
};

View File

@@ -0,0 +1,247 @@
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretReplicationDALFactory } from "./secret-replication-dal";
import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
type TSecretReplicationServiceFactoryDep = {
secretReplicationDAL: TSecretReplicationDALFactory;
secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
};
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
// function getRandomError(): number {
// const minCeiled: number = Math.ceil(0);
// const maxFloored: number = Math.floor(20);
// const val = Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled); // The maximum is exclusive and the minimum is inclusive
// if (val >= 10) throw new Error("Random error point");
// return val;
// }
export const secretReplicationServiceFactory = ({
secretReplicationDAL,
secretDAL,
queueService,
secretVersionDAL,
secretImportDAL,
keyStore,
secretVersionTagDAL,
secretTagDAL,
folderDAL
}: TSecretReplicationServiceFactoryDep) => {
queueService.start(QueueName.SecretReplication, async (job) => {
logger.info(job.data, "Replication started");
const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
const secretImports = await secretImportDAL.find({
importPath: secretPath,
importEnv: environmentId,
isReplication: true
});
console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
if (!secretImports.length) return;
// unfiltered secrets to be replicated
console.log(secrets.length);
const toBeReplicatedSecrets = await secretReplicationDAL.findSecrets({ folderId, secrets });
const replicatedSecrets = toBeReplicatedSecrets.filter(
({ version, latestReplicatedVersion, secretBlindIndex }) =>
secretBlindIndex && (version === 1 || latestReplicatedVersion <= version)
);
const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId);
console.log("replicated ", replicatedSecretsGroupBySecretId);
const lock = await keyStore.acquireLock(
replicatedSecrets.map(({ id }) => id),
5000
);
try {
/* eslint-disable no-await-in-loop */
for (const secretImport of secretImports) {
const importFolderId = secretImport.folderId;
const localSecrets = await secretDAL.find({
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
folderId: importFolderId,
isReplicated: true
});
const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string);
const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
return (
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
});
const locallyUpdatedSecrets = secrets.filter(
({ operation, id }) =>
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
);
console.log(replicatedSecretsGroupBySecretId);
console.log(locallyCreatedSecrets);
console.log("update", locallyUpdatedSecrets);
console.log("local board", localSecrets);
const locallyDeletedSecrets = secrets
.filter(
({ operation, id }) =>
operation === SecretReplicationOperations.Delete &&
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
)
.map(
({ id }) =>
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
);
let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
await secretReplicationDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
const newSecrets = await fnSecretBulkInsert({
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
folderId,
type: doc.type,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
);
}
if (locallyUpdatedSecrets.length) {
const newSecrets = await fnSecretBulkUpdate({
projectId,
folderId: importFolderId,
secretVersionDAL,
secretDAL,
tx,
secretTagDAL,
secretVersionTagDAL,
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
const doc = replicatedSecretsGroupBySecretId[id][0];
return {
filter: {
folderId: importFolderId,
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
},
data: {
keyEncoding: doc.keyEncoding,
algorithm: doc.algorithm,
type: doc.type,
metadata: doc.metadata,
secretKeyIV: doc.secretKeyIV,
secretKeyTag: doc.secretKeyTag,
secretKeyCiphertext: doc.secretKeyCiphertext,
secretValueIV: doc.secretValueIV,
secretValueTag: doc.secretValueTag,
secretValueCiphertext: doc.secretValueCiphertext,
secretBlindIndex: doc.secretBlindIndex,
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
isReplicated: true,
skipMultilineEncoding: doc.skipMultilineEncoding
}
};
})
});
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
);
}
if (locallyDeletedSecrets.length) {
const newSecrets = await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
},
isReplicated: true,
folderId: importFolderId
},
tx
);
nestedImportSecrets = nestedImportSecrets.concat(
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
);
}
});
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
folderId: folder.id,
projectId,
secrets: nestedImportSecrets,
secretPath: folder.path,
environmentId: folder.envId
});
}
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
/* eslint-enable no-await-in-loop */
} finally {
await lock.release();
}
});
queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
logger.error(err, "Failed to replicate secret", job?.data);
});
const replicate = async (data: TSyncSecretReplicationDTO) => {
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
},
removeOnComplete: true,
removeOnFail: true
});
};
return {
replicate
};
};

View File

@@ -0,0 +1,17 @@
export enum SecretReplicationOperations {
Create = "create",
Update = "update",
Delete = "delete"
}
export type TSyncSecretReplicationDTO = {
secretPath: string;
projectId: string;
environmentId: string;
folderId: string;
secrets: {
operation: SecretReplicationOperations;
id: string;
version: number;
}[];
};

View File

@@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => {
};
const deleteMany = async (
data: Array<{ blindIndex: string; type: SecretType }>,
data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>,
folderId: string,
userId: string,
tx?: Knex

View File

@@ -509,7 +509,7 @@ export const fnSecretBulkInsert = async ({
}))
);
const secretVersions = await secretVersionDAL.insertMany(
inputSecrets.map(({ tags, references, ...el }) => ({
inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({
...el,
folderId,
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id

View File

@@ -30,6 +30,8 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
@@ -85,6 +87,7 @@ type TSecretServiceFactoryDep = {
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
};
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -101,7 +104,8 @@ export const secretServiceFactory = ({
projectDAL,
projectBotService,
secretImportDAL,
secretVersionTagDAL
secretVersionTagDAL,
secretReplicationService
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
@@ -285,6 +289,19 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: [
{
operation: SecretReplicationOperations.Create,
id: secret[0].id,
version: 1
}
]
});
return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
};
@@ -414,7 +431,19 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: [
{
operation: SecretReplicationOperations.Update,
id: updatedSecret[0].id,
version: updatedSecret[0].version
}
]
});
return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
};
@@ -482,7 +511,19 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: [
{
operation: SecretReplicationOperations.Delete,
id: deletedSecret[0].id,
version: deletedSecret[0].version
}
]
});
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
};
@@ -768,6 +809,13 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
});
return newSecrets;
};
@@ -868,6 +916,13 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
});
return secrets;
};
@@ -930,6 +985,13 @@ export const secretServiceFactory = ({
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
await secretReplicationService.replicate({
folderId,
projectId,
environmentId: folder.envId,
secretPath: path,
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
});
return secretsDeleted;
};

View File

@@ -9,12 +9,13 @@ export const useCreateSecretImport = () => {
const queryClient = useQueryClient();
return useMutation<{}, {}, TCreateSecretImportDTO>({
mutationFn: async ({ import: secretImport, environment, projectId, path }) => {
mutationFn: async ({ import: secretImport, environment, isReplication, projectId, path }) => {
const { data } = await apiRequest.post("/api/v1/secret-imports", {
import: secretImport,
environment,
workspaceId: projectId,
path
path,
isReplication
});
return data;
},

View File

@@ -60,6 +60,7 @@ export type TCreateSecretImportDTO = {
environment: string;
path: string;
};
isReplication?: boolean;
};
export type TUpdateSecretImportDTO = {

View File

@@ -4,7 +4,15 @@ import { AxiosError } from "axios";
import { z } from "zod";
import { createNotification } from "@app/components/notifications";
import { Button, FormControl, Modal, ModalContent, Select, SelectItem } from "@app/components/v2";
import {
Button,
Checkbox,
FormControl,
Modal,
ModalContent,
Select,
SelectItem
} from "@app/components/v2";
import { SecretPathInput } from "@app/components/v2/SecretPathInput";
import { useWorkspace } from "@app/context";
import { useCreateSecretImport } from "@app/hooks/api";
@@ -16,7 +24,8 @@ const typeSchema = z.object({
.trim()
.transform((val) =>
typeof val === "string" && val.at(-1) === "/" && val.length > 1 ? val.slice(0, -1) : val
)
),
isReplication: z.boolean().default(false)
});
type TFormSchema = z.infer<typeof typeSchema>;
@@ -54,13 +63,15 @@ export const CreateSecretImportForm = ({
const handleFormSubmit = async ({
environment: importedEnv,
secretPath: importedSecPath
secretPath: importedSecPath,
isReplication
}: TFormSchema) => {
try {
await createSecretImport({
environment,
projectId: workspaceId,
path: secretPath,
isReplication,
import: {
environment: importedEnv,
path: importedSecPath
@@ -127,7 +138,17 @@ export const CreateSecretImportForm = ({
</FormControl>
)}
/>
<Controller
name="isReplication"
control={control}
defaultValue={false}
render={({ field }) => (
<Checkbox isChecked={field.value} onCheckedChange={field.onChange} id="isReplication">
The replication mode retrieves secrets when changes occur in the specified
environment and secret path.
</Checkbox>
)}
/>
<div className="mt-7 flex items-center">
<Button
isDisabled={isSubmitting}