mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
feat: poc for secret replication completed
This commit is contained in:
2
backend/src/@types/fastify.d.ts
vendored
2
backend/src/@types/fastify.d.ts
vendored
@@ -52,6 +52,7 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service";
|
||||
import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service";
|
||||
import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
|
||||
import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
|
||||
import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service";
|
||||
import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
|
||||
import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
|
||||
import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
|
||||
@@ -108,6 +109,7 @@ declare module "fastify" {
|
||||
projectKey: TProjectKeyServiceFactory;
|
||||
projectRole: TProjectRoleServiceFactory;
|
||||
secret: TSecretServiceFactory;
|
||||
secretReplication: TSecretReplicationServiceFactory;
|
||||
secretTag: TSecretTagServiceFactory;
|
||||
secretImport: TSecretImportServiceFactory;
|
||||
projectBot: TProjectBotServiceFactory;
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TableName } from "../schemas";
|
||||
|
||||
export async function up(knex: Knex): Promise<void> {
|
||||
if (await knex.schema.hasTable(TableName.SecretImport)) {
|
||||
await knex.schema.alterTable(TableName.SecretImport, (t) => {
|
||||
t.boolean("isReplication").defaultTo(false);
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.Secret)) {
|
||||
await knex.schema.alterTable(TableName.Secret, (t) => {
|
||||
t.boolean("isReplicated");
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.SecretVersion)) {
|
||||
await knex.schema.alterTable(TableName.SecretVersion, (t) => {
|
||||
t.boolean("isReplicated");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(knex: Knex): Promise<void> {
|
||||
if (await knex.schema.hasTable(TableName.SecretImport)) {
|
||||
await knex.schema.alterTable(TableName.SecretImport, (t) => {
|
||||
t.dropColumns("isReplication");
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.Secret)) {
|
||||
await knex.schema.alterTable(TableName.Secret, (t) => {
|
||||
t.dropColumns("isReplicated");
|
||||
});
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable(TableName.SecretVersion)) {
|
||||
await knex.schema.alterTable(TableName.Secret, (t) => {
|
||||
t.dropColumns("isReplicated");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,8 @@ export const SecretImportsSchema = z.object({
|
||||
position: z.number(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date(),
|
||||
folderId: z.string().uuid()
|
||||
folderId: z.string().uuid(),
|
||||
isReplication: z.boolean().default(false).nullable().optional()
|
||||
});
|
||||
|
||||
export type TSecretImports = z.infer<typeof SecretImportsSchema>;
|
||||
|
||||
@@ -32,7 +32,8 @@ export const SecretVersionsSchema = z.object({
|
||||
folderId: z.string().uuid(),
|
||||
userId: z.string().uuid().nullable().optional(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date()
|
||||
updatedAt: z.date(),
|
||||
isReplicated: z.boolean().nullable().optional()
|
||||
});
|
||||
|
||||
export type TSecretVersions = z.infer<typeof SecretVersionsSchema>;
|
||||
|
||||
@@ -30,7 +30,8 @@ export const SecretsSchema = z.object({
|
||||
userId: z.string().uuid().nullable().optional(),
|
||||
folderId: z.string().uuid(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date()
|
||||
updatedAt: z.date(),
|
||||
isReplicated: z.boolean().nullable().optional()
|
||||
});
|
||||
|
||||
export type TSecrets = z.infer<typeof SecretsSchema>;
|
||||
|
||||
@@ -1,20 +1,42 @@
|
||||
import { Redis } from "ioredis";
|
||||
|
||||
import { Redlock, Settings } from "@app/lib/red-lock";
|
||||
|
||||
export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
|
||||
|
||||
// all the key prefixes used must be set here to avoid conflict
|
||||
export enum KeyStorePrefixes {
|
||||
SecretReplication = "secret-replication"
|
||||
}
|
||||
|
||||
export const keyStoreFactory = (redisUrl: string) => {
|
||||
const redis = new Redis(redisUrl);
|
||||
const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
|
||||
|
||||
const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value);
|
||||
const setItem = async (key: string, value: string | number | Buffer, prefix?: string) =>
|
||||
redis.set(prefix ? `${prefix}:${key}` : key, value);
|
||||
|
||||
const getItem = async (key: string) => redis.get(key);
|
||||
|
||||
const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) =>
|
||||
redis.setex(key, exp, value);
|
||||
const setItemWithExpiry = async (
|
||||
key: string,
|
||||
exp: number | string,
|
||||
value: string | number | Buffer,
|
||||
prefix?: string
|
||||
) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value);
|
||||
|
||||
const deleteItem = async (key: string) => redis.del(key);
|
||||
|
||||
const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
|
||||
|
||||
return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy };
|
||||
return {
|
||||
setItem,
|
||||
getItem,
|
||||
setItemWithExpiry,
|
||||
deleteItem,
|
||||
incrementBy,
|
||||
acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
|
||||
return redisLock.acquire(resources, duration, settings);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
682
backend/src/lib/red-lock/index.ts
Normal file
682
backend/src/lib/red-lock/index.ts
Normal file
@@ -0,0 +1,682 @@
|
||||
/* eslint-disable */
|
||||
// Source code credits: https://github.com/mike-marcacci/node-redlock
|
||||
// Taken to avoid external dependency
|
||||
import { randomBytes, createHash } from "crypto";
|
||||
import { EventEmitter } from "events";
|
||||
|
||||
// AbortController became available as a global in node version 16. Once version
|
||||
// 14 reaches its end-of-life, this can be removed.
|
||||
|
||||
import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis";
|
||||
|
||||
type Client = IORedisClient | IORedisCluster;
|
||||
|
||||
// Define script constants.
|
||||
const ACQUIRE_SCRIPT = `
|
||||
-- Return 0 if an entry already exists.
|
||||
for i, key in ipairs(KEYS) do
|
||||
if redis.call("exists", key) == 1 then
|
||||
return 0
|
||||
end
|
||||
end
|
||||
|
||||
-- Create an entry for each provided key.
|
||||
for i, key in ipairs(KEYS) do
|
||||
redis.call("set", key, ARGV[1], "PX", ARGV[2])
|
||||
end
|
||||
|
||||
-- Return the number of entries added.
|
||||
return #KEYS
|
||||
`;
|
||||
|
||||
const EXTEND_SCRIPT = `
|
||||
-- Return 0 if an entry exists with a *different* lock value.
|
||||
for i, key in ipairs(KEYS) do
|
||||
if redis.call("get", key) ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
end
|
||||
|
||||
-- Update the entry for each provided key.
|
||||
for i, key in ipairs(KEYS) do
|
||||
redis.call("set", key, ARGV[1], "PX", ARGV[2])
|
||||
end
|
||||
|
||||
-- Return the number of entries updated.
|
||||
return #KEYS
|
||||
`;
|
||||
|
||||
const RELEASE_SCRIPT = `
|
||||
local count = 0
|
||||
for i, key in ipairs(KEYS) do
|
||||
-- Only remove entries for *this* lock value.
|
||||
if redis.call("get", key) == ARGV[1] then
|
||||
redis.pcall("del", key)
|
||||
count = count + 1
|
||||
end
|
||||
end
|
||||
|
||||
-- Return the number of entries removed.
|
||||
return count
|
||||
`;
|
||||
|
||||
export type ClientExecutionResult =
|
||||
| {
|
||||
client: Client;
|
||||
vote: "for";
|
||||
value: number;
|
||||
}
|
||||
| {
|
||||
client: Client;
|
||||
vote: "against";
|
||||
error: Error;
|
||||
};
|
||||
|
||||
/*
|
||||
* This object contains a summary of results.
|
||||
*/
|
||||
export type ExecutionStats = {
|
||||
readonly membershipSize: number;
|
||||
readonly quorumSize: number;
|
||||
readonly votesFor: Set<Client>;
|
||||
readonly votesAgainst: Map<Client, Error>;
|
||||
};
|
||||
|
||||
/*
|
||||
* This object contains a summary of results. Because the result of an attempt
|
||||
* can sometimes be determined before all requests are finished, each attempt
|
||||
* contains a Promise that will resolve ExecutionStats once all requests are
|
||||
* finished. A rejection of these promises should be considered undefined
|
||||
* behavior and should cause a crash.
|
||||
*/
|
||||
export type ExecutionResult = {
|
||||
attempts: ReadonlyArray<Promise<ExecutionStats>>;
|
||||
start: number;
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
export interface Settings {
|
||||
readonly driftFactor: number;
|
||||
readonly retryCount: number;
|
||||
readonly retryDelay: number;
|
||||
readonly retryJitter: number;
|
||||
readonly automaticExtensionThreshold: number;
|
||||
}
|
||||
|
||||
// Define default settings.
|
||||
const defaultSettings: Readonly<Settings> = {
|
||||
driftFactor: 0.01,
|
||||
retryCount: 10,
|
||||
retryDelay: 200,
|
||||
retryJitter: 100,
|
||||
automaticExtensionThreshold: 500
|
||||
};
|
||||
|
||||
// Modifyng this object is forbidden.
|
||||
Object.freeze(defaultSettings);
|
||||
|
||||
/*
|
||||
* This error indicates a failure due to the existence of another lock for one
|
||||
* or more of the requested resources.
|
||||
*/
|
||||
export class ResourceLockedError extends Error {
|
||||
constructor(public readonly message: string) {
|
||||
super();
|
||||
this.name = "ResourceLockedError";
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This error indicates a failure of an operation to pass with a quorum.
|
||||
*/
|
||||
export class ExecutionError extends Error {
|
||||
constructor(
|
||||
public readonly message: string,
|
||||
public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>
|
||||
) {
|
||||
super();
|
||||
this.name = "ExecutionError";
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* An object of this type is returned when a resource is successfully locked. It
|
||||
* contains convenience methods `release` and `extend` which perform the
|
||||
* associated Redlock method on itself.
|
||||
*/
|
||||
export class Lock {
|
||||
constructor(
|
||||
public readonly redlock: Redlock,
|
||||
public readonly resources: string[],
|
||||
public readonly value: string,
|
||||
public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>,
|
||||
public expiration: number
|
||||
) {}
|
||||
|
||||
async release(): Promise<ExecutionResult> {
|
||||
return this.redlock.release(this);
|
||||
}
|
||||
|
||||
async extend(duration: number): Promise<Lock> {
|
||||
return this.redlock.extend(this, duration);
|
||||
}
|
||||
}
|
||||
|
||||
export type RedlockAbortSignal = AbortSignal & { error?: Error };
|
||||
|
||||
/**
|
||||
* A redlock object is instantiated with an array of at least one redis client
|
||||
* and an optional `options` object. Properties of the Redlock object should NOT
|
||||
* be changed after it is first used, as doing so could have unintended
|
||||
* consequences for live locks.
|
||||
*/
|
||||
export class Redlock extends EventEmitter {
|
||||
public readonly clients: Set<Client>;
|
||||
public readonly settings: Settings;
|
||||
public readonly scripts: {
|
||||
readonly acquireScript: { value: string; hash: string };
|
||||
readonly extendScript: { value: string; hash: string };
|
||||
readonly releaseScript: { value: string; hash: string };
|
||||
};
|
||||
|
||||
public constructor(
|
||||
clients: Iterable<Client>,
|
||||
settings: Partial<Settings> = {},
|
||||
scripts: {
|
||||
readonly acquireScript?: string | ((script: string) => string);
|
||||
readonly extendScript?: string | ((script: string) => string);
|
||||
readonly releaseScript?: string | ((script: string) => string);
|
||||
} = {}
|
||||
) {
|
||||
super();
|
||||
|
||||
// Prevent crashes on error events.
|
||||
this.on("error", () => {
|
||||
// Because redlock is designed for high availability, it does not care if
|
||||
// a minority of redis instances/clusters fail at an operation.
|
||||
//
|
||||
// However, it can be helpful to monitor and log such cases. Redlock emits
|
||||
// an "error" event whenever it encounters an error, even if the error is
|
||||
// ignored in its normal operation.
|
||||
//
|
||||
// This function serves to prevent node's default behavior of crashing
|
||||
// when an "error" event is emitted in the absence of listeners.
|
||||
});
|
||||
|
||||
// Create a new array of client, to ensure no accidental mutation.
|
||||
this.clients = new Set(clients);
|
||||
if (this.clients.size === 0) {
|
||||
throw new Error("Redlock must be instantiated with at least one redis client.");
|
||||
}
|
||||
|
||||
// Customize the settings for this instance.
|
||||
this.settings = {
|
||||
driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor,
|
||||
retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount,
|
||||
retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay,
|
||||
retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter,
|
||||
automaticExtensionThreshold:
|
||||
typeof settings.automaticExtensionThreshold === "number"
|
||||
? settings.automaticExtensionThreshold
|
||||
: defaultSettings.automaticExtensionThreshold
|
||||
};
|
||||
|
||||
// Use custom scripts and script modifiers.
|
||||
const acquireScript =
|
||||
typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT;
|
||||
const extendScript =
|
||||
typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT;
|
||||
const releaseScript =
|
||||
typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT;
|
||||
|
||||
this.scripts = {
|
||||
acquireScript: {
|
||||
value: acquireScript,
|
||||
hash: this._hash(acquireScript)
|
||||
},
|
||||
extendScript: {
|
||||
value: extendScript,
|
||||
hash: this._hash(extendScript)
|
||||
},
|
||||
releaseScript: {
|
||||
value: releaseScript,
|
||||
hash: this._hash(releaseScript)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a sha1 hash compatible with redis evalsha.
|
||||
*/
|
||||
private _hash(value: string): string {
|
||||
return createHash("sha1").update(value).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a cryptographically random string.
|
||||
*/
|
||||
private _random(): string {
|
||||
return randomBytes(16).toString("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* This method runs `.quit()` on all client connections.
|
||||
*/
|
||||
public async quit(): Promise<void> {
|
||||
const results = [];
|
||||
for (const client of this.clients) {
|
||||
results.push(client.quit());
|
||||
}
|
||||
|
||||
await Promise.all(results);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method acquires a locks on the resources for the duration specified by
|
||||
* the `duration`.
|
||||
*/
|
||||
public async acquire(resources: string[], duration: number, settings?: Partial<Settings>): Promise<Lock> {
|
||||
if (Math.floor(duration) !== duration) {
|
||||
throw new Error("Duration must be an integer value in milliseconds.");
|
||||
}
|
||||
|
||||
const value = this._random();
|
||||
|
||||
try {
|
||||
const { attempts, start } = await this._execute(
|
||||
this.scripts.acquireScript,
|
||||
resources,
|
||||
[value, duration],
|
||||
settings
|
||||
);
|
||||
|
||||
// Add 2 milliseconds to the drift to account for Redis expires precision,
|
||||
// which is 1 ms, plus the configured allowable drift factor.
|
||||
const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
|
||||
|
||||
return new Lock(this, resources, value, attempts, start + duration - drift);
|
||||
} catch (error) {
|
||||
// If there was an error acquiring the lock, release any partial lock
|
||||
// state that may exist on a minority of clients.
|
||||
await this._execute(this.scripts.releaseScript, resources, [value], {
|
||||
retryCount: 0
|
||||
}).catch(() => {
|
||||
// Any error here will be ignored.
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method unlocks the provided lock from all servers still persisting it.
|
||||
* It will fail with an error if it is unable to release the lock on a quorum
|
||||
* of nodes, but will make no attempt to restore the lock in the case of a
|
||||
* failure to release. It is safe to re-attempt a release or to ignore the
|
||||
* error, as the lock will automatically expire after its timeout.
|
||||
*/
|
||||
public async release(lock: Lock, settings?: Partial<Settings>): Promise<ExecutionResult> {
|
||||
// Immediately invalidate the lock.
|
||||
lock.expiration = 0;
|
||||
|
||||
// Attempt to release the lock.
|
||||
return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method extends a valid lock by the provided `duration`.
|
||||
*/
|
||||
public async extend(existing: Lock, duration: number, settings?: Partial<Settings>): Promise<Lock> {
|
||||
if (Math.floor(duration) !== duration) {
|
||||
throw new Error("Duration must be an integer value in milliseconds.");
|
||||
}
|
||||
|
||||
// The lock has already expired.
|
||||
if (existing.expiration < Date.now()) {
|
||||
throw new ExecutionError("Cannot extend an already-expired lock.", []);
|
||||
}
|
||||
|
||||
const { attempts, start } = await this._execute(
|
||||
this.scripts.extendScript,
|
||||
existing.resources,
|
||||
[existing.value, duration],
|
||||
settings
|
||||
);
|
||||
|
||||
// Invalidate the existing lock.
|
||||
existing.expiration = 0;
|
||||
|
||||
// Add 2 milliseconds to the drift to account for Redis expires precision,
|
||||
// which is 1 ms, plus the configured allowable drift factor.
|
||||
const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
|
||||
|
||||
const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift);
|
||||
|
||||
return replacement;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a script on all clients. The resulting promise is resolved or
|
||||
* rejected as soon as this quorum is reached; the resolution or rejection
|
||||
* will contains a `stats` property that is resolved once all votes are in.
|
||||
*/
|
||||
private async _execute(
|
||||
script: { value: string; hash: string },
|
||||
keys: string[],
|
||||
args: (string | number)[],
|
||||
_settings?: Partial<Settings>
|
||||
): Promise<ExecutionResult> {
|
||||
const settings = _settings
|
||||
? {
|
||||
...this.settings,
|
||||
..._settings
|
||||
}
|
||||
: this.settings;
|
||||
|
||||
// For the purpose of easy config serialization, we treat a retryCount of
|
||||
// -1 a equivalent to Infinity.
|
||||
const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1;
|
||||
|
||||
const attempts: Promise<ExecutionStats>[] = [];
|
||||
|
||||
while (true) {
|
||||
const { vote, stats, start } = await this._attemptOperation(script, keys, args);
|
||||
|
||||
attempts.push(stats);
|
||||
|
||||
// The operation achieved a quorum in favor.
|
||||
if (vote === "for") {
|
||||
return { attempts, start };
|
||||
}
|
||||
|
||||
// Wait before reattempting.
|
||||
if (attempts.length < maxAttempts) {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(
|
||||
resolve,
|
||||
Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)),
|
||||
undefined
|
||||
);
|
||||
});
|
||||
} else {
|
||||
throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _attemptOperation(
|
||||
script: { value: string; hash: string },
|
||||
keys: string[],
|
||||
args: (string | number)[]
|
||||
): Promise<
|
||||
| { vote: "for"; stats: Promise<ExecutionStats>; start: number }
|
||||
| { vote: "against"; stats: Promise<ExecutionStats>; start: number }
|
||||
> {
|
||||
const start = Date.now();
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
const clientResults = [];
|
||||
for (const client of this.clients) {
|
||||
clientResults.push(this._attemptOperationOnClient(client, script, keys, args));
|
||||
}
|
||||
|
||||
const stats: ExecutionStats = {
|
||||
membershipSize: clientResults.length,
|
||||
quorumSize: Math.floor(clientResults.length / 2) + 1,
|
||||
votesFor: new Set<Client>(),
|
||||
votesAgainst: new Map<Client, Error>()
|
||||
};
|
||||
|
||||
let done: () => void;
|
||||
const statsPromise = new Promise<typeof stats>((resolve) => {
|
||||
done = () => resolve(stats);
|
||||
});
|
||||
|
||||
// This is the expected flow for all successful and unsuccessful requests.
|
||||
const onResultResolve = (clientResult: ClientExecutionResult): void => {
|
||||
switch (clientResult.vote) {
|
||||
case "for":
|
||||
stats.votesFor.add(clientResult.client);
|
||||
break;
|
||||
case "against":
|
||||
stats.votesAgainst.set(clientResult.client, clientResult.error);
|
||||
break;
|
||||
}
|
||||
|
||||
// A quorum has determined a success.
|
||||
if (stats.votesFor.size === stats.quorumSize) {
|
||||
resolve({
|
||||
vote: "for",
|
||||
stats: statsPromise,
|
||||
start
|
||||
});
|
||||
}
|
||||
|
||||
// A quorum has determined a failure.
|
||||
if (stats.votesAgainst.size === stats.quorumSize) {
|
||||
resolve({
|
||||
vote: "against",
|
||||
stats: statsPromise,
|
||||
start
|
||||
});
|
||||
}
|
||||
|
||||
// All votes are in.
|
||||
if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) {
|
||||
done();
|
||||
}
|
||||
};
|
||||
|
||||
// This is unexpected and should crash to prevent undefined behavior.
|
||||
const onResultReject = (error: Error): void => {
|
||||
throw error;
|
||||
};
|
||||
|
||||
for (const result of clientResults) {
|
||||
result.then(onResultResolve, onResultReject);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async _attemptOperationOnClient(
|
||||
client: Client,
|
||||
script: { value: string; hash: string },
|
||||
keys: string[],
|
||||
args: (string | number)[]
|
||||
): Promise<ClientExecutionResult> {
|
||||
try {
|
||||
let result: number;
|
||||
try {
|
||||
// Attempt to evaluate the script by its hash.
|
||||
// @ts-expect-error
|
||||
const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown;
|
||||
|
||||
if (typeof shaResult !== "number") {
|
||||
throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`);
|
||||
}
|
||||
|
||||
result = shaResult;
|
||||
} catch (error) {
|
||||
// If the redis server does not already have the script cached,
|
||||
// reattempt the request with the script's raw text.
|
||||
if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) {
|
||||
throw error;
|
||||
}
|
||||
// @ts-expect-error
|
||||
const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown;
|
||||
|
||||
if (typeof rawResult !== "number") {
|
||||
throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`);
|
||||
}
|
||||
|
||||
result = rawResult;
|
||||
}
|
||||
|
||||
// One or more of the resources was already locked.
|
||||
if (result !== keys.length) {
|
||||
throw new ResourceLockedError(
|
||||
`The operation was applied to: ${result} of the ${keys.length} requested resources.`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
vote: "for",
|
||||
client,
|
||||
value: result
|
||||
};
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error)) {
|
||||
throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`);
|
||||
}
|
||||
|
||||
// Emit the error on the redlock instance for observability.
|
||||
this.emit("error", error);
|
||||
|
||||
return {
|
||||
vote: "against",
|
||||
client,
|
||||
error
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap and execute a routine in the context of an auto-extending lock,
|
||||
* returning a promise of the routine's value. In the case that auto-extension
|
||||
* fails, an AbortSignal will be updated to indicate that abortion of the
|
||||
* routine is in order, and to pass along the encountered error.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => {
|
||||
* const senderBalance = await getBalance(senderId);
|
||||
* const recipientBalance = await getBalance(recipientId);
|
||||
*
|
||||
* if (senderBalance < amountToSend) {
|
||||
* throw new Error("Insufficient balance.");
|
||||
* }
|
||||
*
|
||||
* // The abort signal will be true if:
|
||||
* // 1. the above took long enough that the lock needed to be extended
|
||||
* // 2. redlock was unable to extend the lock
|
||||
* //
|
||||
* // In such a case, exclusivity can no longer be guaranteed for further
|
||||
* // operations, and should be handled as an exceptional case.
|
||||
* if (signal.aborted) {
|
||||
* throw signal.error;
|
||||
* }
|
||||
*
|
||||
* await setBalances([
|
||||
* {id: senderId, balance: senderBalance - amountToSend},
|
||||
* {id: recipientId, balance: recipientBalance + amountToSend},
|
||||
* ]);
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
|
||||
public async using<T>(
|
||||
resources: string[],
|
||||
duration: number,
|
||||
settings: Partial<Settings>,
|
||||
routine?: (signal: RedlockAbortSignal) => Promise<T>
|
||||
): Promise<T>;
|
||||
|
||||
public async using<T>(
|
||||
resources: string[],
|
||||
duration: number,
|
||||
routine: (signal: RedlockAbortSignal) => Promise<T>
|
||||
): Promise<T>;
|
||||
|
||||
public async using<T>(
|
||||
resources: string[],
|
||||
duration: number,
|
||||
settingsOrRoutine: undefined | Partial<Settings> | ((signal: RedlockAbortSignal) => Promise<T>),
|
||||
optionalRoutine?: (signal: RedlockAbortSignal) => Promise<T>
|
||||
): Promise<T> {
|
||||
if (Math.floor(duration) !== duration) {
|
||||
throw new Error("Duration must be an integer value in milliseconds.");
|
||||
}
|
||||
|
||||
const settings =
|
||||
settingsOrRoutine && typeof settingsOrRoutine !== "function"
|
||||
? {
|
||||
...this.settings,
|
||||
...settingsOrRoutine
|
||||
}
|
||||
: this.settings;
|
||||
|
||||
const routine = optionalRoutine ?? settingsOrRoutine;
|
||||
if (typeof routine !== "function") {
|
||||
throw new Error("INVARIANT: routine is not a function.");
|
||||
}
|
||||
|
||||
if (settings.automaticExtensionThreshold > duration - 100) {
|
||||
throw new Error(
|
||||
"A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting."
|
||||
);
|
||||
}
|
||||
|
||||
// The AbortController/AbortSignal pattern allows the routine to be notified
|
||||
// of a failure to extend the lock, and subsequent expiration. In the event
|
||||
// of an abort, the error object will be made available at `signal.error`.
|
||||
const controller = new AbortController();
|
||||
|
||||
const signal = controller.signal as RedlockAbortSignal;
|
||||
|
||||
function queue(): void {
|
||||
timeout = setTimeout(
|
||||
() => (extension = extend()),
|
||||
lock.expiration - Date.now() - settings.automaticExtensionThreshold
|
||||
);
|
||||
}
|
||||
|
||||
async function extend(): Promise<void> {
|
||||
timeout = undefined;
|
||||
|
||||
try {
|
||||
lock = await lock.extend(duration);
|
||||
queue();
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error)) {
|
||||
throw new Error(`Unexpected thrown ${typeof error}: ${error}.`);
|
||||
}
|
||||
|
||||
if (lock.expiration > Date.now()) {
|
||||
return (extension = extend());
|
||||
}
|
||||
|
||||
signal.error = error instanceof Error ? error : new Error(`${error}`);
|
||||
controller.abort();
|
||||
}
|
||||
}
|
||||
|
||||
let timeout: undefined | NodeJS.Timeout;
|
||||
let extension: undefined | Promise<void>;
|
||||
let lock = await this.acquire(resources, duration, settings);
|
||||
queue();
|
||||
|
||||
try {
|
||||
return await routine(signal);
|
||||
} finally {
|
||||
// Clean up the timer.
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
timeout = undefined;
|
||||
}
|
||||
|
||||
// Wait for an in-flight extension to finish.
|
||||
if (extension) {
|
||||
await extension.catch(() => {
|
||||
// An error here doesn't matter at all, because the routine has
|
||||
// already completed, and a release will be attempted regardless. The
|
||||
// only reason for waiting here is to prevent possible contention
|
||||
// between the extension and release.
|
||||
});
|
||||
}
|
||||
|
||||
await lock.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
TScanFullRepoEventPayload,
|
||||
TScanPushEventPayload
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types";
|
||||
|
||||
export enum QueueName {
|
||||
SecretRotation = "secret-rotation",
|
||||
@@ -21,7 +22,8 @@ export enum QueueName {
|
||||
SecretFullRepoScan = "secret-full-repo-scan",
|
||||
SecretPushEventScan = "secret-push-event-scan",
|
||||
UpgradeProjectToGhost = "upgrade-project-to-ghost",
|
||||
DynamicSecretRevocation = "dynamic-secret-revocation"
|
||||
DynamicSecretRevocation = "dynamic-secret-revocation",
|
||||
SecretReplication = "secret-replication"
|
||||
}
|
||||
|
||||
export enum QueueJobs {
|
||||
@@ -37,7 +39,8 @@ export enum QueueJobs {
|
||||
SecretScan = "secret-scan",
|
||||
UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
|
||||
DynamicSecretRevocation = "dynamic-secret-revocation",
|
||||
DynamicSecretPruning = "dynamic-secret-pruning"
|
||||
DynamicSecretPruning = "dynamic-secret-pruning",
|
||||
SecretReplication = "secret-replication"
|
||||
}
|
||||
|
||||
export type TQueueJobTypes = {
|
||||
@@ -116,6 +119,10 @@ export type TQueueJobTypes = {
|
||||
dynamicSecretCfgId: string;
|
||||
};
|
||||
};
|
||||
[QueueName.SecretReplication]: {
|
||||
name: QueueJobs.SecretReplication;
|
||||
payload: TSyncSecretReplicationDTO;
|
||||
};
|
||||
};
|
||||
|
||||
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
|
||||
@@ -132,7 +139,7 @@ export const queueServiceFactory = (redisUrl: string) => {
|
||||
|
||||
const start = <T extends QueueName>(
|
||||
name: T,
|
||||
jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>) => Promise<void>,
|
||||
jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>, token?: string) => Promise<void>,
|
||||
queueSettings: Omit<QueueOptions, "connection"> = {}
|
||||
) => {
|
||||
if (queueContainer[name]) {
|
||||
@@ -166,7 +173,7 @@ export const queueServiceFactory = (redisUrl: string) => {
|
||||
name: T,
|
||||
job: TQueueJobTypes[T]["name"],
|
||||
data: TQueueJobTypes[T]["payload"],
|
||||
opts: JobsOptions & { jobId?: string }
|
||||
opts?: JobsOptions & { jobId?: string }
|
||||
) => {
|
||||
const q = queueContainer[name];
|
||||
|
||||
|
||||
@@ -130,6 +130,8 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f
|
||||
import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
|
||||
import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
|
||||
import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
|
||||
import { secretReplicationDALFactory } from "@app/services/secret-replication/secret-replication-dal";
|
||||
import { secretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service";
|
||||
import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal";
|
||||
import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
|
||||
import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
|
||||
@@ -193,6 +195,7 @@ export const registerRoutes = async (
|
||||
const projectBotDAL = projectBotDALFactory(db);
|
||||
|
||||
const secretDAL = secretDALFactory(db);
|
||||
const secretReplicationDAL = secretReplicationDALFactory(db);
|
||||
const secretTagDAL = secretTagDALFactory(db);
|
||||
const folderDAL = secretFolderDALFactory(db);
|
||||
const folderVersionDAL = secretFolderVersionDALFactory(db);
|
||||
@@ -600,6 +603,17 @@ export const registerRoutes = async (
|
||||
secretDAL,
|
||||
secretBlindIndexDAL
|
||||
});
|
||||
const secretReplicationService = secretReplicationServiceFactory({
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretImportDAL,
|
||||
keyStore,
|
||||
queueService,
|
||||
secretReplicationDAL,
|
||||
folderDAL
|
||||
});
|
||||
const secretService = secretServiceFactory({
|
||||
folderDAL,
|
||||
secretVersionDAL,
|
||||
@@ -611,6 +625,7 @@ export const registerRoutes = async (
|
||||
secretTagDAL,
|
||||
snapshotService,
|
||||
secretQueueService,
|
||||
secretReplicationService,
|
||||
secretImportDAL,
|
||||
projectEnvDAL,
|
||||
projectBotService
|
||||
@@ -826,6 +841,7 @@ export const registerRoutes = async (
|
||||
projectEnv: projectEnvService,
|
||||
projectRole: projectRoleService,
|
||||
secret: secretService,
|
||||
secretReplication: secretReplicationService,
|
||||
secretTag: secretTagService,
|
||||
folder: folderService,
|
||||
secretImport: secretImportService,
|
||||
|
||||
@@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
|
||||
import: z.object({
|
||||
environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment),
|
||||
path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path)
|
||||
})
|
||||
}),
|
||||
isReplication: z.boolean().default(false)
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
@@ -232,11 +233,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
|
||||
200: z.object({
|
||||
message: z.string(),
|
||||
secretImports: SecretImportsSchema.omit({ importEnv: true })
|
||||
.merge(
|
||||
z.object({
|
||||
importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
|
||||
})
|
||||
)
|
||||
.extend({
|
||||
importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
|
||||
})
|
||||
.array()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ export const secretImportServiceFactory = ({
|
||||
actorOrgId,
|
||||
actorAuthMethod,
|
||||
projectId,
|
||||
isReplication,
|
||||
path
|
||||
}: TCreateSecretImportDTO) => {
|
||||
const { permission } = await permissionService.getProjectPermission(
|
||||
@@ -100,7 +101,8 @@ export const secretImportServiceFactory = ({
|
||||
folderId: folder.id,
|
||||
position: lastPos + 1,
|
||||
importEnv: importEnv.id,
|
||||
importPath: data.path
|
||||
importPath: data.path,
|
||||
isReplication
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
@@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = {
|
||||
environment: string;
|
||||
path: string;
|
||||
};
|
||||
isReplication?: boolean;
|
||||
} & TProjectPermission;
|
||||
|
||||
export type TUpdateSecretImportDTO = {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName, TSecretVersions } from "@app/db/schemas";
|
||||
import { ormify, selectAllTableCols } from "@app/lib/knex";
|
||||
|
||||
export type TSecretReplicationDALFactory = ReturnType<typeof secretReplicationDALFactory>;
|
||||
|
||||
export const secretReplicationDALFactory = (db: TDbClient) => {
|
||||
const orm = ormify(db, TableName.SecretVersion);
|
||||
|
||||
const findSecrets = async (filter: { folderId: string; secrets: { id: string; version: number }[] }, tx?: Knex) => {
|
||||
if (!filter.secrets) return [];
|
||||
|
||||
const sqlRawDocs = await (tx || db)(TableName.SecretVersion)
|
||||
.where({ folderId: filter.folderId })
|
||||
.andWhere((bd) => {
|
||||
filter.secrets.forEach((el) => {
|
||||
void bd.orWhere({
|
||||
[`${TableName.SecretVersion}.secretId` as "secretId"]: el.id,
|
||||
[`${TableName.SecretVersion}.version` as "version"]: el.version
|
||||
});
|
||||
});
|
||||
})
|
||||
.leftJoin<TSecretVersions>(
|
||||
(tx || db)(TableName.SecretVersion)
|
||||
.where("isReplicated", true)
|
||||
.groupBy(["secretId", "version"])
|
||||
.max("version")
|
||||
.select("version", "secretId")
|
||||
.as("latestVersion"),
|
||||
(bd) => {
|
||||
bd.on(`${TableName.SecretVersion}.secretId`, "latestVersion.secretId").andOn(
|
||||
`${TableName.SecretVersion}.version`,
|
||||
"latestVersion.max"
|
||||
);
|
||||
}
|
||||
)
|
||||
// .leftJoin<TSecretVersions>(
|
||||
// (tx || db)(TableName.SecretVersion).select("isReplicated", "version", "secretId").as("previousVersion"),
|
||||
// (bd) => {
|
||||
// bd.on(`${TableName.SecretVersion}.secretId`, "previousVersion.secretId").andOn(
|
||||
// "previousVersion.version",
|
||||
// (tx || db).raw(`${TableName.SecretVersion}.version - 1`)
|
||||
// );
|
||||
// }
|
||||
// )
|
||||
.select(db.ref("version").withSchema("latestVersion").as("latestReplicatedVersion"))
|
||||
.select(selectAllTableCols(TableName.SecretVersion));
|
||||
|
||||
return sqlRawDocs;
|
||||
};
|
||||
|
||||
return {
|
||||
findSecrets,
|
||||
...orm
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,247 @@
|
||||
import { TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { groupBy } from "@app/lib/fn";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
|
||||
import { TSecretDALFactory } from "../secret/secret-dal";
|
||||
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns";
|
||||
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
|
||||
import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal";
|
||||
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
|
||||
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
|
||||
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
|
||||
import { TSecretReplicationDALFactory } from "./secret-replication-dal";
|
||||
import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types";
|
||||
|
||||
type TSecretReplicationServiceFactoryDep = {
|
||||
secretReplicationDAL: TSecretReplicationDALFactory;
|
||||
secretDAL: Pick<TSecretDALFactory, "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete">;
|
||||
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
|
||||
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update">;
|
||||
secretTagDAL: Pick<TSecretTagDALFactory, "find" | "saveTagsToSecret" | "deleteTagsManySecret">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
|
||||
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
|
||||
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
|
||||
};
|
||||
|
||||
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
|
||||
|
||||
// function getRandomError(): number {
|
||||
// const minCeiled: number = Math.ceil(0);
|
||||
// const maxFloored: number = Math.floor(20);
|
||||
// const val = Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled); // The maximum is exclusive and the minimum is inclusive
|
||||
// if (val >= 10) throw new Error("Random error point");
|
||||
// return val;
|
||||
// }
|
||||
|
||||
export const secretReplicationServiceFactory = ({
|
||||
secretReplicationDAL,
|
||||
secretDAL,
|
||||
queueService,
|
||||
secretVersionDAL,
|
||||
secretImportDAL,
|
||||
keyStore,
|
||||
secretVersionTagDAL,
|
||||
secretTagDAL,
|
||||
folderDAL
|
||||
}: TSecretReplicationServiceFactoryDep) => {
|
||||
queueService.start(QueueName.SecretReplication, async (job) => {
|
||||
logger.info(job.data, "Replication started");
|
||||
const { secrets, folderId, secretPath, environmentId, projectId } = job.data;
|
||||
const secretImports = await secretImportDAL.find({
|
||||
importPath: secretPath,
|
||||
importEnv: environmentId,
|
||||
isReplication: true
|
||||
});
|
||||
console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId);
|
||||
if (!secretImports.length) return;
|
||||
|
||||
// unfiltered secrets to be replicated
|
||||
console.log(secrets.length);
|
||||
const toBeReplicatedSecrets = await secretReplicationDAL.findSecrets({ folderId, secrets });
|
||||
const replicatedSecrets = toBeReplicatedSecrets.filter(
|
||||
({ version, latestReplicatedVersion, secretBlindIndex }) =>
|
||||
secretBlindIndex && (version === 1 || latestReplicatedVersion <= version)
|
||||
);
|
||||
|
||||
const replicatedSecretsGroupBySecretId = groupBy(replicatedSecrets, (i) => i.secretId);
|
||||
console.log("replicated ", replicatedSecretsGroupBySecretId);
|
||||
const lock = await keyStore.acquireLock(
|
||||
replicatedSecrets.map(({ id }) => id),
|
||||
5000
|
||||
);
|
||||
|
||||
try {
|
||||
/* eslint-disable no-await-in-loop */
|
||||
for (const secretImport of secretImports) {
|
||||
const importFolderId = secretImport.folderId;
|
||||
|
||||
const localSecrets = await secretDAL.find({
|
||||
$in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) },
|
||||
folderId: importFolderId,
|
||||
isReplicated: true
|
||||
});
|
||||
const localSecretsGroupedByBlindIndex = groupBy(localSecrets, (i) => i.secretBlindIndex as string);
|
||||
|
||||
const locallyCreatedSecrets = secrets.filter(({ operation, id }) => {
|
||||
return (
|
||||
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
|
||||
!localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
);
|
||||
});
|
||||
|
||||
const locallyUpdatedSecrets = secrets.filter(
|
||||
({ operation, id }) =>
|
||||
(operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) &&
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
);
|
||||
|
||||
console.log(replicatedSecretsGroupBySecretId);
|
||||
console.log(locallyCreatedSecrets);
|
||||
console.log("update", locallyUpdatedSecrets);
|
||||
console.log("local board", localSecrets);
|
||||
|
||||
const locallyDeletedSecrets = secrets
|
||||
.filter(
|
||||
({ operation, id }) =>
|
||||
operation === SecretReplicationOperations.Delete &&
|
||||
Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) &&
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0]
|
||||
)
|
||||
.map(
|
||||
({ id }) =>
|
||||
localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0]
|
||||
);
|
||||
|
||||
let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = [];
|
||||
await secretReplicationDAL.transaction(async (tx) => {
|
||||
if (locallyCreatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkInsert({
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyCreatedSecrets.map(({ id }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
return {
|
||||
keyEncoding: doc.keyEncoding,
|
||||
algorithm: doc.algorithm,
|
||||
folderId,
|
||||
type: doc.type,
|
||||
metadata: doc.metadata,
|
||||
secretKeyIV: doc.secretKeyIV,
|
||||
secretKeyTag: doc.secretKeyTag,
|
||||
secretKeyCiphertext: doc.secretKeyCiphertext,
|
||||
secretValueIV: doc.secretValueIV,
|
||||
secretValueTag: doc.secretValueTag,
|
||||
secretValueCiphertext: doc.secretValueCiphertext,
|
||||
secretBlindIndex: doc.secretBlindIndex,
|
||||
secretCommentIV: doc.secretCommentIV,
|
||||
secretCommentTag: doc.secretCommentTag,
|
||||
secretCommentCiphertext: doc.secretCommentCiphertext,
|
||||
isReplicated: true,
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyUpdatedSecrets.length) {
|
||||
const newSecrets = await fnSecretBulkUpdate({
|
||||
projectId,
|
||||
folderId: importFolderId,
|
||||
secretVersionDAL,
|
||||
secretDAL,
|
||||
tx,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
inputSecrets: locallyUpdatedSecrets.map(({ id }) => {
|
||||
const doc = replicatedSecretsGroupBySecretId[id][0];
|
||||
return {
|
||||
filter: {
|
||||
folderId: importFolderId,
|
||||
id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
|
||||
},
|
||||
data: {
|
||||
keyEncoding: doc.keyEncoding,
|
||||
algorithm: doc.algorithm,
|
||||
type: doc.type,
|
||||
metadata: doc.metadata,
|
||||
secretKeyIV: doc.secretKeyIV,
|
||||
secretKeyTag: doc.secretKeyTag,
|
||||
secretKeyCiphertext: doc.secretKeyCiphertext,
|
||||
secretValueIV: doc.secretValueIV,
|
||||
secretValueTag: doc.secretValueTag,
|
||||
secretValueCiphertext: doc.secretValueCiphertext,
|
||||
secretBlindIndex: doc.secretBlindIndex,
|
||||
secretCommentIV: doc.secretCommentIV,
|
||||
secretCommentTag: doc.secretCommentTag,
|
||||
secretCommentCiphertext: doc.secretCommentCiphertext,
|
||||
isReplicated: true,
|
||||
skipMultilineEncoding: doc.skipMultilineEncoding
|
||||
}
|
||||
};
|
||||
})
|
||||
});
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id }))
|
||||
);
|
||||
}
|
||||
if (locallyDeletedSecrets.length) {
|
||||
const newSecrets = await secretDAL.delete(
|
||||
{
|
||||
$in: {
|
||||
id: locallyDeletedSecrets.map(({ id }) => id)
|
||||
},
|
||||
isReplicated: true,
|
||||
folderId: importFolderId
|
||||
},
|
||||
tx
|
||||
);
|
||||
nestedImportSecrets = nestedImportSecrets.concat(
|
||||
...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id }))
|
||||
);
|
||||
}
|
||||
});
|
||||
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]);
|
||||
console.log("Environment ID -> slug", folder.envId, folder.environmentSlug);
|
||||
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, {
|
||||
folderId: folder.id,
|
||||
projectId,
|
||||
secrets: nestedImportSecrets,
|
||||
secretPath: folder.path,
|
||||
environmentId: folder.envId
|
||||
});
|
||||
}
|
||||
await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true });
|
||||
/* eslint-enable no-await-in-loop */
|
||||
} finally {
|
||||
await lock.release();
|
||||
}
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => {
|
||||
logger.error(err, "Failed to replicate secret", job?.data);
|
||||
});
|
||||
|
||||
const replicate = async (data: TSyncSecretReplicationDTO) => {
|
||||
await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 1000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
replicate
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
export enum SecretReplicationOperations {
|
||||
Create = "create",
|
||||
Update = "update",
|
||||
Delete = "delete"
|
||||
}
|
||||
|
||||
export type TSyncSecretReplicationDTO = {
|
||||
secretPath: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
folderId: string;
|
||||
secrets: {
|
||||
operation: SecretReplicationOperations;
|
||||
id: string;
|
||||
version: number;
|
||||
}[];
|
||||
};
|
||||
@@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => {
|
||||
};
|
||||
|
||||
const deleteMany = async (
|
||||
data: Array<{ blindIndex: string; type: SecretType }>,
|
||||
data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>,
|
||||
folderId: string,
|
||||
userId: string,
|
||||
tx?: Knex
|
||||
|
||||
@@ -509,7 +509,7 @@ export const fnSecretBulkInsert = async ({
|
||||
}))
|
||||
);
|
||||
const secretVersions = await secretVersionDAL.insertMany(
|
||||
inputSecrets.map(({ tags, references, ...el }) => ({
|
||||
inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({
|
||||
...el,
|
||||
folderId,
|
||||
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
|
||||
|
||||
@@ -30,6 +30,8 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-
|
||||
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
|
||||
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
|
||||
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
|
||||
import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service";
|
||||
import { SecretReplicationOperations } from "../secret-replication/secret-replication-types";
|
||||
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
|
||||
import { TSecretDALFactory } from "./secret-dal";
|
||||
import {
|
||||
@@ -85,6 +87,7 @@ type TSecretServiceFactoryDep = {
|
||||
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
|
||||
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
|
||||
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
|
||||
secretReplicationService: Pick<TSecretReplicationServiceFactory, "replicate">;
|
||||
};
|
||||
|
||||
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
|
||||
@@ -101,7 +104,8 @@ export const secretServiceFactory = ({
|
||||
projectDAL,
|
||||
projectBotService,
|
||||
secretImportDAL,
|
||||
secretVersionTagDAL
|
||||
secretVersionTagDAL,
|
||||
secretReplicationService
|
||||
}: TSecretServiceFactoryDep) => {
|
||||
const getSecretReference = async (projectId: string) => {
|
||||
// if bot key missing means e2e still exist
|
||||
@@ -285,6 +289,19 @@ export const secretServiceFactory = ({
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Create,
|
||||
id: secret[0].id,
|
||||
version: 1
|
||||
}
|
||||
]
|
||||
});
|
||||
return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
|
||||
};
|
||||
|
||||
@@ -414,7 +431,19 @@ export const secretServiceFactory = ({
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Update,
|
||||
id: updatedSecret[0].id,
|
||||
version: updatedSecret[0].version
|
||||
}
|
||||
]
|
||||
});
|
||||
return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
|
||||
};
|
||||
|
||||
@@ -482,7 +511,19 @@ export const secretServiceFactory = ({
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: [
|
||||
{
|
||||
operation: SecretReplicationOperations.Delete,
|
||||
id: deletedSecret[0].id,
|
||||
version: deletedSecret[0].version
|
||||
}
|
||||
]
|
||||
});
|
||||
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
|
||||
return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
|
||||
};
|
||||
@@ -768,6 +809,13 @@ export const secretServiceFactory = ({
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create }))
|
||||
});
|
||||
|
||||
return newSecrets;
|
||||
};
|
||||
@@ -868,6 +916,13 @@ export const secretServiceFactory = ({
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update }))
|
||||
});
|
||||
|
||||
return secrets;
|
||||
};
|
||||
@@ -930,6 +985,13 @@ export const secretServiceFactory = ({
|
||||
|
||||
await snapshotService.performSnapshot(folderId);
|
||||
await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
|
||||
await secretReplicationService.replicate({
|
||||
folderId,
|
||||
projectId,
|
||||
environmentId: folder.envId,
|
||||
secretPath: path,
|
||||
secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete }))
|
||||
});
|
||||
|
||||
return secretsDeleted;
|
||||
};
|
||||
|
||||
@@ -9,12 +9,13 @@ export const useCreateSecretImport = () => {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation<{}, {}, TCreateSecretImportDTO>({
|
||||
mutationFn: async ({ import: secretImport, environment, projectId, path }) => {
|
||||
mutationFn: async ({ import: secretImport, environment, isReplication, projectId, path }) => {
|
||||
const { data } = await apiRequest.post("/api/v1/secret-imports", {
|
||||
import: secretImport,
|
||||
environment,
|
||||
workspaceId: projectId,
|
||||
path
|
||||
path,
|
||||
isReplication
|
||||
});
|
||||
return data;
|
||||
},
|
||||
|
||||
@@ -60,6 +60,7 @@ export type TCreateSecretImportDTO = {
|
||||
environment: string;
|
||||
path: string;
|
||||
};
|
||||
isReplication?: boolean;
|
||||
};
|
||||
|
||||
export type TUpdateSecretImportDTO = {
|
||||
|
||||
@@ -4,7 +4,15 @@ import { AxiosError } from "axios";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createNotification } from "@app/components/notifications";
|
||||
import { Button, FormControl, Modal, ModalContent, Select, SelectItem } from "@app/components/v2";
|
||||
import {
|
||||
Button,
|
||||
Checkbox,
|
||||
FormControl,
|
||||
Modal,
|
||||
ModalContent,
|
||||
Select,
|
||||
SelectItem
|
||||
} from "@app/components/v2";
|
||||
import { SecretPathInput } from "@app/components/v2/SecretPathInput";
|
||||
import { useWorkspace } from "@app/context";
|
||||
import { useCreateSecretImport } from "@app/hooks/api";
|
||||
@@ -16,7 +24,8 @@ const typeSchema = z.object({
|
||||
.trim()
|
||||
.transform((val) =>
|
||||
typeof val === "string" && val.at(-1) === "/" && val.length > 1 ? val.slice(0, -1) : val
|
||||
)
|
||||
),
|
||||
isReplication: z.boolean().default(false)
|
||||
});
|
||||
|
||||
type TFormSchema = z.infer<typeof typeSchema>;
|
||||
@@ -54,13 +63,15 @@ export const CreateSecretImportForm = ({
|
||||
|
||||
const handleFormSubmit = async ({
|
||||
environment: importedEnv,
|
||||
secretPath: importedSecPath
|
||||
secretPath: importedSecPath,
|
||||
isReplication
|
||||
}: TFormSchema) => {
|
||||
try {
|
||||
await createSecretImport({
|
||||
environment,
|
||||
projectId: workspaceId,
|
||||
path: secretPath,
|
||||
isReplication,
|
||||
import: {
|
||||
environment: importedEnv,
|
||||
path: importedSecPath
|
||||
@@ -127,7 +138,17 @@ export const CreateSecretImportForm = ({
|
||||
</FormControl>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
name="isReplication"
|
||||
control={control}
|
||||
defaultValue={false}
|
||||
render={({ field }) => (
|
||||
<Checkbox isChecked={field.value} onCheckedChange={field.onChange} id="isReplication">
|
||||
The replication mode retrieves secrets when changes occur in the specified
|
||||
environment and secret path.
|
||||
</Checkbox>
|
||||
)}
|
||||
/>
|
||||
<div className="mt-7 flex items-center">
|
||||
<Button
|
||||
isDisabled={isSubmitting}
|
||||
|
||||
Reference in New Issue
Block a user