mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-16 10:18:16 +00:00
feat: changed backend-pg to backend
This commit is contained in:
Akhil Mohan
@@ -0,0 +1,2 @@
|
||||
export type { TSecretRotationQueueFactory } from "./secret-rotation-queue";
|
||||
export { DisableRotationErrors, secretRotationQueueFactory } from "./secret-rotation-queue";
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
/* eslint-disable no-param-reassign */
|
||||
import axios from "axios";
|
||||
import jmespath from "jmespath";
|
||||
import knex from "knex";
|
||||
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
|
||||
import {
|
||||
TAssignOp,
|
||||
TDbProviderClients,
|
||||
TDirectAssignOp,
|
||||
THttpProviderFunction
|
||||
} from "../templates/types";
|
||||
import { TSecretRotationData, TSecretRotationDbFn } from "./secret-rotation-queue-types";
|
||||
|
||||
const REGEX = /\${([^}]+)}/g;
|
||||
const EXTERNAL_REQUEST_TIMEOUT = 10 * 1000;
|
||||
|
||||
export const interpolate = (data: any, getValue: (key: string) => unknown) => {
|
||||
if (!data) return;
|
||||
|
||||
if (typeof data === "number") return data;
|
||||
|
||||
if (typeof data === "string") {
|
||||
return data.replace(REGEX, (_a, b) => getValue(b) as string);
|
||||
}
|
||||
|
||||
if (typeof data === "object" && Array.isArray(data)) {
|
||||
data.forEach((el, index) => {
|
||||
// eslint-disable-next-line
|
||||
data[index] = interpolate(el, getValue);
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof data === "object") {
|
||||
if ((data as { ref: string })?.ref) return getValue((data as { ref: string }).ref);
|
||||
const temp = data as Record<string, unknown>; // for converting ts object to record type
|
||||
Object.keys(temp).forEach((key) => {
|
||||
temp[key as keyof typeof temp] = interpolate(data[key as keyof typeof temp], getValue);
|
||||
});
|
||||
}
|
||||
return data;
|
||||
};
|
||||
|
||||
const getInterpolationValue = (variables: TSecretRotationData) => (key: string) => {
|
||||
if (key.includes("|")) {
|
||||
const [keyword, ...arg] = key.split("|").map((el) => el.trim());
|
||||
switch (keyword) {
|
||||
case "random": {
|
||||
return alphaNumericNanoId(parseInt(arg[0], 10));
|
||||
}
|
||||
default: {
|
||||
throw Error(`Interpolation key not found - ${key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
const [type, keyName] = key.split(".").map((el) => el.trim());
|
||||
return variables[type as keyof TSecretRotationData][keyName];
|
||||
};
|
||||
|
||||
export const secretRotationHttpFn = async (
|
||||
func: THttpProviderFunction,
|
||||
variables: TSecretRotationData
|
||||
) => {
|
||||
// string interpolation
|
||||
const headers = interpolate(func.header, getInterpolationValue(variables));
|
||||
const url = interpolate(func.url, getInterpolationValue(variables));
|
||||
const body = interpolate(func.body, getInterpolationValue(variables));
|
||||
// axios will automatically throw error if req status is not between 2xx range
|
||||
return axios({
|
||||
method: func.method,
|
||||
url,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: EXTERNAL_REQUEST_TIMEOUT,
|
||||
signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT)
|
||||
});
|
||||
};
|
||||
|
||||
export const secretRotationDbFn = async ({
|
||||
ca,
|
||||
host,
|
||||
port,
|
||||
query,
|
||||
database,
|
||||
password,
|
||||
username,
|
||||
client,
|
||||
variables
|
||||
}: TSecretRotationDbFn) => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
const ssl = ca ? { rejectUnauthorized: false, ca } : undefined;
|
||||
if (host === "localhost" || host === "127.0.0.1" || appCfg.DB_CONNECTION_URI.includes(host))
|
||||
throw new Error("Invalid db host");
|
||||
|
||||
const db = knex({
|
||||
client,
|
||||
connection: {
|
||||
database,
|
||||
port,
|
||||
host,
|
||||
user: username,
|
||||
password,
|
||||
connectionTimeoutMillis: EXTERNAL_REQUEST_TIMEOUT,
|
||||
ssl,
|
||||
pool: { min: 0, max: 1 }
|
||||
}
|
||||
});
|
||||
const data = await db.raw(query, variables);
|
||||
return data;
|
||||
};
|
||||
|
||||
export const secretRotationPreSetFn = (
|
||||
op: Record<string, TDirectAssignOp>,
|
||||
variables: TSecretRotationData
|
||||
) => {
|
||||
const getValFn = getInterpolationValue(variables);
|
||||
Object.entries(op || {}).forEach(([key, assignFn]) => {
|
||||
const [type, keyName] = key.split(".") as [keyof TSecretRotationData, string];
|
||||
variables[type][keyName] = interpolate(assignFn.value, getValFn);
|
||||
});
|
||||
};
|
||||
|
||||
export const secretRotationHttpSetFn = async (
|
||||
func: THttpProviderFunction,
|
||||
variables: TSecretRotationData
|
||||
) => {
|
||||
const getValFn = getInterpolationValue(variables);
|
||||
// http setter
|
||||
const res = await secretRotationHttpFn(func, variables);
|
||||
Object.entries(func.setter || {}).forEach(([key, assignFn]) => {
|
||||
const [type, keyName] = key.split(".") as [keyof TSecretRotationData, string];
|
||||
if (assignFn.assign === TAssignOp.JmesPath) {
|
||||
variables[type][keyName] = jmespath.search(res.data, assignFn.path);
|
||||
} else if (assignFn.value) {
|
||||
variables[type][keyName] = interpolate(assignFn.value, getValFn);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
export const getDbSetQuery = (
|
||||
db: TDbProviderClients,
|
||||
variables: { username: string; password: string }
|
||||
) => {
|
||||
if (db === TDbProviderClients.Pg) {
|
||||
return {
|
||||
query: `ALTER USER ?? WITH PASSWORD '${variables.password}'`,
|
||||
variables: [variables.username]
|
||||
};
|
||||
}
|
||||
// add more based on client
|
||||
return {
|
||||
query: `ALTER USER ?? IDENTIFIED BY '${variables.password}'`,
|
||||
variables: [variables.username]
|
||||
};
|
||||
};
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import { TDbProviderClients } from "../templates/types";
|
||||
|
||||
export type TSecretRotationEncData = {
|
||||
inputs: Record<string, unknown>;
|
||||
creds: Array<{
|
||||
outputs: Record<string, unknown>;
|
||||
internal: Record<string, unknown>;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type TSecretRotationData = {
|
||||
inputs: Record<string, unknown>;
|
||||
outputs: Record<string, unknown>;
|
||||
internal: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type TSecretRotationDbFn = {
|
||||
client: TDbProviderClients;
|
||||
username: string;
|
||||
password: string;
|
||||
host: string;
|
||||
database: string;
|
||||
port: number;
|
||||
query: string;
|
||||
variables: unknown[];
|
||||
ca?: string;
|
||||
};
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
import { SecretKeyEncoding, SecretType } from "@app/db/schemas";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import {
|
||||
encryptSymmetric128BitHexKeyUTF8,
|
||||
infisicalSymmetricDecrypt,
|
||||
infisicalSymmetricEncypt
|
||||
} from "@app/lib/crypto/encryption";
|
||||
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { alphaNumericNanoId } from "@app/lib/nanoid";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
|
||||
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
|
||||
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
|
||||
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
|
||||
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
|
||||
|
||||
import { TSecretRotationDALFactory } from "../secret-rotation-dal";
|
||||
import { rotationTemplates } from "../templates";
|
||||
import {
|
||||
TDbProviderClients,
|
||||
TProviderFunctionTypes,
|
||||
TSecretRotationProviderTemplate
|
||||
} from "../templates/types";
|
||||
import {
|
||||
getDbSetQuery,
|
||||
secretRotationDbFn,
|
||||
secretRotationHttpFn,
|
||||
secretRotationHttpSetFn,
|
||||
secretRotationPreSetFn
|
||||
} from "./secret-rotation-queue-fn";
|
||||
import {
|
||||
TSecretRotationData,
|
||||
TSecretRotationDbFn,
|
||||
TSecretRotationEncData
|
||||
} from "./secret-rotation-queue-types";
|
||||
|
||||
export type TSecretRotationQueueFactory = ReturnType<typeof secretRotationQueueFactory>;
|
||||
|
||||
type TSecretRotationQueueFactoryDep = {
|
||||
queue: TQueueServiceFactory;
|
||||
secretRotationDAL: TSecretRotationDALFactory;
|
||||
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
|
||||
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "find">;
|
||||
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionMany">;
|
||||
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
|
||||
};
|
||||
|
||||
// These error should stop the repeatable job and ask user to reconfigure rotation
|
||||
export class DisableRotationErrors extends Error {
|
||||
name: string;
|
||||
|
||||
error: unknown;
|
||||
|
||||
constructor({ name, error, message }: { message: string; name?: string; error?: unknown }) {
|
||||
super(message);
|
||||
this.name = name || "DisableRotationErrors";
|
||||
this.error = error;
|
||||
}
|
||||
}
|
||||
|
||||
export const secretRotationQueueFactory = ({
|
||||
queue,
|
||||
secretRotationDAL,
|
||||
projectBotService,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
telemetryService
|
||||
}: TSecretRotationQueueFactoryDep) => {
|
||||
const addToQueue = async (rotationId: string, interval: number) => {
|
||||
const appCfg = getConfig();
|
||||
queue.queue(
|
||||
QueueName.SecretRotation,
|
||||
QueueJobs.SecretRotation,
|
||||
{ rotationId },
|
||||
{
|
||||
jobId: rotationId,
|
||||
repeat: {
|
||||
// on prod it this will be in days, in development this will be second
|
||||
every:
|
||||
appCfg.NODE_ENV === "development"
|
||||
? secondsToMillis(interval)
|
||||
: daysToMillisecond(interval),
|
||||
immediately: true
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const removeFromQueue = async (rotationId: string, interval: number) => {
|
||||
const appCfg = getConfig();
|
||||
await queue.stopRepeatableJob(
|
||||
QueueName.SecretRotation,
|
||||
QueueJobs.SecretRotation,
|
||||
{
|
||||
// on prod it this will be in days, in development this will be second
|
||||
every:
|
||||
appCfg.NODE_ENV === "development"
|
||||
? secondsToMillis(interval)
|
||||
: daysToMillisecond(interval)
|
||||
},
|
||||
rotationId
|
||||
);
|
||||
};
|
||||
|
||||
queue.start(QueueName.SecretRotation, async (job) => {
|
||||
const { rotationId } = job.data;
|
||||
logger.info(`secretRotationQueue.process: [rotationDocument=${rotationId}]`);
|
||||
const secretRotation = await secretRotationDAL.findById(rotationId);
|
||||
const rotationProvider = rotationTemplates.find(
|
||||
({ name }) => name === secretRotation?.provider
|
||||
);
|
||||
|
||||
try {
|
||||
if (!rotationProvider || !secretRotation)
|
||||
throw new DisableRotationErrors({ message: "Provider not found" });
|
||||
|
||||
const rotationOutputs = await secretRotationDAL.findRotationOutputsByRotationId(rotationId);
|
||||
if (!rotationOutputs.length)
|
||||
throw new DisableRotationErrors({ message: "Secrets not found" });
|
||||
|
||||
// deep copy
|
||||
const provider = JSON.parse(
|
||||
JSON.stringify(rotationProvider)
|
||||
) as TSecretRotationProviderTemplate;
|
||||
|
||||
// now get the encrypted variable values
|
||||
// in includes the inputs, the previous outputs
|
||||
// internal mapping variables etc
|
||||
const { encryptedDataTag, encryptedDataIV, encryptedData, keyEncoding } = secretRotation;
|
||||
if (!encryptedDataTag || !encryptedDataIV || !encryptedData || !keyEncoding) {
|
||||
throw new DisableRotationErrors({ message: "No inputs found" });
|
||||
}
|
||||
const decryptedData = infisicalSymmetricDecrypt({
|
||||
keyEncoding: keyEncoding as SecretKeyEncoding,
|
||||
ciphertext: encryptedData,
|
||||
iv: encryptedDataIV,
|
||||
tag: encryptedDataTag
|
||||
});
|
||||
|
||||
const variables = JSON.parse(decryptedData) as TSecretRotationEncData;
|
||||
// rotation set cycle
|
||||
const newCredential: TSecretRotationData = {
|
||||
inputs: variables.inputs,
|
||||
outputs: {},
|
||||
internal: {}
|
||||
};
|
||||
|
||||
// when its a database we keep cycling the variables accordingly
|
||||
if (provider.template.type === TProviderFunctionTypes.DB) {
|
||||
const lastCred = variables.creds.at(-1);
|
||||
if (lastCred && variables.creds.length === 1) {
|
||||
newCredential.internal.username =
|
||||
lastCred.internal.username === variables.inputs.username1
|
||||
? variables.inputs.username2
|
||||
: variables.inputs.username1;
|
||||
} else {
|
||||
newCredential.internal.username = lastCred
|
||||
? lastCred.internal.username
|
||||
: variables.inputs.username1;
|
||||
}
|
||||
// set a random value for new password
|
||||
newCredential.internal.rotated_password = alphaNumericNanoId(32);
|
||||
const {
|
||||
admin_username: username,
|
||||
admin_password: password,
|
||||
host,
|
||||
database,
|
||||
port,
|
||||
ca
|
||||
} = newCredential.inputs;
|
||||
const dbFunctionArg = {
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
database,
|
||||
port,
|
||||
ca: ca as string,
|
||||
client:
|
||||
provider.template.client === TDbProviderClients.MySql
|
||||
? "mysql2"
|
||||
: provider.template.client
|
||||
} as TSecretRotationDbFn;
|
||||
// set function
|
||||
await secretRotationDbFn({
|
||||
...dbFunctionArg,
|
||||
...getDbSetQuery(provider.template.client, {
|
||||
password: newCredential.internal.rotated_password as string,
|
||||
username: newCredential.internal.username as string
|
||||
})
|
||||
});
|
||||
// test function
|
||||
await secretRotationDbFn({
|
||||
...dbFunctionArg,
|
||||
query: "SELECT NOW()",
|
||||
variables: []
|
||||
});
|
||||
newCredential.outputs.db_username = newCredential.internal.username;
|
||||
newCredential.outputs.db_password = newCredential.internal.rotated_password;
|
||||
// clean up
|
||||
if (variables.creds.length === 2) variables.creds.pop();
|
||||
}
|
||||
|
||||
if (provider.template.type === TProviderFunctionTypes.HTTP) {
|
||||
if (provider.template.functions.set?.pre) {
|
||||
secretRotationPreSetFn(provider.template.functions.set.pre, newCredential);
|
||||
}
|
||||
await secretRotationHttpSetFn(provider.template.functions.set, newCredential);
|
||||
// now test
|
||||
await secretRotationHttpFn(provider.template.functions.test, newCredential);
|
||||
if (variables.creds.length === 2) {
|
||||
const deleteCycleCred = variables.creds.pop();
|
||||
if (deleteCycleCred && provider.template.functions.remove) {
|
||||
const deleteCycleVar = { inputs: variables.inputs, ...deleteCycleCred };
|
||||
await secretRotationHttpFn(provider.template.functions.remove, deleteCycleVar);
|
||||
}
|
||||
}
|
||||
}
|
||||
variables.creds.unshift({
|
||||
outputs: newCredential.outputs,
|
||||
internal: newCredential.internal
|
||||
});
|
||||
const encVarData = infisicalSymmetricEncypt(JSON.stringify(variables));
|
||||
const key = await projectBotService.getBotKey(secretRotation.projectId);
|
||||
const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({
|
||||
secretId,
|
||||
value: encryptSymmetric128BitHexKeyUTF8(
|
||||
typeof newCredential.outputs[outputKey] === "object"
|
||||
? JSON.stringify(newCredential.outputs[outputKey])
|
||||
: String(newCredential.outputs[outputKey]),
|
||||
key
|
||||
)
|
||||
}));
|
||||
await secretRotationDAL.transaction(async (tx) => {
|
||||
await secretRotationDAL.updateById(
|
||||
rotationId,
|
||||
{
|
||||
encryptedData: encVarData.ciphertext,
|
||||
encryptedDataIV: encVarData.iv,
|
||||
encryptedDataTag: encVarData.tag,
|
||||
keyEncoding: encVarData.encoding,
|
||||
algorithm: encVarData.algorithm,
|
||||
lastRotatedAt: new Date(),
|
||||
statusMessage: "Rotated successfull",
|
||||
status: "success"
|
||||
},
|
||||
tx
|
||||
);
|
||||
const updatedSecrets = await secretDAL.bulkUpdate(
|
||||
encryptedSecrets.map(({ secretId, value }) => ({
|
||||
// this secret id is validated when user is inserted
|
||||
filter: { id: secretId, type: SecretType.Shared },
|
||||
data: {
|
||||
secretValueCiphertext: value.ciphertext,
|
||||
secretValueIV: value.iv,
|
||||
secretValueTag: value.tag
|
||||
}
|
||||
})),
|
||||
tx
|
||||
);
|
||||
await secretVersionDAL.insertMany(
|
||||
updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({
|
||||
...el,
|
||||
secretId: id
|
||||
})),
|
||||
tx
|
||||
);
|
||||
});
|
||||
|
||||
telemetryService.sendPostHogEvents({
|
||||
event: PostHogEventTypes.SecretRotated,
|
||||
distinctId: "",
|
||||
properties: {
|
||||
numberOfSecrets: encryptedSecrets.length,
|
||||
environment: secretRotation.environment.slug,
|
||||
secretPath: secretRotation.secretPath,
|
||||
workspaceId: secretRotation.projectId
|
||||
}
|
||||
});
|
||||
|
||||
logger.info("Finished rotating: rotation id: ", rotationId);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
if (error instanceof DisableRotationErrors) {
|
||||
if (job.id) {
|
||||
queue.stopRepeatableJobByJobId(QueueName.SecretRotation, job.id);
|
||||
}
|
||||
}
|
||||
|
||||
await secretRotationDAL.updateById(rotationId, {
|
||||
status: "failed",
|
||||
statusMessage: (error as Error).message.slice(0, 500),
|
||||
lastRotatedAt: new Date()
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
addToQueue,
|
||||
removeFromQueue
|
||||
};
|
||||
};
|
||||
Reference in New Issue
Block a user