awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvements: final adjustments/improvements

Browse Source
This commit is contained in:
Scott Wilson
2025-01-21 20:21:29 -08:00
parent 3c1fc024c2
commit 99ca9e04f8
22 changed files with 255 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/lib/api-docs/constants.ts
View File

@@ -1679,6 +1679,9 @@ export const SecretSyncs = {
const destinationName = SECRET_SYNC_NAME_MAP[destination];
return {
syncId: `The ID of the ${destinationName} Sync to be updated.`,
connectionId: `The updated ID of the ${
APP_CONNECTION_NAME_MAP[SECRET_SYNC_CONNECTION_MAP[destination]]
} Connection to use for syncing.`,
name: `The updated name of the ${destinationName} Sync. Must be slug-friendly.`,
environment: `The updated slug of the project environment to sync secrets from.`,
secretPath: `The updated folder path to sync secrets from.`,

1
backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts
View File

@@ -31,6 +31,7 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
isEnabled?: boolean;
}>;
updateSchema: z.ZodType<{
connectionId?: string;
name?: string;
environment?: string;
secretPath?: string;

10
backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts
View File

@@ -129,7 +129,7 @@ const deleteParametersBatch = async (
};
export const AwsParameterStoreSyncFns = {
syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
@@ -138,7 +138,7 @@ export const AwsParameterStoreSyncFns = {
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
for await (const entry of Object.entries(affixedSecretMap)) {
for await (const entry of Object.entries(secretMap)) {
const [key, { value }] = entry;
// skip empty values (not allowed by AWS) or secrets that haven't changed
@@ -167,7 +167,7 @@ export const AwsParameterStoreSyncFns = {
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
const [key, parameter] = entry;
if (!(key in affixedSecretMap) || !affixedSecretMap[key].value) {
if (!(key in secretMap) || !secretMap[key].value) {
parametersToDelete.push(parameter);
}
}
@@ -185,7 +185,7 @@ export const AwsParameterStoreSyncFns = {
Object.entries(awsParameterStoreSecretsRecord).map(([key, value]) => [key, { value: value.Value ?? "" }])
);
},
removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
@@ -197,7 +197,7 @@ export const AwsParameterStoreSyncFns = {
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
const [key, param] = entry;
if (key in affixedSecretMap) {
if (key in secretMap) {
parametersToDelete.push(param);
}
}

56
backend/src/services/secret-sync/github/github-sync-fns.ts
View File

@@ -3,6 +3,7 @@ import sodium from "libsodium-wrappers";
import { getGitHubClient } from "@app/services/app-connection/github";
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
@@ -158,7 +159,33 @@ const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials
};
export const GithubSyncFns = {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
switch (secretSync.destinationConfig.scope) {
case GitHubSyncScope.Organization:
if (Object.values(secretMap).length > 1000) {
throw new SecretSyncError({
message: "GitHub does not support storing more than 1,000 secrets at the organization level.",
shouldRetry: false
});
}
break;
case GitHubSyncScope.Repository:
case GitHubSyncScope.RepositoryEnvironment:
if (Object.values(secretMap).length > 100) {
throw new SecretSyncError({
message: "GitHub does not support storing more than 100 secrets at the repository level.",
shouldRetry: false
});
}
break;
default:
throw new Error(
`Unsupported GitHub Sync scope ${
(secretSync.destinationConfig as TGitHubSyncWithCredentials["destinationConfig"]).scope
}`
);
}
const client = getGitHubClient(secretSync.connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
@@ -166,16 +193,16 @@ export const GithubSyncFns = {
const publicKey = await getPublicKey(client, secretSync);
for await (const encryptedSecret of encryptedSecrets) {
if (!(encryptedSecret.name in affixedSecretMap)) {
if (!(encryptedSecret.name in secretMap)) {
await deleteSecret(client, secretSync, encryptedSecret);
}
}
await sodium.ready.then(async () => {
for await (const key of Object.keys(affixedSecretMap)) {
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(affixedSecretMap[key].value);
const binarySecretValue = sodium.from_string(secretMap[key].value);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
@@ -183,24 +210,31 @@ export const GithubSyncFns = {
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
try {
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
}
});
},
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in affixedSecretMap) {
if (encryptedSecret.name in secretMap) {
await deleteSecret(client, secretSync, encryptedSecret);
}
}

15
backend/src/services/secret-sync/secret-sync-errors.ts
View File

@@ -1,14 +1,23 @@
export class SecretSyncError extends Error {
name: string;
error: unknown;
error?: unknown;
secretKey?: string;
constructor({ name, error, secretKey }: { name?: string; error?: unknown; secretKey?: string } = {}) {
super();
shouldRetry?: boolean;
constructor({
name,
error,
secretKey,
message,
shouldRetry = true
}: { name?: string; error?: unknown; secretKey?: string; shouldRetry?: boolean; message?: string } = {}) {
super(message);
this.name = name || "SecretSyncError";
this.error = error;
this.secretKey = secretKey;
this.shouldRetry = shouldRetry;
}
}

95
backend/src/services/secret-sync/secret-sync-fns.ts
View File

@@ -22,55 +22,55 @@ export const listSecretSyncOptions = () => {
return Object.values(SECRET_SYNC_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
};
const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
let secretMap = { ...unprocessedSecretMap };
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
if (appendSuffix || prependPrefix) {
secretMap = {};
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value;
});
}
return secretMap;
};
const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
let secretMap = { ...unprocessedSecretMap };
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
if (appendSuffix || prependPrefix) {
secretMap = {};
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
let processedKey = key;
if (prependPrefix && processedKey.startsWith(prependPrefix)) {
processedKey = processedKey.slice(prependPrefix.length);
}
if (appendSuffix && processedKey.endsWith(appendSuffix)) {
processedKey = processedKey.slice(0, -appendSuffix.length);
}
secretMap[processedKey] = value;
});
}
return secretMap;
};
// const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
// let secretMap = { ...unprocessedSecretMap };
//
// const { appendSuffix, prependPrefix } = secretSync.syncOptions;
//
// if (appendSuffix || prependPrefix) {
// secretMap = {};
// Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
// secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value;
// });
// }
//
// return secretMap;
// };
//
// const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
// let secretMap = { ...unprocessedSecretMap };
//
// const { appendSuffix, prependPrefix } = secretSync.syncOptions;
//
// if (appendSuffix || prependPrefix) {
// secretMap = {};
// Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
// let processedKey = key;
//
// if (prependPrefix && processedKey.startsWith(prependPrefix)) {
// processedKey = processedKey.slice(prependPrefix.length);
// }
//
// if (appendSuffix && processedKey.endsWith(appendSuffix)) {
// processedKey = processedKey.slice(0, -appendSuffix.length);
// }
//
// secretMap[processedKey] = value;
// });
// }
//
// return secretMap;
// };
export const SecretSyncFns = {
syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
// const affixedSecretMap = addAffixes(secretSync, secretMap);
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.syncSecrets(secretSync, affixedSecretMap);
return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, affixedSecretMap);
return GithubSyncFns.syncSecrets(secretSync, secretMap);
default:
throw new Error(
`Unhandled sync destination for push secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -92,16 +92,17 @@ export const SecretSyncFns = {
);
}
return stripAffixes(secretSync, secretMap);
return secretMap;
// return stripAffixes(secretSync, secretMap);
},
removeSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
// const affixedSecretMap = addAffixes(secretSync, secretMap);
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.removeSecrets(secretSync, affixedSecretMap);
return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, affixedSecretMap);
return GithubSyncFns.removeSecrets(secretSync, secretMap);
default:
throw new Error(
`Unhandled sync destination for removing secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -114,7 +115,7 @@ export const parseSyncErrorMessage = (err: unknown): string => {
if (err instanceof SecretSyncError) {
return JSON.stringify({
secretKey: err.secretKey,
error: parseSyncErrorMessage(err.error)
error: err.message ?? parseSyncErrorMessage(err.error)
});
}

46
backend/src/services/secret-sync/secret-sync-queue.ts
View File

@@ -31,6 +31,7 @@ import {
SecretSyncImportBehavior,
SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import {
@@ -94,6 +95,19 @@ type SecretSyncActionJob = Job<
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;
const getRequeueDelay = (failureCount?: number) => {
if (!failureCount) return 0;
const baseDelay = 1000;
const maxDelay = 30000;
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
const jitter = delay * (0.5 + Math.random() * 0.5);
return jitter;
};
export const secretSyncQueueFactory = ({
queueService,
kmsService,
@@ -164,13 +178,18 @@ export const secretSyncQueueFactory = ({
resourceMetadataDAL
});
const $getSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, includeImports = true) => {
const $getInfisicalSecrets = async (
secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,
includeImports = true
) => {
const { projectId, folderId, environment, folder } = secretSync;
if (!folderId || !environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
throw new SecretSyncError({
message:
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path.",
shouldRetry: false
});
const secretMap: TSecretMap = {};
@@ -247,7 +266,7 @@ export const secretSyncQueueFactory = ({
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
delay: payload.failedToAcquireLockCount ? 1000 : 0, // we don't want to delay initial job
delay: getRequeueDelay(payload.failedToAcquireLockCount), // this is for delaying re-queued jobs if sync is locked
attempts: 5,
backoff: {
type: "exponential",
@@ -309,7 +328,7 @@ export const secretSyncQueueFactory = ({
const importedSecretMap: TSecretMap = {};
const secretMap = await $getSecrets(secretSync, false);
const secretMap = await $getInfisicalSecrets(secretSync, false);
const secretsToCreate: Parameters<typeof $createManySecretsRawFn>[0]["secrets"] = [];
const secretsToUpdate: Parameters<typeof $updateManySecretsRawFn>[0]["secrets"] = [];
@@ -374,7 +393,7 @@ export const secretSyncQueueFactory = ({
let isSynced = false;
let syncMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
let isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
@@ -400,7 +419,7 @@ export const secretSyncQueueFactory = ({
syncOptions: { initialSyncBehavior }
} = secretSyncWithCredentials;
const secretMap = await $getSecrets(secretSync);
const secretMap = await $getInfisicalSecrets(secretSync);
if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
const importedSecretMap = await $importSecrets(
@@ -438,8 +457,12 @@ export const secretSyncQueueFactory = ({
syncMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
if (err instanceof SecretSyncError && !err.shouldRetry) {
isFinalAttempt = true;
} else {
// re-throw so job fails
throw err;
}
} finally {
const ranAt = new Date();
const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
@@ -639,7 +662,7 @@ export const secretSyncQueueFactory = ({
kmsService
});
const secretMap = await $getSecrets(secretSync);
const secretMap = await $getInfisicalSecrets(secretSync);
await SecretSyncFns.removeSecrets(
{
@@ -816,7 +839,6 @@ export const secretSyncQueueFactory = ({
case QueueJobs.SecretSyncSyncSecrets: {
const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;
//
if (failedToAcquireLockCount < 10) {
await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
return;

27
backend/src/services/secret-sync/secret-sync-schemas.ts
View File

@@ -13,19 +13,19 @@ const SyncOptionsSchema = (secretSync: SecretSync, options: TSyncOptionsConfig =
initialSyncBehavior: (options.canImportSecrets
? z.nativeEnum(SecretSyncInitialSyncBehavior)
: z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination)
).describe(SecretSyncs.SYNC_OPTIONS(secretSync).INITIAL_SYNC_BEHAVIOR),
prependPrefix: z
.string()
.trim()
.transform((str) => str.toUpperCase())
.optional()
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).PREPEND_PREFIX),
appendSuffix: z
.string()
.trim()
.transform((str) => str.toUpperCase())
.optional()
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).APPEND_SUFFIX)
).describe(SecretSyncs.SYNC_OPTIONS(secretSync).INITIAL_SYNC_BEHAVIOR)
// prependPrefix: z
// .string()
// .trim()
// .transform((str) => str.toUpperCase())
// .optional()
// .describe(SecretSyncs.SYNC_OPTIONS(secretSync).PREPEND_PREFIX),
// appendSuffix: z
// .string()
// .trim()
// .transform((str) => str.toUpperCase())
// .optional()
// .describe(SecretSyncs.SYNC_OPTIONS(secretSync).APPEND_SUFFIX)
});
export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
@@ -72,6 +72,7 @@ export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syn
export const GenericUpdateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
z.object({
name: slugSchema({ field: "name" }).describe(SecretSyncs.UPDATE(destination).name).optional(),
connectionId: z.string().uuid().describe(SecretSyncs.UPDATE(destination).connectionId).optional(),
description: z
.string()
.trim()

8
backend/src/services/secret-sync/secret-sync-service.ts
View File

@@ -13,6 +13,7 @@ import { OrgServiceActor } from "@app/lib/types";
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { listSecretSyncOptions } from "@app/services/secret-sync/secret-sync-fns";
import {
SecretSyncStatus,
@@ -262,6 +263,13 @@ export const secretSyncServiceFactory = ({
const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => {
let { folderId } = secretSync;
if (params.connectionId) {
const destinationApp = SECRET_SYNC_CONNECTION_MAP[secretSync.destination as SecretSync];
// validates permission to connect and app is valid for sync destination
await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor);
}
if (
(secretPath && secretPath !== secretSync.folder?.path) ||
(environment && environment !== secretSync.environment?.slug)

2
backend/src/services/secret-sync/secret-sync-types.ts
View File

@@ -54,7 +54,7 @@ export type TCreateSecretSyncDTO = Pick<TSecretSync, "syncOptions" | "destinatio
isEnabled?: boolean;
};
export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "connectionId" | "projectId">> & {
export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "projectId">> & {
syncId: string;
destination: SecretSync;
};