feature: aws secrets manager sync

This commit is contained in:
Scott Wilson
2025-02-04 19:58:30 -08:00
parent 4f874734ab
commit 3f02481e78
54 changed files with 1051 additions and 40 deletions
+6
View File
@@ -1719,6 +1719,12 @@ export const SecretSyncs = {
REGION: "The AWS region to sync secrets to.",
PATH: "The Parameter Store path to sync secrets to."
},
AWS_SECRETS_MANAGER: {
REGION: "The AWS region to sync secrets to.",
MAPPING_BEHAVIOR:
"How secrets from Infisical should be mapped to AWS Secrets Manager; one-to-one or many-to-one.",
SECRET_NAME: "The secret name in AWS Secrets Manager to sync to when using mapping behavior many-to-one."
},
GITHUB: {
ORG: "The name of the GitHub organization.",
OWNER: "The name of the GitHub account owner of the repository.",
@@ -0,0 +1,17 @@
import {
AwsSecretsManagerSyncSchema,
CreateAwsSecretsManagerSyncSchema,
UpdateAwsSecretsManagerSyncSchema
} from "@app/services/secret-sync/aws-secrets-manager";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints";
export const registerAwsSecretsManagerSyncRouter = async (server: FastifyZodProvider) =>
registerSyncSecretsEndpoints({
destination: SecretSync.AWSSecretsManager,
server,
responseSchema: AwsSecretsManagerSyncSchema,
createSchema: CreateAwsSecretsManagerSyncSchema,
updateSchema: UpdateAwsSecretsManagerSyncSchema
});
@@ -1,6 +1,7 @@
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { registerAwsParameterStoreSyncRouter } from "./aws-parameter-store-sync-router";
import { registerAwsSecretsManagerSyncRouter } from "./aws-secrets-manager-sync-router";
import { registerGcpSyncRouter } from "./gcp-sync-router";
import { registerGitHubSyncRouter } from "./github-sync-router";
@@ -8,6 +9,7 @@ export * from "./secret-sync-router";
export const SECRET_SYNC_REGISTER_ROUTER_MAP: Record<SecretSync, (server: FastifyZodProvider) => Promise<void>> = {
[SecretSync.AWSParameterStore]: registerAwsParameterStoreSyncRouter,
[SecretSync.AWSSecretsManager]: registerAwsSecretsManagerSyncRouter,
[SecretSync.GitHub]: registerGitHubSyncRouter,
[SecretSync.GCPSecretManager]: registerGcpSyncRouter
};
@@ -9,17 +9,23 @@ import {
AwsParameterStoreSyncListItemSchema,
AwsParameterStoreSyncSchema
} from "@app/services/secret-sync/aws-parameter-store";
import {
AwsSecretsManagerSyncListItemSchema,
AwsSecretsManagerSyncSchema
} from "@app/services/secret-sync/aws-secrets-manager";
import { GcpSyncListItemSchema, GcpSyncSchema } from "@app/services/secret-sync/gcp";
import { GitHubSyncListItemSchema, GitHubSyncSchema } from "@app/services/secret-sync/github";
const SecretSyncSchema = z.discriminatedUnion("destination", [
AwsParameterStoreSyncSchema,
AwsSecretsManagerSyncSchema,
GitHubSyncSchema,
GcpSyncSchema
]);
const SecretSyncOptionsSchema = z.discriminatedUnion("destination", [
AwsParameterStoreSyncListItemSchema,
AwsSecretsManagerSyncListItemSchema,
GitHubSyncListItemSchema,
GcpSyncListItemSchema
]);
@@ -69,6 +69,8 @@ const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise<TAWSPara
attempt += 1;
// eslint-disable-next-line no-await-in-loop
await sleep();
// eslint-disable-next-line no-continue
continue;
}
throw e;
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const AWS_SECRETS_MANAGER_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "AWS Secrets Manager",
destination: SecretSync.AWSSecretsManager,
connection: AppConnection.AWS,
canImportSecrets: true
};
@@ -0,0 +1,4 @@
export enum AwsSecretsManagerSyncMappingBehavior {
OneToOne = "one-to-one",
ManyToOne = "many-to-one"
}
@@ -0,0 +1,346 @@
import {
BatchGetSecretValueCommand,
CreateSecretCommand,
CreateSecretCommandInput,
DeleteSecretCommand,
DeleteSecretResponse,
ListSecretsCommand,
SecretsManagerClient,
UpdateSecretCommand,
UpdateSecretCommandInput
} from "@aws-sdk/client-secrets-manager";
import { AWSError } from "aws-sdk";
import { CreateSecretResponse, SecretListEntry, SecretValueEntry } from "aws-sdk/clients/secretsmanager";
import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
import { AwsSecretsManagerSyncMappingBehavior } from "@app/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAwsSecretsManagerSyncWithCredentials } from "./aws-secrets-manager-sync-types";
type TAwsSecretsRecord = Record<string, SecretListEntry>;
type TAwsSecretValuesRecord = Record<string, SecretValueEntry>;
const MAX_RETRIES = 5;
const BATCH_SIZE = 20;
const getSecretsManagerClient = async (secretSync: TAwsSecretsManagerSyncWithCredentials) => {
const { destinationConfig, connection } = secretSync;
const config = await getAwsConnectionConfig(connection, destinationConfig.region);
const secretsManagerClient = new SecretsManagerClient({
region: config.region,
credentials: config.credentials!
});
return secretsManagerClient;
};
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 1000);
});
const getSecretsRecord = async (client: SecretsManagerClient): Promise<TAwsSecretsRecord> => {
const awsSecretsRecord: TAwsSecretsRecord = {};
let hasNext = true;
let nextToken: string | undefined;
let attempt = 0;
while (hasNext) {
try {
// eslint-disable-next-line no-await-in-loop
const output = await client.send(new ListSecretsCommand({ NextToken: nextToken }));
attempt = 0;
if (output.SecretList) {
output.SecretList.forEach((secretEntry) => {
if (secretEntry.Name) {
awsSecretsRecord[secretEntry.Name] = secretEntry;
}
});
}
hasNext = Boolean(output.NextToken);
nextToken = output.NextToken;
} catch (e) {
if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
attempt += 1;
// eslint-disable-next-line no-await-in-loop
await sleep();
// eslint-disable-next-line no-continue
continue;
}
throw e;
}
}
return awsSecretsRecord;
};
const getSecretValuesRecord = async (
client: SecretsManagerClient,
awsSecretsRecord: TAwsSecretsRecord
): Promise<TAwsSecretValuesRecord> => {
const awsSecretValuesRecord: TAwsSecretValuesRecord = {};
let attempt = 0;
const secretIdList = Object.keys(awsSecretsRecord);
for (let i = 0; i < secretIdList.length; i += BATCH_SIZE) {
const batchSecretIds = secretIdList.slice(i, i + BATCH_SIZE);
let hasNext = true;
let nextToken: string | undefined;
while (hasNext) {
try {
// eslint-disable-next-line no-await-in-loop
const output = await client.send(
new BatchGetSecretValueCommand({
SecretIdList: batchSecretIds,
NextToken: nextToken
})
);
attempt = 0;
if (output.SecretValues) {
output.SecretValues.forEach((secretValueEntry) => {
if (secretValueEntry.Name) {
awsSecretValuesRecord[secretValueEntry.Name] = secretValueEntry;
}
});
}
hasNext = Boolean(output.NextToken);
nextToken = output.NextToken;
} catch (e) {
if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
attempt += 1;
// eslint-disable-next-line no-await-in-loop
await sleep();
// eslint-disable-next-line no-continue
continue;
}
throw e;
}
}
}
return awsSecretValuesRecord;
};
const createSecret = async (
client: SecretsManagerClient,
input: CreateSecretCommandInput,
attempt = 0
): Promise<CreateSecretResponse> => {
try {
return await client.send(new CreateSecretCommand(input));
} catch (error) {
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
await sleep();
// retry
return createSecret(client, input, attempt + 1);
}
throw error;
}
};
const updateSecret = async (
client: SecretsManagerClient,
input: UpdateSecretCommandInput,
attempt = 0
): Promise<CreateSecretResponse> => {
try {
return await client.send(new UpdateSecretCommand(input));
} catch (error) {
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
await sleep();
// retry
return updateSecret(client, input, attempt + 1);
}
throw error;
}
};
const deleteSecret = async (
client: SecretsManagerClient,
secretKey: string,
attempt = 0
): Promise<DeleteSecretResponse> => {
try {
return await client.send(new DeleteSecretCommand({ SecretId: secretKey, ForceDeleteWithoutRecovery: true }));
} catch (error) {
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
await sleep();
// retry
return deleteSecret(client, secretKey, attempt + 1);
}
throw error;
}
};
export const AwsSecretsManagerSyncFns = {
syncSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const client = await getSecretsManagerClient(secretSync);
const awsSecretsRecord = await getSecretsRecord(client);
const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord);
if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
for await (const entry of Object.entries(secretMap)) {
const [key, { value }] = entry;
if (awsSecretsRecord[key]) {
// skip secrets that haven't changed or don't have a value set
if (!value || awsValuesRecord[key]?.SecretString === value) {
// eslint-disable-next-line no-continue
continue;
}
try {
await updateSecret(client, {
SecretId: key,
SecretString: value
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
} else {
try {
await createSecret(client, {
Name: key,
SecretString: value
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
}
}
for await (const secretKey of Object.keys(awsSecretsRecord)) {
if (!(secretKey in secretMap) || !secretMap[secretKey].value) {
try {
await deleteSecret(client, secretKey);
} catch (error) {
throw new SecretSyncError({
error,
secretKey
});
}
}
}
} else {
// Many-To-One Mapping
const secretValue = JSON.stringify(
Object.fromEntries(Object.entries(secretMap).map(([key, secretData]) => [key, secretData.value]))
);
if (awsValuesRecord[destinationConfig.secretName]) {
await updateSecret(client, {
SecretId: destinationConfig.secretName,
SecretString: secretValue
});
} else {
await createSecret(client, {
Name: destinationConfig.secretName,
SecretString: secretValue
});
}
for await (const secretKey of Object.keys(awsSecretsRecord)) {
if (secretKey === destinationConfig.secretName) {
// eslint-disable-next-line no-continue
continue;
}
try {
await deleteSecret(client, secretKey);
} catch (error) {
throw new SecretSyncError({
error,
secretKey
});
}
}
}
},
getSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials): Promise<TSecretMap> => {
const client = await getSecretsManagerClient(secretSync);
const awsSecretsRecord = await getSecretsRecord(client);
const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord);
const { destinationConfig } = secretSync;
if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
return Object.fromEntries(
Object.keys(awsSecretsRecord).map((key) => [key, { value: awsValuesRecord[key].SecretString ?? "" }])
);
}
// Many-To-One Mapping
const secretValueEntry = awsValuesRecord[destinationConfig.secretName];
if (!secretValueEntry) return {};
try {
const parsedValue = (secretValueEntry.SecretString ? JSON.parse(secretValueEntry.SecretString) : {}) as Record<
string,
string
>;
return Object.fromEntries(Object.entries(parsedValue).map(([key, value]) => [key, { value }]));
} catch {
throw new SecretSyncError({
message:
"Failed to import secrets. Invalid format for Many-To-One mapping behavior: requires key/value configuration.",
shouldRetry: false
});
}
},
removeSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const client = await getSecretsManagerClient(secretSync);
const awsSecretsRecord = await getSecretsRecord(client);
if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) {
for await (const secretKey of Object.keys(awsSecretsRecord)) {
if (secretKey in secretMap) {
try {
await deleteSecret(client, secretKey);
} catch (error) {
throw new SecretSyncError({
error,
secretKey
});
}
}
}
} else {
await deleteSecret(client, destinationConfig.secretName);
}
}
};
@@ -0,0 +1,63 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
import { AwsSecretsManagerSyncMappingBehavior } from "@app/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
const AwsSecretsManagerSyncDestinationConfigSchema = z
.discriminatedUnion("mappingBehavior", [
z.object({
mappingBehavior: z
.literal(AwsSecretsManagerSyncMappingBehavior.OneToOne)
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.MAPPING_BEHAVIOR)
}),
z.object({
mappingBehavior: z
.literal(AwsSecretsManagerSyncMappingBehavior.ManyToOne)
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.MAPPING_BEHAVIOR),
secretName: z
.string()
.regex(
/^[a-zA-Z0-9/_+=.@-]+$/,
"Secret name must contain only alphanumeric characters and the characters /_+=.@-"
)
.min(1, "Secret name is required")
.max(256, "Secret name cannot exceed 256 characters")
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.SECRET_NAME)
})
])
.and(
z.object({
region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_SECRETS_MANAGER.REGION)
})
);
export const AwsSecretsManagerSyncSchema = BaseSecretSyncSchema(SecretSync.AWSSecretsManager).extend({
destination: z.literal(SecretSync.AWSSecretsManager),
destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema
});
export const CreateAwsSecretsManagerSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.AWSSecretsManager
).extend({
destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema
});
export const UpdateAwsSecretsManagerSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.AWSSecretsManager
).extend({
destinationConfig: AwsSecretsManagerSyncDestinationConfigSchema.optional()
});
export const AwsSecretsManagerSyncListItemSchema = z.object({
name: z.literal("AWS Secrets Manager"),
connection: z.literal(AppConnection.AWS),
destination: z.literal(SecretSync.AWSSecretsManager),
canImportSecrets: z.literal(true)
});
@@ -0,0 +1,19 @@
import { z } from "zod";
import { TAwsConnection } from "@app/services/app-connection/aws";
import {
AwsSecretsManagerSyncListItemSchema,
AwsSecretsManagerSyncSchema,
CreateAwsSecretsManagerSyncSchema
} from "./aws-secrets-manager-sync-schemas";
export type TAwsSecretsManagerSync = z.infer<typeof AwsSecretsManagerSyncSchema>;
export type TAwsSecretsManagerSyncInput = z.infer<typeof CreateAwsSecretsManagerSyncSchema>;
export type TAwsSecretsManagerSyncListItem = z.infer<typeof AwsSecretsManagerSyncListItemSchema>;
export type TAwsSecretsManagerSyncWithCredentials = TAwsSecretsManagerSync & {
connection: TAwsConnection;
};
@@ -0,0 +1,4 @@
export * from "./aws-secrets-manager-sync-constants";
export * from "./aws-secrets-manager-sync-fns";
export * from "./aws-secrets-manager-sync-schemas";
export * from "./aws-secrets-manager-sync-types";
@@ -1,5 +1,6 @@
export enum SecretSync {
AWSParameterStore = "aws-parameter-store",
AWSSecretsManager = "aws-secrets-manager",
GitHub = "github",
GCPSecretManager = "gcp-secret-manager"
}
@@ -4,6 +4,10 @@ import {
AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
AwsParameterStoreSyncFns
} from "@app/services/secret-sync/aws-parameter-store";
import {
AWS_SECRETS_MANAGER_SYNC_LIST_OPTION,
AwsSecretsManagerSyncFns
} from "@app/services/secret-sync/aws-secrets-manager";
import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
@@ -18,6 +22,7 @@ import { GcpSyncFns } from "./gcp/gcp-sync-fns";
const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
[SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
[SecretSync.AWSSecretsManager]: AWS_SECRETS_MANAGER_SYNC_LIST_OPTION,
[SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION,
[SecretSync.GCPSecretManager]: GCP_SYNC_LIST_OPTION
};
@@ -73,6 +78,8 @@ export const SecretSyncFns = {
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.GCPSecretManager:
@@ -89,6 +96,9 @@ export const SecretSyncFns = {
case SecretSync.AWSParameterStore:
secretMap = await AwsParameterStoreSyncFns.getSecrets(secretSync);
break;
case SecretSync.AWSSecretsManager:
secretMap = await AwsSecretsManagerSyncFns.getSecrets(secretSync);
break;
case SecretSync.GitHub:
secretMap = await GithubSyncFns.getSecrets(secretSync);
break;
@@ -110,6 +120,8 @@ export const SecretSyncFns = {
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.GCPSecretManager:
@@ -3,12 +3,14 @@ import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
[SecretSync.AWSParameterStore]: "AWS Parameter Store",
[SecretSync.AWSSecretsManager]: "AWS Secrets Manager",
[SecretSync.GitHub]: "GitHub",
[SecretSync.GCPSecretManager]: "GCP Secret Manager"
};
export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
[SecretSync.AWSParameterStore]: AppConnection.AWS,
[SecretSync.AWSSecretsManager]: AppConnection.AWS,
[SecretSync.GitHub]: AppConnection.GitHub,
[SecretSync.GCPSecretManager]: AppConnection.GCP
};
@@ -2,6 +2,12 @@ import { Job } from "bullmq";
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
import { QueueJobs } from "@app/queue";
import {
TAwsSecretsManagerSync,
TAwsSecretsManagerSyncInput,
TAwsSecretsManagerSyncListItem,
TAwsSecretsManagerSyncWithCredentials
} from "@app/services/secret-sync/aws-secrets-manager";
import {
TGitHubSync,
TGitHubSyncInput,
@@ -19,16 +25,25 @@ import {
} from "./aws-parameter-store";
import { TGcpSync, TGcpSyncInput, TGcpSyncListItem, TGcpSyncWithCredentials } from "./gcp";
export type TSecretSync = TAwsParameterStoreSync | TGitHubSync | TGcpSync;
export type TSecretSync = TAwsParameterStoreSync | TAwsSecretsManagerSync | TGitHubSync | TGcpSync;
export type TSecretSyncWithCredentials =
| TAwsParameterStoreSyncWithCredentials
| TAwsSecretsManagerSyncWithCredentials
| TGitHubSyncWithCredentials
| TGcpSyncWithCredentials;
export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput | TGcpSyncInput;
export type TSecretSyncInput =
| TAwsParameterStoreSyncInput
| TAwsSecretsManagerSyncInput
| TGitHubSyncInput
| TGcpSyncInput;
export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem | TGcpSyncListItem;
export type TSecretSyncListItem =
| TAwsParameterStoreSyncListItem
| TAwsSecretsManagerSyncListItem
| TGitHubSyncListItem
| TGcpSyncListItem;
export type TSyncOptionsConfig = {
canImportSecrets: boolean;