mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-16 01:05:40 +00:00
feature: secret sync base + aws parameter store & github
This commit is contained in:
Scott Wilson
+10
@@ -0,0 +1,10 @@
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
export const AWS_PARAMETER_STORE_SYNC_LIST_OPTION: TSecretSyncListItem = {
|
||||
name: "AWS Parameter Store",
|
||||
destination: SecretSync.AWSParameterStore,
|
||||
connection: AppConnection.AWS,
|
||||
canImportSecrets: true
|
||||
};
|
||||
@@ -0,0 +1,199 @@
|
||||
import AWS, { AWSError } from "aws-sdk";
|
||||
|
||||
import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
|
||||
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
import { TAwsParameterStoreSyncWithCredentials } from "./aws-parameter-store-sync-types";
|
||||
|
||||
type TAWSParameterStoreRecord = Record<string, AWS.SSM.Parameter>;
|
||||
|
||||
const MAX_RETRIES = 5;
|
||||
const BATCH_SIZE = 10;
|
||||
|
||||
const getSSM = async (secretSync: TAwsParameterStoreSyncWithCredentials) => {
|
||||
const { destinationConfig, connection } = secretSync;
|
||||
|
||||
const config = await getAwsConnectionConfig(connection, destinationConfig.region);
|
||||
|
||||
const ssm = new AWS.SSM({
|
||||
apiVersion: "2014-11-06",
|
||||
region: destinationConfig.region
|
||||
});
|
||||
|
||||
ssm.config.update(config);
|
||||
|
||||
return ssm;
|
||||
};
|
||||
|
||||
const sleep = async () =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, 1000);
|
||||
});
|
||||
|
||||
const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise<TAWSParameterStoreRecord> => {
|
||||
const awsParameterStoreSecretsRecord: TAWSParameterStoreRecord = {};
|
||||
let hasNext = true;
|
||||
let nextToken: string | undefined;
|
||||
let attempt = 0;
|
||||
|
||||
while (hasNext) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const parameters = await ssm
|
||||
.getParametersByPath({
|
||||
Path: path,
|
||||
Recursive: false,
|
||||
WithDecryption: true,
|
||||
MaxResults: BATCH_SIZE,
|
||||
NextToken: nextToken
|
||||
})
|
||||
.promise();
|
||||
|
||||
attempt = 0;
|
||||
|
||||
if (parameters.Parameters) {
|
||||
parameters.Parameters.forEach((parameter) => {
|
||||
if (parameter.Name) {
|
||||
// no leading slash if path is '/'
|
||||
const secKey = path.length > 1 ? parameter.Name.substring(path.length) : parameter.Name;
|
||||
awsParameterStoreSecretsRecord[secKey] = parameter;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
hasNext = Boolean(parameters.NextToken);
|
||||
nextToken = parameters.NextToken;
|
||||
} catch (e) {
|
||||
if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
|
||||
attempt += 1;
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await sleep();
|
||||
}
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
return awsParameterStoreSecretsRecord;
|
||||
};
|
||||
|
||||
const putParameter = async (
|
||||
ssm: AWS.SSM,
|
||||
params: AWS.SSM.PutParameterRequest,
|
||||
attempt = 0
|
||||
): Promise<AWS.SSM.PutParameterResult> => {
|
||||
try {
|
||||
return await ssm.putParameter(params).promise();
|
||||
} catch (error) {
|
||||
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
|
||||
await sleep();
|
||||
|
||||
// retry
|
||||
return putParameter(ssm, params, attempt + 1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const deleteParametersBatch = async (
|
||||
ssm: AWS.SSM,
|
||||
parameters: AWS.SSM.Parameter[],
|
||||
attempt = 0
|
||||
): Promise<AWS.SSM.DeleteParameterResult[]> => {
|
||||
const results: AWS.SSM.DeleteParameterResult[] = [];
|
||||
let remainingParams = [...parameters];
|
||||
|
||||
while (remainingParams.length > 0) {
|
||||
const batch = remainingParams.slice(0, BATCH_SIZE);
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const result = await ssm.deleteParameters({ Names: batch.map((param) => param.Name!) }).promise();
|
||||
results.push(result);
|
||||
remainingParams = remainingParams.slice(BATCH_SIZE);
|
||||
} catch (error) {
|
||||
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await sleep();
|
||||
|
||||
// Retry the current batch
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
return [...results, ...(await deleteParametersBatch(ssm, remainingParams, attempt + 1))];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
|
||||
export const AwsParameterStoreSyncFns = {
|
||||
syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
const ssm = await getSSM(secretSync);
|
||||
|
||||
// TODO(scott): KMS Key ID, Tags
|
||||
|
||||
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
|
||||
|
||||
for await (const entry of Object.entries(affixedSecretMap)) {
|
||||
const [key, { value }] = entry;
|
||||
|
||||
// skip empty values (not allowed by AWS) or secrets that haven't changed
|
||||
if (!value || (key in awsParameterStoreSecretsRecord && awsParameterStoreSecretsRecord[key].Value === value)) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
await putParameter(ssm, {
|
||||
Name: `${destinationConfig.path}${key}`,
|
||||
Type: "SecureString",
|
||||
Value: value,
|
||||
Overwrite: true
|
||||
});
|
||||
}
|
||||
|
||||
const parametersToDelete: AWS.SSM.Parameter[] = [];
|
||||
|
||||
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
|
||||
const [key, parameter] = entry;
|
||||
|
||||
if (!(key in affixedSecretMap) || !affixedSecretMap[key].value) {
|
||||
parametersToDelete.push(parameter);
|
||||
}
|
||||
}
|
||||
|
||||
await deleteParametersBatch(ssm, parametersToDelete);
|
||||
},
|
||||
importSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
const ssm = await getSSM(secretSync);
|
||||
|
||||
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(awsParameterStoreSecretsRecord).map(([key, value]) => [key, { value: value.Value ?? "" }])
|
||||
);
|
||||
},
|
||||
removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
const ssm = await getSSM(secretSync);
|
||||
|
||||
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
|
||||
|
||||
const parametersToDelete: AWS.SSM.Parameter[] = [];
|
||||
|
||||
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
|
||||
const [key, param] = entry;
|
||||
|
||||
if (key in affixedSecretMap) {
|
||||
parametersToDelete.push(param);
|
||||
}
|
||||
}
|
||||
|
||||
await deleteParametersBatch(ssm, parametersToDelete);
|
||||
}
|
||||
};
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretSyncs } from "@app/lib/api-docs";
|
||||
import { wrapWithSlashes } from "@app/lib/fn";
|
||||
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import {
|
||||
BaseSecretSyncSchema,
|
||||
GenericCreateSecretSyncFieldsSchema,
|
||||
GenericUpdateSecretSyncFieldsSchema
|
||||
} from "@app/services/secret-sync/secret-sync-schemas";
|
||||
|
||||
const AwsParameterStoreSyncDestinationConfigSchema = z.object({
|
||||
region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.REGION),
|
||||
path: z
|
||||
.string()
|
||||
.min(1, "Parameter Store Path Required")
|
||||
.transform(wrapWithSlashes)
|
||||
.superRefine((val, ctx) => {
|
||||
if (!/^\/([/]|(([\w-]+\/)+))?$/.test(val)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: `Invalid Parameter Store Path - must follow "/example/path/" format`
|
||||
});
|
||||
}
|
||||
|
||||
if (val.length > 2048) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: `Invalid Parameter Store Path - cannot exceed 2048 characters`
|
||||
});
|
||||
}
|
||||
})
|
||||
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.PATH)
|
||||
});
|
||||
|
||||
export const AwsParameterStoreSyncSchema = BaseSecretSyncSchema(SecretSync.AWSParameterStore).extend({
|
||||
destination: z.literal(SecretSync.AWSParameterStore),
|
||||
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema
|
||||
});
|
||||
|
||||
export const CreateAwsParameterStoreSyncSchema = GenericCreateSecretSyncFieldsSchema(
|
||||
SecretSync.AWSParameterStore
|
||||
).extend({
|
||||
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema
|
||||
});
|
||||
|
||||
export const UpdateAwsParameterStoreSyncSchema = GenericUpdateSecretSyncFieldsSchema(
|
||||
SecretSync.AWSParameterStore
|
||||
).extend({
|
||||
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema.optional()
|
||||
});
|
||||
|
||||
export const AwsParameterStoreSyncListItemSchema = z.object({
|
||||
name: z.literal("AWS Parameter Store"),
|
||||
connection: z.literal(AppConnection.AWS),
|
||||
destination: z.literal(SecretSync.AWSParameterStore),
|
||||
canImportSecrets: z.literal(true)
|
||||
});
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { TAwsConnection } from "@app/services/app-connection/aws";
|
||||
|
||||
import {
|
||||
AwsParameterStoreSyncListItemSchema,
|
||||
AwsParameterStoreSyncSchema,
|
||||
CreateAwsParameterStoreSyncSchema
|
||||
} from "./aws-parameter-store-sync-schemas";
|
||||
|
||||
export type TAwsParameterStoreSync = z.infer<typeof AwsParameterStoreSyncSchema>;
|
||||
|
||||
export type TAwsParameterStoreSyncInput = z.infer<typeof CreateAwsParameterStoreSyncSchema>;
|
||||
|
||||
export type TAwsParameterStoreSyncListItem = z.infer<typeof AwsParameterStoreSyncListItemSchema>;
|
||||
|
||||
export type TAwsParameterStoreSyncWithCredentials = TAwsParameterStoreSync & {
|
||||
connection: TAwsConnection;
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./aws-parameter-store-sync-constants";
|
||||
export * from "./aws-parameter-store-sync-fns";
|
||||
export * from "./aws-parameter-store-sync-schemas";
|
||||
export * from "./aws-parameter-store-sync-types";
|
||||
@@ -0,0 +1,10 @@
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
export const GITHUB_SYNC_LIST_OPTION: TSecretSyncListItem = {
|
||||
name: "GitHub",
|
||||
destination: SecretSync.GitHub,
|
||||
connection: AppConnection.GitHub,
|
||||
canImportSecrets: false
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
export enum GitHubSyncScope {
|
||||
Repository = "repository",
|
||||
Organization = "organization",
|
||||
RepositoryEnvironment = "repository-environment"
|
||||
}
|
||||
|
||||
export enum GitHubSyncVisibility {
|
||||
All = "all",
|
||||
Private = "private",
|
||||
Selected = "selected"
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { Octokit } from "@octokit/rest";
|
||||
import sodium from "libsodium-wrappers";
|
||||
|
||||
import { getGitHubClient } from "@app/services/app-connection/github";
|
||||
import { GitHubSyncScope } from "@app/services/secret-sync/github/github-sync-enums";
|
||||
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
|
||||
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
import { TGitHubSyncWithCredentials } from "./github-sync-types";
|
||||
|
||||
interface GitHubSecret {
|
||||
name: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
visibility?: "all" | "private" | "selected";
|
||||
selected_repositories_url?: string | undefined;
|
||||
}
|
||||
|
||||
// TODO: rate limit handling
|
||||
|
||||
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
|
||||
let encryptedSecrets: GitHubSecret[];
|
||||
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
switch (destinationConfig.scope) {
|
||||
case GitHubSyncScope.Organization: {
|
||||
encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
|
||||
org: destinationConfig.org
|
||||
});
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.Repository: {
|
||||
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.RepositoryEnvironment:
|
||||
default: {
|
||||
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
environment_name: destinationConfig.env
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return encryptedSecrets;
|
||||
};
|
||||
|
||||
interface GitHubPublicKey {
|
||||
key_id: string;
|
||||
key: string;
|
||||
id?: number | undefined;
|
||||
url?: string | undefined;
|
||||
title?: string | undefined;
|
||||
created_at?: string | undefined;
|
||||
}
|
||||
|
||||
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
|
||||
let publicKey: GitHubPublicKey;
|
||||
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
switch (destinationConfig.scope) {
|
||||
case GitHubSyncScope.Organization: {
|
||||
publicKey = (
|
||||
await client.request("GET /orgs/{org}/actions/secrets/public-key", {
|
||||
org: destinationConfig.org
|
||||
})
|
||||
).data;
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.Repository: {
|
||||
publicKey = (
|
||||
await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo
|
||||
})
|
||||
).data;
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.RepositoryEnvironment:
|
||||
default: {
|
||||
publicKey = (
|
||||
await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
environment_name: destinationConfig.env
|
||||
})
|
||||
).data;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return publicKey;
|
||||
};
|
||||
|
||||
const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, encryptedSecret: GitHubSecret) => {
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
switch (destinationConfig.scope) {
|
||||
case GitHubSyncScope.Organization: {
|
||||
await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
|
||||
org: destinationConfig.org,
|
||||
secret_name: encryptedSecret.name
|
||||
});
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.Repository: {
|
||||
await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
secret_name: encryptedSecret.name
|
||||
});
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.RepositoryEnvironment:
|
||||
default: {
|
||||
await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
environment_name: destinationConfig.env,
|
||||
secret_name: encryptedSecret.name
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
interface GitHubSecretPayload {
|
||||
key_id: string;
|
||||
secret_name: string;
|
||||
encrypted_value: string;
|
||||
}
|
||||
|
||||
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: GitHubSecretPayload) => {
|
||||
const { destinationConfig } = secretSync;
|
||||
|
||||
switch (destinationConfig.scope) {
|
||||
case GitHubSyncScope.Organization: {
|
||||
const { visibility, selectedRepositoryIds } = destinationConfig;
|
||||
|
||||
await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
|
||||
org: destinationConfig.org,
|
||||
...payload,
|
||||
visibility,
|
||||
selected_repository_ids: selectedRepositoryIds
|
||||
});
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.Repository: {
|
||||
await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
...payload
|
||||
});
|
||||
break;
|
||||
}
|
||||
case GitHubSyncScope.RepositoryEnvironment:
|
||||
default: {
|
||||
await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
|
||||
owner: destinationConfig.owner,
|
||||
repo: destinationConfig.repo,
|
||||
environment_name: destinationConfig.env,
|
||||
...payload
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const GithubSyncFns = {
|
||||
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
|
||||
const client = getGitHubClient(secretSync.connection);
|
||||
|
||||
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
|
||||
|
||||
const publicKey = await getPublicKey(client, secretSync);
|
||||
|
||||
for await (const encryptedSecret of encryptedSecrets) {
|
||||
if (!(encryptedSecret.name in affixedSecretMap)) {
|
||||
await deleteSecret(client, secretSync, encryptedSecret);
|
||||
}
|
||||
}
|
||||
|
||||
await sodium.ready.then(async () => {
|
||||
for await (const key of Object.keys(affixedSecretMap)) {
|
||||
// convert secret & base64 key to Uint8Array.
|
||||
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
|
||||
const binarySecretValue = sodium.from_string(affixedSecretMap[key].value);
|
||||
|
||||
// encrypt secret using libsodium
|
||||
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
|
||||
|
||||
// convert encrypted Uint8Array to base64
|
||||
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
|
||||
|
||||
await putSecret(client, secretSync, {
|
||||
secret_name: key,
|
||||
encrypted_value: encryptedSecretValue,
|
||||
key_id: publicKey.key_id
|
||||
});
|
||||
}
|
||||
});
|
||||
},
|
||||
importSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
|
||||
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
|
||||
},
|
||||
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
|
||||
const client = getGitHubClient(secretSync.connection);
|
||||
|
||||
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
|
||||
|
||||
for await (const encryptedSecret of encryptedSecrets) {
|
||||
if (encryptedSecret.name in affixedSecretMap) {
|
||||
await deleteSecret(client, secretSync, encryptedSecret);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,72 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretSyncs } from "@app/lib/api-docs";
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import {
|
||||
BaseSecretSyncSchema,
|
||||
GenericCreateSecretSyncFieldsSchema,
|
||||
GenericUpdateSecretSyncFieldsSchema
|
||||
} from "@app/services/secret-sync/secret-sync-schemas";
|
||||
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
const GitHubSyncDestinationConfigSchema = z
|
||||
.discriminatedUnion("scope", [
|
||||
z.object({
|
||||
scope: z.literal(GitHubSyncScope.Organization),
|
||||
org: z.string().min(1, "Organization name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ORG),
|
||||
visibility: z.nativeEnum(GitHubSyncVisibility),
|
||||
selectedRepositoryIds: z.number().array().optional()
|
||||
}),
|
||||
z.object({
|
||||
scope: z.literal(GitHubSyncScope.Repository),
|
||||
owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER),
|
||||
repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO)
|
||||
}),
|
||||
z.object({
|
||||
scope: z.literal(GitHubSyncScope.RepositoryEnvironment),
|
||||
owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER),
|
||||
repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO),
|
||||
env: z.string().min(1, "Environment name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ENV)
|
||||
})
|
||||
])
|
||||
.superRefine((options, ctx) => {
|
||||
if (options.scope !== GitHubSyncScope.Organization) return;
|
||||
|
||||
if (options.visibility === GitHubSyncVisibility.Selected && !options.selectedRepositoryIds?.length) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Select at least 1 repository",
|
||||
path: ["selectedRepositoryIds"]
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const GitHubSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };
|
||||
|
||||
export const GitHubSyncSchema = BaseSecretSyncSchema(SecretSync.GitHub, GitHubSyncOptionsConfig).extend({
|
||||
destination: z.literal(SecretSync.GitHub),
|
||||
destinationConfig: GitHubSyncDestinationConfigSchema
|
||||
});
|
||||
|
||||
export const CreateGitHubSyncSchema = GenericCreateSecretSyncFieldsSchema(
|
||||
SecretSync.GitHub,
|
||||
GitHubSyncOptionsConfig
|
||||
).extend({
|
||||
destinationConfig: GitHubSyncDestinationConfigSchema
|
||||
});
|
||||
|
||||
export const UpdateGitHubSyncSchema = GenericUpdateSecretSyncFieldsSchema(
|
||||
SecretSync.GitHub,
|
||||
GitHubSyncOptionsConfig
|
||||
).extend({
|
||||
destinationConfig: GitHubSyncDestinationConfigSchema.optional()
|
||||
});
|
||||
|
||||
export const GitHubSyncListItemSchema = z.object({
|
||||
name: z.literal("GitHub"),
|
||||
connection: z.literal(AppConnection.GitHub),
|
||||
destination: z.literal(SecretSync.GitHub),
|
||||
canImportSecrets: z.literal(false)
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { TGitHubConnection } from "@app/services/app-connection/github";
|
||||
|
||||
import { CreateGitHubSyncSchema, GitHubSyncListItemSchema, GitHubSyncSchema } from "./github-sync-schemas";
|
||||
|
||||
export type TGitHubSync = z.infer<typeof GitHubSyncSchema>;
|
||||
|
||||
export type TGitHubSyncInput = z.infer<typeof CreateGitHubSyncSchema>;
|
||||
|
||||
export type TGitHubSyncListItem = z.infer<typeof GitHubSyncListItemSchema>;
|
||||
|
||||
export type TGitHubSyncWithCredentials = TGitHubSync & {
|
||||
connection: TGitHubConnection;
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./github-sync-constants";
|
||||
export * from "./github-sync-fns";
|
||||
export * from "./github-sync-schemas";
|
||||
export * from "./github-sync-types";
|
||||
@@ -0,0 +1,201 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName } from "@app/db/schemas";
|
||||
import { TSecretSyncs } from "@app/db/schemas/secret-syncs";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
|
||||
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
|
||||
|
||||
export type TSecretSyncDALFactory = ReturnType<typeof secretSyncDALFactory>;
|
||||
|
||||
type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];
|
||||
|
||||
const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {
|
||||
const query = (tx || db.replicaNode())(TableName.SecretSync)
|
||||
.join(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
|
||||
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
|
||||
.join(TableName.AppConnection, `${TableName.SecretSync}.connectionId`, `${TableName.AppConnection}.id`)
|
||||
.select(selectAllTableCols(TableName.SecretSync))
|
||||
.select(
|
||||
// evironment
|
||||
db.ref("name").withSchema(TableName.Environment).as("envName"),
|
||||
db.ref("id").withSchema(TableName.Environment).as("envId"),
|
||||
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
|
||||
db.ref("projectId").withSchema(TableName.Environment),
|
||||
// entire connection
|
||||
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
|
||||
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
|
||||
db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"),
|
||||
db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"),
|
||||
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
|
||||
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
|
||||
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
|
||||
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
|
||||
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt")
|
||||
);
|
||||
|
||||
// prepends table name to filter keys to avoid ambiguous col references, skipping utility filters like $in, etc.
|
||||
const prependTableName = (filterObj: object): SecretSyncFindFilter =>
|
||||
Object.fromEntries(
|
||||
Object.entries(filterObj).map(([key, value]) =>
|
||||
key.startsWith("$") ? [key, prependTableName(value as object)] : [`${TableName.SecretSync}.${key}`, value]
|
||||
)
|
||||
);
|
||||
|
||||
if (filter) {
|
||||
/* eslint-disable @typescript-eslint/no-misused-promises */
|
||||
void query.where(buildFindFilter(prependTableName(filter)));
|
||||
}
|
||||
|
||||
return query;
|
||||
};
|
||||
|
||||
const expandSecretSync = (
|
||||
secretSync: Awaited<ReturnType<typeof baseSecretSyncQuery>>[number],
|
||||
folder: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
|
||||
) => {
|
||||
const {
|
||||
envId,
|
||||
envName,
|
||||
envSlug,
|
||||
connectionApp,
|
||||
connectionName,
|
||||
connectionId,
|
||||
connectionOrgId,
|
||||
connectionEncryptedCredentials,
|
||||
connectionMethod,
|
||||
connectionDescription,
|
||||
connectionCreatedAt,
|
||||
connectionUpdatedAt,
|
||||
connectionVersion,
|
||||
...el
|
||||
} = secretSync;
|
||||
|
||||
return {
|
||||
...el,
|
||||
connectionId,
|
||||
environment: { id: envId, name: envName, slug: envSlug },
|
||||
connection: {
|
||||
app: connectionApp,
|
||||
id: connectionId,
|
||||
name: connectionName,
|
||||
orgId: connectionOrgId,
|
||||
encryptedCredentials: connectionEncryptedCredentials,
|
||||
method: connectionMethod,
|
||||
description: connectionDescription,
|
||||
createdAt: connectionCreatedAt,
|
||||
updatedAt: connectionUpdatedAt,
|
||||
version: connectionVersion
|
||||
},
|
||||
folder: {
|
||||
id: folder!.id,
|
||||
path: folder!.path
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
export const secretSyncDALFactory = (
|
||||
db: TDbClient,
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">
|
||||
) => {
|
||||
const secretSyncOrm = ormify(db, TableName.SecretSync);
|
||||
|
||||
const findById = async (id: string, tx?: Knex) => {
|
||||
try {
|
||||
const secretSync = await baseSecretSyncQuery({
|
||||
filter: { id },
|
||||
db,
|
||||
tx
|
||||
}).first();
|
||||
|
||||
if (secretSync) {
|
||||
// TODO (scott): replace with cached folder path once implemented
|
||||
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
|
||||
return expandSecretSync(secretSync, folderWithPath);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find by ID - Secret Sync" });
|
||||
}
|
||||
};
|
||||
|
||||
const create = async (data: Parameters<(typeof secretSyncOrm)["create"]>[0]) => {
|
||||
try {
|
||||
const secretSync = (await secretSyncOrm.transaction(async (tx) => {
|
||||
const sync = await secretSyncOrm.create(data, tx);
|
||||
|
||||
return baseSecretSyncQuery({
|
||||
filter: { id: sync.id },
|
||||
db,
|
||||
tx
|
||||
}).first();
|
||||
}))!;
|
||||
|
||||
// TODO (scott): replace with cached folder path once implemented
|
||||
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
|
||||
return expandSecretSync(secretSync, folderWithPath);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Create - Secret Sync" });
|
||||
}
|
||||
};
|
||||
|
||||
const updateById = async (syncId: string, data: Parameters<(typeof secretSyncOrm)["updateById"]>[1]) => {
|
||||
try {
|
||||
const secretSync = (await secretSyncOrm.transaction(async (tx) => {
|
||||
const sync = await secretSyncOrm.updateById(syncId, data, tx);
|
||||
|
||||
return baseSecretSyncQuery({
|
||||
filter: { id: sync.id },
|
||||
db,
|
||||
tx
|
||||
}).first();
|
||||
}))!;
|
||||
|
||||
// TODO (scott): replace with cached folder path once implemented
|
||||
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
|
||||
return expandSecretSync(secretSync, folderWithPath);
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Update by ID - Secret Sync" });
|
||||
}
|
||||
};
|
||||
|
||||
const findOne = async (filter: Parameters<(typeof secretSyncOrm)["findOne"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const secretSync = await baseSecretSyncQuery({ filter, db, tx }).first();
|
||||
|
||||
if (secretSync) {
|
||||
// TODO (scott): replace with cached folder path once implemented
|
||||
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
|
||||
return expandSecretSync(secretSync, folderWithPath);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find One - Secret Sync" });
|
||||
}
|
||||
};
|
||||
|
||||
const find = async (filter: Parameters<(typeof secretSyncOrm)["find"]>[0], tx?: Knex) => {
|
||||
try {
|
||||
const secretSyncs = await baseSecretSyncQuery({ filter, db, tx });
|
||||
|
||||
if (!secretSyncs.length) return [];
|
||||
|
||||
const foldersWithPath = await folderDAL.findSecretPathByFolderIds(
|
||||
secretSyncs[0].projectId,
|
||||
secretSyncs.map((sync) => sync.folderId)
|
||||
);
|
||||
|
||||
// TODO (scott): replace with cached folder path once implemented
|
||||
const folderRecord: Record<string, (typeof foldersWithPath)[number]> = {};
|
||||
|
||||
foldersWithPath.forEach((folder) => {
|
||||
if (folder) folderRecord[folder.id] = folder;
|
||||
});
|
||||
|
||||
return secretSyncs.map((secretSync) => expandSecretSync(secretSync, folderRecord[secretSync.folderId]));
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "Find - Secret Sync" });
|
||||
}
|
||||
};
|
||||
|
||||
return { ...secretSyncOrm, findById, findOne, find, create, updateById };
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
export enum SecretSync {
|
||||
AWSParameterStore = "aws-parameter-store",
|
||||
GitHub = "github"
|
||||
}
|
||||
|
||||
export enum SecretSyncInitialSyncBehavior {
|
||||
OverwriteDestination = "overwrite-destination",
|
||||
ImportPrioritizeSource = "import-prioritize-source",
|
||||
ImportPrioritizeDestination = "import-prioritize-destination"
|
||||
}
|
||||
|
||||
export enum SecretSyncImportBehavior {
|
||||
PrioritizeSource = "prioritize-source",
|
||||
PrioritizeDestination = "prioritize-destination"
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import {
|
||||
AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
|
||||
AwsParameterStoreSyncFns
|
||||
} from "@app/services/secret-sync/aws-parameter-store";
|
||||
import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import {
|
||||
TSecretMap,
|
||||
TSecretSyncListItem,
|
||||
TSecretSyncWithCredentials
|
||||
} from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
|
||||
[SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
|
||||
[SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION
|
||||
};
|
||||
|
||||
export const listSecretSyncOptions = () => {
|
||||
return Object.values(SECRET_SYNC_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
|
||||
};
|
||||
|
||||
const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
|
||||
let secretMap = { ...unprocessedSecretMap };
|
||||
|
||||
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
|
||||
|
||||
if (appendSuffix || prependPrefix) {
|
||||
secretMap = {};
|
||||
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
|
||||
secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value;
|
||||
});
|
||||
}
|
||||
|
||||
return secretMap;
|
||||
};
|
||||
|
||||
const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
|
||||
let secretMap = { ...unprocessedSecretMap };
|
||||
|
||||
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
|
||||
|
||||
if (appendSuffix || prependPrefix) {
|
||||
secretMap = {};
|
||||
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
|
||||
let processedKey = key;
|
||||
|
||||
if (prependPrefix && processedKey.startsWith(prependPrefix)) {
|
||||
processedKey = processedKey.slice(prependPrefix.length);
|
||||
}
|
||||
|
||||
if (appendSuffix && processedKey.endsWith(appendSuffix)) {
|
||||
processedKey = processedKey.slice(0, -appendSuffix.length);
|
||||
}
|
||||
|
||||
secretMap[processedKey] = value;
|
||||
});
|
||||
}
|
||||
|
||||
return secretMap;
|
||||
};
|
||||
|
||||
// TODO(scott): ideally do this in a map to reduce code but requires typescript trickery...
|
||||
|
||||
export const SecretSyncFns = {
|
||||
syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
|
||||
const affixedSecretMap = addAffixes(secretSync, secretMap);
|
||||
|
||||
switch (secretSync.destination) {
|
||||
case SecretSync.AWSParameterStore:
|
||||
return AwsParameterStoreSyncFns.syncSecrets(secretSync, affixedSecretMap);
|
||||
case SecretSync.GitHub:
|
||||
return GithubSyncFns.syncSecrets(secretSync, affixedSecretMap);
|
||||
default:
|
||||
throw new Error(
|
||||
`Unhandled sync destination for push secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
|
||||
);
|
||||
}
|
||||
},
|
||||
importSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
|
||||
let secretMap: TSecretMap;
|
||||
switch (secretSync.destination) {
|
||||
case SecretSync.AWSParameterStore:
|
||||
secretMap = await AwsParameterStoreSyncFns.importSecrets(secretSync);
|
||||
break;
|
||||
case SecretSync.GitHub:
|
||||
secretMap = await GithubSyncFns.importSecrets(secretSync);
|
||||
break;
|
||||
default:
|
||||
throw new Error(
|
||||
`Unhandled sync destination for push secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
|
||||
);
|
||||
}
|
||||
|
||||
return stripAffixes(secretSync, secretMap);
|
||||
},
|
||||
removeSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
|
||||
const affixedSecretMap = addAffixes(secretSync, secretMap);
|
||||
|
||||
switch (secretSync.destination) {
|
||||
case SecretSync.AWSParameterStore:
|
||||
return AwsParameterStoreSyncFns.removeSecrets(secretSync, affixedSecretMap);
|
||||
case SecretSync.GitHub:
|
||||
return GithubSyncFns.removeSecrets(secretSync, affixedSecretMap);
|
||||
default:
|
||||
throw new Error(
|
||||
`Unhandled sync destination for removing secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
|
||||
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
|
||||
|
||||
export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
|
||||
[SecretSync.AWSParameterStore]: "AWS Parameter Store",
|
||||
[SecretSync.GitHub]: "GitHub"
|
||||
};
|
||||
|
||||
export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
|
||||
[SecretSync.AWSParameterStore]: AppConnection.AWS,
|
||||
[SecretSync.GitHub]: AppConnection.GitHub
|
||||
};
|
||||
@@ -0,0 +1,939 @@
|
||||
import opentelemetry from "@opentelemetry/api";
|
||||
import { AxiosError } from "axios";
|
||||
import { Job } from "bullmq";
|
||||
|
||||
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
|
||||
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
|
||||
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { decryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns";
|
||||
import { ActorType } from "@app/services/auth/auth-type";
|
||||
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
|
||||
import { KmsDataKey } from "@app/services/kms/kms-types";
|
||||
import { TProjectDALFactory } from "@app/services/project/project-dal";
|
||||
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
|
||||
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
|
||||
import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
|
||||
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
|
||||
import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
|
||||
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
|
||||
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
|
||||
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
|
||||
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
|
||||
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
|
||||
import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
|
||||
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
|
||||
import {
|
||||
SecretSync,
|
||||
SecretSyncImportBehavior,
|
||||
SecretSyncInitialSyncBehavior
|
||||
} from "@app/services/secret-sync/secret-sync-enums";
|
||||
import { SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
|
||||
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
|
||||
import {
|
||||
SecretSyncAction,
|
||||
SecretSyncStatus,
|
||||
TQueueSecretSyncImportSecretsByIdDTO,
|
||||
TQueueSecretSyncRemoveSecretsByIdDTO,
|
||||
TQueueSecretSyncsByPathDTO,
|
||||
TQueueSecretSyncSyncSecretsByIdDTO,
|
||||
TQueueSendSecretSyncActionFailedNotificationsDTO,
|
||||
TSecretMap,
|
||||
TSecretSyncImportSecretsDTO,
|
||||
TSecretSyncRaw,
|
||||
TSecretSyncRemoveSecretsDTO,
|
||||
TSecretSyncSyncSecretsDTO,
|
||||
TSecretSyncWithCredentials,
|
||||
TSendSecretSyncFailedNotificationsJobDTO
|
||||
} from "@app/services/secret-sync/secret-sync-types";
|
||||
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
|
||||
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
|
||||
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
|
||||
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
|
||||
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
|
||||
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
|
||||
|
||||
export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;
|
||||
|
||||
type TSecretSyncQueueFactoryDep = {
|
||||
queueService: Pick<TQueueServiceFactory, "queue" | "start">;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
|
||||
folderDAL: TSecretFolderDALFactory;
|
||||
secretV2BridgeDAL: Pick<
|
||||
TSecretV2BridgeDALFactory,
|
||||
| "findByFolderId"
|
||||
| "find"
|
||||
| "insertMany"
|
||||
| "upsertSecretReferences"
|
||||
| "findBySecretKeys"
|
||||
| "bulkUpdate"
|
||||
| "deleteMany"
|
||||
>;
|
||||
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
|
||||
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById">;
|
||||
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
|
||||
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
|
||||
projectDAL: TProjectDALFactory;
|
||||
smtpService: Pick<TSmtpService, "sendMail">;
|
||||
projectBotDAL: TProjectBotDALFactory;
|
||||
secretDAL: TSecretDALFactory;
|
||||
secretVersionDAL: TSecretVersionDALFactory;
|
||||
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
|
||||
secretTagDAL: TSecretTagDALFactory;
|
||||
secretVersionTagDAL: TSecretVersionTagDALFactory;
|
||||
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
|
||||
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
|
||||
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
|
||||
};
|
||||
|
||||
type SecretSyncActionJob = Job<
|
||||
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
|
||||
>;
|
||||
|
||||
const getRequeueDelay = (failureCount?: number) => {
|
||||
if (!failureCount) return 0;
|
||||
|
||||
const baseDelay = 1000;
|
||||
const maxDelay = 30000;
|
||||
|
||||
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
|
||||
|
||||
const jitter = delay * (0.5 + Math.random() * 0.5);
|
||||
|
||||
return jitter;
|
||||
};
|
||||
|
||||
export const secretSyncQueueFactory = ({
|
||||
queueService,
|
||||
kmsService,
|
||||
keyStore,
|
||||
folderDAL,
|
||||
secretV2BridgeDAL,
|
||||
secretImportDAL,
|
||||
secretSyncDAL,
|
||||
auditLogService,
|
||||
projectMembershipDAL,
|
||||
projectDAL,
|
||||
smtpService,
|
||||
projectBotDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretBlindIndexDAL,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
secretVersionV2BridgeDAL,
|
||||
secretVersionTagV2BridgeDAL,
|
||||
resourceMetadataDAL
|
||||
}: TSecretSyncQueueFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
const integrationMeter = opentelemetry.metrics.getMeter("SecretSyncs");
|
||||
const syncSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_sync_secrets_errors", {
|
||||
description: "Secret Sync - sync secrets errors",
|
||||
unit: "1"
|
||||
});
|
||||
const importSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_import_secrets_errors", {
|
||||
description: "Secret Sync - import secrets errors",
|
||||
unit: "1"
|
||||
});
|
||||
const removeSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_remove_secrets_errors", {
|
||||
description: "Secret Sync - remove secrets errors",
|
||||
unit: "1"
|
||||
});
|
||||
|
||||
const $createManySecretsRawFn = createManySecretsRawFnFactory({
|
||||
projectDAL,
|
||||
projectBotDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretBlindIndexDAL,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
folderDAL,
|
||||
kmsService,
|
||||
secretVersionV2BridgeDAL,
|
||||
secretV2BridgeDAL,
|
||||
secretVersionTagV2BridgeDAL,
|
||||
resourceMetadataDAL
|
||||
});
|
||||
|
||||
const $updateManySecretsRawFn = updateManySecretsRawFnFactory({
|
||||
projectDAL,
|
||||
projectBotDAL,
|
||||
secretDAL,
|
||||
secretVersionDAL,
|
||||
secretBlindIndexDAL,
|
||||
secretTagDAL,
|
||||
secretVersionTagDAL,
|
||||
folderDAL,
|
||||
kmsService,
|
||||
secretVersionV2BridgeDAL,
|
||||
secretV2BridgeDAL,
|
||||
secretVersionTagV2BridgeDAL,
|
||||
resourceMetadataDAL
|
||||
});
|
||||
|
||||
const $getSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, includeImports = true) => {
|
||||
const {
|
||||
projectId,
|
||||
folderId,
|
||||
environment: { slug: environmentSlug },
|
||||
folder: { path: secretPath }
|
||||
} = secretSync;
|
||||
|
||||
const secretMap: TSecretMap = {};
|
||||
|
||||
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.SecretManager,
|
||||
projectId
|
||||
});
|
||||
|
||||
const decryptSecretValue = (value?: Buffer | undefined | null) =>
|
||||
value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "";
|
||||
|
||||
const { expandSecretReferences } = expandSecretReferencesFactory({
|
||||
decryptSecretValue,
|
||||
secretDAL: secretV2BridgeDAL,
|
||||
folderDAL,
|
||||
projectId,
|
||||
canExpandValue: () => true
|
||||
});
|
||||
|
||||
const secrets = await secretV2BridgeDAL.findByFolderId(folderId);
|
||||
|
||||
await Promise.allSettled(
|
||||
secrets.map(async (secret) => {
|
||||
const secretKey = secret.key;
|
||||
const secretValue = decryptSecretValue(secret.encryptedValue);
|
||||
const expandedSecretValue = await expandSecretReferences({
|
||||
environment: environmentSlug,
|
||||
secretPath,
|
||||
skipMultilineEncoding: secret.skipMultilineEncoding,
|
||||
value: secretValue
|
||||
});
|
||||
secretMap[secretKey] = { value: expandedSecretValue || "" };
|
||||
|
||||
if (secret.encryptedComment) {
|
||||
const commentValue = decryptSecretValue(secret.encryptedComment);
|
||||
secretMap[secretKey].comment = commentValue;
|
||||
}
|
||||
|
||||
secretMap[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
|
||||
})
|
||||
);
|
||||
|
||||
if (!includeImports) return secretMap;
|
||||
|
||||
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
|
||||
|
||||
if (secretImports.length) {
|
||||
const importedSecrets = await fnSecretsV2FromImports({
|
||||
decryptor: decryptSecretValue,
|
||||
folderDAL,
|
||||
secretDAL: secretV2BridgeDAL,
|
||||
expandSecretReferences,
|
||||
secretImportDAL,
|
||||
secretImports,
|
||||
hasSecretAccess: () => true
|
||||
});
|
||||
|
||||
for (let i = importedSecrets.length - 1; i >= 0; i -= 1) {
|
||||
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
|
||||
const importedSecret = importedSecrets[i].secrets[j];
|
||||
if (!secretMap[importedSecret.key]) {
|
||||
secretMap[importedSecret.key] = {
|
||||
skipMultilineEncoding: importedSecret.skipMultilineEncoding,
|
||||
comment: importedSecret.secretComment,
|
||||
value: importedSecret.secretValue || ""
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return secretMap;
|
||||
};
|
||||
|
||||
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) =>
|
||||
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
|
||||
delay: getRequeueDelay(payload.failedToAcquireLockCount),
|
||||
attempts: 5,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
|
||||
const queueSecretSyncImportSecretsById = async (payload: TQueueSecretSyncImportSecretsByIdDTO) =>
|
||||
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncImportSecrets, payload, {
|
||||
attempts: 1,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
|
||||
const queueSecretSyncRemoveSecretsById = async (payload: TQueueSecretSyncRemoveSecretsByIdDTO) =>
|
||||
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncRemoveSecrets, payload, {
|
||||
attempts: 1,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
|
||||
const $queueSendSecretSyncFailedNotifications = async (payload: TQueueSendSecretSyncActionFailedNotificationsDTO) => {
|
||||
if (!appCfg.isSmtpConfigured) return;
|
||||
|
||||
await queueService.queue(
|
||||
QueueName.AppConnectionSecretSync,
|
||||
QueueJobs.SecretSyncSendActionFailedNotifications,
|
||||
payload,
|
||||
{
|
||||
jobId: `secret-sync-${payload.secretSync.id}-failed-notifications`,
|
||||
attempts: 5,
|
||||
delay: 1000 * 60,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
},
|
||||
removeOnFail: true,
|
||||
removeOnComplete: true
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const $importSecrets = async (secretSync: TSecretSyncWithCredentials, importBehavior: SecretSyncImportBehavior) => {
|
||||
const { projectId, environment } = secretSync;
|
||||
|
||||
const importedSecrets = await SecretSyncFns.importSecrets(secretSync);
|
||||
|
||||
if (Object.keys(importedSecrets).length) {
|
||||
const secretMap = await $getSecrets(secretSync, false);
|
||||
|
||||
const secretsToCreate: Parameters<typeof $createManySecretsRawFn>[0]["secrets"] = [];
|
||||
const secretsToUpdate: Parameters<typeof $updateManySecretsRawFn>[0]["secrets"] = [];
|
||||
|
||||
Object.entries(importedSecrets).forEach(([key, { value }]) => {
|
||||
const secret = {
|
||||
secretName: key,
|
||||
secretValue: value,
|
||||
type: SecretType.Shared,
|
||||
secretComment: ""
|
||||
};
|
||||
|
||||
if (Object.hasOwn(secretMap, key)) {
|
||||
secretsToUpdate.push(secret);
|
||||
} else {
|
||||
secretsToCreate.push(secret);
|
||||
}
|
||||
});
|
||||
|
||||
if (secretsToCreate.length) {
|
||||
await $createManySecretsRawFn({
|
||||
projectId,
|
||||
path: secretSync.folder.path,
|
||||
environment: environment.slug,
|
||||
secrets: secretsToCreate
|
||||
});
|
||||
}
|
||||
|
||||
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
|
||||
await $updateManySecretsRawFn({
|
||||
projectId,
|
||||
path: secretSync.folder.path,
|
||||
environment: environment.slug,
|
||||
secrets: secretsToUpdate
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO) => {
|
||||
const {
|
||||
data: { syncId, auditLogInfo }
|
||||
} = job;
|
||||
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
|
||||
|
||||
await secretSyncDAL.updateById(syncId, {
|
||||
syncStatus: SecretSyncStatus.Running
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
let isSynced = false;
|
||||
let syncMessage: string | null = null;
|
||||
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
|
||||
|
||||
try {
|
||||
const {
|
||||
connection: { orgId, encryptedCredentials }
|
||||
} = secretSync;
|
||||
|
||||
const credentials = await decryptAppConnectionCredentials({
|
||||
orgId,
|
||||
encryptedCredentials,
|
||||
kmsService
|
||||
});
|
||||
|
||||
const secretSyncWithCredentials = {
|
||||
...secretSync,
|
||||
connection: {
|
||||
...secretSync.connection,
|
||||
credentials
|
||||
}
|
||||
} as TSecretSyncWithCredentials;
|
||||
|
||||
const {
|
||||
lastSyncedAt,
|
||||
syncOptions: { initialSyncBehavior }
|
||||
} = secretSyncWithCredentials;
|
||||
|
||||
if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
|
||||
await $importSecrets(
|
||||
secretSyncWithCredentials,
|
||||
initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource
|
||||
? SecretSyncImportBehavior.PrioritizeSource
|
||||
: SecretSyncImportBehavior.PrioritizeDestination
|
||||
);
|
||||
}
|
||||
|
||||
const secretMap = await $getSecrets(secretSync);
|
||||
|
||||
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap);
|
||||
|
||||
isSynced = true;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
err,
|
||||
`SecretSync Sync Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
|
||||
syncSecretsErrorHistogram.record(1, {
|
||||
version: 1,
|
||||
destination: secretSync.destination,
|
||||
syncId: secretSync.id,
|
||||
projectId: secretSync.projectId,
|
||||
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
|
||||
status: err instanceof AxiosError ? err.response?.status : undefined,
|
||||
name: err instanceof Error ? err.name : undefined
|
||||
});
|
||||
}
|
||||
|
||||
syncMessage =
|
||||
// eslint-disable-next-line no-nested-ternary
|
||||
(err instanceof AxiosError
|
||||
? err?.response?.data
|
||||
? JSON.stringify(err?.response?.data)
|
||||
: err?.message
|
||||
: (err as Error)?.message) || "An unknown error occurred.";
|
||||
|
||||
// re-throw so job fails
|
||||
throw err;
|
||||
} finally {
|
||||
const ranAt = new Date();
|
||||
const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: secretSync.projectId,
|
||||
...(auditLogInfo ?? {
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
}
|
||||
}),
|
||||
event: {
|
||||
type: EventType.SECRET_SYNC_SYNC_SECRETS,
|
||||
metadata: {
|
||||
syncId: secretSync.id,
|
||||
syncOptions: secretSync.syncOptions,
|
||||
destination: secretSync.destination,
|
||||
destinationConfig: secretSync.destinationConfig,
|
||||
folderId: secretSync.folderId,
|
||||
connectionId: secretSync.connectionId,
|
||||
jobRanAt: ranAt,
|
||||
jobId: job.id!,
|
||||
syncStatus,
|
||||
syncMessage
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (isSynced || isFinalAttempt) {
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
|
||||
syncStatus,
|
||||
lastSyncJobId: job.id,
|
||||
lastSyncMessage: syncMessage,
|
||||
lastSyncedAt: isSynced ? ranAt : undefined
|
||||
});
|
||||
|
||||
if (!isSynced) {
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync: updatedSecretSync,
|
||||
action: SecretSyncAction.SyncSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("SecretSync Sync Job with ID %s Completed", job.id);
|
||||
};
|
||||
|
||||
const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => {
|
||||
const {
|
||||
data: { syncId, auditLogInfo, importBehavior }
|
||||
} = job;
|
||||
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
|
||||
|
||||
await secretSyncDAL.updateById(syncId, {
|
||||
importStatus: SecretSyncStatus.Running
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
let isSuccess = false;
|
||||
let importMessage: string | null = null;
|
||||
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
|
||||
|
||||
try {
|
||||
const {
|
||||
connection: { orgId, encryptedCredentials }
|
||||
} = secretSync;
|
||||
|
||||
const credentials = await decryptAppConnectionCredentials({
|
||||
orgId,
|
||||
encryptedCredentials,
|
||||
kmsService
|
||||
});
|
||||
|
||||
await $importSecrets(
|
||||
{
|
||||
...secretSync,
|
||||
connection: {
|
||||
...secretSync.connection,
|
||||
credentials
|
||||
}
|
||||
} as TSecretSyncWithCredentials,
|
||||
importBehavior
|
||||
);
|
||||
|
||||
isSuccess = true;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
err,
|
||||
`SecretSync Import Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
|
||||
importSecretsErrorHistogram.record(1, {
|
||||
version: 1,
|
||||
destination: secretSync.destination,
|
||||
syncId: secretSync.id,
|
||||
projectId: secretSync.projectId,
|
||||
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
|
||||
status: err instanceof AxiosError ? err.response?.status : undefined,
|
||||
name: err instanceof Error ? err.name : undefined
|
||||
});
|
||||
}
|
||||
|
||||
importMessage =
|
||||
// eslint-disable-next-line no-nested-ternary
|
||||
(err instanceof AxiosError
|
||||
? err?.response?.data
|
||||
? JSON.stringify(err?.response?.data)
|
||||
: err?.message
|
||||
: (err as Error)?.message) || "An unknown error occurred.";
|
||||
|
||||
// re-throw so job fails
|
||||
throw err;
|
||||
} finally {
|
||||
const ranAt = new Date();
|
||||
const importStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: secretSync.projectId,
|
||||
...(auditLogInfo ?? {
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
}
|
||||
}),
|
||||
event: {
|
||||
type: EventType.SECRET_SYNC_IMPORT_SECRETS,
|
||||
metadata: {
|
||||
syncId: secretSync.id,
|
||||
syncOptions: secretSync.syncOptions,
|
||||
destination: secretSync.destination,
|
||||
destinationConfig: secretSync.destinationConfig,
|
||||
folderId: secretSync.folderId,
|
||||
connectionId: secretSync.connectionId,
|
||||
jobRanAt: ranAt,
|
||||
jobId: job.id!,
|
||||
importStatus,
|
||||
importMessage,
|
||||
importBehavior
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (isSuccess || isFinalAttempt) {
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
|
||||
importStatus,
|
||||
lastImportJobId: job.id,
|
||||
lastImportMessage: importMessage,
|
||||
lastImportedAt: isSuccess ? ranAt : undefined
|
||||
});
|
||||
|
||||
if (!isSuccess) {
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync: updatedSecretSync,
|
||||
action: SecretSyncAction.ImportSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("SecretSync Import Job with ID %s Completed", job.id);
|
||||
};
|
||||
|
||||
const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => {
|
||||
const {
|
||||
data: { syncId, auditLogInfo }
|
||||
} = job;
|
||||
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
|
||||
|
||||
await secretSyncDAL.updateById(syncId, {
|
||||
removeStatus: SecretSyncStatus.Running
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
let isSuccess = false;
|
||||
let removeMessage: string | null = null;
|
||||
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
|
||||
|
||||
try {
|
||||
const {
|
||||
connection: { orgId, encryptedCredentials }
|
||||
} = secretSync;
|
||||
|
||||
const credentials = await decryptAppConnectionCredentials({
|
||||
orgId,
|
||||
encryptedCredentials,
|
||||
kmsService
|
||||
});
|
||||
|
||||
const secretMap = await $getSecrets(secretSync);
|
||||
|
||||
await SecretSyncFns.removeSecrets(
|
||||
{
|
||||
...secretSync,
|
||||
connection: {
|
||||
...secretSync.connection,
|
||||
credentials
|
||||
}
|
||||
} as TSecretSyncWithCredentials,
|
||||
secretMap
|
||||
);
|
||||
|
||||
isSuccess = true;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
err,
|
||||
`SecretSync Remove Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
|
||||
);
|
||||
|
||||
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
|
||||
removeSecretsErrorHistogram.record(1, {
|
||||
version: 1,
|
||||
destination: secretSync.destination,
|
||||
syncId: secretSync.id,
|
||||
projectId: secretSync.projectId,
|
||||
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
|
||||
status: err instanceof AxiosError ? err.response?.status : undefined,
|
||||
name: err instanceof Error ? err.name : undefined
|
||||
});
|
||||
}
|
||||
|
||||
removeMessage =
|
||||
// eslint-disable-next-line no-nested-ternary
|
||||
(err instanceof AxiosError
|
||||
? err?.response?.data
|
||||
? JSON.stringify(err?.response?.data)
|
||||
: err?.message
|
||||
: (err as Error)?.message) || "An unknown error occurred.";
|
||||
|
||||
// re-throw so job fails
|
||||
throw err;
|
||||
} finally {
|
||||
const ranAt = new Date();
|
||||
const removeStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: secretSync.projectId,
|
||||
...(auditLogInfo ?? {
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
}
|
||||
}),
|
||||
event: {
|
||||
type: EventType.SECRET_SYNC_REMOVE_SECRETS,
|
||||
metadata: {
|
||||
syncId: secretSync.id,
|
||||
syncOptions: secretSync.syncOptions,
|
||||
destination: secretSync.destination,
|
||||
destinationConfig: secretSync.destinationConfig,
|
||||
folderId: secretSync.folderId,
|
||||
connectionId: secretSync.connectionId,
|
||||
jobRanAt: ranAt,
|
||||
jobId: job.id!,
|
||||
removeStatus,
|
||||
removeMessage
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (isSuccess || isFinalAttempt) {
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
|
||||
removeStatus,
|
||||
lastRemoveJobId: job.id,
|
||||
lastRemoveMessage: removeMessage,
|
||||
lastRemovedAt: isSuccess ? ranAt : undefined
|
||||
});
|
||||
|
||||
if (!isSuccess) {
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync: updatedSecretSync,
|
||||
action: SecretSyncAction.RemoveSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("SecretSync Remove Job with ID %s Completed", job.id);
|
||||
};
|
||||
|
||||
const $sendSecretSyncFailedNotifications = async (job: TSendSecretSyncFailedNotificationsJobDTO) => {
|
||||
const {
|
||||
data: { secretSync, auditLogInfo, action }
|
||||
} = job;
|
||||
|
||||
const { projectId, destination, name, folder, lastSyncMessage, lastRemoveMessage, lastImportMessage, environment } =
|
||||
secretSync;
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
|
||||
let projectAdmins = projectMembers.filter((member) =>
|
||||
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
|
||||
);
|
||||
|
||||
const triggeredByUserId =
|
||||
auditLogInfo && auditLogInfo.actor.type === ActorType.USER && auditLogInfo.actor.metadata.userId;
|
||||
|
||||
// only notify triggering user if triggered by admin
|
||||
if (triggeredByUserId && projectAdmins.map((admin) => admin.userId).includes(triggeredByUserId)) {
|
||||
projectAdmins = projectAdmins.filter((admin) => admin.userId === triggeredByUserId);
|
||||
}
|
||||
|
||||
const syncDestination = SECRET_SYNC_NAME_MAP[destination as SecretSync];
|
||||
|
||||
let actionLabel: string;
|
||||
let failureMessage: string | null | undefined;
|
||||
|
||||
switch (action) {
|
||||
case SecretSyncAction.ImportSecrets:
|
||||
actionLabel = "Import";
|
||||
failureMessage = lastImportMessage;
|
||||
|
||||
break;
|
||||
case SecretSyncAction.RemoveSecrets:
|
||||
actionLabel = "Remove";
|
||||
failureMessage = lastRemoveMessage;
|
||||
|
||||
break;
|
||||
case SecretSyncAction.SyncSecrets:
|
||||
default:
|
||||
actionLabel = `Sync`;
|
||||
failureMessage = lastSyncMessage;
|
||||
break;
|
||||
}
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
|
||||
template: SmtpTemplates.SecretSyncFailed,
|
||||
subjectLine: `Secret Sync Failed to ${actionLabel} Secrets`,
|
||||
substitutions: {
|
||||
syncName: name,
|
||||
syncDestination,
|
||||
content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
|
||||
failureMessage,
|
||||
secretPath: folder.path,
|
||||
environment: environment.name,
|
||||
projectName: project.name,
|
||||
syncUrl: `${appCfg.SITE_URL}/integrations/secret-syncs/${destination}/${secretSync.id}`
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const queueSecretSyncsSyncSecretsByPath = async ({
|
||||
secretPath,
|
||||
projectId,
|
||||
environmentSlug
|
||||
}: TQueueSecretSyncsByPathDTO) => {
|
||||
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
|
||||
|
||||
if (!folder)
|
||||
throw new Error(
|
||||
`Could not find folder at path "${secretPath}" for environment with slug "${environmentSlug}" in project with ID "${projectId}"`
|
||||
);
|
||||
|
||||
const secretSyncs = await secretSyncDAL.find({ folderId: folder.id, isEnabled: true });
|
||||
|
||||
await Promise.all(secretSyncs.map((secretSync) => queueSecretSyncSyncSecretsById({ syncId: secretSync.id })));
|
||||
};
|
||||
|
||||
const $handleAcquireLockFailure = async (job: SecretSyncActionJob) => {
|
||||
const { syncId, auditLogInfo } = job.data;
|
||||
|
||||
switch (job.name) {
|
||||
case QueueJobs.SecretSyncSyncSecrets: {
|
||||
const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;
|
||||
|
||||
//
|
||||
if (failedToAcquireLockCount < 10) {
|
||||
await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
|
||||
return;
|
||||
}
|
||||
|
||||
const secretSync = await secretSyncDAL.updateById(syncId, {
|
||||
syncStatus: SecretSyncStatus.Failed,
|
||||
lastSyncMessage:
|
||||
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
|
||||
lastSyncJobId: job.id
|
||||
});
|
||||
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync,
|
||||
action: SecretSyncAction.SyncSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
// Scott: the two cases below are unlikely to happen as we check the lock at the API level but including this as a fallback
|
||||
case QueueJobs.SecretSyncImportSecrets: {
|
||||
const secretSync = await secretSyncDAL.updateById(syncId, {
|
||||
importStatus: SecretSyncStatus.Failed,
|
||||
lastImportMessage:
|
||||
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
|
||||
lastImportJobId: job.id
|
||||
});
|
||||
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync,
|
||||
action: SecretSyncAction.ImportSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case QueueJobs.SecretSyncRemoveSecrets: {
|
||||
const secretSync = await secretSyncDAL.updateById(syncId, {
|
||||
removeStatus: SecretSyncStatus.Failed,
|
||||
lastRemoveMessage:
|
||||
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
|
||||
lastRemoveJobId: job.id
|
||||
});
|
||||
|
||||
await $queueSendSecretSyncFailedNotifications({
|
||||
secretSync,
|
||||
action: SecretSyncAction.RemoveSecrets,
|
||||
auditLogInfo
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
|
||||
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
|
||||
}
|
||||
};
|
||||
|
||||
queueService.start(QueueName.AppConnectionSecretSync, async (job) => {
|
||||
if (job.name === QueueJobs.SecretSyncSendActionFailedNotifications) {
|
||||
await $sendSecretSyncFailedNotifications(job as TSendSecretSyncFailedNotificationsJobDTO);
|
||||
return;
|
||||
}
|
||||
|
||||
const { syncId } = job.data as
|
||||
| TQueueSecretSyncSyncSecretsByIdDTO
|
||||
| TQueueSecretSyncImportSecretsByIdDTO
|
||||
| TQueueSecretSyncRemoveSecretsByIdDTO;
|
||||
|
||||
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
|
||||
|
||||
try {
|
||||
lock = await keyStore.acquireLock(
|
||||
[KeyStorePrefixes.SecretSyncLock(syncId)],
|
||||
// scott: not sure on this duration; syncs can take excessive amounts of time so we need to keep it locked,
|
||||
// but should always release below...
|
||||
5 * 60 * 1000
|
||||
);
|
||||
} catch (e) {
|
||||
logger.info(`SecretSync Failed to acquire lock [syncId=${syncId}] [job=${job.name}]`);
|
||||
|
||||
await $handleAcquireLockFailure(job as SecretSyncActionJob);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
switch (job.name) {
|
||||
case QueueJobs.SecretSyncSyncSecrets:
|
||||
await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO);
|
||||
break;
|
||||
case QueueJobs.SecretSyncImportSecrets:
|
||||
await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO);
|
||||
break;
|
||||
case QueueJobs.SecretSyncRemoveSecrets:
|
||||
await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO);
|
||||
break;
|
||||
default:
|
||||
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
|
||||
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
|
||||
}
|
||||
} finally {
|
||||
await lock.release();
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
queueSecretSyncSyncSecretsById,
|
||||
queueSecretSyncImportSecretsById,
|
||||
queueSecretSyncRemoveSecretsById,
|
||||
queueSecretSyncsSyncSecretsByPath
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,95 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { SecretSyncsSchema } from "@app/db/schemas/secret-syncs";
|
||||
import { SecretSyncs } from "@app/lib/api-docs";
|
||||
import { removeTrailingSlash } from "@app/lib/fn";
|
||||
import { slugSchema } from "@app/server/lib/schemas";
|
||||
import { SecretSync, SecretSyncInitialSyncBehavior } from "@app/services/secret-sync/secret-sync-enums";
|
||||
import { SECRET_SYNC_CONNECTION_MAP } from "@app/services/secret-sync/secret-sync-maps";
|
||||
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
const SyncOptionsSchema = (secretSync: SecretSync, options: TSyncOptionsConfig = { canImportSecrets: true }) =>
|
||||
z.object({
|
||||
initialSyncBehavior: (options.canImportSecrets
|
||||
? z.nativeEnum(SecretSyncInitialSyncBehavior)
|
||||
: z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination)
|
||||
).describe(SecretSyncs.SYNC_OPTIONS(secretSync).INITIAL_SYNC_BEHAVIOR),
|
||||
prependPrefix: z
|
||||
.string()
|
||||
.trim()
|
||||
.transform((str) => str.toUpperCase())
|
||||
.optional()
|
||||
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).PREPEND_PREFIX),
|
||||
appendSuffix: z
|
||||
.string()
|
||||
.trim()
|
||||
.transform((str) => str.toUpperCase())
|
||||
.optional()
|
||||
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).APPEND_SUFFIX)
|
||||
});
|
||||
|
||||
export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
|
||||
SecretSyncsSchema.omit({
|
||||
destination: true,
|
||||
destinationConfig: true,
|
||||
syncOptions: true
|
||||
}).extend({
|
||||
// destination needs to be on the extended object for type differentiation
|
||||
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig),
|
||||
// join properties
|
||||
projectId: z.string(),
|
||||
connection: z.object({
|
||||
app: z.literal(SECRET_SYNC_CONNECTION_MAP[destination]),
|
||||
name: z.string(),
|
||||
id: z.string().uuid()
|
||||
}),
|
||||
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }),
|
||||
folder: z.object({ id: z.string(), path: z.string() })
|
||||
});
|
||||
|
||||
export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretSyncs.CREATE(destination).name),
|
||||
projectId: z.string().trim().min(1, "Project ID required").describe(SecretSyncs.CREATE(destination).projectId),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretSyncs.CREATE(destination).description),
|
||||
connectionId: z.string().uuid().describe(SecretSyncs.CREATE(destination).connectionId),
|
||||
environment: slugSchema({ field: "environment", max: 64 }).describe(SecretSyncs.CREATE(destination).environment),
|
||||
secretPath: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Secret path required")
|
||||
.transform(removeTrailingSlash)
|
||||
.describe(SecretSyncs.CREATE(destination).secretPath),
|
||||
isEnabled: z.boolean().default(true).describe(SecretSyncs.CREATE(destination).isEnabled),
|
||||
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig).describe(SecretSyncs.CREATE(destination).syncOptions)
|
||||
});
|
||||
|
||||
export const GenericUpdateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
|
||||
z.object({
|
||||
name: slugSchema({ field: "name" }).describe(SecretSyncs.UPDATE(destination).name).optional(),
|
||||
description: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(256, "Description cannot exceed 256 characters")
|
||||
.nullish()
|
||||
.describe(SecretSyncs.UPDATE(destination).description),
|
||||
environment: slugSchema({ field: "environment", max: 64 })
|
||||
.optional()
|
||||
.describe(SecretSyncs.UPDATE(destination).environment),
|
||||
secretPath: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "Invalid secret path")
|
||||
.transform(removeTrailingSlash)
|
||||
.optional()
|
||||
.describe(SecretSyncs.UPDATE(destination).secretPath),
|
||||
isEnabled: z.boolean().optional().describe(SecretSyncs.UPDATE(destination).isEnabled),
|
||||
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig)
|
||||
.optional()
|
||||
.describe(SecretSyncs.UPDATE(destination).syncOptions)
|
||||
});
|
||||
@@ -0,0 +1,485 @@
|
||||
import { ForbiddenError, subject } from "@casl/ability";
|
||||
|
||||
import { ActionProjectType } from "@app/db/schemas";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
|
||||
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { BadRequestError, NotFoundError } from "@app/lib/errors";
|
||||
import { OrgServiceActor } from "@app/lib/types";
|
||||
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
|
||||
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
|
||||
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
|
||||
import { listSecretSyncOptions } from "@app/services/secret-sync/secret-sync-fns";
|
||||
import {
|
||||
SecretSyncStatus,
|
||||
TCreateSecretSyncDTO,
|
||||
TDeleteSecretSyncDTO,
|
||||
TFindSecretSyncByIdDTO,
|
||||
TFindSecretSyncByNameDTO,
|
||||
TListSecretSyncsByProjectId,
|
||||
TSecretSync,
|
||||
TTriggerSecretSyncImportSecretsByIdDTO,
|
||||
TTriggerSecretSyncRemoveSecretsByIdDTO,
|
||||
TTriggerSecretSyncSyncSecretsByIdDTO,
|
||||
TUpdateSecretSyncDTO
|
||||
} from "@app/services/secret-sync/secret-sync-types";
|
||||
|
||||
import { TSecretSyncDALFactory } from "./secret-sync-dal";
|
||||
import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "./secret-sync-maps";
|
||||
import { TSecretSyncQueueFactory } from "./secret-sync-queue";
|
||||
|
||||
type TSecretSyncServiceFactoryDep = {
|
||||
secretSyncDAL: TSecretSyncDALFactory;
|
||||
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
|
||||
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
|
||||
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
|
||||
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId" | "findById" | "findBySecretPath">;
|
||||
keyStore: Pick<TKeyStoreFactory, "getItem">;
|
||||
secretSyncQueue: Pick<
|
||||
TSecretSyncQueueFactory,
|
||||
"queueSecretSyncSyncSecretsById" | "queueSecretSyncImportSecretsById" | "queueSecretSyncRemoveSecretsById"
|
||||
>;
|
||||
};
|
||||
|
||||
export type TSecretSyncServiceFactory = ReturnType<typeof secretSyncServiceFactory>;
|
||||
|
||||
export const secretSyncServiceFactory = ({
|
||||
secretSyncDAL,
|
||||
folderDAL,
|
||||
permissionService,
|
||||
appConnectionService,
|
||||
projectBotService,
|
||||
secretSyncQueue,
|
||||
keyStore
|
||||
}: TSecretSyncServiceFactoryDep) => {
|
||||
const listSecretSyncsByProjectId = async (
|
||||
{ projectId, destination }: TListSecretSyncsByProjectId,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
const folders = await folderDAL.findByProjectId(projectId);
|
||||
|
||||
const secretSyncs = await secretSyncDAL.find({
|
||||
...(destination && { destination }),
|
||||
$in: {
|
||||
folderId: folders.map((folder) => folder.id)
|
||||
}
|
||||
});
|
||||
|
||||
return secretSyncs as TSecretSync[];
|
||||
};
|
||||
|
||||
const findSecretSyncById = async ({ destination, syncId }: TFindSecretSyncByIdDTO, actor: OrgServiceActor) => {
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
return secretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const findSecretSyncByName = async (
|
||||
{ destination, syncName, projectId }: TFindSecretSyncByNameDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const folders = await folderDAL.findByProjectId(projectId);
|
||||
|
||||
// we prevent conflicting names within a project so this will only return one at most
|
||||
const [secretSync] = await secretSyncDAL.find({
|
||||
name: syncName,
|
||||
$in: {
|
||||
folderId: folders.map((folder) => folder.id)
|
||||
}
|
||||
});
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with name "${syncName}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
return secretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const createSecretSync = async (
|
||||
{ projectId, secretPath, environment, ...params }: TCreateSecretSyncDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const { permission: projectPermission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId
|
||||
});
|
||||
|
||||
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
|
||||
|
||||
if (!shouldUseSecretV2Bridge)
|
||||
throw new BadRequestError({ message: "Project version does not support Secret Syncs" });
|
||||
|
||||
ForbiddenError.from(projectPermission).throwUnlessCan(
|
||||
ProjectPermissionActions.Create,
|
||||
ProjectPermissionSub.SecretSyncs
|
||||
);
|
||||
|
||||
ForbiddenError.from(projectPermission).throwUnlessCan(
|
||||
ProjectPermissionActions.Read,
|
||||
subject(ProjectPermissionSub.Secrets, {
|
||||
environment,
|
||||
secretPath
|
||||
})
|
||||
);
|
||||
|
||||
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
|
||||
|
||||
if (!folder)
|
||||
throw new BadRequestError({
|
||||
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${projectId}"`
|
||||
});
|
||||
|
||||
const destinationApp = SECRET_SYNC_CONNECTION_MAP[params.destination];
|
||||
|
||||
// validates permission to connect and app is valid for sync destination
|
||||
await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor);
|
||||
|
||||
const projectFolders = await folderDAL.findByProjectId(folder.projectId);
|
||||
|
||||
const secretSync = await secretSyncDAL.transaction(async (tx) => {
|
||||
const isConflictingName = Boolean(
|
||||
(
|
||||
await secretSyncDAL.find(
|
||||
{
|
||||
name: params.name,
|
||||
$in: {
|
||||
folderId: projectFolders.map((f) => f.id)
|
||||
}
|
||||
},
|
||||
tx
|
||||
)
|
||||
).length
|
||||
);
|
||||
|
||||
if (isConflictingName)
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Sync with the name "${params.name}" already exists for the project with ID "${folder.projectId}"`
|
||||
});
|
||||
|
||||
const sync = await secretSyncDAL.create({
|
||||
folderId: folder.id,
|
||||
...params,
|
||||
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending })
|
||||
});
|
||||
|
||||
return sync;
|
||||
});
|
||||
|
||||
if (secretSync.isEnabled) await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id });
|
||||
|
||||
return secretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const updateSecretSync = async (
|
||||
{ destination, syncId, secretPath, environment, ...params }: TUpdateSecretSyncDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID ${syncId}`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => {
|
||||
if (
|
||||
(secretPath && secretPath !== secretSync.folder.path) ||
|
||||
(environment && environment !== secretSync.environment.slug)
|
||||
) {
|
||||
const updatedEnvironment = environment ?? secretSync.environment.slug;
|
||||
const updatedSecretPath = secretPath ?? secretSync.folder.path;
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
ProjectPermissionActions.Read,
|
||||
subject(ProjectPermissionSub.Secrets, {
|
||||
environment: updatedEnvironment,
|
||||
secretPath: updatedSecretPath
|
||||
})
|
||||
);
|
||||
|
||||
const newFolder = await folderDAL.findBySecretPath(secretSync.projectId, updatedEnvironment, updatedSecretPath);
|
||||
|
||||
if (!newFolder)
|
||||
throw new BadRequestError({
|
||||
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${secretSync.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
if (params.name && secretSync.name !== params.name) {
|
||||
const projectFolders = await folderDAL.findByProjectId(secretSync.projectId);
|
||||
|
||||
const isConflictingName = Boolean(
|
||||
(
|
||||
await secretSyncDAL.find(
|
||||
{
|
||||
name: params.name,
|
||||
$in: {
|
||||
folderId: projectFolders.map((f) => f.id)
|
||||
}
|
||||
},
|
||||
tx
|
||||
)
|
||||
).length
|
||||
);
|
||||
|
||||
if (isConflictingName)
|
||||
throw new BadRequestError({
|
||||
message: `A Secret Sync with the name "${params.name}" already exists for project with ID "${secretSync.projectId}"`
|
||||
});
|
||||
}
|
||||
|
||||
const isEnabled = params.isEnabled ?? secretSync.isEnabled;
|
||||
|
||||
const updatedSync = await secretSyncDAL.updateById(syncId, {
|
||||
...params,
|
||||
...(isEnabled && { syncStatus: SecretSyncStatus.Pending })
|
||||
});
|
||||
|
||||
return updatedSync;
|
||||
});
|
||||
|
||||
if (updatedSecretSync.isEnabled) await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id });
|
||||
|
||||
return updatedSecretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const deleteSecretSync = async ({ destination, syncId }: TDeleteSecretSyncDTO, actor: OrgServiceActor) => {
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
await secretSyncDAL.deleteById(syncId);
|
||||
|
||||
return secretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const triggerSecretSyncSyncSecretsById = async (
|
||||
{ syncId, destination, ...params }: TTriggerSecretSyncSyncSecretsByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
|
||||
|
||||
if (isSyncJobRunning)
|
||||
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
|
||||
|
||||
await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId, ...params });
|
||||
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
|
||||
syncStatus: SecretSyncStatus.Pending
|
||||
});
|
||||
|
||||
return updatedSecretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const triggerSecretSyncImportSecretsById = async (
|
||||
{ syncId, destination, ...params }: TTriggerSecretSyncImportSecretsByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
if (!listSecretSyncOptions().find((option) => option.destination === destination)?.canImportSecrets) {
|
||||
throw new BadRequestError({
|
||||
message: `${SECRET_SYNC_NAME_MAP[destination]} does not support importing secrets.`
|
||||
});
|
||||
}
|
||||
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
|
||||
|
||||
if (isSyncJobRunning)
|
||||
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
|
||||
|
||||
await secretSyncQueue.queueSecretSyncImportSecretsById({ syncId, ...params });
|
||||
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
|
||||
importStatus: SecretSyncStatus.Pending
|
||||
});
|
||||
|
||||
return updatedSecretSync as TSecretSync;
|
||||
};
|
||||
|
||||
const triggerSecretSyncRemoveSecretsById = async (
|
||||
{ syncId, destination, ...params }: TTriggerSecretSyncRemoveSecretsByIdDTO,
|
||||
actor: OrgServiceActor
|
||||
) => {
|
||||
const secretSync = await secretSyncDAL.findById(syncId);
|
||||
|
||||
if (!secretSync)
|
||||
throw new NotFoundError({
|
||||
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
|
||||
});
|
||||
|
||||
const { permission } = await permissionService.getProjectPermission({
|
||||
actor: actor.type,
|
||||
actorId: actor.id,
|
||||
actorAuthMethod: actor.authMethod,
|
||||
actorOrgId: actor.orgId,
|
||||
actionProjectType: ActionProjectType.SecretManager,
|
||||
projectId: secretSync.projectId
|
||||
});
|
||||
|
||||
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
|
||||
|
||||
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
|
||||
throw new BadRequestError({
|
||||
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
|
||||
});
|
||||
|
||||
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
|
||||
|
||||
if (isSyncJobRunning)
|
||||
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
|
||||
|
||||
await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, ...params });
|
||||
|
||||
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
|
||||
removeStatus: SecretSyncStatus.Pending
|
||||
});
|
||||
|
||||
return updatedSecretSync as TSecretSync;
|
||||
};
|
||||
|
||||
return {
|
||||
listSecretSyncOptions,
|
||||
listSecretSyncsByProjectId,
|
||||
findSecretSyncById,
|
||||
findSecretSyncByName,
|
||||
createSecretSync,
|
||||
updateSecretSync,
|
||||
deleteSecretSync,
|
||||
triggerSecretSyncSyncSecretsById,
|
||||
triggerSecretSyncImportSecretsById,
|
||||
triggerSecretSyncRemoveSecretsById
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,146 @@
|
||||
import { Job } from "bullmq";
|
||||
|
||||
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import { QueueJobs } from "@app/queue";
|
||||
import {
|
||||
TGitHubSync,
|
||||
TGitHubSyncInput,
|
||||
TGitHubSyncListItem,
|
||||
TGitHubSyncWithCredentials
|
||||
} from "@app/services/secret-sync/github";
|
||||
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
|
||||
import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums";
|
||||
|
||||
import {
|
||||
TAwsParameterStoreSync,
|
||||
TAwsParameterStoreSyncInput,
|
||||
TAwsParameterStoreSyncListItem,
|
||||
TAwsParameterStoreSyncWithCredentials
|
||||
} from "./aws-parameter-store";
|
||||
|
||||
export type TSecretSync = TAwsParameterStoreSync | TGitHubSync;
|
||||
|
||||
export type TSecretSyncWithCredentials = TAwsParameterStoreSyncWithCredentials | TGitHubSyncWithCredentials;
|
||||
|
||||
export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput;
|
||||
|
||||
export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem;
|
||||
|
||||
export type TSyncOptionsConfig = {
|
||||
canImportSecrets: boolean;
|
||||
};
|
||||
|
||||
export type TListSecretSyncsByProjectId = {
|
||||
projectId: string;
|
||||
destination?: SecretSync;
|
||||
};
|
||||
|
||||
export type TFindSecretSyncByIdDTO = {
|
||||
syncId: string;
|
||||
destination: SecretSync;
|
||||
};
|
||||
|
||||
export type TFindSecretSyncByNameDTO = {
|
||||
syncName: string;
|
||||
projectId: string;
|
||||
destination: SecretSync;
|
||||
};
|
||||
|
||||
export type TCreateSecretSyncDTO = Pick<TSecretSync, "syncOptions" | "destinationConfig" | "name" | "connectionId"> & {
|
||||
destination: SecretSync;
|
||||
projectId: string;
|
||||
secretPath: string;
|
||||
environment: string;
|
||||
isEnabled?: boolean;
|
||||
};
|
||||
|
||||
export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "connectionId" | "projectId">> & {
|
||||
syncId: string;
|
||||
destination: SecretSync;
|
||||
};
|
||||
|
||||
export type TDeleteSecretSyncDTO = {
|
||||
destination: SecretSync;
|
||||
syncId: string;
|
||||
};
|
||||
|
||||
type AuditLogInfo = Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
|
||||
|
||||
export enum SecretSyncStatus {
|
||||
Pending = "pending",
|
||||
Running = "running",
|
||||
Succeeded = "succeeded",
|
||||
Failed = "failed"
|
||||
}
|
||||
|
||||
export enum SecretSyncAction {
|
||||
SyncSecrets = "sync-secrets",
|
||||
ImportSecrets = "import-secrets",
|
||||
RemoveSecrets = "remove-secrets"
|
||||
}
|
||||
|
||||
export type TSecretSyncRaw = NonNullable<Awaited<ReturnType<TSecretSyncDALFactory["findById"]>>>;
|
||||
|
||||
export type TQueueSecretSyncsByPathDTO = {
|
||||
secretPath: string;
|
||||
environmentSlug: string;
|
||||
projectId: string;
|
||||
};
|
||||
|
||||
export type TQueueSecretSyncSyncSecretsByIdDTO = {
|
||||
syncId: string;
|
||||
failedToAcquireLockCount?: number;
|
||||
auditLogInfo?: AuditLogInfo;
|
||||
};
|
||||
|
||||
export type TTriggerSecretSyncSyncSecretsByIdDTO = {
|
||||
destination: SecretSync;
|
||||
} & TQueueSecretSyncSyncSecretsByIdDTO;
|
||||
|
||||
export type TQueueSecretSyncImportSecretsByIdDTO = {
|
||||
syncId: string;
|
||||
importBehavior: SecretSyncImportBehavior;
|
||||
auditLogInfo?: AuditLogInfo;
|
||||
};
|
||||
|
||||
export type TTriggerSecretSyncImportSecretsByIdDTO = {
|
||||
destination: SecretSync;
|
||||
} & TQueueSecretSyncImportSecretsByIdDTO;
|
||||
|
||||
export type TQueueSecretSyncRemoveSecretsByIdDTO = {
|
||||
syncId: string;
|
||||
auditLogInfo?: AuditLogInfo;
|
||||
};
|
||||
|
||||
export type TTriggerSecretSyncRemoveSecretsByIdDTO = {
|
||||
destination: SecretSync;
|
||||
} & TQueueSecretSyncRemoveSecretsByIdDTO;
|
||||
|
||||
export type TQueueSendSecretSyncActionFailedNotificationsDTO = {
|
||||
secretSync: TSecretSyncRaw;
|
||||
auditLogInfo?: AuditLogInfo;
|
||||
action: SecretSyncAction;
|
||||
};
|
||||
|
||||
export type TSecretSyncSyncSecretsDTO = Job<TQueueSecretSyncSyncSecretsByIdDTO, void, QueueJobs.SecretSyncSyncSecrets>;
|
||||
export type TSecretSyncImportSecretsDTO = Job<
|
||||
TQueueSecretSyncImportSecretsByIdDTO,
|
||||
void,
|
||||
QueueJobs.SecretSyncSyncSecrets
|
||||
>;
|
||||
export type TSecretSyncRemoveSecretsDTO = Job<
|
||||
TQueueSecretSyncRemoveSecretsByIdDTO,
|
||||
void,
|
||||
QueueJobs.SecretSyncSyncSecrets
|
||||
>;
|
||||
|
||||
export type TSendSecretSyncFailedNotificationsJobDTO = Job<
|
||||
TQueueSendSecretSyncActionFailedNotificationsDTO,
|
||||
void,
|
||||
QueueJobs.SecretSyncSendActionFailedNotifications
|
||||
>;
|
||||
|
||||
export type TSecretMap = Record<
|
||||
string,
|
||||
{ value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined }
|
||||
>;
|
||||
Reference in New Issue
Block a user