awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 02:51:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvements: address feedback

Browse Source
This commit is contained in:
Scott Wilson
2025-01-20 22:17:20 -08:00
parent 6341b7e989
commit 3c1fc024c2
57 changed files with 981 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts
+15 -7
View File
@@ -1,6 +1,7 @@
import AWS, { AWSError } from "aws-sdk";
import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAwsParameterStoreSyncWithCredentials } from "./aws-parameter-store-sync-types";
@@ -146,12 +147,19 @@ export const AwsParameterStoreSyncFns = {
continue;
}
await putParameter(ssm, {
Name: `${destinationConfig.path}${key}`,
Type: "SecureString",
Value: value,
Overwrite: true
});
try {
await putParameter(ssm, {
Name: `${destinationConfig.path}${key}`,
Type: "SecureString",
Value: value,
Overwrite: true
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
}
const parametersToDelete: AWS.SSM.Parameter[] = [];
@@ -166,7 +174,7 @@ export const AwsParameterStoreSyncFns = {
await deleteParametersBatch(ssm, parametersToDelete);
},
importSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
getSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-schemas.ts
+4 -18
View File
@@ -1,7 +1,6 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { wrapWithSlashes } from "@app/lib/fn";
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
@@ -14,23 +13,10 @@ const AwsParameterStoreSyncDestinationConfigSchema = z.object({
region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.REGION),
path: z
.string()
.min(1, "Parameter Store Path Required")
.transform(wrapWithSlashes)
.superRefine((val, ctx) => {
if (!/^\/([/]|(([\w-]+\/)+))?$/.test(val)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - must follow "/example/path/" format`
});
}
if (val.length > 2048) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - cannot exceed 2048 characters`
});
}
})
.trim()
.min(1, "Parameter Store Path required")
.max(2048, "Cannot exceed 2048 characters")
.regex(/^\/([/]|(([\w-]+\/)+))?$/)
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.PATH)
});
backend/src/services/secret-sync/github/github-sync-fns.ts
+10 -29
View File
@@ -6,20 +6,12 @@ import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TGitHubSyncWithCredentials } from "./github-sync-types";
interface GitHubSecret {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
}
import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithCredentials } from "./github-sync-types";
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: GitHubSecret[];
let encryptedSecrets: TGitHubSecret[];
const { destinationConfig } = secretSync;
@@ -52,17 +44,8 @@ const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithC
return encryptedSecrets;
};
interface GitHubPublicKey {
key_id: string;
key: string;
id?: number | undefined;
url?: string | undefined;
title?: string | undefined;
created_at?: string | undefined;
}
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: GitHubPublicKey;
let publicKey: TGitHubPublicKey;
const { destinationConfig } = secretSync;
@@ -100,7 +83,11 @@ const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredenti
return publicKey;
};
const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, encryptedSecret: GitHubSecret) => {
const deleteSecret = async (
client: Octokit,
secretSync: TGitHubSyncWithCredentials,
encryptedSecret: TGitHubSecret
) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
@@ -132,13 +119,7 @@ const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredenti
}
};
interface GitHubSecretPayload {
key_id: string;
secret_name: string;
encrypted_value: string;
}
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: GitHubSecretPayload) => {
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
@@ -210,7 +191,7 @@ export const GithubSyncFns = {
}
});
},
importSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
backend/src/services/secret-sync/github/github-sync-schemas.ts
+13 -13
View File
@@ -32,24 +32,24 @@ const GitHubSyncDestinationConfigSchema = z
})
])
.superRefine((options, ctx) => {
if (options.scope !== GitHubSyncScope.Organization) return;
if (options.scope === GitHubSyncScope.Organization) {
if (options.visibility === GitHubSyncVisibility.Selected) {
if (!options.selectedRepositoryIds?.length)
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Select at least 1 repository",
path: ["selectedRepositoryIds"]
});
return;
}
if (options.visibility === GitHubSyncVisibility.Selected) {
if (!options.selectedRepositoryIds?.length)
if (options.selectedRepositoryIds?.length) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Select at least 1 repository",
message: `Selected repositories is only supported for visibility "Selected"`,
path: ["selectedRepositoryIds"]
});
return;
}
if (options.selectedRepositoryIds?.length) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Selected repositories is only supported for visibility "Selected"`,
path: ["selectedRepositoryIds"]
});
}
}
});
backend/src/services/secret-sync/github/github-sync-types.ts
+23
View File
@@ -13,3 +13,26 @@ export type TGitHubSyncListItem = z.infer<typeof GitHubSyncListItemSchema>;
export type TGitHubSyncWithCredentials = TGitHubSync & {
connection: TGitHubConnection;
};
export type TGitHubSecret = {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
};
export type TGitHubPublicKey = {
key_id: string;
key: string;
id?: number | undefined;
url?: string | undefined;
title?: string | undefined;
created_at?: string | undefined;
};
export type TGitHubSecretPayload = {
key_id: string;
secret_name: string;
encrypted_value: string;
};
backend/src/services/secret-sync/secret-sync-dal.ts
+27 -16
View File
@@ -13,16 +13,15 @@ type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];
const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {
const query = (tx || db.replicaNode())(TableName.SecretSync)
.join(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
.leftJoin(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(TableName.AppConnection, `${TableName.SecretSync}.connectionId`, `${TableName.AppConnection}.id`)
.select(selectAllTableCols(TableName.SecretSync))
.select(
// evironment
// environment
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
// entire connection
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
@@ -53,7 +52,7 @@ const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: Secre
const expandSecretSync = (
secretSync: Awaited<ReturnType<typeof baseSecretSyncQuery>>[number],
folder: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
folder?: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
) => {
const {
envId,
@@ -75,7 +74,7 @@ const expandSecretSync = (
return {
...el,
connectionId,
environment: { id: envId, name: envName, slug: envSlug },
environment: envId ? { id: envId, name: envName, slug: envSlug } : null,
connection: {
app: connectionApp,
id: connectionId,
@@ -88,10 +87,12 @@ const expandSecretSync = (
updatedAt: connectionUpdatedAt,
version: connectionVersion
},
folder: {
id: folder!.id,
path: folder!.path
}
folder: folder
? {
id: folder.id,
path: folder.path
}
: null
};
};
@@ -111,7 +112,9 @@ export const secretSyncDALFactory = (
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
@@ -132,7 +135,9 @@ export const secretSyncDALFactory = (
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Create - Secret Sync" });
@@ -152,7 +157,9 @@ export const secretSyncDALFactory = (
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Update by ID - Secret Sync" });
@@ -165,7 +172,9 @@ export const secretSyncDALFactory = (
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
@@ -181,7 +190,7 @@ export const secretSyncDALFactory = (
const foldersWithPath = await folderDAL.findSecretPathByFolderIds(
secretSyncs[0].projectId,
secretSyncs.map((sync) => sync.folderId)
secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)
);
// TODO (scott): replace with cached folder path once implemented
@@ -191,7 +200,9 @@ export const secretSyncDALFactory = (
if (folder) folderRecord[folder.id] = folder;
});
return secretSyncs.map((secretSync) => expandSecretSync(secretSync, folderRecord[secretSync.folderId]));
return secretSyncs.map((secretSync) =>
expandSecretSync(secretSync, secretSync.folderId ? folderRecord[secretSync.folderId] : undefined)
);
} catch (error) {
throw new DatabaseError({ error, name: "Find - Secret Sync" });
}
backend/src/services/secret-sync/secret-sync-errors.ts
+14
View File
@@ -0,0 +1,14 @@
export class SecretSyncError extends Error {
name: string;
error: unknown;
secretKey?: string;
constructor({ name, error, secretKey }: { name?: string; error?: unknown; secretKey?: string } = {}) {
super();
this.name = name || "SecretSyncError";
this.error = error;
this.secretKey = secretKey;
}
}
backend/src/services/secret-sync/secret-sync-fns.ts
+21 -5
View File
@@ -1,9 +1,12 @@
import { AxiosError } from "axios";
import {
AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
AwsParameterStoreSyncFns
} from "@app/services/secret-sync/aws-parameter-store";
import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import {
TSecretMap,
TSecretSyncListItem,
@@ -59,8 +62,6 @@ const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretM
return secretMap;
};
// TODO(scott): ideally do this in a map to reduce code but requires typescript trickery...
export const SecretSyncFns = {
syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
@@ -76,14 +77,14 @@ export const SecretSyncFns = {
);
}
},
importSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
getSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
let secretMap: TSecretMap;
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
secretMap = await AwsParameterStoreSyncFns.importSecrets(secretSync);
secretMap = await AwsParameterStoreSyncFns.getSecrets(secretSync);
break;
case SecretSync.GitHub:
secretMap = await GithubSyncFns.importSecrets(secretSync);
secretMap = await GithubSyncFns.getSecrets(secretSync);
break;
default:
throw new Error(
@@ -108,3 +109,18 @@ export const SecretSyncFns = {
}
}
};
export const parseSyncErrorMessage = (err: unknown): string => {
if (err instanceof SecretSyncError) {
return JSON.stringify({
secretKey: err.secretKey,
error: parseSyncErrorMessage(err.error)
});
}
if (err instanceof AxiosError) {
return err?.response?.data ? JSON.stringify(err?.response?.data) : err?.message ?? "An unknown error occurred.";
}
return (err as Error)?.message || "An unknown error occurred.";
};
backend/src/services/secret-sync/secret-sync-queue.ts
+42 -64
View File
@@ -31,7 +31,7 @@ import {
SecretSyncImportBehavior,
SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import {
SecretSyncAction,
@@ -74,7 +74,7 @@ type TSecretSyncQueueFactoryDep = {
| "deleteMany"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById">;
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById" | "deleteById">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: TProjectDALFactory;
@@ -94,19 +94,6 @@ type SecretSyncActionJob = Job<
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;
const getRequeueDelay = (failureCount?: number) => {
if (!failureCount) return 0;
const baseDelay = 1000;
const maxDelay = 30000;
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
const jitter = delay * (0.5 + Math.random() * 0.5);
return jitter;
};
export const secretSyncQueueFactory = ({
queueService,
kmsService,
@@ -178,12 +165,12 @@ export const secretSyncQueueFactory = ({
});
const $getSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, includeImports = true) => {
const {
projectId,
folderId,
environment: { slug: environmentSlug },
folder: { path: secretPath }
} = secretSync;
const { projectId, folderId, environment, folder } = secretSync;
if (!folderId || !environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
const secretMap: TSecretMap = {};
@@ -210,8 +197,8 @@ export const secretSyncQueueFactory = ({
const secretKey = secret.key;
const secretValue = decryptSecretValue(secret.encryptedValue);
const expandedSecretValue = await expandSecretReferences({
environment: environmentSlug,
secretPath,
environment: environment.slug,
secretPath: folder.path,
skipMultilineEncoding: secret.skipMultilineEncoding,
value: secretValue
});
@@ -260,7 +247,7 @@ export const secretSyncQueueFactory = ({
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
delay: getRequeueDelay(payload.failedToAcquireLockCount),
delay: payload.failedToAcquireLockCount ? 1000 : 0, // we don't want to delay initial job
attempts: 5,
backoff: {
type: "exponential",
@@ -309,9 +296,14 @@ export const secretSyncQueueFactory = ({
secretSync: TSecretSyncWithCredentials,
importBehavior: SecretSyncImportBehavior
): Promise<TSecretMap> => {
const { projectId, environment } = secretSync;
const { projectId, environment, folder } = secretSync;
const importedSecrets = await SecretSyncFns.importSecrets(secretSync);
if (!environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
const importedSecrets = await SecretSyncFns.getSecrets(secretSync);
if (!Object.keys(importedSecrets).length) return {};
@@ -345,7 +337,7 @@ export const secretSyncQueueFactory = ({
if (secretsToCreate.length) {
await $createManySecretsRawFn({
projectId,
path: secretSync.folder.path,
path: folder.path,
environment: environment.slug,
secrets: secretsToCreate
});
@@ -354,7 +346,7 @@ export const secretSyncQueueFactory = ({
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
await $updateManySecretsRawFn({
projectId,
path: secretSync.folder.path,
path: folder.path,
environment: environment.slug,
secrets: secretsToUpdate
});
@@ -444,13 +436,7 @@ export const secretSyncQueueFactory = ({
});
}
syncMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
syncMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -566,13 +552,7 @@ export const secretSyncQueueFactory = ({
});
}
importMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
importMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -629,7 +609,7 @@ export const secretSyncQueueFactory = ({
const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => {
const {
data: { syncId, auditLogInfo }
data: { syncId, auditLogInfo, deleteSyncOnComplete }
} = job;
const secretSync = await secretSyncDAL.findById(syncId);
@@ -691,13 +671,7 @@ export const secretSyncQueueFactory = ({
});
}
removeMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
removeMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -731,19 +705,23 @@ export const secretSyncQueueFactory = ({
});
if (isSuccess || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
if (isSuccess && deleteSyncOnComplete) {
await secretSyncDAL.deleteById(secretSync.id);
} else {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
}
}
}
}
@@ -806,8 +784,8 @@ export const secretSyncQueueFactory = ({
syncDestination,
content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
failureMessage,
secretPath: folder.path,
environment: environment.name,
secretPath: folder?.path,
environment: environment?.name,
projectName: project.name,
syncUrl: `${appCfg.SITE_URL}/integrations/secret-syncs/${destination}/${secretSync.id}`
}
backend/src/services/secret-sync/secret-sync-schemas.ts
+2 -2
View File
@@ -43,8 +43,8 @@ export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?
name: z.string(),
id: z.string().uuid()
}),
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }),
folder: z.object({ id: z.string(), path: z.string() })
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }).nullable(),
folder: z.object({ id: z.string(), path: z.string() }).nullable()
});
export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
backend/src/services/secret-sync/secret-sync-service.ts
+100 -32
View File
@@ -2,7 +2,11 @@ import { ForbiddenError, subject } from "@casl/ability";
import { ActionProjectType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import {
ProjectPermissionActions,
ProjectPermissionSecretSyncActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
@@ -65,15 +69,14 @@ export const secretSyncServiceFactory = ({
projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
const folders = await folderDAL.findByProjectId(projectId);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
const secretSyncs = await secretSyncDAL.find({
...(destination && { destination }),
$in: {
folderId: folders.map((folder) => folder.id)
}
projectId
});
return secretSyncs as TSecretSync[];
@@ -96,7 +99,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -134,7 +140,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -163,7 +172,7 @@ export const secretSyncServiceFactory = ({
throw new BadRequestError({ message: "Project version does not support Secret Syncs" });
ForbiddenError.from(projectPermission).throwUnlessCan(
ProjectPermissionActions.Create,
ProjectPermissionSecretSyncActions.Create,
ProjectPermissionSub.SecretSyncs
);
@@ -187,17 +196,13 @@ export const secretSyncServiceFactory = ({
// validates permission to connect and app is valid for sync destination
await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor);
const projectFolders = await folderDAL.findByProjectId(folder.projectId);
const secretSync = await secretSyncDAL.transaction(async (tx) => {
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
projectId
},
tx
)
@@ -212,7 +217,8 @@ export const secretSyncServiceFactory = ({
const sync = await secretSyncDAL.create({
folderId: folder.id,
...params,
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending })
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending }),
projectId
});
return sync;
@@ -243,7 +249,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Edit,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -251,12 +260,17 @@ export const secretSyncServiceFactory = ({
});
const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => {
let { folderId } = secretSync;
if (
(secretPath && secretPath !== secretSync.folder.path) ||
(environment && environment !== secretSync.environment.slug)
(secretPath && secretPath !== secretSync.folder?.path) ||
(environment && environment !== secretSync.environment?.slug)
) {
const updatedEnvironment = environment ?? secretSync.environment.slug;
const updatedSecretPath = secretPath ?? secretSync.folder.path;
const updatedEnvironment = environment ?? secretSync.environment?.slug;
const updatedSecretPath = secretPath ?? secretSync.folder?.path;
if (!updatedEnvironment || !updatedSecretPath)
throw new BadRequestError({ message: "Must specify both source environment and secret path" });
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
@@ -272,19 +286,17 @@ export const secretSyncServiceFactory = ({
throw new BadRequestError({
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${secretSync.projectId}"`
});
folderId = newFolder.id;
}
if (params.name && secretSync.name !== params.name) {
const projectFolders = await folderDAL.findByProjectId(secretSync.projectId);
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
projectId: secretSync.projectId
},
tx
)
@@ -301,7 +313,8 @@ export const secretSyncServiceFactory = ({
const updatedSync = await secretSyncDAL.updateById(syncId, {
...params,
...(isEnabled && { syncStatus: SecretSyncStatus.Pending })
...(isEnabled && folderId && { syncStatus: SecretSyncStatus.Pending }),
folderId
});
return updatedSync;
@@ -312,7 +325,10 @@ export const secretSyncServiceFactory = ({
return updatedSecretSync as TSecretSync;
};
const deleteSecretSync = async ({ destination, syncId }: TDeleteSecretSyncDTO, actor: OrgServiceActor) => {
const deleteSecretSync = async (
{ destination, syncId, removeSecrets }: TDeleteSecretSyncDTO,
actor: OrgServiceActor
) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
@@ -329,13 +345,41 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Delete,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (removeSecrets) {
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.RemoveSecrets,
ProjectPermissionSub.SecretSyncs
);
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, deleteSyncOnComplete: true });
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Pending
});
return updatedSecretSync;
}
await secretSyncDAL.deleteById(syncId);
return secretSync as TSecretSync;
@@ -361,13 +405,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.SyncSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
@@ -408,13 +460,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.ImportSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
@@ -449,13 +509,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.RemoveSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
backend/src/services/secret-sync/secret-sync-types.ts
+2
View File
@@ -62,6 +62,7 @@ export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "connectio
export type TDeleteSecretSyncDTO = {
destination: SecretSync;
syncId: string;
removeSecrets: boolean;
};
type AuditLogInfo = Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
@@ -110,6 +111,7 @@ export type TTriggerSecretSyncImportSecretsByIdDTO = {
export type TQueueSecretSyncRemoveSecretsByIdDTO = {
syncId: string;
auditLogInfo?: AuditLogInfo;
deleteSyncOnComplete?: boolean;
};
export type TTriggerSecretSyncRemoveSecretsByIdDTO = {
backend/src/services/smtp/templates/secretSyncFailed.handlebars
+6 -2
View File
@@ -21,8 +21,12 @@
<p><strong>Name</strong>: {{syncName}}</p>
<p><strong>Destination</strong>: {{syncDestination}}</p>
<p><strong>Project</strong>: {{projectName}}</p>
<p><strong>Environment</strong>: {{environment}}</p>
<p><strong>Secret Path</strong>: {{secretPath}}</p>
{{#if environment}}
<p><strong>Environment</strong>: {{environment}}</p>
{{/if}}
{{#if secretPath}}
<p><strong>Secret Path</strong>: {{secretPath}}</p>
{{/if}}
</div>
{{#if failureMessage}}