improvements: address feedback

This commit is contained in:
Scott Wilson
2025-01-20 22:17:20 -08:00
parent 6341b7e989
commit 3c1fc024c2
57 changed files with 981 additions and 432 deletions

View File

@@ -14,8 +14,12 @@ export async function up(knex: Knex): Promise<void> {
t.integer("version").defaultTo(1).notNullable();
t.jsonb("destinationConfig").notNullable();
t.jsonb("syncOptions").notNullable();
t.uuid("folderId").notNullable();
t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE");
// we're including projectId in addition to folder ID because we allow folderId to be null (if the folder
// is deleted), to preserve sync configuration
t.string("projectId").notNullable();
t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
t.uuid("folderId");
t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("SET NULL");
t.uuid("connectionId").notNullable();
t.foreign("connectionId").references("id").inTable(TableName.AppConnection);
t.timestamps(true, true, true);

View File

@@ -16,7 +16,8 @@ export const SecretSyncsSchema = z.object({
version: z.number().default(1),
destinationConfig: z.unknown(),
syncOptions: z.unknown(),
folderId: z.string().uuid(),
projectId: z.string(),
folderId: z.string().uuid().nullable().optional(),
connectionId: z.string().uuid(),
createdAt: z.date(),
updatedAt: z.date(),

View File

@@ -1,7 +1,6 @@
import { z } from "zod";
import { OrgMembershipRole, OrgMembershipsSchema, OrgRolesSchema } from "@app/db/schemas";
import { OrgPermissionSchema } from "@app/ee/services/permission/org-permission";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { slugSchema } from "@app/server/lib/schemas";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
@@ -25,7 +24,8 @@ export const registerOrgRoleRouter = async (server: FastifyZodProvider) => {
),
name: z.string().trim(),
description: z.string().trim().nullish(),
permissions: OrgPermissionSchema.array()
// TODO(scott): once UI refactored permissions: OrgPermissionSchema.array()
permissions: z.any().array()
}),
response: {
200: z.object({
@@ -97,7 +97,8 @@ export const registerOrgRoleRouter = async (server: FastifyZodProvider) => {
.optional(),
name: z.string().trim().optional(),
description: z.string().trim().nullish(),
permissions: OrgPermissionSchema.array().optional()
// TODO(scott): once UI refactored permissions: OrgPermissionSchema.array().optional()
permissions: z.any().array().optional()
}),
response: {
200: z.object({

View File

@@ -34,6 +34,16 @@ export enum ProjectPermissionDynamicSecretActions {
Lease = "lease"
}
export enum ProjectPermissionSecretSyncActions {
Read = "read",
Create = "create",
Edit = "edit",
Delete = "delete",
SyncSecrets = "sync-secrets",
ImportSecrets = "import-secrets",
RemoveSecrets = "remove-secrets"
}
export enum ProjectPermissionSub {
Role = "role",
Member = "member",
@@ -145,7 +155,7 @@ export type ProjectPermissionSet =
| [ProjectPermissionActions, ProjectPermissionSub.SshCertificateTemplates]
| [ProjectPermissionActions, ProjectPermissionSub.PkiAlerts]
| [ProjectPermissionActions, ProjectPermissionSub.PkiCollections]
| [ProjectPermissionActions, ProjectPermissionSub.SecretSyncs]
| [ProjectPermissionSecretSyncActions, ProjectPermissionSub.SecretSyncs]
| [ProjectPermissionCmekActions, ProjectPermissionSub.Cmek]
| [ProjectPermissionActions.Delete, ProjectPermissionSub.Project]
| [ProjectPermissionActions.Edit, ProjectPermissionSub.Project]
@@ -396,7 +406,7 @@ const GeneralPermissionSchema = [
}),
z.object({
subject: z.literal(ProjectPermissionSub.SecretSyncs).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionActions).describe(
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretSyncActions).describe(
"Describe what action an entity can take."
)
})
@@ -514,8 +524,7 @@ const buildAdminPermissionRules = () => {
ProjectPermissionSub.PkiCollections,
ProjectPermissionSub.SshCertificateAuthorities,
ProjectPermissionSub.SshCertificates,
ProjectPermissionSub.SshCertificateTemplates,
ProjectPermissionSub.SecretSyncs
ProjectPermissionSub.SshCertificateTemplates
].forEach((el) => {
can(
[
@@ -553,6 +562,18 @@ const buildAdminPermissionRules = () => {
],
ProjectPermissionSub.Cmek
);
can(
[
ProjectPermissionSecretSyncActions.Create,
ProjectPermissionSecretSyncActions.Edit,
ProjectPermissionSecretSyncActions.Delete,
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSecretSyncActions.SyncSecrets,
ProjectPermissionSecretSyncActions.ImportSecrets,
ProjectPermissionSecretSyncActions.RemoveSecrets
],
ProjectPermissionSub.SecretSyncs
);
return rules;
};
@@ -719,10 +740,13 @@ const buildMemberPermissionRules = () => {
can(
[
ProjectPermissionActions.Read,
ProjectPermissionActions.Edit,
ProjectPermissionActions.Create,
ProjectPermissionActions.Delete
ProjectPermissionSecretSyncActions.Create,
ProjectPermissionSecretSyncActions.Edit,
ProjectPermissionSecretSyncActions.Delete,
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSecretSyncActions.SyncSecrets,
ProjectPermissionSecretSyncActions.ImportSecrets,
ProjectPermissionSecretSyncActions.RemoveSecrets
],
ProjectPermissionSub.SecretSyncs
);
@@ -760,7 +784,7 @@ const buildViewerPermissionRules = () => {
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateAuthorities);
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates);
can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates);
can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs);
return rules;
};

View File

@@ -1688,7 +1688,8 @@ export const SecretSyncs = {
};
},
DELETE: (destination: SecretSync) => ({
syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to be deleted.`
syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to be deleted.`,
removeSecrets: `Whether previously synced secrets should be removed prior to deletion.`
}),
SYNC_SECRETS: (destination: SecretSync) => ({
syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to trigger a sync for.`

View File

@@ -16,7 +16,3 @@ export const prefixWithSlash = (str: string) => {
};
export const startsWithVowel = (str: string) => /^[aeiou]/i.test(str);
export const wrapWithSlashes = (str: string) => {
return `${str.startsWith("/") ? "" : "/"}${str}${str.endsWith("/") ? "" : `/`}`;
};

View File

@@ -143,7 +143,7 @@ export const registerAppConnectionEndpoints = <T extends TAppConnection, I exten
server.route({
method: "GET",
url: `/name/:connectionName`,
url: `/connection-name/:connectionName`,
config: {
rateLimit: readLimit
},

View File

@@ -127,7 +127,7 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
server.route({
method: "GET",
url: `/name/:syncName`,
url: `/sync-name/:syncName`,
config: {
rateLimit: readLimit
},
@@ -219,7 +219,7 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
rateLimit: writeLimit
},
schema: {
description: `Update the specified ${destinationName} Connection.`,
description: `Update the specified ${destinationName} Sync.`,
params: z.object({
syncId: z.string().uuid().describe(SecretSyncs.UPDATE(destination).syncId)
}),
@@ -261,10 +261,17 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
rateLimit: writeLimit
},
schema: {
description: `Delete the specified ${destinationName} Connection.`,
description: `Delete the specified ${destinationName} Sync.`,
params: z.object({
syncId: z.string().uuid().describe(SecretSyncs.DELETE(destination).syncId)
}),
querystring: z.object({
removeSecrets: z
.enum(["true", "false"])
.default("false")
.transform((value) => value === "true")
.describe(SecretSyncs.DELETE(destination).removeSecrets)
}),
response: {
200: z.object({ secretSync: responseSchema })
}
@@ -272,9 +279,10 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { syncId } = req.params;
const { removeSecrets } = req.query;
const secretSync = (await server.services.secretSync.deleteSecretSync(
{ destination, syncId },
{ destination, syncId, removeSecrets },
req.permission
)) as T;
@@ -285,7 +293,8 @@ export const registerSyncSecretsEndpoints = <T extends TSecretSync, I extends TS
type: EventType.DELETE_SECRET_SYNC,
metadata: {
destination,
syncId
syncId,
removeSecrets
}
}
});

View File

@@ -1,6 +1,7 @@
import AWS, { AWSError } from "aws-sdk";
import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAwsParameterStoreSyncWithCredentials } from "./aws-parameter-store-sync-types";
@@ -146,12 +147,19 @@ export const AwsParameterStoreSyncFns = {
continue;
}
await putParameter(ssm, {
Name: `${destinationConfig.path}${key}`,
Type: "SecureString",
Value: value,
Overwrite: true
});
try {
await putParameter(ssm, {
Name: `${destinationConfig.path}${key}`,
Type: "SecureString",
Value: value,
Overwrite: true
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
}
const parametersToDelete: AWS.SSM.Parameter[] = [];
@@ -166,7 +174,7 @@ export const AwsParameterStoreSyncFns = {
await deleteParametersBatch(ssm, parametersToDelete);
},
importSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
getSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);

View File

@@ -1,7 +1,6 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { wrapWithSlashes } from "@app/lib/fn";
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
@@ -14,23 +13,10 @@ const AwsParameterStoreSyncDestinationConfigSchema = z.object({
region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.REGION),
path: z
.string()
.min(1, "Parameter Store Path Required")
.transform(wrapWithSlashes)
.superRefine((val, ctx) => {
if (!/^\/([/]|(([\w-]+\/)+))?$/.test(val)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - must follow "/example/path/" format`
});
}
if (val.length > 2048) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - cannot exceed 2048 characters`
});
}
})
.trim()
.min(1, "Parameter Store Path required")
.max(2048, "Cannot exceed 2048 characters")
.regex(/^\/([/]|(([\w-]+\/)+))?$/)
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.PATH)
});

View File

@@ -6,20 +6,12 @@ import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TGitHubSyncWithCredentials } from "./github-sync-types";
interface GitHubSecret {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
}
import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithCredentials } from "./github-sync-types";
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: GitHubSecret[];
let encryptedSecrets: TGitHubSecret[];
const { destinationConfig } = secretSync;
@@ -52,17 +44,8 @@ const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithC
return encryptedSecrets;
};
interface GitHubPublicKey {
key_id: string;
key: string;
id?: number | undefined;
url?: string | undefined;
title?: string | undefined;
created_at?: string | undefined;
}
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: GitHubPublicKey;
let publicKey: TGitHubPublicKey;
const { destinationConfig } = secretSync;
@@ -100,7 +83,11 @@ const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredenti
return publicKey;
};
const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, encryptedSecret: GitHubSecret) => {
const deleteSecret = async (
client: Octokit,
secretSync: TGitHubSyncWithCredentials,
encryptedSecret: TGitHubSecret
) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
@@ -132,13 +119,7 @@ const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredenti
}
};
interface GitHubSecretPayload {
key_id: string;
secret_name: string;
encrypted_value: string;
}
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: GitHubSecretPayload) => {
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
@@ -210,7 +191,7 @@ export const GithubSyncFns = {
}
});
},
importSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {

View File

@@ -32,24 +32,24 @@ const GitHubSyncDestinationConfigSchema = z
})
])
.superRefine((options, ctx) => {
if (options.scope !== GitHubSyncScope.Organization) return;
if (options.scope === GitHubSyncScope.Organization) {
if (options.visibility === GitHubSyncVisibility.Selected) {
if (!options.selectedRepositoryIds?.length)
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Select at least 1 repository",
path: ["selectedRepositoryIds"]
});
return;
}
if (options.visibility === GitHubSyncVisibility.Selected) {
if (!options.selectedRepositoryIds?.length)
if (options.selectedRepositoryIds?.length) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Select at least 1 repository",
message: `Selected repositories is only supported for visibility "Selected"`,
path: ["selectedRepositoryIds"]
});
return;
}
if (options.selectedRepositoryIds?.length) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Selected repositories is only supported for visibility "Selected"`,
path: ["selectedRepositoryIds"]
});
}
}
});

View File

@@ -13,3 +13,26 @@ export type TGitHubSyncListItem = z.infer<typeof GitHubSyncListItemSchema>;
export type TGitHubSyncWithCredentials = TGitHubSync & {
connection: TGitHubConnection;
};
export type TGitHubSecret = {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
};
export type TGitHubPublicKey = {
key_id: string;
key: string;
id?: number | undefined;
url?: string | undefined;
title?: string | undefined;
created_at?: string | undefined;
};
export type TGitHubSecretPayload = {
key_id: string;
secret_name: string;
encrypted_value: string;
};

View File

@@ -13,16 +13,15 @@ type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];
const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {
const query = (tx || db.replicaNode())(TableName.SecretSync)
.join(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
.leftJoin(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(TableName.AppConnection, `${TableName.SecretSync}.connectionId`, `${TableName.AppConnection}.id`)
.select(selectAllTableCols(TableName.SecretSync))
.select(
// evironment
// environment
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
// entire connection
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
@@ -53,7 +52,7 @@ const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: Secre
const expandSecretSync = (
secretSync: Awaited<ReturnType<typeof baseSecretSyncQuery>>[number],
folder: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
folder?: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
) => {
const {
envId,
@@ -75,7 +74,7 @@ const expandSecretSync = (
return {
...el,
connectionId,
environment: { id: envId, name: envName, slug: envSlug },
environment: envId ? { id: envId, name: envName, slug: envSlug } : null,
connection: {
app: connectionApp,
id: connectionId,
@@ -88,10 +87,12 @@ const expandSecretSync = (
updatedAt: connectionUpdatedAt,
version: connectionVersion
},
folder: {
id: folder!.id,
path: folder!.path
}
folder: folder
? {
id: folder.id,
path: folder.path
}
: null
};
};
@@ -111,7 +112,9 @@ export const secretSyncDALFactory = (
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
@@ -132,7 +135,9 @@ export const secretSyncDALFactory = (
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Create - Secret Sync" });
@@ -152,7 +157,9 @@ export const secretSyncDALFactory = (
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Update by ID - Secret Sync" });
@@ -165,7 +172,9 @@ export const secretSyncDALFactory = (
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
const [folderWithPath] = secretSync.folderId
? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
: [];
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
@@ -181,7 +190,7 @@ export const secretSyncDALFactory = (
const foldersWithPath = await folderDAL.findSecretPathByFolderIds(
secretSyncs[0].projectId,
secretSyncs.map((sync) => sync.folderId)
secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)
);
// TODO (scott): replace with cached folder path once implemented
@@ -191,7 +200,9 @@ export const secretSyncDALFactory = (
if (folder) folderRecord[folder.id] = folder;
});
return secretSyncs.map((secretSync) => expandSecretSync(secretSync, folderRecord[secretSync.folderId]));
return secretSyncs.map((secretSync) =>
expandSecretSync(secretSync, secretSync.folderId ? folderRecord[secretSync.folderId] : undefined)
);
} catch (error) {
throw new DatabaseError({ error, name: "Find - Secret Sync" });
}

View File

@@ -0,0 +1,14 @@
export class SecretSyncError extends Error {
name: string;
error: unknown;
secretKey?: string;
constructor({ name, error, secretKey }: { name?: string; error?: unknown; secretKey?: string } = {}) {
super();
this.name = name || "SecretSyncError";
this.error = error;
this.secretKey = secretKey;
}
}

View File

@@ -1,9 +1,12 @@
import { AxiosError } from "axios";
import {
AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
AwsParameterStoreSyncFns
} from "@app/services/secret-sync/aws-parameter-store";
import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import {
TSecretMap,
TSecretSyncListItem,
@@ -59,8 +62,6 @@ const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretM
return secretMap;
};
// TODO(scott): ideally do this in a map to reduce code but requires typescript trickery...
export const SecretSyncFns = {
syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
@@ -76,14 +77,14 @@ export const SecretSyncFns = {
);
}
},
importSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
getSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
let secretMap: TSecretMap;
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
secretMap = await AwsParameterStoreSyncFns.importSecrets(secretSync);
secretMap = await AwsParameterStoreSyncFns.getSecrets(secretSync);
break;
case SecretSync.GitHub:
secretMap = await GithubSyncFns.importSecrets(secretSync);
secretMap = await GithubSyncFns.getSecrets(secretSync);
break;
default:
throw new Error(
@@ -108,3 +109,18 @@ export const SecretSyncFns = {
}
}
};
export const parseSyncErrorMessage = (err: unknown): string => {
if (err instanceof SecretSyncError) {
return JSON.stringify({
secretKey: err.secretKey,
error: parseSyncErrorMessage(err.error)
});
}
if (err instanceof AxiosError) {
return err?.response?.data ? JSON.stringify(err?.response?.data) : err?.message ?? "An unknown error occurred.";
}
return (err as Error)?.message || "An unknown error occurred.";
};

View File

@@ -31,7 +31,7 @@ import {
SecretSyncImportBehavior,
SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import {
SecretSyncAction,
@@ -74,7 +74,7 @@ type TSecretSyncQueueFactoryDep = {
| "deleteMany"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById">;
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById" | "deleteById">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: TProjectDALFactory;
@@ -94,19 +94,6 @@ type SecretSyncActionJob = Job<
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;
const getRequeueDelay = (failureCount?: number) => {
if (!failureCount) return 0;
const baseDelay = 1000;
const maxDelay = 30000;
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
const jitter = delay * (0.5 + Math.random() * 0.5);
return jitter;
};
export const secretSyncQueueFactory = ({
queueService,
kmsService,
@@ -178,12 +165,12 @@ export const secretSyncQueueFactory = ({
});
const $getSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, includeImports = true) => {
const {
projectId,
folderId,
environment: { slug: environmentSlug },
folder: { path: secretPath }
} = secretSync;
const { projectId, folderId, environment, folder } = secretSync;
if (!folderId || !environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
const secretMap: TSecretMap = {};
@@ -210,8 +197,8 @@ export const secretSyncQueueFactory = ({
const secretKey = secret.key;
const secretValue = decryptSecretValue(secret.encryptedValue);
const expandedSecretValue = await expandSecretReferences({
environment: environmentSlug,
secretPath,
environment: environment.slug,
secretPath: folder.path,
skipMultilineEncoding: secret.skipMultilineEncoding,
value: secretValue
});
@@ -260,7 +247,7 @@ export const secretSyncQueueFactory = ({
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
delay: getRequeueDelay(payload.failedToAcquireLockCount),
delay: payload.failedToAcquireLockCount ? 1000 : 0, // we don't want to delay initial job
attempts: 5,
backoff: {
type: "exponential",
@@ -309,9 +296,14 @@ export const secretSyncQueueFactory = ({
secretSync: TSecretSyncWithCredentials,
importBehavior: SecretSyncImportBehavior
): Promise<TSecretMap> => {
const { projectId, environment } = secretSync;
const { projectId, environment, folder } = secretSync;
const importedSecrets = await SecretSyncFns.importSecrets(secretSync);
if (!environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
const importedSecrets = await SecretSyncFns.getSecrets(secretSync);
if (!Object.keys(importedSecrets).length) return {};
@@ -345,7 +337,7 @@ export const secretSyncQueueFactory = ({
if (secretsToCreate.length) {
await $createManySecretsRawFn({
projectId,
path: secretSync.folder.path,
path: folder.path,
environment: environment.slug,
secrets: secretsToCreate
});
@@ -354,7 +346,7 @@ export const secretSyncQueueFactory = ({
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
await $updateManySecretsRawFn({
projectId,
path: secretSync.folder.path,
path: folder.path,
environment: environment.slug,
secrets: secretsToUpdate
});
@@ -444,13 +436,7 @@ export const secretSyncQueueFactory = ({
});
}
syncMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
syncMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -566,13 +552,7 @@ export const secretSyncQueueFactory = ({
});
}
importMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
importMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -629,7 +609,7 @@ export const secretSyncQueueFactory = ({
const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => {
const {
data: { syncId, auditLogInfo }
data: { syncId, auditLogInfo, deleteSyncOnComplete }
} = job;
const secretSync = await secretSyncDAL.findById(syncId);
@@ -691,13 +671,7 @@ export const secretSyncQueueFactory = ({
});
}
removeMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
removeMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
@@ -731,19 +705,23 @@ export const secretSyncQueueFactory = ({
});
if (isSuccess || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
if (isSuccess && deleteSyncOnComplete) {
await secretSyncDAL.deleteById(secretSync.id);
} else {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
}
}
}
}
@@ -806,8 +784,8 @@ export const secretSyncQueueFactory = ({
syncDestination,
content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
failureMessage,
secretPath: folder.path,
environment: environment.name,
secretPath: folder?.path,
environment: environment?.name,
projectName: project.name,
syncUrl: `${appCfg.SITE_URL}/integrations/secret-syncs/${destination}/${secretSync.id}`
}

View File

@@ -43,8 +43,8 @@ export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?
name: z.string(),
id: z.string().uuid()
}),
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }),
folder: z.object({ id: z.string(), path: z.string() })
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }).nullable(),
folder: z.object({ id: z.string(), path: z.string() }).nullable()
});
export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>

View File

@@ -2,7 +2,11 @@ import { ForbiddenError, subject } from "@casl/ability";
import { ActionProjectType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import {
ProjectPermissionActions,
ProjectPermissionSecretSyncActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
@@ -65,15 +69,14 @@ export const secretSyncServiceFactory = ({
projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
const folders = await folderDAL.findByProjectId(projectId);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
const secretSyncs = await secretSyncDAL.find({
...(destination && { destination }),
$in: {
folderId: folders.map((folder) => folder.id)
}
projectId
});
return secretSyncs as TSecretSync[];
@@ -96,7 +99,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -134,7 +140,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Read,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -163,7 +172,7 @@ export const secretSyncServiceFactory = ({
throw new BadRequestError({ message: "Project version does not support Secret Syncs" });
ForbiddenError.from(projectPermission).throwUnlessCan(
ProjectPermissionActions.Create,
ProjectPermissionSecretSyncActions.Create,
ProjectPermissionSub.SecretSyncs
);
@@ -187,17 +196,13 @@ export const secretSyncServiceFactory = ({
// validates permission to connect and app is valid for sync destination
await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor);
const projectFolders = await folderDAL.findByProjectId(folder.projectId);
const secretSync = await secretSyncDAL.transaction(async (tx) => {
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
projectId
},
tx
)
@@ -212,7 +217,8 @@ export const secretSyncServiceFactory = ({
const sync = await secretSyncDAL.create({
folderId: folder.id,
...params,
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending })
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending }),
projectId
});
return sync;
@@ -243,7 +249,10 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Edit,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
@@ -251,12 +260,17 @@ export const secretSyncServiceFactory = ({
});
const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => {
let { folderId } = secretSync;
if (
(secretPath && secretPath !== secretSync.folder.path) ||
(environment && environment !== secretSync.environment.slug)
(secretPath && secretPath !== secretSync.folder?.path) ||
(environment && environment !== secretSync.environment?.slug)
) {
const updatedEnvironment = environment ?? secretSync.environment.slug;
const updatedSecretPath = secretPath ?? secretSync.folder.path;
const updatedEnvironment = environment ?? secretSync.environment?.slug;
const updatedSecretPath = secretPath ?? secretSync.folder?.path;
if (!updatedEnvironment || !updatedSecretPath)
throw new BadRequestError({ message: "Must specify both source environment and secret path" });
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
@@ -272,19 +286,17 @@ export const secretSyncServiceFactory = ({
throw new BadRequestError({
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${secretSync.projectId}"`
});
folderId = newFolder.id;
}
if (params.name && secretSync.name !== params.name) {
const projectFolders = await folderDAL.findByProjectId(secretSync.projectId);
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
projectId: secretSync.projectId
},
tx
)
@@ -301,7 +313,8 @@ export const secretSyncServiceFactory = ({
const updatedSync = await secretSyncDAL.updateById(syncId, {
...params,
...(isEnabled && { syncStatus: SecretSyncStatus.Pending })
...(isEnabled && folderId && { syncStatus: SecretSyncStatus.Pending }),
folderId
});
return updatedSync;
@@ -312,7 +325,10 @@ export const secretSyncServiceFactory = ({
return updatedSecretSync as TSecretSync;
};
const deleteSecretSync = async ({ destination, syncId }: TDeleteSecretSyncDTO, actor: OrgServiceActor) => {
const deleteSecretSync = async (
{ destination, syncId, removeSecrets }: TDeleteSecretSyncDTO,
actor: OrgServiceActor
) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
@@ -329,13 +345,41 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.Delete,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (removeSecrets) {
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.RemoveSecrets,
ProjectPermissionSub.SecretSyncs
);
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, deleteSyncOnComplete: true });
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Pending
});
return updatedSecretSync;
}
await secretSyncDAL.deleteById(syncId);
return secretSync as TSecretSync;
@@ -361,13 +405,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.SyncSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
@@ -408,13 +460,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.ImportSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
@@ -449,13 +509,21 @@ export const secretSyncServiceFactory = ({
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretSyncActions.RemoveSecrets,
ProjectPermissionSub.SecretSyncs
);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
if (!secretSync.folderId)
throw new BadRequestError({
message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)

View File

@@ -62,6 +62,7 @@ export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "connectio
export type TDeleteSecretSyncDTO = {
destination: SecretSync;
syncId: string;
removeSecrets: boolean;
};
type AuditLogInfo = Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
@@ -110,6 +111,7 @@ export type TTriggerSecretSyncImportSecretsByIdDTO = {
export type TQueueSecretSyncRemoveSecretsByIdDTO = {
syncId: string;
auditLogInfo?: AuditLogInfo;
deleteSyncOnComplete?: boolean;
};
export type TTriggerSecretSyncRemoveSecretsByIdDTO = {

View File

@@ -21,8 +21,12 @@
<p><strong>Name</strong>: {{syncName}}</p>
<p><strong>Destination</strong>: {{syncDestination}}</p>
<p><strong>Project</strong>: {{projectName}}</p>
<p><strong>Environment</strong>: {{environment}}</p>
<p><strong>Secret Path</strong>: {{secretPath}}</p>
{{#if environment}}
<p><strong>Environment</strong>: {{environment}}</p>
{{/if}}
{{#if secretPath}}
<p><strong>Secret Path</strong>: {{secretPath}}</p>
{{/if}}
</div>
{{#if failureMessage}}