Merge remote-tracking branch 'origin' into ssh-certs

This commit is contained in:
Tuan Dang
2024-12-18 09:20:13 -08:00
56 changed files with 1381 additions and 524 deletions
+3 -1
View File
@@ -22,8 +22,10 @@ export const mockQueue = (): TQueueServiceFactory => {
listen: (name, event) => {
events[name] = event;
},
getRepeatableJobs: async () => [],
clearQueue: async () => {},
stopJobById: async () => {},
stopRepeatableJobByJobId: async () => true
stopRepeatableJobByJobId: async () => true,
stopRepeatableJobByKey: async () => true
};
};
+1
View File
@@ -1137,6 +1137,7 @@ export const INTEGRATION = {
shouldAutoRedeploy: "Used by Render to trigger auto deploy.",
secretGCPLabel: "The label for GCP secrets.",
secretAWSTag: "The tags for AWS secrets.",
azureLabel: "Define which label to assign to secrets created in Azure App Configuration.",
githubVisibility:
"Define where the secrets from the Github Integration should be visible. Option 'selected' lets you directly define which repositories to sync secrets to.",
githubVisibilityRepoIds:
+9 -1
View File
@@ -20,11 +20,12 @@ export const withTransaction = <K extends object>(db: Knex, dal: K) => ({
export type TFindFilter<R extends object = object> = Partial<R> & {
$in?: Partial<{ [k in keyof R]: R[k][] }>;
$notNull?: Array<keyof R>;
$search?: Partial<{ [k in keyof R]: R[k] }>;
$complex?: TKnexDynamicOperator<R>;
};
export const buildFindFilter =
<R extends object = object>({ $in, $search, $complex, ...filter }: TFindFilter<R>) =>
<R extends object = object>({ $in, $notNull, $search, $complex, ...filter }: TFindFilter<R>) =>
(bd: Knex.QueryBuilder<R, R>) => {
void bd.where(filter);
if ($in) {
@@ -34,6 +35,13 @@ export const buildFindFilter =
}
});
}
if ($notNull?.length) {
$notNull.forEach((key) => {
void bd.whereNotNull(key as never);
});
}
if ($search) {
Object.entries($search).forEach(([key, val]) => {
if (val) {
+14
View File
@@ -317,6 +317,13 @@ export const queueServiceFactory = (
}
};
const getRepeatableJobs = (name: QueueName, startOffset?: number, endOffset?: number) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
return q.getRepeatableJobs(startOffset, endOffset);
};
const stopRepeatableJobByJobId = async <T extends QueueName>(name: T, jobId: string) => {
const q = queueContainer[name];
const job = await q.getJob(jobId);
@@ -326,6 +333,11 @@ export const queueServiceFactory = (
return q.removeRepeatableByKey(job.repeatJobKey);
};
const stopRepeatableJobByKey = async <T extends QueueName>(name: T, repeatJobKey: string) => {
const q = queueContainer[name];
return q.removeRepeatableByKey(repeatJobKey);
};
const stopJobById = async <T extends QueueName>(name: T, jobId: string) => {
const q = queueContainer[name];
const job = await q.getJob(jobId);
@@ -349,8 +361,10 @@ export const queueServiceFactory = (
shutdown,
stopRepeatableJob,
stopRepeatableJobByJobId,
stopRepeatableJobByKey,
clearQueue,
stopJobById,
getRepeatableJobs,
startPg,
queuePg
};
+54 -42
View File
@@ -551,7 +551,11 @@ export const registerRoutes = async (
const orgService = orgServiceFactory({
userAliasDAL,
queueService,
identityMetadataDAL,
secretDAL,
secretV2BridgeDAL,
folderDAL,
licenseService,
samlConfigDAL,
orgRoleDAL,
@@ -572,6 +576,7 @@ export const registerRoutes = async (
groupDAL,
orgBotDAL,
oidcConfigDAL,
loginService,
projectBotService
});
const signupService = authSignupServiceFactory({
@@ -805,10 +810,58 @@ export const registerRoutes = async (
projectTemplateDAL
});
const integrationAuthService = integrationAuthServiceFactory({
integrationAuthDAL,
integrationDAL,
permissionService,
projectBotService,
kmsService
});
const secretQueueService = secretQueueFactory({
keyStore,
queueService,
secretDAL,
folderDAL,
integrationAuthService,
projectBotService,
integrationDAL,
secretImportDAL,
projectEnvDAL,
webhookDAL,
orgDAL,
auditLogService,
userDAL,
projectMembershipDAL,
smtpService,
projectDAL,
projectBotDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
secretRotationDAL,
integrationAuthDAL,
snapshotDAL,
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL,
projectKeyDAL,
projectUserMembershipRoleDAL,
orgService
});
const projectService = projectServiceFactory({
permissionService,
projectDAL,
secretDAL,
secretV2BridgeDAL,
queueService,
projectQueue: projectQueueService,
projectBotService,
identityProjectDAL,
identityOrgMembershipDAL,
projectKeyDAL,
@@ -891,48 +944,6 @@ export const registerRoutes = async (
projectDAL
});
const integrationAuthService = integrationAuthServiceFactory({
integrationAuthDAL,
integrationDAL,
permissionService,
projectBotService,
kmsService
});
const secretQueueService = secretQueueFactory({
keyStore,
queueService,
secretDAL,
folderDAL,
integrationAuthService,
projectBotService,
integrationDAL,
secretImportDAL,
projectEnvDAL,
webhookDAL,
orgDAL,
auditLogService,
userDAL,
projectMembershipDAL,
smtpService,
projectDAL,
projectBotDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
secretRotationDAL,
integrationAuthDAL,
snapshotDAL,
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL,
projectKeyDAL,
projectUserMembershipRoleDAL,
orgService
});
const secretImportService = secretImportServiceFactory({
licenseService,
projectBotService,
@@ -1261,6 +1272,7 @@ export const registerRoutes = async (
auditLogDAL,
queueService,
secretVersionDAL,
secretDAL,
secretFolderVersionDAL: folderVersionDAL,
snapshotDAL,
identityAccessTokenDAL,
@@ -1185,4 +1185,50 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider)
return { spaces };
}
});
server.route({
method: "GET",
url: "/:integrationAuthId/circleci/organizations",
config: {
rateLimit: readLimit
},
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
params: z.object({
integrationAuthId: z.string().trim()
}),
response: {
200: z.object({
organizations: z
.object({
name: z.string(),
slug: z.string(),
projects: z
.object({
name: z.string(),
id: z.string()
})
.array(),
contexts: z
.object({
name: z.string(),
id: z.string()
})
.array()
})
.array()
})
}
},
handler: async (req) => {
const organizations = await server.services.integrationAuth.getCircleCIOrganizations({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
id: req.params.integrationAuthId
});
return { organizations };
}
});
};
@@ -16,6 +16,7 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { slugSchema } from "@app/server/lib/schemas";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode, MfaMethod } from "@app/services/auth/auth-type";
import { sanitizedOrganizationSchema } from "@app/services/org/org-schema";
import { integrationAuthPubSchema } from "../sanitizedSchemas";
@@ -29,9 +30,11 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
schema: {
response: {
200: z.object({
organizations: OrganizationsSchema.extend({
orgAuthMethod: z.string()
}).array()
organizations: sanitizedOrganizationSchema
.extend({
orgAuthMethod: z.string()
})
.array()
})
}
},
@@ -10,6 +10,7 @@ import {
UsersSchema
} from "@app/db/schemas";
import { ORGANIZATIONS } from "@app/lib/api-docs";
import { getConfig } from "@app/lib/config/env";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
@@ -363,21 +364,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
}),
response: {
200: z.object({
organization: OrganizationsSchema
organization: OrganizationsSchema,
accessToken: z.string()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]),
handler: async (req) => {
handler: async (req, res) => {
if (req.auth.actor !== ActorType.USER) return;
const organization = await server.services.org.deleteOrganizationById(
req.permission.id,
req.params.organizationId,
req.permission.authMethod,
req.permission.orgId
);
return { organization };
const cfg = getConfig();
const { organization, tokens } = await server.services.org.deleteOrganizationById({
userId: req.permission.id,
orgId: req.params.organizationId,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
authorizationHeader: req.headers.authorization,
userAgentHeader: req.headers["user-agent"],
ipAddress: req.realIp
});
void res.setCookie("jid", tokens.refreshToken, {
httpOnly: true,
path: "/",
sameSite: "strict",
secure: cfg.HTTPS_ENABLED
});
return { organization, accessToken: tokens.accessToken };
}
});
};
+3 -2
View File
@@ -1,10 +1,11 @@
import { z } from "zod";
import { AuthTokenSessionsSchema, OrganizationsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas";
import { AuthTokenSessionsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas";
import { ApiKeysSchema } from "@app/db/schemas/api-keys";
import { authRateLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMethod, AuthMode, MfaMethod } from "@app/services/auth/auth-type";
import { sanitizedOrganizationSchema } from "@app/services/org/org-schema";
export const registerUserRouter = async (server: FastifyZodProvider) => {
server.route({
@@ -134,7 +135,7 @@ export const registerUserRouter = async (server: FastifyZodProvider) => {
description: "Return organizations that current user is part of",
response: {
200: z.object({
organizations: OrganizationsSchema.array()
organizations: sanitizedOrganizationSchema.array()
})
}
},
@@ -12,9 +12,12 @@ export type TTokenDALFactory = ReturnType<typeof tokenDALFactory>;
export const tokenDALFactory = (db: TDbClient) => {
const authOrm = ormify(db, TableName.AuthTokens);
const findOneTokenSession = async (filter: Partial<TAuthTokenSessions>): Promise<TAuthTokenSessions | undefined> => {
const findOneTokenSession = async (
filter: Partial<TAuthTokenSessions>,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first();
const doc = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter).first();
return doc;
} catch (error) {
throw new DatabaseError({ error, name: "FindOneTokenSession" });
@@ -54,10 +57,11 @@ export const tokenDALFactory = (db: TDbClient) => {
const insertTokenSession = async (
userId: string,
ip: string,
userAgent: string
userAgent: string,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const [session] = await db(TableName.AuthTokenSession)
const [session] = await (tx || db)(TableName.AuthTokenSession)
.insert({
userId,
ip,
@@ -1,6 +1,7 @@
import crypto from "node:crypto";
import bcrypt from "bcrypt";
import { Knex } from "knex";
import { TAuthTokens, TAuthTokenSessions } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
@@ -123,14 +124,13 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu
return deletedToken?.[0];
};
const getUserTokenSession = async ({
userId,
ip,
userAgent
}: TIssueAuthTokenDTO): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent });
const getUserTokenSession = async (
{ userId, ip, userAgent }: TIssueAuthTokenDTO,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }, tx);
if (!session) {
session = await tokenDAL.insertTokenSession(userId, ip, userAgent);
session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx);
}
return session;
};
+32 -25
View File
@@ -1,5 +1,6 @@
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import { Knex } from "knex";
import { TUsers, UserDeviceSchema } from "@app/db/schemas";
import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns";
@@ -50,13 +51,13 @@ export const authLoginServiceFactory = ({
* Not exported. This is to update user device list
* If new device is found. Will be saved and a mail will be send
*/
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string) => {
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string, tx?: Knex) => {
const devices = await UserDeviceSchema.parseAsync(user.devices || []);
const isDeviceSeen = devices.some((device) => device.ip === ip && device.userAgent === userAgent);
if (!isDeviceSeen) {
const newDeviceList = devices.concat([{ ip, userAgent }]);
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) });
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }, tx);
if (user.email) {
await smtpService.sendMail({
template: SmtpTemplates.NewDeviceJoin,
@@ -97,30 +98,36 @@ export const authLoginServiceFactory = ({
* Check user device and send mail if new device
* generate the auth and refresh token. fn shared by mfa verification and login verification with mfa disabled
*/
const generateUserTokens = async ({
user,
ip,
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
}) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent);
const tokenSession = await tokenService.getUserTokenSession({
userAgent,
const generateUserTokens = async (
{
user,
ip,
userId: user.id
});
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
},
tx?: Knex
) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent, tx);
const tokenSession = await tokenService.getUserTokenSession(
{
userAgent,
ip,
userId: user.id
},
tx
);
if (!tokenSession) throw new Error("Failed to create token");
const accessToken = jwt.sign(
@@ -0,0 +1,5 @@
export type TCircleCIContext = {
id: string;
name: string;
created_at: string;
};
@@ -17,6 +17,8 @@ import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError, InternalServerError, NotFoundError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { TGenericPermission, TProjectPermission } from "@app/lib/types";
import { TIntegrationDALFactory } from "../integration/integration-dal";
@@ -24,6 +26,7 @@ import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { getApps } from "./integration-app-list";
import { TCircleCIContext } from "./integration-app-types";
import { TIntegrationAuthDALFactory } from "./integration-auth-dal";
import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema";
import {
@@ -31,6 +34,7 @@ import {
TBitbucketEnvironment,
TBitbucketWorkspace,
TChecklyGroups,
TCircleCIOrganization,
TDeleteIntegrationAuthByIdDTO,
TDeleteIntegrationAuthsDTO,
TDuplicateGithubIntegrationAuthDTO,
@@ -42,6 +46,7 @@ import {
TIntegrationAuthBitbucketEnvironmentsDTO,
TIntegrationAuthBitbucketWorkspaceDTO,
TIntegrationAuthChecklyGroupsDTO,
TIntegrationAuthCircleCIOrganizationDTO,
TIntegrationAuthGithubEnvsDTO,
TIntegrationAuthGithubOrgsDTO,
TIntegrationAuthHerokuPipelinesDTO,
@@ -1578,6 +1583,120 @@ export const integrationAuthServiceFactory = ({
return [];
};
const getCircleCIOrganizations = async ({
actorId,
actor,
actorOrgId,
actorAuthMethod,
id
}: TIntegrationAuthCircleCIOrganizationDTO) => {
const integrationAuth = await integrationAuthDAL.findById(id);
if (!integrationAuth) throw new NotFoundError({ message: `Integration auth with ID '${id}' not found` });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
integrationAuth.projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId);
const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey);
const { data: organizations }: { data: TCircleCIOrganization[] } = await request.get(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
{
headers: {
"Circle-Token": `${accessToken}`,
"Accept-Encoding": "application/json"
}
}
);
let projects: {
orgName: string;
projectName: string;
projectId?: string;
}[] = [];
try {
const projectRes = (
await request.get<{ reponame: string; username: string; vcs_url: string }[]>(
`${IntegrationUrls.CIRCLECI_API_URL}/v1.1/projects`,
{
headers: {
"Circle-Token": accessToken,
"Accept-Encoding": "application/json"
}
}
)
).data;
projects = projectRes.map((a) => ({
orgName: a.username, // username maps to unique organization name in CircleCI
projectName: a.reponame, // reponame maps to project name within an organization in CircleCI
projectId: a.vcs_url.split("/").pop() // vcs_url maps to the project id in CircleCI
}));
} catch (error) {
logger.error(error);
}
const projectsByOrg = groupBy(
projects.map((p) => ({
orgName: p.orgName,
name: p.projectName,
id: p.projectId as string
})),
(p) => p.orgName
);
const getOrgContexts = async (orgSlug: string) => {
type NextPageToken = string | null | undefined;
try {
const contexts: TCircleCIContext[] = [];
let nextPageToken: NextPageToken;
while (nextPageToken !== null) {
// eslint-disable-next-line no-await-in-loop
const { data } = await request.get<{
items: TCircleCIContext[];
next_page_token: NextPageToken;
}>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context`, {
headers: {
"Circle-Token": accessToken,
"Accept-Encoding": "application/json"
},
params: new URLSearchParams({
"owner-slug": orgSlug,
...(nextPageToken ? { "page-token": nextPageToken } : {})
})
});
contexts.push(...data.items);
nextPageToken = data.next_page_token;
}
return contexts?.map((context) => ({
name: context.name,
id: context.id
}));
} catch (error) {
logger.error(error);
}
};
return Promise.all(
organizations.map(async (org) => ({
name: org.name,
slug: org.slug,
projects: projectsByOrg[org.name] ?? [],
contexts: (await getOrgContexts(org.slug)) ?? []
}))
);
};
const deleteIntegrationAuths = async ({
projectId,
integration,
@@ -1790,6 +1909,7 @@ export const integrationAuthServiceFactory = ({
getTeamcityBuildConfigs,
getBitbucketWorkspaces,
getBitbucketEnvironments,
getCircleCIOrganizations,
getIntegrationAccessToken,
duplicateIntegrationAuth,
getOctopusDeploySpaces,
@@ -128,6 +128,10 @@ export type TGetIntegrationAuthTeamCityBuildConfigDTO = {
appId: string;
} & Omit<TProjectPermission, "projectId">;
export type TIntegrationAuthCircleCIOrganizationDTO = {
id: string;
} & Omit<TProjectPermission, "projectId">;
export type TVercelBranches = {
ref: string;
lastCommit: string;
@@ -189,6 +193,14 @@ export type TTeamCityBuildConfig = {
webUrl: string;
};
export type TCircleCIOrganization = {
id: string;
vcsType: string;
name: string;
avatarUrl: string;
slug: string;
};
export type TIntegrationsWithEnvironment = TIntegrations & {
environment?:
| {
@@ -215,6 +227,11 @@ export enum OctopusDeployScope {
// add tenant, variable set, etc.
}
export enum CircleCiScope {
Project = "project",
Context = "context"
}
export type TOctopusDeployVariableSet = {
Id: string;
OwnerId: string;
@@ -76,7 +76,6 @@ export enum IntegrationUrls {
RAILWAY_API_URL = "https://backboard.railway.app/graphql/v2",
FLYIO_API_URL = "https://api.fly.io/graphql",
CIRCLECI_API_URL = "https://circleci.com/api",
DATABRICKS_API_URL = "https:/xxxx.com/api",
TRAVISCI_API_URL = "https://api.travis-ci.com",
SUPABASE_API_URL = "https://api.supabase.com",
LARAVELFORGE_API_URL = "https://forge.laravel.com",
@@ -218,9 +217,9 @@ export const getIntegrationOptions = async () => {
docsLink: ""
},
{
name: "Circle CI",
name: "CircleCI",
slug: "circleci",
image: "Circle CI.png",
image: "CircleCI.png",
isAvailable: true,
type: "pat",
clientId: "",
@@ -0,0 +1,35 @@
export const isAzureKeyVaultReference = (uri: string) => {
const tryJsonDecode = () => {
try {
return (JSON.parse(uri) as { uri: string }).uri || uri;
} catch {
return uri;
}
};
const cleanUri = tryJsonDecode();
if (!cleanUri.startsWith("https://")) {
return false;
}
if (!cleanUri.includes(".vault.azure.net/secrets/")) {
return false;
}
// 3. Check for non-empty string between https:// and .vault.azure.net/secrets/
const parts = cleanUri.split(".vault.azure.net/secrets/");
const vaultName = parts[0].replace("https://", "");
if (!vaultName) {
return false;
}
// 4. Check for non-empty secret name
const secretParts = parts[1].split("/");
const secretName = secretParts[0];
if (!secretName) {
return false;
}
return true;
};
@@ -39,13 +39,19 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { IntegrationMetadataSchema } from "../integration/integration-schema";
import { IntegrationAuthMetadataSchema } from "./integration-auth-schema";
import { OctopusDeployScope, TIntegrationsWithEnvironment, TOctopusDeployVariableSet } from "./integration-auth-types";
import {
CircleCiScope,
OctopusDeployScope,
TIntegrationsWithEnvironment,
TOctopusDeployVariableSet
} from "./integration-auth-types";
import {
IntegrationInitialSyncBehavior,
IntegrationMappingBehavior,
Integrations,
IntegrationUrls
} from "./integration-list";
import { isAzureKeyVaultReference } from "./integration-sync-secret-fns";
const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -320,11 +326,12 @@ const syncSecretsAzureAppConfig = async ({
};
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
const azureAppConfigSecrets = (
await getCompleteAzureAppConfigValues(
`${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix || ""}*`
)
).reduce(
const azureAppConfigValuesUrl = `${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix}*${
metadata.azureLabel ? `&label=${metadata.azureLabel}` : ""
}`;
const azureAppConfigSecrets = (await getCompleteAzureAppConfigValues(azureAppConfigValuesUrl)).reduce(
(accum, entry) => {
accum[entry.key] = entry.value;
@@ -405,14 +412,24 @@ const syncSecretsAzureAppConfig = async ({
}
// create or update secrets on Azure App Config
for await (const key of Object.keys(secrets)) {
if (!(key in azureAppConfigSecrets) || secrets[key]?.value !== azureAppConfigSecrets[key]) {
await request.put(
`${integration.app}/kv/${key}?api-version=2023-11-01`,
{
value: secrets[key]?.value
value: secrets[key]?.value,
...(isAzureKeyVaultReference(secrets[key]?.value || "") && {
content_type: "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8"
})
},
{
...(metadata.azureLabel && {
params: {
label: metadata.azureLabel
}
}),
headers: {
Authorization: `Bearer ${accessToken}`
},
@@ -432,6 +449,11 @@ const syncSecretsAzureAppConfig = async ({
headers: {
Authorization: `Bearer ${accessToken}`
},
...(metadata.azureLabel && {
params: {
label: metadata.azureLabel
}
}),
// we force IPV4 because docker setup fails with ipv6
httpsAgent: new https.Agent({
family: 4
@@ -2245,102 +2267,174 @@ const syncSecretsCircleCI = async ({
secrets: Record<string, { value: string; comment?: string }>;
accessToken: string;
}) => {
const getProjectSlug = async () => {
const requestConfig = {
headers: {
"Circle-Token": accessToken,
"Accept-Encoding": "application/json"
}
};
try {
const projectDetails = (
await request.get<{ slug: string }>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
requestConfig
if (integration.scope === CircleCiScope.Context) {
// sync secrets to CircleCI
await Promise.all(
Object.keys(secrets).map(async (key) =>
request.put(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${key}`,
{
value: secrets[key].value
},
{
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
}
}
)
).data;
)
);
return projectDetails.slug;
} catch (err) {
if (err instanceof AxiosError) {
if (err.response?.data?.message !== "Not Found") {
throw new Error("Failed to get project slug from CircleCI during first attempt.");
}
}
}
// get secrets from CircleCI
const getSecretsRes = async () => {
type EnvVars = {
variable: string;
created_at: string;
updated_at: string;
context_id: string;
};
// For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
try {
const circleCiOrganization = (
await request.get<{ slug: string; name: string }[]>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
requestConfig
)
).data;
let nextPageToken: string | null | undefined;
const envVars: EnvVars[] = [];
// Case 1: This is a new integration where the organization name is stored under `integration.owner`
if (integration.owner) {
const org = circleCiOrganization.find((o) => o.name === integration.owner);
if (org) {
return `${org.slug}/${integration.app}`;
}
}
// Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
return `${circleCiOrganization[0].slug}/${integration.app}`;
} catch (err) {
throw new Error("Failed to get project slug from CircleCI during second attempt.");
}
};
const projectSlug = await getProjectSlug();
// sync secrets to CircleCI
await Promise.all(
Object.keys(secrets).map(async (key) =>
request.post(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
{
name: key,
value: secrets[key].value
},
{
while (nextPageToken !== null) {
const res = await request.get<{
items: EnvVars[];
next_page_token: string | null;
}>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable`, {
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
}
}
)
)
);
"Accept-Encoding": "application/json"
},
params: nextPageToken
? new URLSearchParams({
"page-token": nextPageToken
})
: undefined
});
// get secrets from CircleCI
const getSecretsRes = (
await request.get<{ items: { name: string }[] }>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
{
envVars.push(...res.data.items);
nextPageToken = res.data.next_page_token;
}
return envVars;
};
// delete secrets from CircleCI
await Promise.all(
(await getSecretsRes()).map(async (sec) => {
if (!(sec.variable in secrets)) {
return request.delete(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${sec.variable}`,
{
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
}
}
);
}
})
);
} else {
const getProjectSlug = async () => {
const requestConfig = {
headers: {
"Circle-Token": accessToken,
"Accept-Encoding": "application/json"
}
}
)
).data?.items;
};
// delete secrets from CircleCI
await Promise.all(
getSecretsRes.map(async (sec) => {
if (!(sec.name in secrets)) {
return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
try {
const projectDetails = (
await request.get<{ slug: string }>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
requestConfig
)
).data;
return projectDetails.slug;
} catch (err) {
if (err instanceof AxiosError) {
if (err.response?.data?.message !== "Not Found") {
throw new Error("Failed to get project slug from CircleCI during first attempt.");
}
}
}
// For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
try {
const circleCiOrganization = (
await request.get<{ slug: string; name: string }[]>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
requestConfig
)
).data;
// Case 1: This is a new integration where the organization name is stored under `integration.owner`
if (integration.owner) {
const org = circleCiOrganization.find((o) => o.name === integration.owner);
if (org) {
return `${org.slug}/${integration.app}`;
}
}
// Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
return `${circleCiOrganization[0].slug}/${integration.app}`;
} catch (err) {
throw new Error("Failed to get project slug from CircleCI during second attempt.");
}
};
const projectSlug = await getProjectSlug();
// sync secrets to CircleCI
await Promise.all(
Object.keys(secrets).map(async (key) =>
request.post(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
{
name: key,
value: secrets[key].value
},
{
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
}
}
)
)
);
// get secrets from CircleCI
const getSecretsRes = (
await request.get<{ items: { name: string }[] }>(
`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
{
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
"Accept-Encoding": "application/json"
}
});
}
})
);
}
)
).data?.items;
// delete secrets from CircleCI
await Promise.all(
getSecretsRes.map(async (sec) => {
if (!(sec.name in secrets)) {
return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
headers: {
"Circle-Token": accessToken,
"Content-Type": "application/json"
}
});
}
})
);
}
};
/**
@@ -35,6 +35,8 @@ export const IntegrationMetadataSchema = z.object({
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
azureLabel: z.string().optional().describe(INTEGRATION.CREATE.metadata.azureLabel),
githubVisibility: z
.union([z.literal("selected"), z.literal("private"), z.literal("all")])
.optional()
+16
View File
@@ -0,0 +1,16 @@
import { OrganizationsSchema } from "@app/db/schemas";
export const sanitizedOrganizationSchema = OrganizationsSchema.pick({
id: true,
name: true,
customerId: true,
slug: true,
createdAt: true,
updatedAt: true,
authEnforced: true,
scimEnabled: true,
kmsDefaultKeyId: true,
defaultMembershipRole: true,
enforceMfa: true,
selectedMfaMethod: true
});
+94 -14
View File
@@ -31,11 +31,13 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { isDisposableEmail } from "@app/lib/validator";
import { TQueueServiceFactory } from "@app/queue";
import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal";
import { ActorAuthMethod, ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type";
import { TAuthLoginFactory } from "../auth/auth-login-service";
import { ActorAuthMethod, ActorType, AuthMethod, AuthModeJwtTokenPayload, AuthTokenType } from "../auth/auth-type";
import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service";
import { TokenType } from "../auth-token/auth-token-types";
import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal";
@@ -47,6 +49,10 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TUserDALFactory } from "../user/user-dal";
import { TIncidentContactsDALFactory } from "./incident-contacts-dal";
@@ -69,6 +75,9 @@ import {
type TOrgServiceFactoryDep = {
userAliasDAL: Pick<TUserAliasDALFactory, "delete">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
orgDAL: TOrgDALFactory;
orgBotDAL: TOrgBotDALFactory;
orgRoleDAL: TOrgRoleDALFactory;
@@ -97,6 +106,8 @@ type TOrgServiceFactoryDep = {
projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "updateById">;
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany" | "create">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
loginService: Pick<TAuthLoginFactory, "generateUserTokens">;
};
export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
@@ -104,6 +115,9 @@ export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
export const orgServiceFactory = ({
userAliasDAL,
orgDAL,
secretDAL,
secretV2BridgeDAL,
folderDAL,
userDAL,
groupDAL,
orgRoleDAL,
@@ -124,7 +138,9 @@ export const orgServiceFactory = ({
projectBotDAL,
projectUserMembershipRoleDAL,
identityMetadataDAL,
projectBotService
projectBotService,
queueService,
loginService
}: TOrgServiceFactoryDep) => {
/*
* Get organization details by the organization id
@@ -419,24 +435,88 @@ export const orgServiceFactory = ({
/*
* Delete organization by id
* */
const deleteOrganizationById = async (
userId: string,
orgId: string,
actorAuthMethod: ActorAuthMethod,
actorOrgId: string | undefined
) => {
const deleteOrganizationById = async ({
userId,
authorizationHeader,
userAgentHeader,
ipAddress,
orgId,
actorAuthMethod,
actorOrgId
}: {
userId: string;
authorizationHeader?: string;
userAgentHeader?: string;
ipAddress: string;
orgId: string;
actorAuthMethod: ActorAuthMethod;
actorOrgId: string | undefined;
}) => {
const { membership } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId);
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin)
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) {
throw new ForbiddenRequestError({
name: "DeleteOrganizationById",
message: "Insufficient privileges"
});
const organization = await orgDAL.deleteById(orgId);
if (organization.customerId) {
await licenseService.removeOrgCustomer(organization.customerId);
}
return organization;
if (!authorizationHeader) {
throw new UnauthorizedError({ name: "Authorization header not set on request." });
}
if (!userAgentHeader) {
throw new BadRequestError({ name: "User agent not set on request." });
}
const cfg = getConfig();
const authToken = authorizationHeader.replace("Bearer ", "");
const decodedToken = jwt.verify(authToken, cfg.AUTH_SECRET) as AuthModeJwtTokenPayload;
if (!decodedToken.authMethod) throw new UnauthorizedError({ name: "Auth method not found on existing token" });
const response = await orgDAL.transaction(async (tx) => {
const projects = await projectDAL.find({ orgId }, { tx });
for await (const project of projects) {
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
}
const deletedOrg = await orgDAL.deleteById(orgId, tx);
if (deletedOrg.customerId) {
await licenseService.removeOrgCustomer(deletedOrg.customerId);
}
// Generate new tokens without the organization ID present
const user = await userDAL.findById(userId, tx);
const { access: accessToken, refresh: refreshToken } = await loginService.generateUserTokens(
{
user,
authMethod: decodedToken.authMethod,
ip: ipAddress,
userAgent: userAgentHeader,
isMfaVerified: decodedToken.isMfaVerified,
mfaMethod: decodedToken.mfaMethod
},
tx
);
return {
organization: deletedOrg,
tokens: {
accessToken,
refreshToken
}
};
});
return response;
};
/*
* Org membership management
+1 -1
View File
@@ -51,7 +51,7 @@ export const projectDALFactory = (db: TDbClient) => {
.join(TableName.Project, `${TableName.GroupProjectMembership}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Project}.orgId`, orgId)
.andWhere((qb) => {
if (projectType) {
if (projectType !== "all") {
void qb.where(`${TableName.Project}.type`, projectType);
}
})
@@ -17,6 +17,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TProjectPermission } from "@app/lib/types";
import { TQueueServiceFactory } from "@app/queue";
import { ActorType } from "../auth/auth-type";
import { TCertificateDALFactory } from "../certificate/certificate-dal";
@@ -31,13 +32,17 @@ import { TOrgServiceFactory } from "../org/org-service";
import { TPkiAlertDALFactory } from "../pki-alert/pki-alert-dal";
import { TPkiCollectionDALFactory } from "../pki-collection/pki-collection-dal";
import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { getPredefinedRoles } from "../project-role/project-role-fns";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal";
import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal";
import { TUserDALFactory } from "../user/user-dal";
@@ -80,7 +85,10 @@ type TProjectServiceFactoryDep = {
projectDAL: TProjectDALFactory;
projectQueue: TProjectQueueFactory;
userDAL: TUserDALFactory;
folderDAL: TSecretFolderDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "insertMany" | "findByProjectId">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "insertMany" | "find">;
identityOrgMembershipDAL: TIdentityOrgDALFactory;
identityProjectDAL: TIdentityProjectDALFactory;
@@ -101,6 +109,8 @@ type TProjectServiceFactoryDep = {
permissionService: TPermissionServiceFactory;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
orgDAL: Pick<TOrgDALFactory, "findOne">;
keyStore: Pick<TKeyStoreFactory, "deleteItem">;
projectBotDAL: Pick<TProjectBotDALFactory, "create">;
@@ -121,9 +131,13 @@ export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
export const projectServiceFactory = ({
projectDAL,
secretDAL,
secretV2BridgeDAL,
projectQueue,
projectKeyDAL,
permissionService,
queueService,
projectBotService,
orgDAL,
userDAL,
folderDAL,
@@ -436,6 +450,14 @@ export const projectServiceFactory = ({
await userDAL.deleteById(projectGhostUser.id, tx);
}
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
return delProject;
});
@@ -453,7 +475,12 @@ export const projectServiceFactory = ({
const workspaces = await projectDAL.findAllProjects(actorId, actorOrgId, type);
if (includeRoles) {
const { permission } = await permissionService.getUserOrgPermission(actorId, actorOrgId, actorAuthMethod);
const { permission } = await permissionService.getUserOrgPermission(
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
// `includeRoles` is specifically used by organization admins when inviting new users to the organizations to avoid looping redundant api calls.
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member);
@@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
@@ -16,6 +17,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
identityUniversalAuthClientSecretDAL: Pick<TIdentityUaClientSecretDALFactory, "removeExpiredClientSecrets">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretVersionV2DAL: Pick<TSecretVersionV2DALFactory, "pruneExcessVersions">;
secretDAL: Pick<TSecretDALFactory, "pruneSecretReminders">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
@@ -30,6 +32,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
secretDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL,
@@ -37,6 +40,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
+92 -1
View File
@@ -5,6 +5,8 @@ import { TDbClient } from "@app/db";
import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName, TQueueServiceFactory } from "@app/queue";
export type TSecretDALFactory = ReturnType<typeof secretDALFactory>;
@@ -339,6 +341,94 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const pruneSecretReminders = async (queueService: TQueueServiceFactory) => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
if (reminderJobs.length === 0) {
logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId);
const payload = {
$in: {
id: batchIds
}
};
const opts = {
limit: REMINDER_PRUNE_BATCH_SIZE
};
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const [secrets, secretsV2] = await Promise.all([
ormify(db, TableName.Secret).find(payload, opts),
ormify(db, TableName.SecretV2).find(payload, opts)
]);
const foundSecretIds = new Set([
...secrets.map((secret) => secret.id),
...secretsV2.map((secret) => secret.id)
]);
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
};
return {
...secretOrm,
update,
@@ -352,6 +442,7 @@ export const secretDALFactory = (db: TDbClient) => {
findByBlindIndexes,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
findAllProjectSecretValues,
pruneSecretReminders
};
};
+50
View File
@@ -19,9 +19,11 @@ import {
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
@@ -31,8 +33,10 @@ import {
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
import { KmsDataKey } from "../kms/kms-types";
import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
TCreateManySecretsRawFn,
@@ -1138,3 +1142,49 @@ export const decryptSecretWithBot = (
secretComment
};
};
type TFnDeleteProjectSecretReminders = {
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
};
export const fnDeleteProjectSecretReminders = async (
projectId: string,
{ secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders
) => {
const projectFolders = await folderDAL.findByProjectId(projectId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false);
const projectSecrets = shouldUseSecretV2Bridge
? await secretV2BridgeDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["reminderRepeatDays"]
})
: await secretDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["secretReminderRepeatDays"]
});
const appCfg = getConfig();
for await (const secret of projectSecrets) {
const repeatDays = shouldUseSecretV2Bridge
? (secret as { reminderRepeatDays: number }).reminderRepeatDays
: (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays;
// We're using the queue service directly to get around conflicting imports.
if (repeatDays) {
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays)
},
`reminder-${secret.id}`
);
}
}
};
+3 -1
View File
@@ -248,7 +248,9 @@ export const secretQueueFactory = ({
? secondsToMillis(newSecret.secretReminderRepeatDays)
: daysToMillisecond(newSecret.secretReminderRepeatDays),
immediately: true
}
},
removeOnComplete: true,
removeOnFail: true
}
);
} catch (err) {
+30 -8
View File
@@ -491,8 +491,8 @@ export const secretServiceFactory = ({
secretDAL
});
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const deletedSecret = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
projectId,
folderId,
actorId,
@@ -505,8 +505,19 @@ export const secretServiceFactory = ({
}
],
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
@@ -971,8 +982,8 @@ export const secretServiceFactory = ({
secretDAL
});
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const secretsDeleted = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
secretDAL,
secretQueueService,
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
@@ -983,8 +994,19 @@ export const secretServiceFactory = ({
folderId,
actorId,
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
Binary file not shown.

Before

Width:  |  Height:  |  Size: 162 KiB

After

Width:  |  Height:  |  Size: 494 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 537 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 538 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 339 KiB

After

Width:  |  Height:  |  Size: 555 KiB

+16 -7
View File
@@ -11,21 +11,30 @@ Prerequisites:
<Step title="Authorize Infisical for CircleCI">
Obtain an API token in User Settings > Personal API Tokens
![integrations circleci token](../../images/integrations/circleci/integrations-circleci-token.png)
![integrations circleci token](/images/integrations/circleci/integrations-circleci-token.png)
Navigate to your project's integrations tab in Infisical.
![integrations](../../images/integrations.png)
![integrations](/images/integrations.png)
Press on the CircleCI tile and input your CircleCI API token to grant Infisical access to your CircleCI account.
![integrations circleci authorization](../../images/integrations/circleci/integrations-circleci-auth.png)
![integrations circleci authorization](/images/integrations/circleci/integrations-circleci-auth.png)
</Step>
<Step title="Start integration">
Select which Infisical environment secrets you want to sync to which CircleCI project and press create integration to start syncing secrets to CircleCI.
Select which Infisical environment secrets you want to sync to which CircleCI project or context.
<Tabs>
<Tab title="Project">
![integrations circle ci project](/images/integrations/circleci/integrations-circleci-create-project.png)
</Tab>
<Tab title="Context">
![integrations circle ci project](/images/integrations/circleci/integrations-circleci-create-context.png)
</Tab>
</Tabs>
Finally, press create integration to start syncing secrets to CircleCI.
![integrations circleci](/images/integrations/circleci/integrations-circleci.png)
![create integration circleci](../../images/integrations/circleci/integrations-circleci-create.png)
![integrations circleci](../../images/integrations/circleci/integrations-circleci.png)
</Step>
</Steps>
</Steps>
+33 -7
View File
@@ -43,13 +43,28 @@ The operator can be install via [Helm](https://helm.sh) or [kubectl](https://git
**Namespace-scoped Installation**
The operator can be configured to watch and manage secrets in a specific namespace instead of having cluster-wide access.
The operator can be configured to watch and manage secrets in a specific namespace instead of having cluster-wide access. This is useful for:
- **Enhanced Security**: Limit the operator's permissions to only specific namespaces instead of cluster-wide access
- **Multi-tenant Clusters**: Run separate operator instances for different teams or applications
- **Resource Isolation**: Ensure operators in different namespaces don't interfere with each other
- **Development & Testing**: Run development and production operators side by side in isolated namespaces
**Note**: For multiple namespace-scoped installations, only the first installation should install CRDs. Subsequent installations should set `installCRDs: false` to avoid conflicts.
```bash
helm install operator infisical-helm-charts/secrets-operator \
--namespace your-namespace \
--set scopedNamespace=your-namespace \
# First namespace installation (with CRDs)
helm install operator-namespace1 infisical-helm-charts/secrets-operator \
--namespace first-namespace \
--set scopedNamespace=first-namespace \
--set scopedRBAC=true
# Subsequent namespace installations
helm install operator-namespace2 infisical-helm-charts/secrets-operator \
--namespace another-namespace \
--set scopedNamespace=another-namespace \
--set scopedRBAC=true \
--set installCRDs=false
```
When scoped to a namespace, the operator will:
@@ -61,14 +76,19 @@ The operator can be install via [Helm](https://helm.sh) or [kubectl](https://git
The default configuration gives cluster-wide access:
```yaml
installCRDs: true # Install CRDs (set to false for additional namespace installations)
scopedNamespace: "" # Empty for cluster-wide access
scopedRBAC: false # Cluster-wide permissions
```
If you want to install operators in multiple namespaces simultaneously:
- Make sure to set `installCRDs: false` for all but one of the installations to avoid conflicts, as CRDs are cluster-wide resources.
- Use unique release names for each installation (e.g., operator-namespace1, operator-namespace2).
</Tab>
<Tab title="Kubectl">
For production deployments, it is highly recommended to set the version of the Kubernetes operator manually instead of pointing to the latest version.
Doing so will help you avoid accidental updates to the newest release which may introduce unintended breaking changes. View all application versions [here](https://hub.docker.com/r/infisical/kubernetes-operator/tags).
<Tab title="Kubectl">
For production deployments, it is highly recommended to set the version of the Kubernetes operator manually instead of pointing to the latest version.
Doing so will help you avoid accidental updates to the newest release which may introduce unintended breaking changes. View all application versions [here](https://hub.docker.com/r/infisical/kubernetes-operator/tags).
The command below will install the most recent version of the Kubernetes operator.
However, to set the version manually, download the manifest and set the image tag version of `infisical/kubernetes-operator` according to your desired version.
@@ -714,6 +734,7 @@ Define secret keys and their corresponding templates.
Each data value uses a Golang template with access to all secrets retrieved from the specified scope.
Secrets are structured as follows:
```golang
type TemplateSecret struct {
Value string `json:"value"`
@@ -722,6 +743,7 @@ type TemplateSecret struct {
```
#### Example template configuration:
```golang
managedSecretReference:
secretName: managed-secret
@@ -733,19 +755,23 @@ type TemplateSecret struct {
```
When you run the following command:
```bash
kubectl get secret managed-secret -o jsonpath='{.data}'
```
You'll receive Kubernetes secrets output that includes the NEW_KEY:
```bash
{... "KEY":"d29ybGQ=","NEW_KEY":"LyBoZWxsbw=="}
```
When you set `includeAllSecrets` as `false` the Kubernetes secrets outputs will be:
```bash
{"NEW_KEY":"LyBoZWxsbw=="}
```
</Accordion>
<Accordion title="managedSecretReference.creationPolicy">
Creation polices allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator.

Before

Width:  |  Height:  |  Size: 7.8 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

@@ -9,6 +9,7 @@ import { twMerge } from "tailwind-merge";
import { useOrganization, useWorkspace } from "@app/context";
import { useToggle } from "@app/hooks";
import { ProjectType } from "@app/hooks/api/workspace/types";
import { createNotification } from "../notifications";
import { IconButton, Select, SelectItem, Tooltip } from "../v2";
@@ -69,7 +70,11 @@ export default function NavHeader({
<div className="mr-2 flex h-5 w-5 min-w-[1.25rem] items-center justify-center rounded-md bg-primary text-sm text-black">
{currentOrg?.name?.charAt(0)}
</div>
<Link passHref legacyBehavior href={`/org/${currentOrg?.id}/overview`}>
<Link
passHref
legacyBehavior
href={`/org/${currentOrg?.id}/${ProjectType.SecretManager}/overview`}
>
<a className="truncate pl-0.5 text-sm font-semibold text-primary/80 hover:text-primary">
{currentOrg?.name}
</a>
@@ -93,7 +98,10 @@ export default function NavHeader({
<Link
passHref
legacyBehavior
href={{ pathname: "/project/[id]/secrets/overview", query: { id: router.query.id } }}
href={{
pathname: `/${ProjectType.SecretManager}/[id]/secrets/overview`,
query: { id: router.query.id }
}}
>
<a className="text-sm font-semibold text-primary/80 hover:text-primary">{pageName}</a>
</Link>
@@ -130,7 +138,7 @@ export default function NavHeader({
passHref
legacyBehavior
href={{
pathname: "/project/[id]/secrets/[env]",
pathname: `/${ProjectType.SecretManager}/[id]/secrets/[env]`,
query: { id: router.query.id, env: router.query.env }
}}
>
@@ -199,7 +207,10 @@ export default function NavHeader({
<Link
passHref
legacyBehavior
href={{ pathname: "/project/[id]/secrets/[env]", query }}
href={{
pathname: `/${ProjectType.SecretManager}/[id]/secrets/[env]`,
query
}}
>
<a
className={twMerge(
+1 -1
View File
@@ -72,7 +72,7 @@ ModalContent.displayName = "ModalContent";
export type ModalProps = Omit<DialogPrimitive.DialogProps, "open"> & { isOpen?: boolean };
export const Modal = ({ isOpen, ...props }: ModalProps) => (
<DialogPrimitive.Root open={isOpen} {...props} />
<DialogPrimitive.Root open={isOpen} {...props} modal/>
);
export const ModalTrigger = DialogPrimitive.Trigger;
+7 -2
View File
@@ -77,11 +77,16 @@ export const selectOrganization = async (data: {
export const useSelectOrganization = () => {
const queryClient = useQueryClient();
return useMutation({
mutationFn: async (details: { organizationId: string; userAgent?: UserAgentType }) => {
mutationFn: async (details: {
organizationId: string;
userAgent?: UserAgentType;
forceSetCredentials?: boolean;
}) => {
const data = await selectOrganization(details);
// If a custom user agent is set, then this session is meant for another consuming application, not the web application.
if (!details.userAgent && !data.isMfaEnabled) {
if ((!details.userAgent && !data.isMfaEnabled) || details.forceSetCredentials) {
localStorage.setItem("orgData.id", details.organizationId);
SecurityClient.setToken(data.token);
SecurityClient.setProviderAuthToken("");
}
@@ -7,6 +7,7 @@ export {
useGetIntegrationAuthBitBucketWorkspaces,
useGetIntegrationAuthById,
useGetIntegrationAuthChecklyGroups,
useGetIntegrationAuthCircleCIOrganizations,
useGetIntegrationAuthGithubEnvs,
useGetIntegrationAuthGithubOrgs,
useGetIntegrationAuthNorthflankSecretGroups,
@@ -8,6 +8,7 @@ import {
BitBucketEnvironment,
BitBucketWorkspace,
ChecklyGroup,
CircleCIOrganization,
Environment,
HerokuPipelineCoupling,
IntegrationAuth,
@@ -128,7 +129,9 @@ const integrationAuthKeys = {
integrationAuthId,
...params
}: TGetIntegrationAuthOctopusDeployScopeValuesDTO) =>
[{ integrationAuthId }, "getIntegrationAuthOctopusDeployScopeValues", params] as const
[{ integrationAuthId }, "getIntegrationAuthOctopusDeployScopeValues", params] as const,
getIntegrationAuthCircleCIOrganizations: (integrationAuthId: string) =>
[{ integrationAuthId }, "getIntegrationAuthCircleCIOrganizations"] as const
};
const fetchIntegrationAuthById = async (integrationAuthId: string) => {
@@ -510,6 +513,15 @@ const fetchIntegrationAuthOctopusDeployScopeValues = async ({
return data;
};
const fetchIntegrationAuthCircleCIOrganizations = async (integrationAuthId: string) => {
const {
data: { organizations }
} = await apiRequest.get<{
organizations: CircleCIOrganization[];
}>(`/api/v1/integration-auth/${integrationAuthId}/circleci/organizations`);
return organizations;
};
export const useGetIntegrationAuthById = (integrationAuthId: string) => {
return useQuery({
queryKey: integrationAuthKeys.getIntegrationAuthById(integrationAuthId),
@@ -884,6 +896,13 @@ export const useGetIntegrationAuthTeamCityBuildConfigs = ({
});
};
export const useGetIntegrationAuthCircleCIOrganizations = (integrationAuthId: string) => {
return useQuery({
queryKey: integrationAuthKeys.getIntegrationAuthCircleCIOrganizations(integrationAuthId),
queryFn: () => fetchIntegrationAuthCircleCIOrganizations(integrationAuthId)
});
};
export const useAuthorizeIntegration = () => {
const queryClient = useQueryClient();
@@ -105,6 +105,19 @@ export enum OctopusDeployScope {
// tenant, variable set
}
export type CircleCIOrganization = {
name: string;
slug: string;
projects: {
name: string;
id: string;
}[];
contexts: {
name: string;
id: string;
}[];
};
export type TGetIntegrationAuthOctopusDeployScopeValuesDTO = {
integrationAuthId: string;
spaceId: string;
@@ -125,3 +138,8 @@ export type TOctopusDeployVariableSetScopeValues = {
Name: string;
}[];
};
export enum CircleCiScope {
Context = "context",
Project = "project"
}
@@ -80,6 +80,7 @@ export const useCreateIntegration = () => {
key: string;
value: string;
}[];
azureLabel?: string;
githubVisibility?: string;
githubVisibilityRepoIds?: string[];
kmsKeyId?: string;
@@ -41,6 +41,7 @@ export type TIntegration = {
key: string;
value: string;
}[];
azureLabel?: string;
kmsKeyId?: string;
secretSuffix?: string;
@@ -1,5 +1,6 @@
import { useMutation, useQuery, useQueryClient, UseQueryOptions } from "@tanstack/react-query";
import SecurityClient from "@app/components/utilities/SecurityClient";
import { apiRequest } from "@app/config/request";
import { OrderByDirection } from "@app/hooks/api/generic/types";
@@ -67,7 +68,7 @@ export const useCreateOrg = (options: { invalidate: boolean } = { invalidate: tr
mutationFn: async ({ name }: { name: string }) => {
const {
data: { organization }
} = await apiRequest.post("/api/v2/organizations", {
} = await apiRequest.post<{ organization: { id: string } }>("/api/v2/organizations", {
name
});
@@ -437,10 +438,13 @@ export const useDeleteOrgById = () => {
return useMutation({
mutationFn: async ({ organizationId }: { organizationId: string }) => {
const {
data: { organization }
} = await apiRequest.delete<{ organization: Organization }>(
data: { organization, accessToken }
} = await apiRequest.delete<{ organization: Organization; accessToken: string }>(
`/api/v2/organizations/${organizationId}`
);
SecurityClient.setToken(accessToken);
localStorage.removeItem("orgData.id");
return organization;
},
onSuccess(_, dto) {
@@ -10,6 +10,7 @@ import { zodResolver } from "@hookform/resolvers/zod";
import queryString from "query-string";
import { z } from "zod";
import { createNotification } from "@app/components/notifications";
import { SecretPathInput } from "@app/components/v2/SecretPathInput";
import { useCreateIntegration } from "@app/hooks/api";
import { IntegrationSyncBehavior } from "@app/hooks/api/integrations/types";
@@ -19,9 +20,11 @@ import {
Card,
CardTitle,
FormControl,
FormLabel,
Input,
Select,
SelectItem
SelectItem,
Switch
} from "../../../components/v2";
import { useGetIntegrationAuthById } from "../../../hooks/api/integrationAuth";
import { useGetWorkspaceById } from "../../../hooks/api/workspace";
@@ -39,7 +42,9 @@ const schema = z.object({
secretPath: z.string().trim().min(1, { message: "Secret path is required" }),
sourceEnvironment: z.string().trim().min(1, { message: "Source environment is required" }),
initialSyncBehavior: z.nativeEnum(IntegrationSyncBehavior),
secretPrefix: z.string().default("")
secretPrefix: z.string().default(""),
useLabels: z.boolean().default(false),
azureLabel: z.string().min(1).optional()
});
type TFormSchema = z.infer<typeof schema>;
@@ -60,6 +65,7 @@ export default function AzureAppConfigurationCreateIntegration() {
const router = useRouter();
const {
control,
watch,
setValue,
handleSubmit,
formState: { isSubmitting }
@@ -85,16 +91,28 @@ export default function AzureAppConfigurationCreateIntegration() {
}
}, [workspace]);
const shouldUseLabels = watch("useLabels");
const handleIntegrationSubmit = async ({
secretPath,
useLabels,
sourceEnvironment,
baseUrl,
initialSyncBehavior,
secretPrefix
secretPrefix,
azureLabel
}: TFormSchema) => {
try {
if (!integrationAuth?.id) return;
if (useLabels && !azureLabel) {
createNotification({
type: "error",
text: "Label must be provided when 'Use Labels' is enabled"
});
return;
}
await mutateAsync({
integrationAuthId: integrationAuth?.id,
isActive: true,
@@ -103,7 +121,8 @@ export default function AzureAppConfigurationCreateIntegration() {
secretPath,
metadata: {
initialSyncBehavior,
secretPrefix
secretPrefix,
...(useLabels && { azureLabel })
}
});
@@ -155,35 +174,70 @@ export default function AzureAppConfigurationCreateIntegration() {
</div>
</CardTitle>
<div className="px-6">
<Controller
control={control}
name="sourceEnvironment"
render={({ field, fieldState: { error } }) => (
<FormControl
label="Project Environment"
errorText={error?.message}
isError={Boolean(error)}
>
<Select
className="w-full border border-mineshaft-500"
dropdownContainerClassName="max-w-full"
value={field.value}
onValueChange={(val) => {
field.onChange(val);
}}
<div className="">
<Controller
control={control}
name="sourceEnvironment"
render={({ field, fieldState: { error } }) => (
<FormControl
label="Project Environment"
errorText={error?.message}
isError={Boolean(error)}
>
{workspace?.environments.map((sourceEnvironment) => (
<SelectItem
value={sourceEnvironment.slug}
key={`source-environment-${sourceEnvironment.slug}`}
<Select
className="w-full border border-mineshaft-500"
dropdownContainerClassName="max-w-full"
value={field.value}
onValueChange={(val) => {
field.onChange(val);
}}
>
{workspace?.environments.map((sourceEnvironment) => (
<SelectItem
value={sourceEnvironment.slug}
key={`source-environment-${sourceEnvironment.slug}`}
>
{sourceEnvironment.name}
</SelectItem>
))}
</Select>
</FormControl>
)}
/>
<div className="mb-2 flex w-full flex-col gap-1">
<Controller
control={control}
name="useLabels"
render={({ field: { onChange, value } }) => (
<Switch
id="use-environment-labels"
onCheckedChange={(isChecked) => onChange(isChecked)}
isChecked={value}
>
<FormLabel label="Use Labels" />
</Switch>
)}
/>
{shouldUseLabels && (
<Controller
control={control}
name="azureLabel"
render={({ field, fieldState: { error } }) => (
<FormControl
className=""
// label="Label"
errorText={error?.message}
isError={Boolean(error)}
>
{sourceEnvironment.name}
</SelectItem>
))}
</Select>
</FormControl>
)}
/>
<Input {...field} placeholder="pre-prod" />
</FormControl>
)}
/>
)}
</div>
</div>
<Controller
control={control}
name="secretPath"
@@ -56,7 +56,7 @@ export default function CircleCICreateIntegrationPage() {
<div className="flex flex-row items-center">
<div className="inline flex items-center pb-0.5">
<Image
src="/images/integrations/Circle CI.png"
src="/images/integrations/CircleCI.png"
height={30}
width={30}
alt="CircleCI logo"
@@ -1,293 +1,309 @@
import { useEffect, useMemo, useState } from "react";
import Head from "next/head";
import { Controller, useForm } from "react-hook-form";
import Image from "next/image";
import Link from "next/link";
import { useRouter } from "next/router";
import {
faArrowUpRightFromSquare,
faBookOpen,
faBugs,
faCircleInfo
} from "@fortawesome/free-solid-svg-icons";
import { faArrowUpRightFromSquare, faBookOpen } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import queryString from "query-string";
import { zodResolver } from "@hookform/resolvers/zod";
import { z } from "zod";
import { createNotification } from "@app/components/notifications";
import { useCreateIntegration } from "@app/hooks/api";
import {
Button,
Card,
CardTitle,
FilterableSelect,
FormControl,
Input,
Select,
SelectItem
} from "../../../components/v2";
import {
useGetIntegrationAuthApps,
useGetIntegrationAuthById
} from "../../../hooks/api/integrationAuth";
import { useGetWorkspaceById } from "../../../hooks/api/workspace";
SelectItem,
Spinner
} from "@app/components/v2";
import { SecretPathInput } from "@app/components/v2/SecretPathInput";
import { useWorkspace } from "@app/context";
import { useCreateIntegration } from "@app/hooks/api";
import { useGetIntegrationAuthCircleCIOrganizations } from "@app/hooks/api/integrationAuth";
import { CircleCiScope } from "@app/hooks/api/integrationAuth/types";
const formSchema = z.discriminatedUnion("scope", [
z.object({
scope: z.literal(CircleCiScope.Context),
secretPath: z.string().default("/"),
sourceEnvironment: z.object({ name: z.string(), slug: z.string() }),
targetOrg: z.object({ name: z.string().min(1), slug: z.string().min(1) }),
targetContext: z.object({ name: z.string().min(1), id: z.string().min(1) })
}),
z.object({
scope: z.literal(CircleCiScope.Project),
secretPath: z.string().default("/"),
sourceEnvironment: z.object({ name: z.string(), slug: z.string() }),
targetOrg: z.object({ name: z.string().min(1), slug: z.string().min(1) }),
targetProject: z.object({ name: z.string().min(1), id: z.string().min(1) })
})
]);
type TFormData = z.infer<typeof formSchema>;
export default function CircleCICreateIntegrationPage() {
const router = useRouter();
const { mutateAsync } = useCreateIntegration();
const { mutateAsync, isLoading: isCreatingIntegration } = useCreateIntegration();
const { currentWorkspace, isLoading: isProjectLoading } = useWorkspace();
const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]);
const integrationAuthId = router.query.integrationAuthId as string;
const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? "");
const { data: integrationAuth, isLoading: isintegrationAuthLoading } = useGetIntegrationAuthById(
(integrationAuthId as string) ?? ""
);
const { data: integrationAuthApps, isLoading: isIntegrationAuthAppsLoading } =
useGetIntegrationAuthApps({
integrationAuthId: (integrationAuthId as string) ?? ""
});
const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
const [targetOrganization, setTargetOrganization] = useState("");
const [secretPath, setSecretPath] = useState("/");
const [targetProjectId, setTargetProjectId] = useState("");
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
if (workspace) {
setSelectedSourceEnvironment(workspace.environments[0].slug);
const { control, watch, handleSubmit, setValue } = useForm<TFormData>({
resolver: zodResolver(formSchema),
defaultValues: {
secretPath: "/",
sourceEnvironment: currentWorkspace?.environments[0],
scope: CircleCiScope.Project
}
}, [workspace]);
});
const handleButtonClick = async () => {
const selectedScope = watch("scope");
const selectedOrg = watch("targetOrg");
const { data: circleCIOrganizations, isLoading: isCircleCIOrganizationsLoading } =
useGetIntegrationAuthCircleCIOrganizations(integrationAuthId);
const selectedOrganizationEntry = selectedOrg
? circleCIOrganizations?.find((org) => org.slug === selectedOrg.slug)
: undefined;
const onSubmit = async (data: TFormData) => {
try {
if (!integrationAuth?.id) return;
if (!targetProjectId || targetOrganization === "none") {
createNotification({
type: "error",
text: "Please select a project"
if (data.scope === CircleCiScope.Context) {
await mutateAsync({
scope: data.scope,
integrationAuthId,
isActive: true,
sourceEnvironment: data.sourceEnvironment.slug,
app: data.targetContext.name,
appId: data.targetContext.id,
owner: data.targetOrg.name,
secretPath: data.secretPath
});
} else {
await mutateAsync({
scope: data.scope,
integrationAuthId,
isActive: true,
app: data.targetProject.name, // project name
owner: data.targetOrg.name, // organization name
appId: data.targetProject.id, // project id (used for syncing)
sourceEnvironment: data.sourceEnvironment.slug,
secretPath: data.secretPath
});
setIsLoading(false);
return;
}
setIsLoading(true);
const selectedApp = integrationAuthApps?.find(
(integrationAuthApp) => integrationAuthApp.appId === targetProjectId
);
if (!selectedApp) {
createNotification({
type: "error",
text: "Invalid project selected"
});
setIsLoading(false);
return;
}
await mutateAsync({
integrationAuthId: integrationAuth?.id,
isActive: true,
app: selectedApp.name, // project name
owner: selectedApp.owner, // organization name
appId: selectedApp.appId, // project id (used for syncing)
sourceEnvironment: selectedSourceEnvironment,
secretPath
createNotification({
type: "success",
text: "Successfully created integration"
});
setIsLoading(false);
router.push(`/integrations/${localStorage.getItem("projectData.id")}`);
router.push(`/integrations/${currentWorkspace?.id}`);
} catch (err) {
createNotification({
type: "error",
text: "Failed to create integration"
});
console.error(err);
}
};
const filteredProjects = useMemo(() => {
if (!integrationAuthApps) return [];
if (isProjectLoading || isCircleCIOrganizationsLoading)
return (
<div className="flex h-full w-full items-center justify-center p-24">
<Spinner />
</div>
);
return integrationAuthApps.filter((integrationAuthApp) => {
return integrationAuthApp.owner === targetOrganization;
});
}, [integrationAuthApps, targetOrganization]);
const filteredOrganizations = useMemo(() => {
const organizations = new Set<string>();
if (integrationAuthApps) {
integrationAuthApps.forEach((integrationAuthApp) => {
if (!integrationAuthApp.owner) return;
organizations.add(integrationAuthApp.owner);
});
}
return Array.from(organizations);
}, [integrationAuthApps]);
return integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps ? (
<div className="flex h-full w-full flex-col items-center justify-center">
<Head>
<title>Set Up CircleCI Integration</title>
<link rel="icon" href="/infisical.ico" />
</Head>
<Card className="max-w-lg rounded-md border border-mineshaft-600">
return (
<form
onSubmit={handleSubmit(onSubmit)}
className="flex h-full w-full items-center justify-center"
>
<Card className="max-w-lg rounded-md p-8 pt-4">
<CardTitle
className="px-6 text-left text-xl"
subTitle="Choose which environment or folder in Infisical you want to sync to CircleCI environment variables."
className="w-full px-0 text-left text-xl"
subTitle="Choose which environment or folder in Infisical you want to sync to CircleCI."
>
<div className="flex flex-row items-center">
<div className="flex items-center pb-0.5">
<div className="flex w-full flex-row items-center justify-between">
<div className="flex flex-row items-center gap-1.5">
<Image
src="/images/integrations/Circle CI.png"
src="/images/integrations/CircleCI.png"
height={30}
width={30}
alt="CircleCI logo"
/>
<span className="">CircleCI Context Integration </span>
</div>
<span className="ml-1.5">CircleCI Integration </span>
<Link href="https://infisical.com/docs/integrations/cicd/circleci" passHref>
<a target="_blank" rel="noopener noreferrer">
<div className="ml-2 mb-1 inline-block cursor-default rounded-md bg-yellow/20 px-1.5 pb-[0.03rem] pt-[0.04rem] text-sm text-yellow opacity-80 hover:opacity-100">
<FontAwesomeIcon icon={faBookOpen} className="mr-1.5" />
Docs
<FontAwesomeIcon
icon={faArrowUpRightFromSquare}
className="ml-1.5 mb-[0.07rem] text-xxs"
/>
</div>
</a>
<Link
href="https://infisical.com/docs/integrations/cicd/circleci"
target="_blank"
rel="noopener noreferrer"
passHref
>
<div className="ml-2 mb-1 flex cursor-pointer flex-row items-center gap-0.5 rounded-md bg-yellow/20 px-1.5 pb-[0.03rem] pt-[0.04rem] text-sm text-yellow opacity-80 hover:opacity-100">
<FontAwesomeIcon icon={faBookOpen} className="mr-1.5" />
Docs
<FontAwesomeIcon
icon={faArrowUpRightFromSquare}
className="ml-1.5 mb-[0.07rem] text-xxs"
/>
</div>
</Link>
</div>
</CardTitle>
<FormControl label="Project Environment" className="px-6">
<Select
value={selectedSourceEnvironment}
onValueChange={(val) => setSelectedSourceEnvironment(val)}
className="w-full border border-mineshaft-500"
>
{workspace?.environments.map((sourceEnvironment) => (
<SelectItem
value={sourceEnvironment.slug}
key={`source-environment-${sourceEnvironment.slug}`}
>
{sourceEnvironment.name}
</SelectItem>
))}
</Select>
</FormControl>
<FormControl label="Secrets Path" className="px-6">
<Input
value={secretPath}
onChange={(evt) => setSecretPath(evt.target.value)}
placeholder="Provide a path, default is /"
/>
</FormControl>
<FormControl label="CircleCI Organization" className="px-6">
<Select
value={targetOrganization}
onValueChange={(val) => {
setTargetOrganization(val);
setTargetProjectId("none");
}}
className="w-full border border-mineshaft-500"
isDisabled={filteredOrganizations.length === 0}
>
{filteredOrganizations.length > 0 ? (
filteredOrganizations.map((org) => (
<SelectItem value={org} key={`target-org-${org}`}>
{org}
</SelectItem>
))
) : (
<SelectItem value="none" key="target-app-none">
No organizations found
</SelectItem>
)}
</Select>
</FormControl>
{targetOrganization && (
<FormControl label="CircleCI Project ID" className="px-6">
<Select
value={targetProjectId}
onValueChange={(val) => {
setTargetProjectId(val);
}}
className="w-full border border-mineshaft-500"
isDisabled={filteredProjects.length === 0}
<Controller
control={control}
name="sourceEnvironment"
render={({ field: { value, onChange }, fieldState: { error } }) => (
<FormControl
errorText={error?.message}
isError={Boolean(error)}
label="Project Environment"
>
{filteredProjects.length > 0 ? (
filteredProjects.map((project) => (
<SelectItem value={project.appId!} key={`target-project-${project.owner}`}>
{project.name}
</SelectItem>
))
) : (
<SelectItem value="none" key="target-app-none">
No projects found
</SelectItem>
)}
</Select>
</FormControl>
<FilterableSelect
getOptionValue={(option) => option.slug}
value={value}
getOptionLabel={(option) => option.name}
onChange={onChange}
options={currentWorkspace?.environments}
placeholder="Select a project environment"
isDisabled={!currentWorkspace?.environments.length}
/>
</FormControl>
)}
/>
<Controller
control={control}
name="secretPath"
render={({ field, fieldState: { error } }) => (
<FormControl label="Secrets Path" errorText={error?.message} isError={Boolean(error)}>
<SecretPathInput {...field} />
</FormControl>
)}
/>
<Controller
control={control}
name="targetOrg"
render={({ field: { value, onChange }, fieldState: { error } }) => (
<FormControl
errorText={error?.message}
isError={Boolean(error)}
label="CircleCI Organization"
>
<FilterableSelect
getOptionValue={(option) => option.slug}
value={value}
getOptionLabel={(option) => option.name}
onChange={(e) => {
setValue("targetProject", {
name: "",
id: ""
});
setValue("targetContext", {
name: "",
id: ""
});
onChange(e);
}}
options={circleCIOrganizations}
placeholder={
circleCIOrganizations?.length
? "Select an organization..."
: "No organizations found..."
}
isDisabled={!circleCIOrganizations?.length}
/>
</FormControl>
)}
/>
<Controller
control={control}
name="scope"
render={({ field: { onChange, ...field }, fieldState: { error } }) => (
<FormControl label="Scope" errorText={error?.message} isError={Boolean(error)}>
<Select
defaultValue={field.value}
onValueChange={(e) => {
onChange(e);
}}
className="w-full border border-mineshaft-500"
>
<SelectItem value={CircleCiScope.Project}>Project</SelectItem>
<SelectItem value={CircleCiScope.Context}>Context</SelectItem>
</Select>
</FormControl>
)}
/>
{selectedScope === CircleCiScope.Context && selectedOrganizationEntry && (
<Controller
control={control}
name="targetContext"
render={({ field: { value, onChange }, fieldState: { error } }) => (
<FormControl
errorText={error?.message}
isError={Boolean(error)}
label="CircleCI Context"
>
<FilterableSelect
value={value}
getOptionValue={(option) => option.id!}
getOptionLabel={(option) => option.name}
onChange={onChange}
options={selectedOrganizationEntry?.contexts}
placeholder={
selectedOrganizationEntry.contexts?.length
? "Select a context..."
: "No contexts found..."
}
isDisabled={!selectedOrganizationEntry.contexts?.length}
/>
</FormControl>
)}
/>
)}
{selectedScope === CircleCiScope.Project && selectedOrganizationEntry && (
<Controller
control={control}
name="targetProject"
render={({ field: { value, onChange }, fieldState: { error } }) => (
<FormControl
errorText={error?.message}
isError={Boolean(error)}
label="CircleCI Project"
>
<FilterableSelect
value={value}
getOptionValue={(option) => option.id!}
getOptionLabel={(option) => option.name}
onChange={onChange}
options={selectedOrganizationEntry?.projects}
placeholder={
selectedOrganizationEntry.projects?.length
? "Select a project..."
: "No projects found..."
}
isDisabled={!selectedOrganizationEntry.projects?.length}
/>
</FormControl>
)}
/>
)}
<Button
onClick={handleButtonClick}
type="submit"
colorSchema="primary"
variant="outline_bg"
className="mb-6 mt-2 ml-auto mr-6 w-min"
isLoading={isLoading}
isDisabled={integrationAuthApps.length === 0}
className="mt-4"
isLoading={isCreatingIntegration}
isDisabled={isCreatingIntegration}
>
Create Integration
</Button>
</Card>
<div className="mt-6 w-full max-w-md border-t border-mineshaft-800" />
<div className="mt-6 flex w-full max-w-lg flex-col rounded-md border border-mineshaft-600 bg-mineshaft-800 p-4">
<div className="flex flex-row items-center">
<FontAwesomeIcon icon={faCircleInfo} className="text-xl text-mineshaft-200" />{" "}
<span className="text-md ml-3 text-mineshaft-100">Pro Tip</span>
</div>
<span className="mt-4 text-sm text-mineshaft-300">
After creating an integration, your secrets will start syncing immediately. This might
cause an unexpected override of current secrets in CircleCI with secrets from Infisical.
</span>
</div>
</div>
) : (
<div className="flex h-full w-full items-center justify-center">
<Head>
<title>Set Up CircleCI Integration</title>
<link rel="icon" href="/infisical.ico" />
</Head>
{isIntegrationAuthAppsLoading || isintegrationAuthLoading ? (
<img
src="/images/loading/loading.gif"
height={70}
width={120}
alt="infisical loading indicator"
/>
) : (
<div className="flex h-max max-w-md flex-col rounded-md border border-mineshaft-600 bg-mineshaft-800 p-6 text-center text-mineshaft-200">
<FontAwesomeIcon icon={faBugs} className="inlineli my-2 text-6xl" />
<p>
Something went wrong. Please contact{" "}
<a
className="inline cursor-pointer text-mineshaft-100 underline decoration-primary-500 underline-offset-4 opacity-80 duration-200 hover:opacity-100"
target="_blank"
rel="noopener noreferrer"
href="mailto:[email protected]"
>
support@infisical.com
</a>{" "}
if the issue persists.
</p>
</div>
)}
</div>
</form>
);
}
@@ -1,6 +1,7 @@
import { integrationSlugNameMapping } from "public/data/frequentConstants";
import { FormLabel } from "@app/components/v2";
import { CircleCiScope } from "@app/hooks/api/integrationAuth/types";
import { IntegrationMappingBehavior, TIntegrationWithEnv } from "@app/hooks/api/integrations/types";
type Props = {
@@ -46,6 +47,11 @@ export const IntegrationConnectionSection = ({ integration }: Props) => {
case "qovery":
return integration.scope;
case "circleci":
if (integration.scope === CircleCiScope.Context) {
return "Context";
}
return "Project";
case "terraform-cloud":
return "Project";
case "aws-secret-manager":
@@ -77,7 +83,6 @@ export const IntegrationConnectionSection = ({ integration }: Props) => {
return `${integration.owner}`;
}
return `${integration.owner}/${integration.app}`;
case "aws-parameter-store":
case "rundeck":
return `${integration.path}`;
@@ -14,6 +14,7 @@ const metadataMappings: Record<keyof NonNullable<TIntegrationWithEnv["metadata"]
githubVisibilityRepoIds: "Github Visibility Repo Ids",
shouldAutoRedeploy: "Auto Redeploy Target Application When Secrets Change",
secretAWSTag: "Tags For Secrets Stored In AWS",
azureLabel: "Azure Label",
kmsKeyId: "AWS KMS Key ID",
secretSuffix: "Secret Suffix",
secretPrefix: "Secret Prefix",
@@ -86,7 +87,7 @@ export const IntegrationSettingsSection = ({ integration }: Props) => {
Object.entries(integration.metadata).map(([key, value]) => (
<div key={key} className="flex flex-col">
<p className="text-sm text-gray-400">
{metadataMappings[key as keyof typeof metadataMappings]}
{!!value && metadataMappings[key as keyof typeof metadataMappings]}
</p>
<p className="text-sm text-gray-200">{renderValue(key as MetadataKey, value)}</p>
</div>
@@ -1,4 +1,5 @@
import { FormLabel } from "@app/components/v2";
import { CircleCiScope } from "@app/hooks/api/integrationAuth/types";
import { IntegrationMappingBehavior, TIntegration } from "@app/hooks/api/integrations/types";
type Props = {
@@ -52,7 +53,8 @@ export const IntegrationDetails = ({ integration }: Props) => {
<FormLabel
label={
(integration.integration === "qovery" && integration?.scope) ||
(integration.integration === "circleci" && "Project") ||
(integration.integration === "circleci" &&
(integration.scope === CircleCiScope.Context ? "Context" : "Project")) ||
(integration.integration === "bitbucket" && "Repository") ||
(integration.integration === "octopus-deploy" && "Project") ||
(integration.integration === "aws-secret-manager" && "Secret") ||
+1 -1
View File
@@ -7,7 +7,7 @@ import { ProjectType } from "@app/hooks/api/workspace/types";
import { queryClient } from "@app/reactQuery";
export const navigateUserToOrg = async (router: NextRouter, organizationId?: string) => {
const userOrgs = await fetchOrganizations();
const userOrgs = await fetchOrganizations().catch(() => []);
const nonAuthEnforcedOrgs = userOrgs.filter((org) => !org.authEnforced);
@@ -6,7 +6,7 @@ import z from "zod";
import { createNotification } from "@app/components/notifications";
import { Button, FormControl, Input, Modal, ModalContent } from "@app/components/v2";
import { useCreateOrg, useSelectOrganization } from "@app/hooks/api";
import { useCreateOrg, useGetOrganizations, useSelectOrganization } from "@app/hooks/api";
import { ProjectType } from "@app/hooks/api/workspace/types";
const schema = z
@@ -23,9 +23,10 @@ interface CreateOrgModalProps {
}
export const CreateOrgModal: FC<CreateOrgModalProps> = ({ isOpen, onClose }) => {
const router = useRouter();
const { refetch: refetchOrganizations } = useGetOrganizations();
const {
control,
handleSubmit,
@@ -50,19 +51,21 @@ export const CreateOrgModal: FC<CreateOrgModalProps> = ({ isOpen, onClose }) =>
});
await selectOrg({
organizationId: organization.id
organizationId: organization.id,
forceSetCredentials: true
});
await refetchOrganizations();
createNotification({
text: "Successfully created organization",
type: "success"
});
if (router.isReady) router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`);
if (router.isReady)
router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`);
else window.location.href = `/org/${organization.id}/${ProjectType.SecretManager}/overview`;
localStorage.setItem("orgData.id", organization.id);
reset();
onClose();
} catch (err) {
@@ -1125,6 +1125,7 @@ export const SecretOverviewPage = () => {
bodyClassName="overflow-visible"
title="Create Secrets"
subTitle="Create a secret across multiple environments"
onPointerDownOutside={(e) => e.preventDefault()}
>
<CreateSecretForm
secretPath={secretPath}
+2 -2
View File
@@ -13,9 +13,9 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: v0.7.6
version: v0.7.7
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v0.7.6"
appVersion: "v0.7.7"
@@ -1,3 +1,4 @@
{{- if .Values.installCRDs }}
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
@@ -424,4 +425,5 @@ status:
kind: ""
plural: ""
conditions: []
storedVersions: []
storedVersions: []
{{- end }}
+2 -1
View File
@@ -32,7 +32,7 @@ controllerManager:
- ALL
image:
repository: infisical/kubernetes-operator
tag: v0.7.6
tag: v0.7.7
resources:
limits:
cpu: 500m
@@ -48,6 +48,7 @@ controllerManager:
kubernetesClusterDomain: cluster.local
scopedNamespace: ""
scopedRBAC: false
installCRDs: true
metricsService:
ports:
- name: https