diff --git a/backend/e2e-test/mocks/queue.ts b/backend/e2e-test/mocks/queue.ts index 0028381bd..99e3999e1 100644 --- a/backend/e2e-test/mocks/queue.ts +++ b/backend/e2e-test/mocks/queue.ts @@ -22,8 +22,10 @@ export const mockQueue = (): TQueueServiceFactory => { listen: (name, event) => { events[name] = event; }, + getRepeatableJobs: async () => [], clearQueue: async () => {}, stopJobById: async () => {}, - stopRepeatableJobByJobId: async () => true + stopRepeatableJobByJobId: async () => true, + stopRepeatableJobByKey: async () => true }; }; diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 473c20924..4e655420d 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -1137,6 +1137,7 @@ export const INTEGRATION = { shouldAutoRedeploy: "Used by Render to trigger auto deploy.", secretGCPLabel: "The label for GCP secrets.", secretAWSTag: "The tags for AWS secrets.", + azureLabel: "Define which label to assign to secrets created in Azure App Configuration.", githubVisibility: "Define where the secrets from the Github Integration should be visible. Option 'selected' lets you directly define which repositories to sync secrets to.", githubVisibilityRepoIds: diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index f55d8e6e6..0022ee8ea 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -20,11 +20,12 @@ export const withTransaction = (db: Knex, dal: K) => ({ export type TFindFilter = Partial & { $in?: Partial<{ [k in keyof R]: R[k][] }>; + $notNull?: Array; $search?: Partial<{ [k in keyof R]: R[k] }>; $complex?: TKnexDynamicOperator; }; export const buildFindFilter = - ({ $in, $search, $complex, ...filter }: TFindFilter) => + ({ $in, $notNull, $search, $complex, ...filter }: TFindFilter) => (bd: Knex.QueryBuilder) => { void bd.where(filter); if ($in) { @@ -34,6 +35,13 @@ export const buildFindFilter = } }); } + + if ($notNull?.length) { + $notNull.forEach((key) => { + void bd.whereNotNull(key as never); + }); + } + if ($search) { Object.entries($search).forEach(([key, val]) => { if (val) { diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 051fe9cbd..330193052 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -317,6 +317,13 @@ export const queueServiceFactory = ( } }; + const getRepeatableJobs = (name: QueueName, startOffset?: number, endOffset?: number) => { + const q = queueContainer[name]; + if (!q) throw new Error(`Queue '${name}' not initialized`); + + return q.getRepeatableJobs(startOffset, endOffset); + }; + const stopRepeatableJobByJobId = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); @@ -326,6 +333,11 @@ export const queueServiceFactory = ( return q.removeRepeatableByKey(job.repeatJobKey); }; + const stopRepeatableJobByKey = async (name: T, repeatJobKey: string) => { + const q = queueContainer[name]; + return q.removeRepeatableByKey(repeatJobKey); + }; + const stopJobById = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); @@ -349,8 +361,10 @@ export const queueServiceFactory = ( shutdown, stopRepeatableJob, stopRepeatableJobByJobId, + stopRepeatableJobByKey, clearQueue, stopJobById, + getRepeatableJobs, startPg, queuePg }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1449a0e24..2c39ef4b5 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -551,7 +551,11 @@ export const registerRoutes = async ( const orgService = orgServiceFactory({ userAliasDAL, + queueService, identityMetadataDAL, + secretDAL, + secretV2BridgeDAL, + folderDAL, licenseService, samlConfigDAL, orgRoleDAL, @@ -572,6 +576,7 @@ export const registerRoutes = async ( groupDAL, orgBotDAL, oidcConfigDAL, + loginService, projectBotService }); const signupService = authSignupServiceFactory({ @@ -805,10 +810,58 @@ export const registerRoutes = async ( projectTemplateDAL }); + const integrationAuthService = integrationAuthServiceFactory({ + integrationAuthDAL, + integrationDAL, + permissionService, + projectBotService, + kmsService + }); + + const secretQueueService = secretQueueFactory({ + keyStore, + queueService, + secretDAL, + folderDAL, + integrationAuthService, + projectBotService, + integrationDAL, + secretImportDAL, + projectEnvDAL, + webhookDAL, + orgDAL, + auditLogService, + userDAL, + projectMembershipDAL, + smtpService, + projectDAL, + projectBotDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + secretRotationDAL, + integrationAuthDAL, + snapshotDAL, + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL, + projectKeyDAL, + projectUserMembershipRoleDAL, + orgService + }); + const projectService = projectServiceFactory({ permissionService, projectDAL, + secretDAL, + secretV2BridgeDAL, + queueService, projectQueue: projectQueueService, + projectBotService, identityProjectDAL, identityOrgMembershipDAL, projectKeyDAL, @@ -891,48 +944,6 @@ export const registerRoutes = async ( projectDAL }); - const integrationAuthService = integrationAuthServiceFactory({ - integrationAuthDAL, - integrationDAL, - permissionService, - projectBotService, - kmsService - }); - const secretQueueService = secretQueueFactory({ - keyStore, - queueService, - secretDAL, - folderDAL, - integrationAuthService, - projectBotService, - integrationDAL, - secretImportDAL, - projectEnvDAL, - webhookDAL, - orgDAL, - auditLogService, - userDAL, - projectMembershipDAL, - smtpService, - projectDAL, - projectBotDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - kmsService, - secretVersionV2BridgeDAL, - secretV2BridgeDAL, - secretVersionTagV2BridgeDAL, - secretRotationDAL, - integrationAuthDAL, - snapshotDAL, - snapshotSecretV2BridgeDAL, - secretApprovalRequestDAL, - projectKeyDAL, - projectUserMembershipRoleDAL, - orgService - }); const secretImportService = secretImportServiceFactory({ licenseService, projectBotService, @@ -1261,6 +1272,7 @@ export const registerRoutes = async ( auditLogDAL, queueService, secretVersionDAL, + secretDAL, secretFolderVersionDAL: folderVersionDAL, snapshotDAL, identityAccessTokenDAL, diff --git a/backend/src/server/routes/v1/integration-auth-router.ts b/backend/src/server/routes/v1/integration-auth-router.ts index e00e06b77..ffeb748b8 100644 --- a/backend/src/server/routes/v1/integration-auth-router.ts +++ b/backend/src/server/routes/v1/integration-auth-router.ts @@ -1185,4 +1185,50 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider) return { spaces }; } }); + + server.route({ + method: "GET", + url: "/:integrationAuthId/circleci/organizations", + config: { + rateLimit: readLimit + }, + onRequest: verifyAuth([AuthMode.JWT]), + schema: { + params: z.object({ + integrationAuthId: z.string().trim() + }), + response: { + 200: z.object({ + organizations: z + .object({ + name: z.string(), + slug: z.string(), + projects: z + .object({ + name: z.string(), + id: z.string() + }) + .array(), + contexts: z + .object({ + name: z.string(), + id: z.string() + }) + .array() + }) + .array() + }) + } + }, + handler: async (req) => { + const organizations = await server.services.integrationAuth.getCircleCIOrganizations({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + id: req.params.integrationAuthId + }); + return { organizations }; + } + }); }; diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index 1327faeb1..104898099 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -16,6 +16,7 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { slugSchema } from "@app/server/lib/schemas"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode, MfaMethod } from "@app/services/auth/auth-type"; +import { sanitizedOrganizationSchema } from "@app/services/org/org-schema"; import { integrationAuthPubSchema } from "../sanitizedSchemas"; @@ -29,9 +30,11 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { schema: { response: { 200: z.object({ - organizations: OrganizationsSchema.extend({ - orgAuthMethod: z.string() - }).array() + organizations: sanitizedOrganizationSchema + .extend({ + orgAuthMethod: z.string() + }) + .array() }) } }, diff --git a/backend/src/server/routes/v2/organization-router.ts b/backend/src/server/routes/v2/organization-router.ts index cb630b143..8ca105ad4 100644 --- a/backend/src/server/routes/v2/organization-router.ts +++ b/backend/src/server/routes/v2/organization-router.ts @@ -10,6 +10,7 @@ import { UsersSchema } from "@app/db/schemas"; import { ORGANIZATIONS } from "@app/lib/api-docs"; +import { getConfig } from "@app/lib/config/env"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; @@ -363,21 +364,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - organization: OrganizationsSchema + organization: OrganizationsSchema, + accessToken: z.string() }) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), - handler: async (req) => { + handler: async (req, res) => { if (req.auth.actor !== ActorType.USER) return; - const organization = await server.services.org.deleteOrganizationById( - req.permission.id, - req.params.organizationId, - req.permission.authMethod, - req.permission.orgId - ); - return { organization }; + const cfg = getConfig(); + + const { organization, tokens } = await server.services.org.deleteOrganizationById({ + userId: req.permission.id, + orgId: req.params.organizationId, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + authorizationHeader: req.headers.authorization, + userAgentHeader: req.headers["user-agent"], + ipAddress: req.realIp + }); + + void res.setCookie("jid", tokens.refreshToken, { + httpOnly: true, + path: "/", + sameSite: "strict", + secure: cfg.HTTPS_ENABLED + }); + + return { organization, accessToken: tokens.accessToken }; } }); }; diff --git a/backend/src/server/routes/v2/user-router.ts b/backend/src/server/routes/v2/user-router.ts index a52a45fa9..851d9c4ff 100644 --- a/backend/src/server/routes/v2/user-router.ts +++ b/backend/src/server/routes/v2/user-router.ts @@ -1,10 +1,11 @@ import { z } from "zod"; -import { AuthTokenSessionsSchema, OrganizationsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; +import { AuthTokenSessionsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; import { ApiKeysSchema } from "@app/db/schemas/api-keys"; import { authRateLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMethod, AuthMode, MfaMethod } from "@app/services/auth/auth-type"; +import { sanitizedOrganizationSchema } from "@app/services/org/org-schema"; export const registerUserRouter = async (server: FastifyZodProvider) => { server.route({ @@ -134,7 +135,7 @@ export const registerUserRouter = async (server: FastifyZodProvider) => { description: "Return organizations that current user is part of", response: { 200: z.object({ - organizations: OrganizationsSchema.array() + organizations: sanitizedOrganizationSchema.array() }) } }, diff --git a/backend/src/services/auth-token/auth-token-dal.ts b/backend/src/services/auth-token/auth-token-dal.ts index c058c13e8..221b691cf 100644 --- a/backend/src/services/auth-token/auth-token-dal.ts +++ b/backend/src/services/auth-token/auth-token-dal.ts @@ -12,9 +12,12 @@ export type TTokenDALFactory = ReturnType; export const tokenDALFactory = (db: TDbClient) => { const authOrm = ormify(db, TableName.AuthTokens); - const findOneTokenSession = async (filter: Partial): Promise => { + const findOneTokenSession = async ( + filter: Partial, + tx?: Knex + ): Promise => { try { - const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first(); + const doc = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter).first(); return doc; } catch (error) { throw new DatabaseError({ error, name: "FindOneTokenSession" }); @@ -54,10 +57,11 @@ export const tokenDALFactory = (db: TDbClient) => { const insertTokenSession = async ( userId: string, ip: string, - userAgent: string + userAgent: string, + tx?: Knex ): Promise => { try { - const [session] = await db(TableName.AuthTokenSession) + const [session] = await (tx || db)(TableName.AuthTokenSession) .insert({ userId, ip, diff --git a/backend/src/services/auth-token/auth-token-service.ts b/backend/src/services/auth-token/auth-token-service.ts index 321abb5b3..c0bb7dc17 100644 --- a/backend/src/services/auth-token/auth-token-service.ts +++ b/backend/src/services/auth-token/auth-token-service.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import bcrypt from "bcrypt"; +import { Knex } from "knex"; import { TAuthTokens, TAuthTokenSessions } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; @@ -123,14 +124,13 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu return deletedToken?.[0]; }; - const getUserTokenSession = async ({ - userId, - ip, - userAgent - }: TIssueAuthTokenDTO): Promise => { - let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }); + const getUserTokenSession = async ( + { userId, ip, userAgent }: TIssueAuthTokenDTO, + tx?: Knex + ): Promise => { + let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }, tx); if (!session) { - session = await tokenDAL.insertTokenSession(userId, ip, userAgent); + session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx); } return session; }; diff --git a/backend/src/services/auth/auth-login-service.ts b/backend/src/services/auth/auth-login-service.ts index dea41e60b..8dfe69643 100644 --- a/backend/src/services/auth/auth-login-service.ts +++ b/backend/src/services/auth/auth-login-service.ts @@ -1,5 +1,6 @@ import bcrypt from "bcrypt"; import jwt from "jsonwebtoken"; +import { Knex } from "knex"; import { TUsers, UserDeviceSchema } from "@app/db/schemas"; import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns"; @@ -50,13 +51,13 @@ export const authLoginServiceFactory = ({ * Not exported. This is to update user device list * If new device is found. Will be saved and a mail will be send */ - const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string) => { + const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string, tx?: Knex) => { const devices = await UserDeviceSchema.parseAsync(user.devices || []); const isDeviceSeen = devices.some((device) => device.ip === ip && device.userAgent === userAgent); if (!isDeviceSeen) { const newDeviceList = devices.concat([{ ip, userAgent }]); - await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }); + await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }, tx); if (user.email) { await smtpService.sendMail({ template: SmtpTemplates.NewDeviceJoin, @@ -97,30 +98,36 @@ export const authLoginServiceFactory = ({ * Check user device and send mail if new device * generate the auth and refresh token. fn shared by mfa verification and login verification with mfa disabled */ - const generateUserTokens = async ({ - user, - ip, - userAgent, - organizationId, - authMethod, - isMfaVerified, - mfaMethod - }: { - user: TUsers; - ip: string; - userAgent: string; - organizationId?: string; - authMethod: AuthMethod; - isMfaVerified?: boolean; - mfaMethod?: MfaMethod; - }) => { - const cfg = getConfig(); - await updateUserDeviceSession(user, ip, userAgent); - const tokenSession = await tokenService.getUserTokenSession({ - userAgent, + const generateUserTokens = async ( + { + user, ip, - userId: user.id - }); + userAgent, + organizationId, + authMethod, + isMfaVerified, + mfaMethod + }: { + user: TUsers; + ip: string; + userAgent: string; + organizationId?: string; + authMethod: AuthMethod; + isMfaVerified?: boolean; + mfaMethod?: MfaMethod; + }, + tx?: Knex + ) => { + const cfg = getConfig(); + await updateUserDeviceSession(user, ip, userAgent, tx); + const tokenSession = await tokenService.getUserTokenSession( + { + userAgent, + ip, + userId: user.id + }, + tx + ); if (!tokenSession) throw new Error("Failed to create token"); const accessToken = jwt.sign( diff --git a/backend/src/services/integration-auth/integration-app-types.ts b/backend/src/services/integration-auth/integration-app-types.ts new file mode 100644 index 000000000..1ddd2e4d2 --- /dev/null +++ b/backend/src/services/integration-auth/integration-app-types.ts @@ -0,0 +1,5 @@ +export type TCircleCIContext = { + id: string; + name: string; + created_at: string; +}; diff --git a/backend/src/services/integration-auth/integration-auth-service.ts b/backend/src/services/integration-auth/integration-auth-service.ts index b4bbbd7cb..e2957be98 100644 --- a/backend/src/services/integration-auth/integration-auth-service.ts +++ b/backend/src/services/integration-auth/integration-auth-service.ts @@ -17,6 +17,8 @@ import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, InternalServerError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; import { TGenericPermission, TProjectPermission } from "@app/lib/types"; import { TIntegrationDALFactory } from "../integration/integration-dal"; @@ -24,6 +26,7 @@ import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { getApps } from "./integration-app-list"; +import { TCircleCIContext } from "./integration-app-types"; import { TIntegrationAuthDALFactory } from "./integration-auth-dal"; import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema"; import { @@ -31,6 +34,7 @@ import { TBitbucketEnvironment, TBitbucketWorkspace, TChecklyGroups, + TCircleCIOrganization, TDeleteIntegrationAuthByIdDTO, TDeleteIntegrationAuthsDTO, TDuplicateGithubIntegrationAuthDTO, @@ -42,6 +46,7 @@ import { TIntegrationAuthBitbucketEnvironmentsDTO, TIntegrationAuthBitbucketWorkspaceDTO, TIntegrationAuthChecklyGroupsDTO, + TIntegrationAuthCircleCIOrganizationDTO, TIntegrationAuthGithubEnvsDTO, TIntegrationAuthGithubOrgsDTO, TIntegrationAuthHerokuPipelinesDTO, @@ -1578,6 +1583,120 @@ export const integrationAuthServiceFactory = ({ return []; }; + const getCircleCIOrganizations = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + id + }: TIntegrationAuthCircleCIOrganizationDTO) => { + const integrationAuth = await integrationAuthDAL.findById(id); + if (!integrationAuth) throw new NotFoundError({ message: `Integration auth with ID '${id}' not found` }); + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + integrationAuth.projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); + + const { data: organizations }: { data: TCircleCIOrganization[] } = await request.get( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, + { + headers: { + "Circle-Token": `${accessToken}`, + "Accept-Encoding": "application/json" + } + } + ); + + let projects: { + orgName: string; + projectName: string; + projectId?: string; + }[] = []; + + try { + const projectRes = ( + await request.get<{ reponame: string; username: string; vcs_url: string }[]>( + `${IntegrationUrls.CIRCLECI_API_URL}/v1.1/projects`, + { + headers: { + "Circle-Token": accessToken, + "Accept-Encoding": "application/json" + } + } + ) + ).data; + + projects = projectRes.map((a) => ({ + orgName: a.username, // username maps to unique organization name in CircleCI + projectName: a.reponame, // reponame maps to project name within an organization in CircleCI + projectId: a.vcs_url.split("/").pop() // vcs_url maps to the project id in CircleCI + })); + } catch (error) { + logger.error(error); + } + + const projectsByOrg = groupBy( + projects.map((p) => ({ + orgName: p.orgName, + name: p.projectName, + id: p.projectId as string + })), + (p) => p.orgName + ); + + const getOrgContexts = async (orgSlug: string) => { + type NextPageToken = string | null | undefined; + + try { + const contexts: TCircleCIContext[] = []; + let nextPageToken: NextPageToken; + + while (nextPageToken !== null) { + // eslint-disable-next-line no-await-in-loop + const { data } = await request.get<{ + items: TCircleCIContext[]; + next_page_token: NextPageToken; + }>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context`, { + headers: { + "Circle-Token": accessToken, + "Accept-Encoding": "application/json" + }, + params: new URLSearchParams({ + "owner-slug": orgSlug, + ...(nextPageToken ? { "page-token": nextPageToken } : {}) + }) + }); + + contexts.push(...data.items); + nextPageToken = data.next_page_token; + } + + return contexts?.map((context) => ({ + name: context.name, + id: context.id + })); + } catch (error) { + logger.error(error); + } + }; + + return Promise.all( + organizations.map(async (org) => ({ + name: org.name, + slug: org.slug, + projects: projectsByOrg[org.name] ?? [], + contexts: (await getOrgContexts(org.slug)) ?? [] + })) + ); + }; + const deleteIntegrationAuths = async ({ projectId, integration, @@ -1790,6 +1909,7 @@ export const integrationAuthServiceFactory = ({ getTeamcityBuildConfigs, getBitbucketWorkspaces, getBitbucketEnvironments, + getCircleCIOrganizations, getIntegrationAccessToken, duplicateIntegrationAuth, getOctopusDeploySpaces, diff --git a/backend/src/services/integration-auth/integration-auth-types.ts b/backend/src/services/integration-auth/integration-auth-types.ts index 3ffa6959a..68d7bf5b9 100644 --- a/backend/src/services/integration-auth/integration-auth-types.ts +++ b/backend/src/services/integration-auth/integration-auth-types.ts @@ -128,6 +128,10 @@ export type TGetIntegrationAuthTeamCityBuildConfigDTO = { appId: string; } & Omit; +export type TIntegrationAuthCircleCIOrganizationDTO = { + id: string; +} & Omit; + export type TVercelBranches = { ref: string; lastCommit: string; @@ -189,6 +193,14 @@ export type TTeamCityBuildConfig = { webUrl: string; }; +export type TCircleCIOrganization = { + id: string; + vcsType: string; + name: string; + avatarUrl: string; + slug: string; +}; + export type TIntegrationsWithEnvironment = TIntegrations & { environment?: | { @@ -215,6 +227,11 @@ export enum OctopusDeployScope { // add tenant, variable set, etc. } +export enum CircleCiScope { + Project = "project", + Context = "context" +} + export type TOctopusDeployVariableSet = { Id: string; OwnerId: string; diff --git a/backend/src/services/integration-auth/integration-list.ts b/backend/src/services/integration-auth/integration-list.ts index 45cbdaea9..d6da2194d 100644 --- a/backend/src/services/integration-auth/integration-list.ts +++ b/backend/src/services/integration-auth/integration-list.ts @@ -76,7 +76,6 @@ export enum IntegrationUrls { RAILWAY_API_URL = "https://backboard.railway.app/graphql/v2", FLYIO_API_URL = "https://api.fly.io/graphql", CIRCLECI_API_URL = "https://circleci.com/api", - DATABRICKS_API_URL = "https:/xxxx.com/api", TRAVISCI_API_URL = "https://api.travis-ci.com", SUPABASE_API_URL = "https://api.supabase.com", LARAVELFORGE_API_URL = "https://forge.laravel.com", @@ -218,9 +217,9 @@ export const getIntegrationOptions = async () => { docsLink: "" }, { - name: "Circle CI", + name: "CircleCI", slug: "circleci", - image: "Circle CI.png", + image: "CircleCI.png", isAvailable: true, type: "pat", clientId: "", diff --git a/backend/src/services/integration-auth/integration-sync-secret-fns.ts b/backend/src/services/integration-auth/integration-sync-secret-fns.ts new file mode 100644 index 000000000..df8b990af --- /dev/null +++ b/backend/src/services/integration-auth/integration-sync-secret-fns.ts @@ -0,0 +1,35 @@ +export const isAzureKeyVaultReference = (uri: string) => { + const tryJsonDecode = () => { + try { + return (JSON.parse(uri) as { uri: string }).uri || uri; + } catch { + return uri; + } + }; + + const cleanUri = tryJsonDecode(); + + if (!cleanUri.startsWith("https://")) { + return false; + } + + if (!cleanUri.includes(".vault.azure.net/secrets/")) { + return false; + } + + // 3. Check for non-empty string between https:// and .vault.azure.net/secrets/ + const parts = cleanUri.split(".vault.azure.net/secrets/"); + const vaultName = parts[0].replace("https://", ""); + if (!vaultName) { + return false; + } + + // 4. Check for non-empty secret name + const secretParts = parts[1].split("/"); + const secretName = secretParts[0]; + if (!secretName) { + return false; + } + + return true; +}; diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index c147150f0..4fd139608 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -39,13 +39,19 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/ import { TIntegrationDALFactory } from "../integration/integration-dal"; import { IntegrationMetadataSchema } from "../integration/integration-schema"; import { IntegrationAuthMetadataSchema } from "./integration-auth-schema"; -import { OctopusDeployScope, TIntegrationsWithEnvironment, TOctopusDeployVariableSet } from "./integration-auth-types"; +import { + CircleCiScope, + OctopusDeployScope, + TIntegrationsWithEnvironment, + TOctopusDeployVariableSet +} from "./integration-auth-types"; import { IntegrationInitialSyncBehavior, IntegrationMappingBehavior, Integrations, IntegrationUrls } from "./integration-list"; +import { isAzureKeyVaultReference } from "./integration-sync-secret-fns"; const getSecretKeyValuePair = (secrets: Record) => Object.keys(secrets).reduce>((prev, key) => { @@ -320,11 +326,12 @@ const syncSecretsAzureAppConfig = async ({ }; const metadata = IntegrationMetadataSchema.parse(integration.metadata); - const azureAppConfigSecrets = ( - await getCompleteAzureAppConfigValues( - `${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix || ""}*` - ) - ).reduce( + + const azureAppConfigValuesUrl = `${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix}*${ + metadata.azureLabel ? `&label=${metadata.azureLabel}` : "" + }`; + + const azureAppConfigSecrets = (await getCompleteAzureAppConfigValues(azureAppConfigValuesUrl)).reduce( (accum, entry) => { accum[entry.key] = entry.value; @@ -405,14 +412,24 @@ const syncSecretsAzureAppConfig = async ({ } // create or update secrets on Azure App Config + for await (const key of Object.keys(secrets)) { if (!(key in azureAppConfigSecrets) || secrets[key]?.value !== azureAppConfigSecrets[key]) { await request.put( `${integration.app}/kv/${key}?api-version=2023-11-01`, { - value: secrets[key]?.value + value: secrets[key]?.value, + ...(isAzureKeyVaultReference(secrets[key]?.value || "") && { + content_type: "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" + }) }, { + ...(metadata.azureLabel && { + params: { + label: metadata.azureLabel + } + }), + headers: { Authorization: `Bearer ${accessToken}` }, @@ -432,6 +449,11 @@ const syncSecretsAzureAppConfig = async ({ headers: { Authorization: `Bearer ${accessToken}` }, + ...(metadata.azureLabel && { + params: { + label: metadata.azureLabel + } + }), // we force IPV4 because docker setup fails with ipv6 httpsAgent: new https.Agent({ family: 4 @@ -2245,102 +2267,174 @@ const syncSecretsCircleCI = async ({ secrets: Record; accessToken: string; }) => { - const getProjectSlug = async () => { - const requestConfig = { - headers: { - "Circle-Token": accessToken, - "Accept-Encoding": "application/json" - } - }; - - try { - const projectDetails = ( - await request.get<{ slug: string }>( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`, - requestConfig + if (integration.scope === CircleCiScope.Context) { + // sync secrets to CircleCI + await Promise.all( + Object.keys(secrets).map(async (key) => + request.put( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${key}`, + { + value: secrets[key].value + }, + { + headers: { + "Circle-Token": accessToken, + "Content-Type": "application/json" + } + } ) - ).data; + ) + ); - return projectDetails.slug; - } catch (err) { - if (err instanceof AxiosError) { - if (err.response?.data?.message !== "Not Found") { - throw new Error("Failed to get project slug from CircleCI during first attempt."); - } - } - } + // get secrets from CircleCI + const getSecretsRes = async () => { + type EnvVars = { + variable: string; + created_at: string; + updated_at: string; + context_id: string; + }; - // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization - try { - const circleCiOrganization = ( - await request.get<{ slug: string; name: string }[]>( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, - requestConfig - ) - ).data; + let nextPageToken: string | null | undefined; + const envVars: EnvVars[] = []; - // Case 1: This is a new integration where the organization name is stored under `integration.owner` - if (integration.owner) { - const org = circleCiOrganization.find((o) => o.name === integration.owner); - if (org) { - return `${org.slug}/${integration.app}`; - } - } - - // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one - return `${circleCiOrganization[0].slug}/${integration.app}`; - } catch (err) { - throw new Error("Failed to get project slug from CircleCI during second attempt."); - } - }; - - const projectSlug = await getProjectSlug(); - - // sync secrets to CircleCI - await Promise.all( - Object.keys(secrets).map(async (key) => - request.post( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`, - { - name: key, - value: secrets[key].value - }, - { + while (nextPageToken !== null) { + const res = await request.get<{ + items: EnvVars[]; + next_page_token: string | null; + }>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable`, { headers: { "Circle-Token": accessToken, - "Content-Type": "application/json" - } - } - ) - ) - ); + "Accept-Encoding": "application/json" + }, + params: nextPageToken + ? new URLSearchParams({ + "page-token": nextPageToken + }) + : undefined + }); - // get secrets from CircleCI - const getSecretsRes = ( - await request.get<{ items: { name: string }[] }>( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`, - { + envVars.push(...res.data.items); + nextPageToken = res.data.next_page_token; + } + + return envVars; + }; + + // delete secrets from CircleCI + await Promise.all( + (await getSecretsRes()).map(async (sec) => { + if (!(sec.variable in secrets)) { + return request.delete( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${sec.variable}`, + { + headers: { + "Circle-Token": accessToken, + "Content-Type": "application/json" + } + } + ); + } + }) + ); + } else { + const getProjectSlug = async () => { + const requestConfig = { headers: { "Circle-Token": accessToken, "Accept-Encoding": "application/json" } - } - ) - ).data?.items; + }; - // delete secrets from CircleCI - await Promise.all( - getSecretsRes.map(async (sec) => { - if (!(sec.name in secrets)) { - return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, { + try { + const projectDetails = ( + await request.get<{ slug: string }>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`, + requestConfig + ) + ).data; + + return projectDetails.slug; + } catch (err) { + if (err instanceof AxiosError) { + if (err.response?.data?.message !== "Not Found") { + throw new Error("Failed to get project slug from CircleCI during first attempt."); + } + } + } + + // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization + try { + const circleCiOrganization = ( + await request.get<{ slug: string; name: string }[]>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, + requestConfig + ) + ).data; + + // Case 1: This is a new integration where the organization name is stored under `integration.owner` + if (integration.owner) { + const org = circleCiOrganization.find((o) => o.name === integration.owner); + if (org) { + return `${org.slug}/${integration.app}`; + } + } + + // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one + return `${circleCiOrganization[0].slug}/${integration.app}`; + } catch (err) { + throw new Error("Failed to get project slug from CircleCI during second attempt."); + } + }; + + const projectSlug = await getProjectSlug(); + + // sync secrets to CircleCI + await Promise.all( + Object.keys(secrets).map(async (key) => + request.post( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`, + { + name: key, + value: secrets[key].value + }, + { + headers: { + "Circle-Token": accessToken, + "Content-Type": "application/json" + } + } + ) + ) + ); + + // get secrets from CircleCI + const getSecretsRes = ( + await request.get<{ items: { name: string }[] }>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`, + { headers: { "Circle-Token": accessToken, - "Content-Type": "application/json" + "Accept-Encoding": "application/json" } - }); - } - }) - ); + } + ) + ).data?.items; + + // delete secrets from CircleCI + await Promise.all( + getSecretsRes.map(async (sec) => { + if (!(sec.name in secrets)) { + return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, { + headers: { + "Circle-Token": accessToken, + "Content-Type": "application/json" + } + }); + } + }) + ); + } }; /** diff --git a/backend/src/services/integration/integration-schema.ts b/backend/src/services/integration/integration-schema.ts index d047a0c11..de4790188 100644 --- a/backend/src/services/integration/integration-schema.ts +++ b/backend/src/services/integration/integration-schema.ts @@ -35,6 +35,8 @@ export const IntegrationMetadataSchema = z.object({ .optional() .describe(INTEGRATION.CREATE.metadata.secretAWSTag), + azureLabel: z.string().optional().describe(INTEGRATION.CREATE.metadata.azureLabel), + githubVisibility: z .union([z.literal("selected"), z.literal("private"), z.literal("all")]) .optional() diff --git a/backend/src/services/org/org-schema.ts b/backend/src/services/org/org-schema.ts new file mode 100644 index 000000000..8f5b85403 --- /dev/null +++ b/backend/src/services/org/org-schema.ts @@ -0,0 +1,16 @@ +import { OrganizationsSchema } from "@app/db/schemas"; + +export const sanitizedOrganizationSchema = OrganizationsSchema.pick({ + id: true, + name: true, + customerId: true, + slug: true, + createdAt: true, + updatedAt: true, + authEnforced: true, + scimEnabled: true, + kmsDefaultKeyId: true, + defaultMembershipRole: true, + enforceMfa: true, + selectedMfaMethod: true +}); diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 33931bf26..73b8c04e5 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -31,11 +31,13 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro import { groupBy } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { isDisposableEmail } from "@app/lib/validator"; +import { TQueueServiceFactory } from "@app/queue"; import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal"; -import { ActorAuthMethod, ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type"; +import { TAuthLoginFactory } from "../auth/auth-login-service"; +import { ActorAuthMethod, ActorType, AuthMethod, AuthModeJwtTokenPayload, AuthTokenType } from "../auth/auth-type"; import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service"; import { TokenType } from "../auth-token/auth-token-types"; import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal"; @@ -47,6 +49,10 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnDeleteProjectSecretReminders } from "../secret/secret-fns"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { TUserDALFactory } from "../user/user-dal"; import { TIncidentContactsDALFactory } from "./incident-contacts-dal"; @@ -69,6 +75,9 @@ import { type TOrgServiceFactoryDep = { userAliasDAL: Pick; + secretDAL: Pick; + secretV2BridgeDAL: Pick; + folderDAL: Pick; orgDAL: TOrgDALFactory; orgBotDAL: TOrgBotDALFactory; orgRoleDAL: TOrgRoleDALFactory; @@ -97,6 +106,8 @@ type TOrgServiceFactoryDep = { projectBotDAL: Pick; projectUserMembershipRoleDAL: Pick; projectBotService: Pick; + queueService: Pick; + loginService: Pick; }; export type TOrgServiceFactory = ReturnType; @@ -104,6 +115,9 @@ export type TOrgServiceFactory = ReturnType; export const orgServiceFactory = ({ userAliasDAL, orgDAL, + secretDAL, + secretV2BridgeDAL, + folderDAL, userDAL, groupDAL, orgRoleDAL, @@ -124,7 +138,9 @@ export const orgServiceFactory = ({ projectBotDAL, projectUserMembershipRoleDAL, identityMetadataDAL, - projectBotService + projectBotService, + queueService, + loginService }: TOrgServiceFactoryDep) => { /* * Get organization details by the organization id @@ -419,24 +435,88 @@ export const orgServiceFactory = ({ /* * Delete organization by id * */ - const deleteOrganizationById = async ( - userId: string, - orgId: string, - actorAuthMethod: ActorAuthMethod, - actorOrgId: string | undefined - ) => { + const deleteOrganizationById = async ({ + userId, + authorizationHeader, + userAgentHeader, + ipAddress, + orgId, + actorAuthMethod, + actorOrgId + }: { + userId: string; + authorizationHeader?: string; + userAgentHeader?: string; + ipAddress: string; + orgId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string | undefined; + }) => { const { membership } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId); - if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) + if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ name: "DeleteOrganizationById", message: "Insufficient privileges" }); - - const organization = await orgDAL.deleteById(orgId); - if (organization.customerId) { - await licenseService.removeOrgCustomer(organization.customerId); } - return organization; + + if (!authorizationHeader) { + throw new UnauthorizedError({ name: "Authorization header not set on request." }); + } + + if (!userAgentHeader) { + throw new BadRequestError({ name: "User agent not set on request." }); + } + + const cfg = getConfig(); + const authToken = authorizationHeader.replace("Bearer ", ""); + + const decodedToken = jwt.verify(authToken, cfg.AUTH_SECRET) as AuthModeJwtTokenPayload; + if (!decodedToken.authMethod) throw new UnauthorizedError({ name: "Auth method not found on existing token" }); + + const response = await orgDAL.transaction(async (tx) => { + const projects = await projectDAL.find({ orgId }, { tx }); + + for await (const project of projects) { + await fnDeleteProjectSecretReminders(project.id, { + secretDAL, + secretV2BridgeDAL, + queueService, + projectBotService, + folderDAL + }); + } + + const deletedOrg = await orgDAL.deleteById(orgId, tx); + + if (deletedOrg.customerId) { + await licenseService.removeOrgCustomer(deletedOrg.customerId); + } + + // Generate new tokens without the organization ID present + const user = await userDAL.findById(userId, tx); + const { access: accessToken, refresh: refreshToken } = await loginService.generateUserTokens( + { + user, + authMethod: decodedToken.authMethod, + ip: ipAddress, + userAgent: userAgentHeader, + isMfaVerified: decodedToken.isMfaVerified, + mfaMethod: decodedToken.mfaMethod + }, + tx + ); + + return { + organization: deletedOrg, + tokens: { + accessToken, + refreshToken + } + }; + }); + + return response; }; /* * Org membership management diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 992789da2..896e1b858 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -51,7 +51,7 @@ export const projectDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.GroupProjectMembership}.projectId`, `${TableName.Project}.id`) .where(`${TableName.Project}.orgId`, orgId) .andWhere((qb) => { - if (projectType) { + if (projectType !== "all") { void qb.where(`${TableName.Project}.type`, projectType); } }) diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index f2b8f829e..fc302c4c8 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -17,6 +17,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/ import { groupBy } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TProjectPermission } from "@app/lib/types"; +import { TQueueServiceFactory } from "@app/queue"; import { ActorType } from "../auth/auth-type"; import { TCertificateDALFactory } from "../certificate/certificate-dal"; @@ -31,13 +32,17 @@ import { TOrgServiceFactory } from "../org/org-service"; import { TPkiAlertDALFactory } from "../pki-alert/pki-alert-dal"; import { TPkiCollectionDALFactory } from "../pki-collection/pki-collection-dal"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; import { getPredefinedRoles } from "../project-role/project-role-fns"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnDeleteProjectSecretReminders } from "../secret/secret-fns"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal"; import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal"; import { TUserDALFactory } from "../user/user-dal"; @@ -80,7 +85,10 @@ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; projectQueue: TProjectQueueFactory; userDAL: TUserDALFactory; - folderDAL: TSecretFolderDALFactory; + projectBotService: Pick; + folderDAL: Pick; + secretDAL: Pick; + secretV2BridgeDAL: Pick; projectEnvDAL: Pick; identityOrgMembershipDAL: TIdentityOrgDALFactory; identityProjectDAL: TIdentityProjectDALFactory; @@ -101,6 +109,8 @@ type TProjectServiceFactoryDep = { permissionService: TPermissionServiceFactory; orgService: Pick; licenseService: Pick; + queueService: Pick; + orgDAL: Pick; keyStore: Pick; projectBotDAL: Pick; @@ -121,9 +131,13 @@ export type TProjectServiceFactory = ReturnType; export const projectServiceFactory = ({ projectDAL, + secretDAL, + secretV2BridgeDAL, projectQueue, projectKeyDAL, permissionService, + queueService, + projectBotService, orgDAL, userDAL, folderDAL, @@ -436,6 +450,14 @@ export const projectServiceFactory = ({ await userDAL.deleteById(projectGhostUser.id, tx); } + await fnDeleteProjectSecretReminders(project.id, { + secretDAL, + secretV2BridgeDAL, + queueService, + projectBotService, + folderDAL + }); + return delProject; }); @@ -453,7 +475,12 @@ export const projectServiceFactory = ({ const workspaces = await projectDAL.findAllProjects(actorId, actorOrgId, type); if (includeRoles) { - const { permission } = await permissionService.getUserOrgPermission(actorId, actorOrgId, actorAuthMethod); + const { permission } = await permissionService.getUserOrgPermission( + actorId, + actorOrgId, + actorAuthMethod, + actorOrgId + ); // `includeRoles` is specifically used by organization admins when inviting new users to the organizations to avoid looping redundant api calls. ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member); diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index dab70806f..aa1ed9d25 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal"; import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; @@ -16,6 +17,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = { identityUniversalAuthClientSecretDAL: Pick; secretVersionDAL: Pick; secretVersionV2DAL: Pick; + secretDAL: Pick; secretFolderVersionDAL: Pick; snapshotDAL: Pick; secretSharingDAL: Pick; @@ -30,6 +32,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ snapshotDAL, secretVersionDAL, secretFolderVersionDAL, + secretDAL, identityAccessTokenDAL, secretSharingDAL, secretVersionV2DAL, @@ -37,6 +40,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ }: TDailyResourceCleanUpQueueServiceFactoryDep) => { queueService.start(QueueName.DailyResourceCleanUp, async () => { logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); + await secretDAL.pruneSecretReminders(queueService); await auditLogDAL.pruneAuditLog(); await identityAccessTokenDAL.removeExpiredTokens(); await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets(); diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 0d4ae0cda..cbaf7ddcd 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -5,6 +5,8 @@ import { TDbClient } from "@app/db"; import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { QueueName, TQueueServiceFactory } from "@app/queue"; export type TSecretDALFactory = ReturnType; @@ -339,6 +341,94 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const pruneSecretReminders = async (queueService: TQueueServiceFactory) => { + const REMINDER_PRUNE_BATCH_SIZE = 5_000; + const MAX_RETRY_ON_FAILURE = 3; + let numberOfRetryOnFailure = 0; + let deletedReminderCount = 0; + + logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`); + + try { + const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder); + const reminderJobs = repeatableJobs + .map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key })) + .filter(Boolean); + + if (reminderJobs.length === 0) { + logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`); + return; + } + + for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) { + try { + const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId); + + const payload = { + $in: { + id: batchIds + } + }; + + const opts = { + limit: REMINDER_PRUNE_BATCH_SIZE + }; + + // Find existing secrets with pagination + // eslint-disable-next-line no-await-in-loop + const [secrets, secretsV2] = await Promise.all([ + ormify(db, TableName.Secret).find(payload, opts), + ormify(db, TableName.SecretV2).find(payload, opts) + ]); + + const foundSecretIds = new Set([ + ...secrets.map((secret) => secret.id), + ...secretsV2.map((secret) => secret.id) + ]); + + // Find IDs that don't exist in either table + const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId)); + + // Delete reminders for non-existent secrets + for (const secretId of secretIdsNotFound) { + const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey; + + if (jobKey) { + // eslint-disable-next-line no-await-in-loop + await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey); + deletedReminderCount += 1; + } + } + + numberOfRetryOnFailure = 0; + } catch (error) { + numberOfRetryOnFailure += 1; + logger.error(error, `Failed to process batch at offset ${offset}`); + + if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) { + break; + } + + // Retry the current batch + offset -= REMINDER_PRUNE_BATCH_SIZE; + + // eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure)); + } + + // Small delay between batches + // eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } catch (error) { + logger.error(error, "Failed to complete secret reminder pruning"); + } finally { + logger.info( + `${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders` + ); + } + }; + return { ...secretOrm, update, @@ -352,6 +442,7 @@ export const secretDALFactory = (db: TDbClient) => { findByBlindIndexes, upsertSecretReferences, findReferencedSecretReferences, - findAllProjectSecretValues + findAllProjectSecretValues, + pruneSecretReminders }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 65691fcbb..6336c479d 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -19,9 +19,11 @@ import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, @@ -31,8 +33,10 @@ import { import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { KmsDataKey } from "../kms/kms-types"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretDALFactory } from "./secret-dal"; import { TCreateManySecretsRawFn, @@ -1138,3 +1142,49 @@ export const decryptSecretWithBot = ( secretComment }; }; + +type TFnDeleteProjectSecretReminders = { + secretDAL: Pick; + secretV2BridgeDAL: Pick; + queueService: Pick; + projectBotService: Pick; + folderDAL: Pick; +}; + +export const fnDeleteProjectSecretReminders = async ( + projectId: string, + { secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders +) => { + const projectFolders = await folderDAL.findByProjectId(projectId); + const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false); + + const projectSecrets = shouldUseSecretV2Bridge + ? await secretV2BridgeDAL.find({ + $in: { folderId: projectFolders.map((folder) => folder.id) }, + $notNull: ["reminderRepeatDays"] + }) + : await secretDAL.find({ + $in: { folderId: projectFolders.map((folder) => folder.id) }, + $notNull: ["secretReminderRepeatDays"] + }); + + const appCfg = getConfig(); + for await (const secret of projectSecrets) { + const repeatDays = shouldUseSecretV2Bridge + ? (secret as { reminderRepeatDays: number }).reminderRepeatDays + : (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays; + + // We're using the queue service directly to get around conflicting imports. + if (repeatDays) { + await queueService.stopRepeatableJob( + QueueName.SecretReminder, + QueueJobs.SecretReminder, + { + // on prod it this will be in days, in development this will be second + every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays) + }, + `reminder-${secret.id}` + ); + } + } +}; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 84a8584ee..57d32ab11 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -248,7 +248,9 @@ export const secretQueueFactory = ({ ? secondsToMillis(newSecret.secretReminderRepeatDays) : daysToMillisecond(newSecret.secretReminderRepeatDays), immediately: true - } + }, + removeOnComplete: true, + removeOnFail: true } ); } catch (err) { diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 6f058e023..fbf90a7f8 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -491,8 +491,8 @@ export const secretServiceFactory = ({ secretDAL }); - const deletedSecret = await secretDAL.transaction(async (tx) => - fnSecretBulkDelete({ + const deletedSecret = await secretDAL.transaction(async (tx) => { + const secrets = await fnSecretBulkDelete({ projectId, folderId, actorId, @@ -505,8 +505,19 @@ export const secretServiceFactory = ({ } ], tx - }) - ); + }); + + for await (const secret of secrets) { + if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) { + await secretQueueService.removeSecretReminder({ + repeatDays: secret.secretReminderRepeatDays, + secretId: secret.id + }); + } + } + + return secrets; + }); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); @@ -971,8 +982,8 @@ export const secretServiceFactory = ({ secretDAL }); - const secretsDeleted = await secretDAL.transaction(async (tx) => - fnSecretBulkDelete({ + const secretsDeleted = await secretDAL.transaction(async (tx) => { + const secrets = await fnSecretBulkDelete({ secretDAL, secretQueueService, inputSecrets: inputSecrets.map(({ type, secretName }) => ({ @@ -983,8 +994,19 @@ export const secretServiceFactory = ({ folderId, actorId, tx - }) - ); + }); + + for await (const secret of secrets) { + if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) { + await secretQueueService.removeSecretReminder({ + repeatDays: secret.secretReminderRepeatDays, + secretId: secret.id + }); + } + } + + return secrets; + }); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ diff --git a/docs/images/integrations/circleci/integrations-circleci-auth.png b/docs/images/integrations/circleci/integrations-circleci-auth.png index 055ebbf4a..73a5fd686 100644 Binary files a/docs/images/integrations/circleci/integrations-circleci-auth.png and b/docs/images/integrations/circleci/integrations-circleci-auth.png differ diff --git a/docs/images/integrations/circleci/integrations-circleci-create-context.png b/docs/images/integrations/circleci/integrations-circleci-create-context.png new file mode 100644 index 000000000..9d911953e Binary files /dev/null and b/docs/images/integrations/circleci/integrations-circleci-create-context.png differ diff --git a/docs/images/integrations/circleci/integrations-circleci-create-project.png b/docs/images/integrations/circleci/integrations-circleci-create-project.png new file mode 100644 index 000000000..73ab1e75a Binary files /dev/null and b/docs/images/integrations/circleci/integrations-circleci-create-project.png differ diff --git a/docs/images/integrations/circleci/integrations-circleci.png b/docs/images/integrations/circleci/integrations-circleci.png index 5ee9a9df0..cde74678d 100644 Binary files a/docs/images/integrations/circleci/integrations-circleci.png and b/docs/images/integrations/circleci/integrations-circleci.png differ diff --git a/docs/integrations/cicd/circleci.mdx b/docs/integrations/cicd/circleci.mdx index 0753f40f7..5bf04822d 100644 --- a/docs/integrations/cicd/circleci.mdx +++ b/docs/integrations/cicd/circleci.mdx @@ -11,21 +11,30 @@ Prerequisites: Obtain an API token in User Settings > Personal API Tokens - ![integrations circleci token](../../images/integrations/circleci/integrations-circleci-token.png) + ![integrations circleci token](/images/integrations/circleci/integrations-circleci-token.png) Navigate to your project's integrations tab in Infisical. - ![integrations](../../images/integrations.png) + ![integrations](/images/integrations.png) Press on the CircleCI tile and input your CircleCI API token to grant Infisical access to your CircleCI account. - ![integrations circleci authorization](../../images/integrations/circleci/integrations-circleci-auth.png) + ![integrations circleci authorization](/images/integrations/circleci/integrations-circleci-auth.png) - Select which Infisical environment secrets you want to sync to which CircleCI project and press create integration to start syncing secrets to CircleCI. + Select which Infisical environment secrets you want to sync to which CircleCI project or context. + + + ![integrations circle ci project](/images/integrations/circleci/integrations-circleci-create-project.png) + + + ![integrations circle ci project](/images/integrations/circleci/integrations-circleci-create-context.png) + + + + Finally, press create integration to start syncing secrets to CircleCI. + ![integrations circleci](/images/integrations/circleci/integrations-circleci.png) - ![create integration circleci](../../images/integrations/circleci/integrations-circleci-create.png) - ![integrations circleci](../../images/integrations/circleci/integrations-circleci.png) - \ No newline at end of file + diff --git a/docs/integrations/platforms/kubernetes.mdx b/docs/integrations/platforms/kubernetes.mdx index 6defea119..ae4ee8071 100644 --- a/docs/integrations/platforms/kubernetes.mdx +++ b/docs/integrations/platforms/kubernetes.mdx @@ -43,13 +43,28 @@ The operator can be install via [Helm](https://helm.sh) or [kubectl](https://git **Namespace-scoped Installation** - The operator can be configured to watch and manage secrets in a specific namespace instead of having cluster-wide access. + The operator can be configured to watch and manage secrets in a specific namespace instead of having cluster-wide access. This is useful for: + + - **Enhanced Security**: Limit the operator's permissions to only specific namespaces instead of cluster-wide access + - **Multi-tenant Clusters**: Run separate operator instances for different teams or applications + - **Resource Isolation**: Ensure operators in different namespaces don't interfere with each other + - **Development & Testing**: Run development and production operators side by side in isolated namespaces + + **Note**: For multiple namespace-scoped installations, only the first installation should install CRDs. Subsequent installations should set `installCRDs: false` to avoid conflicts. ```bash - helm install operator infisical-helm-charts/secrets-operator \ - --namespace your-namespace \ - --set scopedNamespace=your-namespace \ + # First namespace installation (with CRDs) + helm install operator-namespace1 infisical-helm-charts/secrets-operator \ + --namespace first-namespace \ + --set scopedNamespace=first-namespace \ --set scopedRBAC=true + + # Subsequent namespace installations + helm install operator-namespace2 infisical-helm-charts/secrets-operator \ + --namespace another-namespace \ + --set scopedNamespace=another-namespace \ + --set scopedRBAC=true \ + --set installCRDs=false ``` When scoped to a namespace, the operator will: @@ -61,14 +76,19 @@ The operator can be install via [Helm](https://helm.sh) or [kubectl](https://git The default configuration gives cluster-wide access: ```yaml + installCRDs: true # Install CRDs (set to false for additional namespace installations) scopedNamespace: "" # Empty for cluster-wide access scopedRBAC: false # Cluster-wide permissions ``` + If you want to install operators in multiple namespaces simultaneously: + - Make sure to set `installCRDs: false` for all but one of the installations to avoid conflicts, as CRDs are cluster-wide resources. + - Use unique release names for each installation (e.g., operator-namespace1, operator-namespace2). + - - For production deployments, it is highly recommended to set the version of the Kubernetes operator manually instead of pointing to the latest version. - Doing so will help you avoid accidental updates to the newest release which may introduce unintended breaking changes. View all application versions [here](https://hub.docker.com/r/infisical/kubernetes-operator/tags). + + For production deployments, it is highly recommended to set the version of the Kubernetes operator manually instead of pointing to the latest version. + Doing so will help you avoid accidental updates to the newest release which may introduce unintended breaking changes. View all application versions [here](https://hub.docker.com/r/infisical/kubernetes-operator/tags). The command below will install the most recent version of the Kubernetes operator. However, to set the version manually, download the manifest and set the image tag version of `infisical/kubernetes-operator` according to your desired version. @@ -714,6 +734,7 @@ Define secret keys and their corresponding templates. Each data value uses a Golang template with access to all secrets retrieved from the specified scope. Secrets are structured as follows: + ```golang type TemplateSecret struct { Value string `json:"value"` @@ -722,6 +743,7 @@ type TemplateSecret struct { ``` #### Example template configuration: + ```golang managedSecretReference: secretName: managed-secret @@ -733,19 +755,23 @@ type TemplateSecret struct { ``` When you run the following command: + ```bash kubectl get secret managed-secret -o jsonpath='{.data}' ``` You'll receive Kubernetes secrets output that includes the NEW_KEY: + ```bash {... "KEY":"d29ybGQ=","NEW_KEY":"LyBoZWxsbw=="} ``` When you set `includeAllSecrets` as `false` the Kubernetes secrets outputs will be: + ```bash {"NEW_KEY":"LyBoZWxsbw=="} ``` + Creation polices allow you to control whether or not owner references should be added to the managed Kubernetes secret that is generated by the Infisical operator. diff --git a/frontend/public/images/integrations/Circle CI.png b/frontend/public/images/integrations/CircleCI.png similarity index 100% rename from frontend/public/images/integrations/Circle CI.png rename to frontend/public/images/integrations/CircleCI.png diff --git a/frontend/src/components/navigation/NavHeader.tsx b/frontend/src/components/navigation/NavHeader.tsx index 973feeb73..76a60d082 100644 --- a/frontend/src/components/navigation/NavHeader.tsx +++ b/frontend/src/components/navigation/NavHeader.tsx @@ -9,6 +9,7 @@ import { twMerge } from "tailwind-merge"; import { useOrganization, useWorkspace } from "@app/context"; import { useToggle } from "@app/hooks"; +import { ProjectType } from "@app/hooks/api/workspace/types"; import { createNotification } from "../notifications"; import { IconButton, Select, SelectItem, Tooltip } from "../v2"; @@ -69,7 +70,11 @@ export default function NavHeader({
{currentOrg?.name?.charAt(0)}
- + {currentOrg?.name} @@ -93,7 +98,10 @@ export default function NavHeader({ {pageName} @@ -130,7 +138,7 @@ export default function NavHeader({ passHref legacyBehavior href={{ - pathname: "/project/[id]/secrets/[env]", + pathname: `/${ProjectType.SecretManager}/[id]/secrets/[env]`, query: { id: router.query.id, env: router.query.env } }} > @@ -199,7 +207,10 @@ export default function NavHeader({ & { isOpen?: boolean }; export const Modal = ({ isOpen, ...props }: ModalProps) => ( - + ); export const ModalTrigger = DialogPrimitive.Trigger; diff --git a/frontend/src/hooks/api/auth/queries.tsx b/frontend/src/hooks/api/auth/queries.tsx index de3c60d46..0c3dbe0d9 100644 --- a/frontend/src/hooks/api/auth/queries.tsx +++ b/frontend/src/hooks/api/auth/queries.tsx @@ -77,11 +77,16 @@ export const selectOrganization = async (data: { export const useSelectOrganization = () => { const queryClient = useQueryClient(); return useMutation({ - mutationFn: async (details: { organizationId: string; userAgent?: UserAgentType }) => { + mutationFn: async (details: { + organizationId: string; + userAgent?: UserAgentType; + forceSetCredentials?: boolean; + }) => { const data = await selectOrganization(details); // If a custom user agent is set, then this session is meant for another consuming application, not the web application. - if (!details.userAgent && !data.isMfaEnabled) { + if ((!details.userAgent && !data.isMfaEnabled) || details.forceSetCredentials) { + localStorage.setItem("orgData.id", details.organizationId); SecurityClient.setToken(data.token); SecurityClient.setProviderAuthToken(""); } diff --git a/frontend/src/hooks/api/integrationAuth/index.tsx b/frontend/src/hooks/api/integrationAuth/index.tsx index 0ae3511de..e7ee5928a 100644 --- a/frontend/src/hooks/api/integrationAuth/index.tsx +++ b/frontend/src/hooks/api/integrationAuth/index.tsx @@ -7,6 +7,7 @@ export { useGetIntegrationAuthBitBucketWorkspaces, useGetIntegrationAuthById, useGetIntegrationAuthChecklyGroups, + useGetIntegrationAuthCircleCIOrganizations, useGetIntegrationAuthGithubEnvs, useGetIntegrationAuthGithubOrgs, useGetIntegrationAuthNorthflankSecretGroups, diff --git a/frontend/src/hooks/api/integrationAuth/queries.tsx b/frontend/src/hooks/api/integrationAuth/queries.tsx index e5f928158..84a50ae1f 100644 --- a/frontend/src/hooks/api/integrationAuth/queries.tsx +++ b/frontend/src/hooks/api/integrationAuth/queries.tsx @@ -8,6 +8,7 @@ import { BitBucketEnvironment, BitBucketWorkspace, ChecklyGroup, + CircleCIOrganization, Environment, HerokuPipelineCoupling, IntegrationAuth, @@ -128,7 +129,9 @@ const integrationAuthKeys = { integrationAuthId, ...params }: TGetIntegrationAuthOctopusDeployScopeValuesDTO) => - [{ integrationAuthId }, "getIntegrationAuthOctopusDeployScopeValues", params] as const + [{ integrationAuthId }, "getIntegrationAuthOctopusDeployScopeValues", params] as const, + getIntegrationAuthCircleCIOrganizations: (integrationAuthId: string) => + [{ integrationAuthId }, "getIntegrationAuthCircleCIOrganizations"] as const }; const fetchIntegrationAuthById = async (integrationAuthId: string) => { @@ -510,6 +513,15 @@ const fetchIntegrationAuthOctopusDeployScopeValues = async ({ return data; }; +const fetchIntegrationAuthCircleCIOrganizations = async (integrationAuthId: string) => { + const { + data: { organizations } + } = await apiRequest.get<{ + organizations: CircleCIOrganization[]; + }>(`/api/v1/integration-auth/${integrationAuthId}/circleci/organizations`); + return organizations; +}; + export const useGetIntegrationAuthById = (integrationAuthId: string) => { return useQuery({ queryKey: integrationAuthKeys.getIntegrationAuthById(integrationAuthId), @@ -884,6 +896,13 @@ export const useGetIntegrationAuthTeamCityBuildConfigs = ({ }); }; +export const useGetIntegrationAuthCircleCIOrganizations = (integrationAuthId: string) => { + return useQuery({ + queryKey: integrationAuthKeys.getIntegrationAuthCircleCIOrganizations(integrationAuthId), + queryFn: () => fetchIntegrationAuthCircleCIOrganizations(integrationAuthId) + }); +}; + export const useAuthorizeIntegration = () => { const queryClient = useQueryClient(); diff --git a/frontend/src/hooks/api/integrationAuth/types.ts b/frontend/src/hooks/api/integrationAuth/types.ts index 58a643dff..e2dee6067 100644 --- a/frontend/src/hooks/api/integrationAuth/types.ts +++ b/frontend/src/hooks/api/integrationAuth/types.ts @@ -105,6 +105,19 @@ export enum OctopusDeployScope { // tenant, variable set } +export type CircleCIOrganization = { + name: string; + slug: string; + projects: { + name: string; + id: string; + }[]; + contexts: { + name: string; + id: string; + }[]; +}; + export type TGetIntegrationAuthOctopusDeployScopeValuesDTO = { integrationAuthId: string; spaceId: string; @@ -125,3 +138,8 @@ export type TOctopusDeployVariableSetScopeValues = { Name: string; }[]; }; + +export enum CircleCiScope { + Context = "context", + Project = "project" +} diff --git a/frontend/src/hooks/api/integrations/queries.tsx b/frontend/src/hooks/api/integrations/queries.tsx index 11d42631a..5c059ae98 100644 --- a/frontend/src/hooks/api/integrations/queries.tsx +++ b/frontend/src/hooks/api/integrations/queries.tsx @@ -80,6 +80,7 @@ export const useCreateIntegration = () => { key: string; value: string; }[]; + azureLabel?: string; githubVisibility?: string; githubVisibilityRepoIds?: string[]; kmsKeyId?: string; diff --git a/frontend/src/hooks/api/integrations/types.ts b/frontend/src/hooks/api/integrations/types.ts index 0346b065a..7054befc7 100644 --- a/frontend/src/hooks/api/integrations/types.ts +++ b/frontend/src/hooks/api/integrations/types.ts @@ -41,6 +41,7 @@ export type TIntegration = { key: string; value: string; }[]; + azureLabel?: string; kmsKeyId?: string; secretSuffix?: string; diff --git a/frontend/src/hooks/api/organization/queries.tsx b/frontend/src/hooks/api/organization/queries.tsx index 4923177ba..82894d988 100644 --- a/frontend/src/hooks/api/organization/queries.tsx +++ b/frontend/src/hooks/api/organization/queries.tsx @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient, UseQueryOptions } from "@tanstack/react-query"; +import SecurityClient from "@app/components/utilities/SecurityClient"; import { apiRequest } from "@app/config/request"; import { OrderByDirection } from "@app/hooks/api/generic/types"; @@ -67,7 +68,7 @@ export const useCreateOrg = (options: { invalidate: boolean } = { invalidate: tr mutationFn: async ({ name }: { name: string }) => { const { data: { organization } - } = await apiRequest.post("/api/v2/organizations", { + } = await apiRequest.post<{ organization: { id: string } }>("/api/v2/organizations", { name }); @@ -437,10 +438,13 @@ export const useDeleteOrgById = () => { return useMutation({ mutationFn: async ({ organizationId }: { organizationId: string }) => { const { - data: { organization } - } = await apiRequest.delete<{ organization: Organization }>( + data: { organization, accessToken } + } = await apiRequest.delete<{ organization: Organization; accessToken: string }>( `/api/v2/organizations/${organizationId}` ); + SecurityClient.setToken(accessToken); + localStorage.removeItem("orgData.id"); + return organization; }, onSuccess(_, dto) { diff --git a/frontend/src/pages/integrations/azure-app-configuration/create.tsx b/frontend/src/pages/integrations/azure-app-configuration/create.tsx index c9fe4d1db..9b647c2a2 100644 --- a/frontend/src/pages/integrations/azure-app-configuration/create.tsx +++ b/frontend/src/pages/integrations/azure-app-configuration/create.tsx @@ -10,6 +10,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import queryString from "query-string"; import { z } from "zod"; +import { createNotification } from "@app/components/notifications"; import { SecretPathInput } from "@app/components/v2/SecretPathInput"; import { useCreateIntegration } from "@app/hooks/api"; import { IntegrationSyncBehavior } from "@app/hooks/api/integrations/types"; @@ -19,9 +20,11 @@ import { Card, CardTitle, FormControl, + FormLabel, Input, Select, - SelectItem + SelectItem, + Switch } from "../../../components/v2"; import { useGetIntegrationAuthById } from "../../../hooks/api/integrationAuth"; import { useGetWorkspaceById } from "../../../hooks/api/workspace"; @@ -39,7 +42,9 @@ const schema = z.object({ secretPath: z.string().trim().min(1, { message: "Secret path is required" }), sourceEnvironment: z.string().trim().min(1, { message: "Source environment is required" }), initialSyncBehavior: z.nativeEnum(IntegrationSyncBehavior), - secretPrefix: z.string().default("") + secretPrefix: z.string().default(""), + useLabels: z.boolean().default(false), + azureLabel: z.string().min(1).optional() }); type TFormSchema = z.infer; @@ -60,6 +65,7 @@ export default function AzureAppConfigurationCreateIntegration() { const router = useRouter(); const { control, + watch, setValue, handleSubmit, formState: { isSubmitting } @@ -85,16 +91,28 @@ export default function AzureAppConfigurationCreateIntegration() { } }, [workspace]); + const shouldUseLabels = watch("useLabels"); + const handleIntegrationSubmit = async ({ secretPath, + useLabels, sourceEnvironment, baseUrl, initialSyncBehavior, - secretPrefix + secretPrefix, + azureLabel }: TFormSchema) => { try { if (!integrationAuth?.id) return; + if (useLabels && !azureLabel) { + createNotification({ + type: "error", + text: "Label must be provided when 'Use Labels' is enabled" + }); + return; + } + await mutateAsync({ integrationAuthId: integrationAuth?.id, isActive: true, @@ -103,7 +121,8 @@ export default function AzureAppConfigurationCreateIntegration() { secretPath, metadata: { initialSyncBehavior, - secretPrefix + secretPrefix, + ...(useLabels && { azureLabel }) } }); @@ -155,35 +174,70 @@ export default function AzureAppConfigurationCreateIntegration() {
- ( - - + + )} + /> + +
+ ( + onChange(isChecked)} + isChecked={value} + > + + + )} + /> + + {shouldUseLabels && ( + ( + - {sourceEnvironment.name} - - ))} - - - )} - /> + + + )} + /> + )} +
+
CircleCI logo; export default function CircleCICreateIntegrationPage() { const router = useRouter(); - const { mutateAsync } = useCreateIntegration(); + const { mutateAsync, isLoading: isCreatingIntegration } = useCreateIntegration(); + const { currentWorkspace, isLoading: isProjectLoading } = useWorkspace(); - const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]); + const integrationAuthId = router.query.integrationAuthId as string; - const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? ""); - const { data: integrationAuth, isLoading: isintegrationAuthLoading } = useGetIntegrationAuthById( - (integrationAuthId as string) ?? "" - ); - const { data: integrationAuthApps, isLoading: isIntegrationAuthAppsLoading } = - useGetIntegrationAuthApps({ - integrationAuthId: (integrationAuthId as string) ?? "" - }); - - const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState(""); - const [targetOrganization, setTargetOrganization] = useState(""); - const [secretPath, setSecretPath] = useState("/"); - - const [targetProjectId, setTargetProjectId] = useState(""); - - const [isLoading, setIsLoading] = useState(false); - - useEffect(() => { - if (workspace) { - setSelectedSourceEnvironment(workspace.environments[0].slug); + const { control, watch, handleSubmit, setValue } = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + secretPath: "/", + sourceEnvironment: currentWorkspace?.environments[0], + scope: CircleCiScope.Project } - }, [workspace]); + }); - const handleButtonClick = async () => { + const selectedScope = watch("scope"); + const selectedOrg = watch("targetOrg"); + + const { data: circleCIOrganizations, isLoading: isCircleCIOrganizationsLoading } = + useGetIntegrationAuthCircleCIOrganizations(integrationAuthId); + + const selectedOrganizationEntry = selectedOrg + ? circleCIOrganizations?.find((org) => org.slug === selectedOrg.slug) + : undefined; + + const onSubmit = async (data: TFormData) => { try { - if (!integrationAuth?.id) return; - - if (!targetProjectId || targetOrganization === "none") { - createNotification({ - type: "error", - text: "Please select a project" + if (data.scope === CircleCiScope.Context) { + await mutateAsync({ + scope: data.scope, + integrationAuthId, + isActive: true, + sourceEnvironment: data.sourceEnvironment.slug, + app: data.targetContext.name, + appId: data.targetContext.id, + owner: data.targetOrg.name, + secretPath: data.secretPath + }); + } else { + await mutateAsync({ + scope: data.scope, + integrationAuthId, + isActive: true, + app: data.targetProject.name, // project name + owner: data.targetOrg.name, // organization name + appId: data.targetProject.id, // project id (used for syncing) + sourceEnvironment: data.sourceEnvironment.slug, + secretPath: data.secretPath }); - setIsLoading(false); - return; } - setIsLoading(true); - - const selectedApp = integrationAuthApps?.find( - (integrationAuthApp) => integrationAuthApp.appId === targetProjectId - ); - - if (!selectedApp) { - createNotification({ - type: "error", - text: "Invalid project selected" - }); - setIsLoading(false); - return; - } - - await mutateAsync({ - integrationAuthId: integrationAuth?.id, - isActive: true, - app: selectedApp.name, // project name - owner: selectedApp.owner, // organization name - appId: selectedApp.appId, // project id (used for syncing) - sourceEnvironment: selectedSourceEnvironment, - secretPath + createNotification({ + type: "success", + text: "Successfully created integration" }); - - setIsLoading(false); - - router.push(`/integrations/${localStorage.getItem("projectData.id")}`); + router.push(`/integrations/${currentWorkspace?.id}`); } catch (err) { + createNotification({ + type: "error", + text: "Failed to create integration" + }); console.error(err); } }; - const filteredProjects = useMemo(() => { - if (!integrationAuthApps) return []; + if (isProjectLoading || isCircleCIOrganizationsLoading) + return ( +
+ +
+ ); - return integrationAuthApps.filter((integrationAuthApp) => { - return integrationAuthApp.owner === targetOrganization; - }); - }, [integrationAuthApps, targetOrganization]); - - const filteredOrganizations = useMemo(() => { - const organizations = new Set(); - - if (integrationAuthApps) { - integrationAuthApps.forEach((integrationAuthApp) => { - if (!integrationAuthApp.owner) return; - organizations.add(integrationAuthApp.owner); - }); - } - - return Array.from(organizations); - }, [integrationAuthApps]); - - return integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps ? ( -
- - Set Up CircleCI Integration - - - + return ( +
+ -
-
+ - - - - - - setSecretPath(evt.target.value)} - placeholder="Provide a path, default is /" - /> - - - - - - - {targetOrganization && ( - - - + option.slug} + value={value} + getOptionLabel={(option) => option.name} + onChange={onChange} + options={currentWorkspace?.environments} + placeholder="Select a project environment" + isDisabled={!currentWorkspace?.environments.length} + /> + + )} + /> + ( + + + + )} + /> + ( + + option.slug} + value={value} + getOptionLabel={(option) => option.name} + onChange={(e) => { + setValue("targetProject", { + name: "", + id: "" + }); + setValue("targetContext", { + name: "", + id: "" + }); + + onChange(e); + }} + options={circleCIOrganizations} + placeholder={ + circleCIOrganizations?.length + ? "Select an organization..." + : "No organizations found..." + } + isDisabled={!circleCIOrganizations?.length} + /> + + )} + /> + ( + + + + )} + /> + {selectedScope === CircleCiScope.Context && selectedOrganizationEntry && ( + ( + + option.id!} + getOptionLabel={(option) => option.name} + onChange={onChange} + options={selectedOrganizationEntry?.contexts} + placeholder={ + selectedOrganizationEntry.contexts?.length + ? "Select a context..." + : "No contexts found..." + } + isDisabled={!selectedOrganizationEntry.contexts?.length} + /> + + )} + /> + )} + {selectedScope === CircleCiScope.Project && selectedOrganizationEntry && ( + ( + + option.id!} + getOptionLabel={(option) => option.name} + onChange={onChange} + options={selectedOrganizationEntry?.projects} + placeholder={ + selectedOrganizationEntry.projects?.length + ? "Select a project..." + : "No projects found..." + } + isDisabled={!selectedOrganizationEntry.projects?.length} + /> + + )} + /> )} -
-
-
- {" "} - Pro Tip -
- - After creating an integration, your secrets will start syncing immediately. This might - cause an unexpected override of current secrets in CircleCI with secrets from Infisical. - -
-
- ) : ( -
- - Set Up CircleCI Integration - - - {isIntegrationAuthAppsLoading || isintegrationAuthLoading ? ( - infisical loading indicator - ) : ( -
- -

- Something went wrong. Please contact{" "} - - support@infisical.com - {" "} - if the issue persists. -

-
- )} -
+ ); } diff --git a/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationConnectionSection.tsx b/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationConnectionSection.tsx index 778cdd00a..465760996 100644 --- a/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationConnectionSection.tsx +++ b/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationConnectionSection.tsx @@ -1,6 +1,7 @@ import { integrationSlugNameMapping } from "public/data/frequentConstants"; import { FormLabel } from "@app/components/v2"; +import { CircleCiScope } from "@app/hooks/api/integrationAuth/types"; import { IntegrationMappingBehavior, TIntegrationWithEnv } from "@app/hooks/api/integrations/types"; type Props = { @@ -46,6 +47,11 @@ export const IntegrationConnectionSection = ({ integration }: Props) => { case "qovery": return integration.scope; case "circleci": + if (integration.scope === CircleCiScope.Context) { + return "Context"; + } + + return "Project"; case "terraform-cloud": return "Project"; case "aws-secret-manager": @@ -77,7 +83,6 @@ export const IntegrationConnectionSection = ({ integration }: Props) => { return `${integration.owner}`; } return `${integration.owner}/${integration.app}`; - case "aws-parameter-store": case "rundeck": return `${integration.path}`; diff --git a/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationSettingsSection.tsx b/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationSettingsSection.tsx index f537baec6..0b8174c18 100644 --- a/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationSettingsSection.tsx +++ b/frontend/src/views/IntegrationsPage/IntegrationDetailsPage/components/IntegrationSettingsSection.tsx @@ -14,6 +14,7 @@ const metadataMappings: Record { Object.entries(integration.metadata).map(([key, value]) => (

- {metadataMappings[key as keyof typeof metadataMappings]} + {!!value && metadataMappings[key as keyof typeof metadataMappings]}

{renderValue(key as MetadataKey, value)}

diff --git a/frontend/src/views/IntegrationsPage/components/IntegrationsSection/components/IntegrationDetails.tsx b/frontend/src/views/IntegrationsPage/components/IntegrationsSection/components/IntegrationDetails.tsx index f785ca745..4e31cdedb 100644 --- a/frontend/src/views/IntegrationsPage/components/IntegrationsSection/components/IntegrationDetails.tsx +++ b/frontend/src/views/IntegrationsPage/components/IntegrationsSection/components/IntegrationDetails.tsx @@ -1,4 +1,5 @@ import { FormLabel } from "@app/components/v2"; +import { CircleCiScope } from "@app/hooks/api/integrationAuth/types"; import { IntegrationMappingBehavior, TIntegration } from "@app/hooks/api/integrations/types"; type Props = { @@ -52,7 +53,8 @@ export const IntegrationDetails = ({ integration }: Props) => { { - const userOrgs = await fetchOrganizations(); + const userOrgs = await fetchOrganizations().catch(() => []); const nonAuthEnforcedOrgs = userOrgs.filter((org) => !org.authEnforced); diff --git a/frontend/src/views/Org/components/CreateOrgModal.tsx b/frontend/src/views/Org/components/CreateOrgModal.tsx index d1baf2c9c..faa02eccd 100644 --- a/frontend/src/views/Org/components/CreateOrgModal.tsx +++ b/frontend/src/views/Org/components/CreateOrgModal.tsx @@ -6,7 +6,7 @@ import z from "zod"; import { createNotification } from "@app/components/notifications"; import { Button, FormControl, Input, Modal, ModalContent } from "@app/components/v2"; -import { useCreateOrg, useSelectOrganization } from "@app/hooks/api"; +import { useCreateOrg, useGetOrganizations, useSelectOrganization } from "@app/hooks/api"; import { ProjectType } from "@app/hooks/api/workspace/types"; const schema = z @@ -23,9 +23,10 @@ interface CreateOrgModalProps { } export const CreateOrgModal: FC = ({ isOpen, onClose }) => { - const router = useRouter(); + const { refetch: refetchOrganizations } = useGetOrganizations(); + const { control, handleSubmit, @@ -50,19 +51,21 @@ export const CreateOrgModal: FC = ({ isOpen, onClose }) => }); await selectOrg({ - organizationId: organization.id + organizationId: organization.id, + forceSetCredentials: true }); + await refetchOrganizations(); + createNotification({ text: "Successfully created organization", type: "success" }); - if (router.isReady) router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`); + if (router.isReady) + router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`); else window.location.href = `/org/${organization.id}/${ProjectType.SecretManager}/overview`; - localStorage.setItem("orgData.id", organization.id); - reset(); onClose(); } catch (err) { diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index 1162485cf..eee9adfee 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -1125,6 +1125,7 @@ export const SecretOverviewPage = () => { bodyClassName="overflow-visible" title="Create Secrets" subTitle="Create a secret across multiple environments" + onPointerDownOutside={(e) => e.preventDefault()} >