feat: gcp secret sync management

backend/src/server/routes/v1/app-connection-routers/gcp-connection-router.ts

@@ -1,13 +1,18 @@
+import z from "zod";
+
+import { readLimit } from "@app/server/config/rateLimiter";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 import {
   CreateGcpConnectionSchema,
   SanitizedGcpConnectionSchema,
   UpdateGcpConnectionSchema
 } from "@app/services/app-connection/gcp";
+import { AuthMode } from "@app/services/auth/auth-type";
 
 import { registerAppConnectionEndpoints } from "./app-connection-endpoints";
 
-export const registerGcpConnectionRouter = async (server: FastifyZodProvider) =>
+export const registerGcpConnectionRouter = async (server: FastifyZodProvider) => {
   registerAppConnectionEndpoints({
     app: AppConnection.GCP,
     server,
@@ -15,3 +20,29 @@ export const registerGcpConnectionRouter = async (server: FastifyZodProvider) =>
     createSchema: CreateGcpConnectionSchema,
     updateSchema: UpdateGcpConnectionSchema
   });
+
+  // The below endpoints are not exposed and for Infisical App use
+  server.route({
+    method: "GET",
+    url: `/:connectionId/secret-manager-projects`,
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      params: z.object({
+        connectionId: z.string().uuid()
+      }),
+      response: {
+        200: z.object({ id: z.string(), name: z.string() }).array()
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { connectionId } = req.params;
+
+      const projects = await server.services.appConnection.gcp.listSecretManagerProjects(connectionId, req.permission);
+
+      return projects;
+    }
+  });
+};

backend/src/server/routes/v1/secret-sync-routers/gcp-sync-router.ts

@@ -0,0 +1,13 @@
+import { CreateGcpSyncSchema, GcpSyncSchema, UpdateGcpSyncSchema } from "@app/services/secret-sync/gcp";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+
+import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints";
+
+export const registerGcpSyncRouter = async (server: FastifyZodProvider) =>
+  registerSyncSecretsEndpoints({
+    destination: SecretSync.GCP,
+    server,
+    responseSchema: GcpSyncSchema,
+    createSchema: CreateGcpSyncSchema,
+    updateSchema: UpdateGcpSyncSchema
+  });

backend/src/server/routes/v1/secret-sync-routers/index.ts

@@ -1,11 +1,13 @@
 import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
 
 import { registerAwsParameterStoreSyncRouter } from "./aws-parameter-store-sync-router";
+import { registerGcpSyncRouter } from "./gcp-sync-router";
 import { registerGitHubSyncRouter } from "./github-sync-router";
 
 export * from "./secret-sync-router";
 
 export const SECRET_SYNC_REGISTER_ROUTER_MAP: Record<SecretSync, (server: FastifyZodProvider) => Promise<void>> = {
   [SecretSync.AWSParameterStore]: registerAwsParameterStoreSyncRouter,
-  [SecretSync.GitHub]: registerGitHubSyncRouter
+  [SecretSync.GitHub]: registerGitHubSyncRouter,
+  [SecretSync.GCP]: registerGcpSyncRouter
 };

backend/src/server/routes/v1/secret-sync-routers/secret-sync-router.ts

@@ -9,13 +9,19 @@ import {
   AwsParameterStoreSyncListItemSchema,
   AwsParameterStoreSyncSchema
 } from "@app/services/secret-sync/aws-parameter-store";
+import { GcpSyncListItemSchema, GcpSyncSchema } from "@app/services/secret-sync/gcp";
 import { GitHubSyncListItemSchema, GitHubSyncSchema } from "@app/services/secret-sync/github";
 
-const SecretSyncSchema = z.discriminatedUnion("destination", [AwsParameterStoreSyncSchema, GitHubSyncSchema]);
+const SecretSyncSchema = z.discriminatedUnion("destination", [
+  AwsParameterStoreSyncSchema,
+  GitHubSyncSchema,
+  GcpSyncSchema
+]);
 
 const SecretSyncOptionsSchema = z.discriminatedUnion("destination", [
   AwsParameterStoreSyncListItemSchema,
-  GitHubSyncListItemSchema
+  GitHubSyncListItemSchema,
+  GcpSyncListItemSchema
 ]);
 
 export const registerSecretSyncRouter = async (server: FastifyZodProvider) => {

backend/src/services/app-connection/app-connection-maps.ts

@@ -2,5 +2,6 @@ import { AppConnection } from "./app-connection-enums";
 
 export const APP_CONNECTION_NAME_MAP: Record<AppConnection, string> = {
   [AppConnection.AWS]: "AWS",
-  [AppConnection.GitHub]: "GitHub"
+  [AppConnection.GitHub]: "GitHub",
+  [AppConnection.GCP]: "GCP"
 };

backend/src/services/app-connection/app-connection-service.ts

@@ -27,6 +27,7 @@ import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 
 import { TAppConnectionDALFactory } from "./app-connection-dal";
 import { ValidateGcpConnectionCredentialsSchema } from "./gcp";
+import { gcpConnectionService } from "./gcp/gcp-connection-service";
 
 export type TAppConnectionServiceFactoryDep = {
   appConnectionDAL: TAppConnectionDALFactory;
@@ -384,6 +385,7 @@ export const appConnectionServiceFactory = ({
     deleteAppConnection,
     connectAppConnectionById,
     listAvailableAppConnectionsForUser,
-    github: githubConnectionService(connectAppConnectionById)
+    github: githubConnectionService(connectAppConnectionById),
+    gcp: gcpConnectionService(connectAppConnectionById)
   };
 };

backend/src/services/app-connection/gcp/gcp-connection-fns.ts

@@ -2,12 +2,20 @@ import { gaxios, Impersonated, JWT } from "google-auth-library";
 import { GetAccessTokenResponse } from "google-auth-library/build/src/auth/oauth2client";
 
 import { getConfig } from "@app/lib/config/env";
+import { request } from "@app/lib/config/request";
 import { BadRequestError, InternalServerError } from "@app/lib/errors";
+import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
 
 import { AppConnection } from "../app-connection-enums";
 import { getAppConnectionMethodName } from "../app-connection-fns";
 import { GcpConnectionMethod } from "./gcp-connection-enums";
-import { TGcpConnectionConfig } from "./gcp-connection-types";
+import {
+  GCPApp,
+  GCPGetProjectsRes,
+  GCPGetServiceRes,
+  TGcpConnection,
+  TGcpConnectionConfig
+} from "./gcp-connection-types";
 
 export const getGcpAppConnectionListItem = () => {
   return {
@@ -17,9 +25,8 @@ export const getGcpAppConnectionListItem = () => {
   };
 };
 
-export const validateGcpConnectionCredentials = async (appConnection: TGcpConnectionConfig) => {
+export const getAuthToken = async (appConnection: TGcpConnectionConfig) => {
   const appCfg = getConfig();
-
   if (!appCfg.INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL) {
     throw new InternalServerError({
       message: `Environment variables have not been configured for GCP ${getAppConnectionMethodName(
@@ -67,5 +74,79 @@ export const validateGcpConnectionCredentials = async (appConnection: TGcpConnec
     });
   }
 
+  return tokenResponse.token;
+};
+
+export const getGcpSecretManagerProjects = async (appConnection: TGcpConnection) => {
+  const accessToken = await getAuthToken(appConnection);
+
+  let gcpApps: GCPApp[] = [];
+
+  const pageSize = 100;
+  let pageToken: string | undefined;
+  let hasMorePages = true;
+
+  const projects: {
+    name: string;
+    id: string;
+  }[] = [];
+
+  while (hasMorePages) {
+    const params = new URLSearchParams({
+      pageSize: String(pageSize),
+      ...(pageToken ? { pageToken } : {})
+    });
+
+    // eslint-disable-next-line no-await-in-loop
+    const { data } = await request.get<GCPGetProjectsRes>(`${IntegrationUrls.GCP_API_URL}/v1/projects`, {
+      params,
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Accept-Encoding": "application/json"
+      }
+    });
+
+    gcpApps = gcpApps.concat(data.projects);
+
+    if (!data.nextPageToken) {
+      hasMorePages = false;
+    }
+
+    pageToken = data.nextPageToken;
+  }
+
+  // eslint-disable-next-line
+  for await (const gcpApp of gcpApps) {
+    try {
+      const res = (
+        await request.get<GCPGetServiceRes>(
+          `${IntegrationUrls.GCP_SERVICE_USAGE_URL}/v1/projects/${gcpApp.projectId}/services/${IntegrationUrls.GCP_SECRET_MANAGER_SERVICE_NAME}`,
+          {
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              "Accept-Encoding": "application/json"
+            }
+          }
+        )
+      ).data;
+
+      if (res.state === "ENABLED") {
+        projects.push({
+          name: gcpApp.name,
+          id: gcpApp.projectId
+        });
+      }
+    } catch {
+      // eslint-disable-next-line
+      continue;
+    }
+  }
+
+  return projects;
+};
+
+export const validateGcpConnectionCredentials = async (appConnection: TGcpConnectionConfig) => {
+  await getAuthToken(appConnection);
+
   return appConnection.credentials;
 };

backend/src/services/app-connection/gcp/gcp-connection-service.ts

@@ -0,0 +1,29 @@
+import { OrgServiceActor } from "@app/lib/types";
+
+import { AppConnection } from "../app-connection-enums";
+import { getGcpSecretManagerProjects } from "./gcp-connection-fns";
+import { TGcpConnection } from "./gcp-connection-types";
+
+type TGetAppConnectionFunc = (
+  app: AppConnection,
+  connectionId: string,
+  actor: OrgServiceActor
+) => Promise<TGcpConnection>;
+
+export const gcpConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+  const listSecretManagerProjects = async (connectionId: string, actor: OrgServiceActor) => {
+    const appConnection = await getAppConnection(AppConnection.GCP, connectionId, actor);
+
+    try {
+      const projects = await getGcpSecretManagerProjects(appConnection);
+
+      return projects;
+    } catch (error) {
+      return [];
+    }
+  };
+
+  return {
+    listSecretManagerProjects
+  };
+};

backend/src/services/app-connection/gcp/gcp-connection-types.ts

@@ -20,3 +20,26 @@ export type TValidateGcpConnectionCredentials = typeof ValidateGcpConnectionCred
 export type TGcpConnectionConfig = DiscriminativePick<TGcpConnectionInput, "method" | "app" | "credentials"> & {
   orgId: string;
 };
+
+export interface GCPApp {
+  projectNumber: string;
+  projectId: string;
+  lifecycleState: "ACTIVE" | "LIFECYCLE_STATE_UNSPECIFIED" | "DELETE_REQUESTED" | "DELETE_IN_PROGRESS";
+  name: string;
+  createTime: string;
+  parent: {
+    type: "organization" | "folder" | "project";
+    id: string;
+  };
+}
+
+export interface GCPGetProjectsRes {
+  projects: GCPApp[];
+  nextPageToken?: string;
+}
+
+export interface GCPGetServiceRes {
+  name: string;
+  parent: string;
+  state: "ENABLED" | "DISABLED" | "STATE_UNSPECIFIED";
+}

backend/src/services/secret-sync/gcp/gcp-sync-constants.ts

@@ -0,0 +1,10 @@
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
+
+export const GCP_SYNC_LIST_OPTION: TSecretSyncListItem = {
+  name: "GCP",
+  destination: SecretSync.GCP,
+  connection: AppConnection.GCP,
+  canImportSecrets: false
+};

backend/src/services/secret-sync/gcp/gcp-sync-schemas.ts

@@ -0,0 +1,37 @@
+import z from "zod";
+
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import {
+  BaseSecretSyncSchema,
+  GenericCreateSecretSyncFieldsSchema,
+  GenericUpdateSecretSyncFieldsSchema
+} from "@app/services/secret-sync/secret-sync-schemas";
+import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
+
+import { SecretSync } from "../secret-sync-enums";
+
+const GcpSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };
+
+const GcpSyncDestinationConfigSchema = z.object({
+  projectId: z.string().min(1, "Project ID is required")
+});
+
+export const GcpSyncSchema = BaseSecretSyncSchema(SecretSync.GCP, GcpSyncOptionsConfig).extend({
+  destination: z.literal(SecretSync.GCP),
+  destinationConfig: GcpSyncDestinationConfigSchema
+});
+
+export const CreateGcpSyncSchema = GenericCreateSecretSyncFieldsSchema(SecretSync.GCP, GcpSyncOptionsConfig).extend({
+  destinationConfig: GcpSyncDestinationConfigSchema
+});
+
+export const UpdateGcpSyncSchema = GenericUpdateSecretSyncFieldsSchema(SecretSync.GCP, GcpSyncOptionsConfig).extend({
+  destinationConfig: GcpSyncDestinationConfigSchema.optional()
+});
+
+export const GcpSyncListItemSchema = z.object({
+  name: z.literal("GCP"),
+  connection: z.literal(AppConnection.GCP),
+  destination: z.literal(SecretSync.GCP),
+  canImportSecrets: z.literal(false)
+});

backend/src/services/secret-sync/gcp/gcp-sync-types.ts

@@ -0,0 +1,15 @@
+import z from "zod";
+
+import { TGcpConnection } from "@app/services/app-connection/gcp";
+
+import { CreateGcpSyncSchema, GcpSyncListItemSchema, GcpSyncSchema } from "./gcp-sync-schemas";
+
+export type TGcpSyncListItem = z.infer<typeof GcpSyncListItemSchema>;
+
+export type TGcpSync = z.infer<typeof GcpSyncSchema>;
+
+export type TGcpSyncInput = z.infer<typeof CreateGcpSyncSchema>;
+
+export type TGcpSyncWithCredentials = TGcpSync & {
+  connection: TGcpConnection;
+};

backend/src/services/secret-sync/gcp/index.ts

@@ -0,0 +1,3 @@
+export * from "./gcp-sync-constants";
+export * from "./gcp-sync-schemas";
+export * from "./gcp-sync-types";

backend/src/services/secret-sync/secret-sync-enums.ts

@@ -1,6 +1,7 @@
 export enum SecretSync {
   AWSParameterStore = "aws-parameter-store",
-  GitHub = "github"
+  GitHub = "github",
+  GCP = "gcp"
 }
 
 export enum SecretSyncInitialSyncBehavior {

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -13,9 +13,12 @@ import {
   TSecretSyncWithCredentials
 } from "@app/services/secret-sync/secret-sync-types";
 
+import { GCP_SYNC_LIST_OPTION } from "./gcp";
+
 const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
-  [SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION
+  [SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION,
+  [SecretSync.GCP]: GCP_SYNC_LIST_OPTION
 };
 
 export const listSecretSyncOptions = () => {

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -3,10 +3,12 @@ import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
 
 export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.AWSParameterStore]: "AWS Parameter Store",
-  [SecretSync.GitHub]: "GitHub"
+  [SecretSync.GitHub]: "GitHub",
+  [SecretSync.GCP]: "GCP Secret Manager"
 };
 
 export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
   [SecretSync.AWSParameterStore]: AppConnection.AWS,
-  [SecretSync.GitHub]: AppConnection.GitHub
+  [SecretSync.GitHub]: AppConnection.GitHub,
+  [SecretSync.GCP]: AppConnection.GCP
 };

backend/src/services/secret-sync/secret-sync-types.ts

@@ -17,14 +17,18 @@ import {
   TAwsParameterStoreSyncListItem,
   TAwsParameterStoreSyncWithCredentials
 } from "./aws-parameter-store";
+import { TGcpSync, TGcpSyncInput, TGcpSyncListItem, TGcpSyncWithCredentials } from "./gcp";
 
-export type TSecretSync = TAwsParameterStoreSync | TGitHubSync;
+export type TSecretSync = TAwsParameterStoreSync | TGitHubSync | TGcpSync;
 
-export type TSecretSyncWithCredentials = TAwsParameterStoreSyncWithCredentials | TGitHubSyncWithCredentials;
+export type TSecretSyncWithCredentials =
+  | TAwsParameterStoreSyncWithCredentials
+  | TGitHubSyncWithCredentials
+  | TGcpSyncWithCredentials;
 
-export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput;
+export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput | TGcpSyncInput;
 
-export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem;
+export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem | TGcpSyncListItem;
 
 export type TSyncOptionsConfig = {
   canImportSecrets: boolean;