MVP preliminary idea for initial sync behavior

backend/src/services/integration-auth/integration-auth-dal.ts

@@ -1,10 +1,35 @@
+import { Knex } from "knex";
+
 import { TDbClient } from "@app/db";
-import { TableName } from "@app/db/schemas";
+import { TableName, TIntegrationAuths, TIntegrationAuthsUpdate } from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
 import { ormify } from "@app/lib/knex";
 
 export type TIntegrationAuthDALFactory = ReturnType<typeof integrationAuthDALFactory>;
 
 export const integrationAuthDALFactory = (db: TDbClient) => {
   const integrationAuthOrm = ormify(db, TableName.IntegrationAuth);
-  return integrationAuthOrm;
+
+  const bulkUpdate = async (
+    data: Array<{ filter: Partial<TIntegrationAuths>; data: TIntegrationAuthsUpdate }>,
+    tx?: Knex
+  ) => {
+    try {
+      const integrationAuths = await Promise.all(
+        data.map(async ({ filter, data: updateData }) => {
+          const [doc] = await (tx || db)(TableName.IntegrationAuth).where(filter).update(updateData).returning("*");
+          if (!doc) throw new BadRequestError({ message: "Failed to update document" });
+          return doc;
+        })
+      );
+      return integrationAuths;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
+  return {
+    ...integrationAuthOrm,
+    bulkUpdate
+  };
 };

backend/src/services/integration-auth/integration-auth-service.ts

@@ -43,6 +43,7 @@ import {
 import { getIntegrationOptions, Integrations, IntegrationUrls } from "./integration-list";
 import { getTeams } from "./integration-team";
 import { exchangeCode, exchangeRefresh } from "./integration-token";
+import { access } from "node:fs";
 
 type TIntegrationAuthServiceFactoryDep = {
   integrationAuthDAL: TIntegrationAuthDALFactory;
@@ -591,6 +592,7 @@ export const integrationAuthServiceFactory = ({
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
     const botKey = await projectBotService.getBotKey(integrationAuth.projectId);
     const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey);
+
     const { data } = await request.get<THerokuPipelineCoupling[]>(
       `${IntegrationUrls.HEROKU_API_URL}/pipeline-couplings`,
       {
@@ -601,7 +603,7 @@ export const integrationAuthServiceFactory = ({
         }
       }
     );
-
+    
     return data.map(({ app: { id: appId }, stage, pipeline: { id: pipelineId, name } }) => ({
       app: { appId },
       stage,

backend/src/services/integration-auth/integration-list.ts

@@ -37,6 +37,12 @@ export enum IntegrationType {
   OAUTH2 = "oauth2"
 }
 
+export enum IntegrationSyncBehavior {
+  OVERWRITE_TARGET = "overwrite-target",
+  PREFER_TARGET = "prefer-target",
+  PREFER_SOURCE = "prefer-source"
+}
+
 export enum IntegrationUrls {
   // integration oauth endpoints
   GCP_TOKEN_URL = "https://oauth2.googleapis.com/token",

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -20,11 +20,20 @@ import sodium from "libsodium-wrappers";
 import isEqual from "lodash.isequal";
 import { z } from "zod";
 
-import { TIntegrationAuths, TIntegrations } from "@app/db/schemas";
+import { TIntegrationAuths, TIntegrations, SecretType } from "@app/db/schemas";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
 
-import { Integrations, IntegrationUrls } from "./integration-list";
+import { Integrations, IntegrationUrls, IntegrationSyncBehavior } from "./integration-list";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+
+import { createManySecretsRawHelper } from "../secret/secret-fns";
 
 const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
   Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -441,16 +450,19 @@ const syncSecretsAWSParameterStore = async ({
 }) => {
   if (!accessId) return;
 
-  AWS.config.update({
+  const config = new AWS.Config({
     region: integration.region as string,
-    accessKeyId: accessId,
-    secretAccessKey: accessToken
+    credentials: {
+      accessKeyId: accessId,
+      secretAccessKey: accessToken
+    }
   });
 
   const ssm = new AWS.SSM({
     apiVersion: "2014-11-06",
     region: integration.region as string
   });
+  ssm.config.update(config);
 
   const params = {
     Path: integration.path as string,
@@ -514,12 +526,6 @@ const syncSecretsAWSParameterStore = async ({
       }
     })
   );
-
-  AWS.config.update({
-    region: undefined,
-    accessKeyId: undefined,
-    secretAccessKey: undefined
-  });
 };
 
 /**
@@ -541,12 +547,6 @@ const syncSecretsAWSSecretManager = async ({
   try {
     if (!accessId) return;
 
-    AWS.config.update({
-      region: integration.region as string,
-      accessKeyId: accessId,
-      secretAccessKey: accessToken
-    });
-
     secretsManager = new SecretsManagerClient({
       region: integration.region as string,
       credentials: {
@@ -575,12 +575,6 @@ const syncSecretsAWSSecretManager = async ({
         })
       );
     }
-
-    AWS.config.update({
-      region: undefined,
-      accessKeyId: undefined,
-      secretAccessKey: undefined
-    });
   } catch (err) {
     if (err instanceof ResourceNotFoundException && secretsManager) {
       await secretsManager.send(
@@ -590,26 +584,45 @@ const syncSecretsAWSSecretManager = async ({
         })
       );
     }
-    AWS.config.update({
-      region: undefined,
-      accessKeyId: undefined,
-      secretAccessKey: undefined
-    });
   }
 };
 
 /**
  * Sync/push [secrets] to Heroku app named [integration.app]
+ * server.services... 
  */
 const syncSecretsHeroku = async ({
+  projectDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL,
+  botKey, // TODO: consider getting botKey inside this fn
+  projectId,
+  environment,
+  secretPath,
   integration,
   secrets,
   accessToken
 }: {
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+  botKey: string;
+  projectId: string;
+  environment: string;
+  secretPath: string;
   integration: TIntegrations;
   secrets: Record<string, { value: string; comment?: string } | null>;
   accessToken: string;
 }) => {
+
   const herokuSecrets = (
     await request.get(`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`, {
       headers: {
@@ -619,13 +632,63 @@ const syncSecretsHeroku = async ({
       }
     })
   ).data;
+  
+  let secretsToAdd: { [key: string]: string } = {};
+  let secretsToUpdate: { [key: string]: string } = {};
+
+  const metadata = z.record(z.any()).parse(integration.metadata);
 
   Object.keys(herokuSecrets).forEach((key) => {
-    if (!(key in secrets)) {
-      secrets[key] = null;
+    switch (metadata.syncBehavior) {
+      case IntegrationSyncBehavior.OVERWRITE_TARGET: {
+        if (!(key in secrets)) secrets[key] = null;
+        break;
+      };
+      case IntegrationSyncBehavior.PREFER_TARGET: {
+        secrets[key] = herokuSecrets[key];
+        if (!(key in secrets)) {
+          secretsToAdd[key] = herokuSecrets[key];
+        } else {
+          if (secrets[key] !== herokuSecrets[key]) {
+            secretsToUpdate[key] = secrets[key]?.value as string;
+          }
+        }
+        break;
+      };
+      case IntegrationSyncBehavior.PREFER_SOURCE: {
+        if(!(key in secrets)) {
+          secrets[key] = herokuSecrets[key];
+          secretsToAdd[key] = herokuSecrets[key];
+        }
+        break;
+      };
+      default: {
+        if (!(key in secrets)) secrets[key] = null;
+        break;
+      };
     }
   });
 
+  await createManySecretsRawHelper({
+    botKey,
+    projectDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL,
+    projectId,
+    environment,
+    path: secretPath,
+    secrets: Object.keys(secretsToAdd).map((key) => ({
+      secretName: key,
+      secretValue: secretsToAdd[key],
+      type: SecretType.Shared,
+      secretComment: ""
+    }))
+  });
+
   await request.patch(
     `${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`,
     getSecretKeyValuePair(secrets),
@@ -2950,8 +3013,22 @@ const syncSecretsHasuraCloud = async ({
 
 /**
  * Sync/push [secrets] to [app] in integration named [integration]
+ * 
+ * Do this in terms of DAL
+ * 
  */
 export const syncIntegrationSecrets = async ({
+  projectDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL,
+  botKey,
+  projectId,
+  environment,
+  secretPath,
   integration,
   integrationAuth,
   secrets,
@@ -2959,6 +3036,17 @@ export const syncIntegrationSecrets = async ({
   accessToken,
   appendices
 }: {
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+  botKey: string;
+  projectId: string;
+  environment: string;
+  secretPath: string;
   integration: TIntegrations;
   integrationAuth: TIntegrationAuths;
   secrets: Record<string, { value: string; comment?: string }>;
@@ -2999,9 +3087,20 @@ export const syncIntegrationSecrets = async ({
       break;
     case Integrations.HEROKU:
       await syncSecretsHeroku({
+        projectDAL,
+        secretDAL,
+        secretVersionDAL,
+        secretBlindIndexDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        folderDAL,
+        botKey,
+        projectId,
+        environment,
+        secretPath,
         integration,
         secrets,
-        accessToken
+        accessToken,
       });
       break;
     case Integrations.VERCEL: