add integ tests for fetching/secrets secrets with jwt/service token

.github/workflows/docker-image.yml

@@ -9,6 +9,12 @@ jobs:
     steps:
       - name: ☁️ Checkout source
         uses: actions/checkout@v3
+      - name: 📦 Install dependencies to test all dependencies
+        run: npm ci --only-production
+        working-directory: backend
+      - name: 🧪 Run tests
+        run: npm run test:ci
+        working-directory: backend
       - name: Save commit hashes for tag
         id: commit
         uses: pr-mpt/actions-commit-hash@v2
@@ -45,8 +51,8 @@ jobs:
           token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
           push: true
           context: backend
-          tags: infisical/backend:${{ steps.commit.outputs.short }}, 
+          tags: infisical/backend:${{ steps.commit.outputs.short }},
-                infisical/backend:latest
+            infisical/backend:latest
           platforms: linux/amd64,linux/arm64
 
   frontend-image:
@@ -94,8 +100,8 @@ jobs:
           push: true
           token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
           context: frontend
-          tags: infisical/frontend:${{ steps.commit.outputs.short }}, 
+          tags: infisical/frontend:${{ steps.commit.outputs.short }},
-                infisical/frontend:latest
+            infisical/frontend:latest
           platforms: linux/amd64,linux/arm64
           build-args: |
             POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
@@ -122,7 +128,7 @@ jobs:
           token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
       - name: Save DigitalOcean kubeconfig with short-lived credentials
         run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 k8s-1-25-4-do-0-nyc1-1670645170179
-      - name: switch to gamma namespace 
+      - name: switch to gamma namespace
         run: kubectl config set-context --current --namespace=gamma
       - name: test kubectl
         run: kubectl get ingress
@@ -135,4 +141,4 @@ jobs:
             exit 1
           else
             echo "Helm upgrade was successful"
-          fi
+          fi

backend/package-lock.json

@@ -62,7 +62,7 @@
         "@types/cookie-parser": "^1.4.3",
         "@types/cors": "^2.8.12",
         "@types/express": "^4.17.14",
-        "@types/jest": "^29.2.4",
+        "@types/jest": "^29.5.0",
         "@types/jsonwebtoken": "^8.5.9",
         "@types/lodash": "^4.14.191",
         "@types/node": "^18.11.3",
@@ -3629,9 +3629,9 @@
       }
     },
     "node_modules/@types/jest": {
-      "version": "29.4.0",
+      "version": "29.5.0",
-      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.0.tgz",
-      "integrity": "sha512-VaywcGQ9tPorCX/Jkkni7RWGFfI11whqzs8dvxF41P17Z+z872thvEvlIbznjPJ02kl1HMX3LmLOonsj2n7HeQ==",
+      "integrity": "sha512-3Emr5VOl/aoBwnWcH/EFQvlSAmjV+XtV9GGu5mwdYew5vhQh0IUZx/60x0TzHDu09Bi7HMx10t/namdJw5QIcg==",
       "dev": true,
       "dependencies": {
         "expect": "^29.0.0",
@@ -15642,9 +15642,9 @@
       }
     },
     "@types/jest": {
-      "version": "29.4.0",
+      "version": "29.5.0",
-      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.0.tgz",
-      "integrity": "sha512-VaywcGQ9tPorCX/Jkkni7RWGFfI11whqzs8dvxF41P17Z+z872thvEvlIbznjPJ02kl1HMX3LmLOonsj2n7HeQ==",
+      "integrity": "sha512-3Emr5VOl/aoBwnWcH/EFQvlSAmjV+XtV9GGu5mwdYew5vhQh0IUZx/60x0TzHDu09Bi7HMx10t/namdJw5QIcg==",
       "dev": true,
       "requires": {
         "expect": "^29.0.0",

backend/package.json

@@ -3,8 +3,8 @@
     "@aws-sdk/client-secrets-manager": "^3.287.0",
     "@godaddy/terminus": "^4.11.2",
     "@octokit/rest": "^19.0.5",
-    "@sentry/tracing": "^7.39.0",
     "@sentry/node": "^7.40.0",
+    "@sentry/tracing": "^7.39.0",
     "@types/crypto-js": "^4.1.1",
     "@types/libsodium-wrappers": "^0.7.10",
     "await-to-js": "^3.0.0",
@@ -57,7 +57,7 @@
     "lint-and-fix": "eslint . --ext .ts --fix",
     "lint-staged": "lint-staged",
     "pretest": "docker compose -f test-resources/docker-compose.test.yml up -d",
-    "test": "cross-env NODE_ENV=test jest --verbose --testTimeout=10000 --detectOpenHandles",
+    "test": "cross-env NODE_ENV=test jest --verbose --testTimeout=10000 --detectOpenHandles; npm run posttest",
     "test:ci": "npm test -- --watchAll=false --ci --reporters=default --reporters=jest-junit --reporters=github-actions --coverage --testLocationInResults --json --outputFile=coverage/report.json",
     "posttest": "docker compose -f test-resources/docker-compose.test.yml down"
   },
@@ -80,7 +80,7 @@
     "@types/cookie-parser": "^1.4.3",
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.14",
-    "@types/jest": "^29.2.4",
+    "@types/jest": "^29.5.0",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/lodash": "^4.14.191",
     "@types/node": "^18.11.3",

backend/src/index.ts

@@ -48,18 +48,18 @@ import {
     integrationAuth as v1IntegrationAuthRouter
 } from './routes/v1';
 import {
-  signup as v2SignupRouter,
+    signup as v2SignupRouter,
-  auth as v2AuthRouter,
+    auth as v2AuthRouter,
-  users as v2UsersRouter,
+    users as v2UsersRouter,
-  organizations as v2OrganizationsRouter,
+    organizations as v2OrganizationsRouter,
-  workspace as v2WorkspaceRouter,
+    workspace as v2WorkspaceRouter,
-  secret as v2SecretRouter, // begin to phase out
+    secret as v2SecretRouter, // begin to phase out
-  secrets as v2SecretsRouter,
+    secrets as v2SecretsRouter,
-  serviceTokenData as v2ServiceTokenDataRouter,
+    serviceTokenData as v2ServiceTokenDataRouter,
-  serviceAccounts as v2ServiceAccountsRouter,
+    serviceAccounts as v2ServiceAccountsRouter,
-  apiKeyData as v2APIKeyDataRouter,
+    apiKeyData as v2APIKeyDataRouter,
-  environment as v2EnvironmentRouter,
+    environment as v2EnvironmentRouter,
-  tags as v2TagsRouter,
+    tags as v2TagsRouter,
 } from './routes/v2';
 import { healthCheck } from './routes/status';
 import { getLogger } from './utils/logger';
@@ -172,7 +172,7 @@ const main = async () => {
         getLogger("backend-main").info(`Server started listening at port ${getPort()}`)
     });
 
-    createTestUserForDevelopment();
+    await createTestUserForDevelopment();
     setUpHealthEndpoint(server);
 
     server.on('close', async () => {

backend/src/utils/addDevelopmentUser.ts

@@ -8,17 +8,18 @@ import { Key, Membership, MembershipOrg, Organization, User, Workspace } from ".
 import { Types } from 'mongoose';
 import { getNodeEnv } from '../config';
 
-export const createTestUserForDevelopment = async () => {
+export const testUserEmail = "[email protected]"
-  if (getNodeEnv() === "development") {
+export const testUserPassword = "testInfisical1"
-    const testUserEmail = "[email protected]"
+export const testUserId = "63cefa6ec8d3175601cfa980"
-    const testUserPassword = "testInfisical1"
+export const testWorkspaceId = "63cefb15c8d3175601cfa989"
-    const testUserId = "63cefa6ec8d3175601cfa980"
+export const testOrgId = "63cefb15c8d3175601cfa985"
-    const testWorkspaceId = "63cefb15c8d3175601cfa989"
+export const testMembershipId = "63cefb159185d9aa3ef0cf35"
-    const testOrgId = "63cefb15c8d3175601cfa985"
+export const testMembershipOrgId = "63cefb159185d9aa3ef0cf31"
-    const testMembershipId = "63cefb159185d9aa3ef0cf35"
+export const testWorkspaceKeyId = "63cf48f0225e6955acec5eff"
-    const testMembershipOrgId = "63cefb159185d9aa3ef0cf31"
+export const plainTextWorkspaceKey = "543fef8224813a46230b0a50a46c5fb2"
-    const testWorkspaceKeyId = "63cf48f0225e6955acec5eff"
 
+export const createTestUserForDevelopment = async () => {
+  if (getNodeEnv() === "development" || getNodeEnv() === "test") {
     const testUser = {
       _id: testUserId,
       email: testUserEmail,

backend/test-resources/env-vars.js

@@ -3,3 +3,8 @@ process.env.MONGO_URL =
   'mongodb://test:test1234@localhost:27018/?authSource=admin';
 process.env.MONGO_USERNAME = 'test';
 process.env.MONGO_PASSWORD = 'test1234';
+process.env.NODE_ENV = 'test';
+process.env.JWT_SIGNUP_SECRET= "38ea90fb7998b92176080f457d890392"
+process.env.JWT_REFRESH_SECRET= "7764c7bbf3928ad501591a3e005eb364"
+process.env.JWT_AUTH_SECRET= "5239fea3a4720c0e524f814a540e14a2"
+process.env.JWT_SERVICE_SECRET= "8509fb8b90c9b53e9e61d1e35826dcb5"

backend/tests/data/batch-secrets.json

@@ -0,0 +1,56 @@
+[
+  {
+    "method": "POST",
+    "secret": {
+        "workspace": "63cefb15c8d3175601cfa989",
+        "type": "shared",
+        "tags": [],
+        "environment": "dev",
+        "secretKeyCiphertext": "eaX9a2g=",
+        "secretKeyIV": "YJ4adgI/wEHifGdtT9reaA==",
+        "secretKeyTag": "dP73x3wrq7pqxzAHo+bfPA==",
+        "secretValueCiphertext": "cw==",
+        "secretValueIV": "7ksYWWZ3+9rzLG5NpEbEgg==",
+        "secretValueTag": "H0YQ8vrhiVJ0XSW4nBJdQA==",
+        "secretCommentCiphertext": "",
+        "secretCommentIV": "yXhMdLdA9q7Vaw4UUaeBYA==",
+        "secretCommentTag": "qMj7SHESM5Jn+C2qpbw2pA=="
+    }
+  },
+  {
+    "method": "POST",
+    "secret": {
+        "workspace": "63cefb15c8d3175601cfa989",
+        "type": "shared",
+        "tags": [],
+        "environment": "dev",
+        "secretKeyCiphertext": "eaX9a2g=",
+        "secretKeyIV": "YJ4adgI/wEHifGdtT9reaA==",
+        "secretKeyTag": "dP73x3wrq7pqxzAHo+bfPA==",
+        "secretValueCiphertext": "cw==",
+        "secretValueIV": "7ksYWWZ3+9rzLG5NpEbEgg==",
+        "secretValueTag": "H0YQ8vrhiVJ0XSW4nBJdQA==",
+        "secretCommentCiphertext": "",
+        "secretCommentIV": "yXhMdLdA9q7Vaw4UUaeBYA==",
+        "secretCommentTag": "qMj7SHESM5Jn+C2qpbw2pA=="
+    }
+  },
+  {
+    "method": "POST",
+    "secret": {
+        "workspace": "63cefb15c8d3175601cfa989",
+        "type": "shared",
+        "tags": [],
+        "environment": "dev",
+        "secretKeyCiphertext": "eaX9a2g=",
+        "secretKeyIV": "YJ4adgI/wEHifGdtT9reaA==",
+        "secretKeyTag": "dP73x3wrq7pqxzAHo+bfPA==",
+        "secretValueCiphertext": "cw==",
+        "secretValueIV": "7ksYWWZ3+9rzLG5NpEbEgg==",
+        "secretValueTag": "H0YQ8vrhiVJ0XSW4nBJdQA==",
+        "secretCommentCiphertext": "",
+        "secretCommentIV": "yXhMdLdA9q7Vaw4UUaeBYA==",
+        "secretCommentTag": "qMj7SHESM5Jn+C2qpbw2pA=="
+    }
+  }
+]

backend/tests/helper/helper.ts

@@ -0,0 +1,91 @@
+// Helper functions for integration tests
+
+import { Secret } from "../../src/models";
+import { testUserEmail, testUserPassword } from "../../src/utils/addDevelopmentUser";
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const crypto = require('crypto')
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const jsrp = require('jsrp');
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const axios = require('axios');
+import { plainTextWorkspaceKey, testWorkspaceId } from "../../src/utils/addDevelopmentUser";
+import { encryptSymmetric } from "../../src/utils/crypto";
+
+interface TokenData {
+  token: string;
+  publicKey: string;
+  encryptedPrivateKey: string;
+  iv: string;
+  tag: string;
+}
+
+export const getJWTFromTestUser = (): Promise<TokenData> => {
+  return new Promise((resolve, reject) => {
+    const client = new jsrp.client();
+    const EMAIL = testUserEmail
+    const PASSWORD = testUserPassword
+
+    client.init({
+      username: EMAIL,
+      password: PASSWORD,
+    }, async () => {
+      const clientPublicKey = client.getPublicKey();
+
+      // POST: /login1 
+      const reqBody = {
+        email: EMAIL,
+        clientPublicKey
+      }
+
+
+      const loginOneRes = await axios.post('http://localhost:4000/api/v1/auth/login1', reqBody);
+      const serverPublicKey = loginOneRes.data.serverPublicKey;
+      const salt = loginOneRes.data.salt;
+
+      client.setSalt(salt);
+      client.setServerPublicKey(serverPublicKey);
+      const clientSharedKey = client.getSharedKey(); // shared Key
+      const clientProof = client.getProof(); // called M1
+
+      // POST: /login2
+      const reqBody2 = {
+        email: EMAIL,
+        clientProof
+      }
+
+      const response2 = await axios.post('http://localhost:4000/api/v1/auth/login2', reqBody2);
+
+      resolve(response2.data)
+    })
+  });
+}
+
+export const getServiceTokenFromTestUser = async () => {
+  const loggedInUserDetails = await getJWTFromTestUser()
+  const randomBytes = crypto.randomBytes(16).toString('hex');
+  const { ciphertext, iv, tag } = encryptSymmetric({
+    plaintext: plainTextWorkspaceKey,
+    key: randomBytes,
+  });
+
+  const newServiceToken = await axios.post('http://localhost:4000/api/v2/service-token/', {
+    'name': "test service token",
+    'workspaceId': testWorkspaceId,
+    'environment': "dev",
+    'encryptedKey': ciphertext,
+    'iv': iv,
+    'tag': tag,
+    'expiresIn': Date.now() + 90000,
+    'permissions': ["read"]
+  }, {
+    headers: {
+      'Authorization': `Bearer ${loggedInUserDetails.token}`
+    }
+  });
+
+  return `${newServiceToken.data.serviceToken}.${randomBytes}`
+}
+
+export const deleteAllSecrets = async () => {
+  await Secret.deleteMany()
+}

backend/tests/integration-tests/routes/v2/secrets.test.ts

@@ -0,0 +1,198 @@
+import request from 'supertest'
+import main from '../../../../src/index'
+import { testWorkspaceId } from '../../../../src/utils/addDevelopmentUser';
+import { deleteAllSecrets, getJWTFromTestUser, getServiceTokenFromTestUser } from '../../../helper/helper';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const batchSecretRequest = require('../../../data/batch-secrets.json');
+
+let server: any;
+
+beforeAll(async () => {
+  server = await main;
+});
+
+afterAll(async () => {
+  server.close();
+});
+
+describe("GET /api/v2/secrets", () => {
+  describe("Get secrets via JTW with no personal secrets", () => {
+    test("should respond with a 200 status code", async () => {
+      try {
+        // get login details 
+        const loginResponse = await getJWTFromTestUser()
+
+        // create creates 
+        const createSecretsResponse = await request(server)
+          .post("/api/v2/secrets/batch")
+          .set('Authorization', `Bearer ${loginResponse.token}`)
+          .send({
+            workspaceId: testWorkspaceId,
+            environment: "dev",
+            requests: batchSecretRequest
+          })
+
+        expect(createSecretsResponse.statusCode).toBe(200)
+
+
+        const getSecrets = await request(server)
+          .get("/api/v2/secrets")
+          .set('Authorization', `Bearer ${loginResponse.token}`)
+          .query({
+            workspaceId: testWorkspaceId,
+            environment: "dev"
+          })
+
+        expect(getSecrets.statusCode).toBe(200)
+        expect(getSecrets.body).toHaveProperty("secrets")
+        expect(getSecrets.body.secrets).toHaveLength(3)
+        expect(getSecrets.body.secrets).toBeInstanceOf(Array);
+
+        getSecrets.body.secrets.forEach((secret: any) => {
+          expect(secret).toHaveProperty('_id');
+          expect(secret._id).toBeTruthy();
+
+          expect(secret).toHaveProperty('version');
+          expect(secret.version).toBeTruthy();
+
+          expect(secret).toHaveProperty('workspace');
+          expect(secret.workspace).toBeTruthy();
+
+          expect(secret).toHaveProperty('type');
+          expect(secret.type).toBeTruthy();
+
+          expect(secret).toHaveProperty('tags');
+          expect(secret.tags).toHaveLength(0);
+
+          expect(secret).toHaveProperty('environment');
+          expect(secret.environment).toEqual("dev");
+
+          expect(secret).toHaveProperty('secretKeyCiphertext');
+          expect(secret.secretKeyCiphertext).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretKeyIV');
+          expect(secret.secretKeyIV).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretKeyTag');
+          expect(secret.secretKeyTag).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretValueCiphertext');
+          expect(secret.secretValueCiphertext).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretValueIV');
+          expect(secret.secretValueIV).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretValueTag');
+          expect(secret.secretValueTag).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretCommentCiphertext');
+          expect(secret.secretCommentCiphertext).toBeFalsy();
+
+          expect(secret).toHaveProperty('secretCommentIV');
+          expect(secret.secretCommentIV).toBeTruthy();
+
+          expect(secret).toHaveProperty('secretCommentTag');
+          expect(secret.secretCommentTag).toBeTruthy();
+
+          expect(secret).toHaveProperty('createdAt');
+          expect(secret.createdAt).toBeTruthy();
+
+          expect(secret).toHaveProperty('updatedAt');
+          expect(secret.updatedAt).toBeTruthy();
+        });
+      } finally {
+        // clean up
+        await deleteAllSecrets()
+      }
+    })
+  })
+
+  describe("fetch secrets via service token with no personal secrets", () => {
+    test("should respond with a 200 status code", async () => {
+      // get login details 
+      const loginResponse = await getJWTFromTestUser()
+
+      // create creates 
+      const createSecretsResponse = await request(server)
+        .post("/api/v2/secrets/batch")
+        .set('Authorization', `Bearer ${loginResponse.token}`)
+        .send({
+          workspaceId: testWorkspaceId,
+          environment: "dev",
+          requests: batchSecretRequest
+        })
+
+      expect(createSecretsResponse.statusCode).toBe(200)
+
+
+      // now use the service token to fetch secrets
+      const serviceToken = await getServiceTokenFromTestUser()
+
+      const getSecrets = await request(server)
+        .get("/api/v2/secrets")
+        .set('Authorization', `Bearer ${serviceToken}`)
+        .query({
+          workspaceId: testWorkspaceId,
+          environment: "dev"
+        })
+
+      expect(getSecrets.statusCode).toBe(200)
+      expect(getSecrets.body).toHaveProperty("secrets")
+      expect(getSecrets.body.secrets).toHaveLength(3)
+      expect(getSecrets.body.secrets).toBeInstanceOf(Array);
+
+      getSecrets.body.secrets.forEach((secret: any) => {
+        expect(secret).toHaveProperty('_id');
+        expect(secret._id).toBeTruthy();
+
+        expect(secret).toHaveProperty('version');
+        expect(secret.version).toBeTruthy();
+
+        expect(secret).toHaveProperty('workspace');
+        expect(secret.workspace).toBeTruthy();
+
+        expect(secret).toHaveProperty('type');
+        expect(secret.type).toBeTruthy();
+
+        expect(secret).toHaveProperty('tags');
+        expect(secret.tags).toHaveLength(0);
+
+        expect(secret).toHaveProperty('environment');
+        expect(secret.environment).toEqual("dev");
+
+        expect(secret).toHaveProperty('secretKeyCiphertext');
+        expect(secret.secretKeyCiphertext).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretKeyIV');
+        expect(secret.secretKeyIV).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretKeyTag');
+        expect(secret.secretKeyTag).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretValueCiphertext');
+        expect(secret.secretValueCiphertext).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretValueIV');
+        expect(secret.secretValueIV).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretValueTag');
+        expect(secret.secretValueTag).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretCommentCiphertext');
+        expect(secret.secretCommentCiphertext).toBeFalsy();
+
+        expect(secret).toHaveProperty('secretCommentIV');
+        expect(secret.secretCommentIV).toBeTruthy();
+
+        expect(secret).toHaveProperty('secretCommentTag');
+        expect(secret.secretCommentTag).toBeTruthy();
+
+        expect(secret).toHaveProperty('createdAt');
+        expect(secret.createdAt).toBeTruthy();
+
+        expect(secret).toHaveProperty('updatedAt');
+        expect(secret.updatedAt).toBeTruthy();
+      });
+    })
+  })
+})

backend/tests/unit-tests/utils/crypto.test.ts

@@ -4,7 +4,7 @@ import {
   decryptSymmetric,
   encryptAsymmetric,
   encryptSymmetric
-} from '../../src/utils/crypto';
+} from '../../../src/utils/crypto';
 
 describe('Crypto', () => {
   describe('encryptAsymmetric', () => {

backend/tests/unit-tests/utils/posthog.test.ts

@@ -1,5 +1,5 @@
 import { describe, test, expect } from '@jest/globals';
-import { getChannelFromUserAgent } from '../../src/utils/posthog';
+import { getChannelFromUserAgent } from '../../../src/utils/posthog';
 
 describe('posthog getChannelFromUserAgent', () => {
   test("should return 'web' when userAgent includes 'mozilla'", () => {