Add unittest to firebase cloud functions. Fix tsc node_modules resolution.

Ensure all auth_code guarded functions generate 401s on invalid auth_code.

Do basic test of transcript reading/writing from fake datastore.

Also fix issue where tsc in the functions directory would look up the parent
node_modules for type resolution in ../common.
microsoft/TypeScript#30124 (comment)

This yielded all sorts of hilarity.

Author: awong-dev

app/[category]/v/[prefix]/[videoId]/[lang]/page.tsx

@@ -1,7 +1,8 @@
+import * as FirebaseUtils from 'utilities/firebase'
 import BoardMeeting from 'components/BoardMeeting'
 import TranscriptControlProvider from 'components/TranscriptControlProvider'
-import { DiarizedTranscript } from "common/transcript"
 import { CloudStorageAccessor } from "common/storage"
+import { DiarizedTranscript } from "common/transcript"
 import { Metadata, ResolvingMetadata } from "next"
 import { getMetadata } from "utilities/metadata-utils"
 import { loadSpeakerControlInfo } from 'utilities/client/speaker'
@@ -18,7 +19,9 @@ type Props = {
   searchParams: { [key: string]: string | string[] | undefined }
 };
 
-const cloudStorageAccessor = new CloudStorageAccessor();
+const cloudStorageAccessor = new CloudStorageAccessor({
+    bucket: FirebaseUtils.storage.bucket(Constants.STORAGE_BUCKET)
+  });
 
 export async function generateMetadata(
     { params, searchParams }: Props,

common/storage.ts

@@ -1,6 +1,3 @@
-import * as Constants from 'config/constants';
-import * as fs from 'node:fs';
-import { Storage } from '@google-cloud/storage';
 import { Stream } from "stream";
 import { arrayBuffer } from 'node:stream/consumers';
 import { createGzip, createGunzip } from 'zlib';
@@ -21,7 +18,7 @@ export interface StorageAccessor {
 }
 
 type CloudStorageAccessorOptions = {
-  keyfile?: string;
+  bucket: Bucket;
   makeLzmaDecompressor?: (() => object);
   makeLzmaCompressor?: (() => object);
 };
@@ -35,19 +32,12 @@ type CloudStorageAccessorOptions = {
 // files so adding the lzma library in is just bloat. This is slightly
 // ugly but it works out as an okay compromise.
 export class CloudStorageAccessor implements StorageAccessor {
-  readonly storage : Storage;
   readonly bucket : Bucket;
   readonly makeLzmaDecompressor?: (() => object);
   readonly makeLzmaCompressor?: (() => object);
 
-  constructor(options : CloudStorageAccessorOptions = {}) {
-    if (options.keyfile) {
-      const serviceAccount = JSON.parse(fs.readFileSync(options.keyfile, {encoding: 'utf8'}));
-      this.storage = new Storage(serviceAccount);
-    } else {
-      this.storage = new Storage();
-    }
-    this.bucket = this.storage.bucket(Constants.STORAGE_BUCKET);
+  constructor(options : CloudStorageAccessorOptions) {
+    this.bucket = options.bucket;
     this.makeLzmaDecompressor = options.makeLzmaDecompressor;
     this.makeLzmaCompressor = options.makeLzmaCompressor;
   }

functions/jest.config.ts

@@ -141,7 +141,7 @@ const config: Config = {
   // runner: "jest-runner",
 
   // The paths to modules that run some code to configure or set up the testing environment before each test
-  // setupFiles: [],
+  setupFiles: [ '<rootDir>/jest.setup.js' ],
 
   // A list of paths to modules that run some code to configure or set up the testing framework before each test
   // setupFilesAfterEnv: [],

functions/package-lock.json

@@ -3,17 +3,20 @@
   "type": "module",
   "scripts": {
     "lint": "eslint --ext .js,.ts .",
-    "build": "esbuild src/index.ts --bundle --platform=node --outfile=lib/index.js --format=esm --external:./node_modules/* --packages=external",
-    "dev:tsc": "tsc --watch --preserveWatchOutput",
-    "dev:node": "firebase emulators:start --only functions",
-    "dev:esbuild": "npm run build --watch",
-    "dev": "run-p dev:*",
+    "build": "run-s build:tsc build:esbuild",
+    "build:esbuild": "esbuild src/index.ts --bundle --platform=node --outfile=lib/index.js --format=esm --external:./node_modules/* --packages=external",
+    "build:tsc": "tsc",
+    "build:watch": "run-p 'build:tsc -- --watch --preserveWatchOutput' 'build:esbuild -- --watch'",
+    "dev": "run-p build:watch dev:*",
+    "dev:emulators": "firebase emulators:start --only functions,storage,database",
     "serve": "npm run build && firebase emulators:start --only functions,storage,database",
     "shell": "npm run build && firebase functions:shell",
     "start": "npm run shell",
     "deploy": "firebase deploy --only functions",
-    "test": "firebase emulators:exec --only storage,database 'npx jest'",
-    "test:watch": "firebase emulators:exec --only storage,database 'npx jest --watch'",
+    "test": "run-s build test:jest",
+    "test:jest": "firebase emulators:exec --only functions,storage,database 'npx jest'",
+    "test:jest:watch": "firebase emulators:exec --ui --only functions,storage,database 'npx jest --watch'",
+    "test:watch": "run-p build:watch test:jest:watch",
     "logs": "firebase functions:log"
   },
   "main": "lib/index.js",
@@ -33,6 +36,8 @@
     "youtubei.js": "^9.4.0"
   },
   "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/jest": "^29.5.12",
     "@typescript-eslint/eslint-plugin": "^5.12.0",
     "@typescript-eslint/parser": "^5.12.0",
     "esbuild": "^0.21.5",

functions/package.json

@@ -0,0 +1,56 @@
+import * as TestingUtils from 'utils/testing';
+
+// List of endpoints that should be guarded by user_id/auth_code params.
+const AUTH_CODE_ENDPOINTS = [
+  [ 'GET', 'video_queue' ],
+  [ 'PATCH', 'video_queue' ],
+  [ 'DELETE', 'video_queue' ],
+  [ 'GET', 'video_queue' ],
+
+  [ 'DELETE', 'vast' ],
+
+  [ 'PUT', 'transcript' ],
+];
+
+describe('auth_code endpoints', () => {
+  beforeAll(TestingUtils.beforeAll);
+
+  for (const [method, endpoint] of AUTH_CODE_ENDPOINTS) {
+    it(`${method} ${endpoint} needs user_id`, async () => {
+      const response = await TestingUtils.fetchEndpoint(
+          endpoint,
+          method,
+          {auth_code: TestingUtils.FAKE_AUTH_CODE});
+      expect(response.status).toStrictEqual(401);
+      const responseJson = await response.json();
+      expect(responseJson.ok).toStrictEqual(false);
+      expect(responseJson.message).toMatch(/missing user_id/);
+    });
+
+    it(`${method} ${endpoint} needs auth_code`, async () => {
+      const response = await TestingUtils.fetchEndpoint(
+          endpoint,
+          method,
+          { user_id: TestingUtils.FAKE_USER_ID });
+      expect(response.status).toStrictEqual(401);
+      const responseJson = await response.json();
+      expect(responseJson.ok).toStrictEqual(false);
+      expect(responseJson.message).toMatch(/invalid auth_code/);
+    });
+
+    it(`${method} ${endpoint} rejects invalid auth_code`, async () => {
+      const response = await TestingUtils.fetchEndpoint(
+          endpoint,
+          method,
+          {
+            user_id: TestingUtils.FAKE_USER_ID,
+            auth_code: (TestingUtils.FAKE_AUTH_CODE + TestingUtils.FAKE_AUTH_CODE)
+          });
+      expect(response.status).toStrictEqual(401);
+      const responseJson = await response.json();
+      expect(responseJson.ok).toStrictEqual(false);
+      expect(responseJson.message).toMatch(/invalid auth_code/);
+    });
+  }
+});
+

functions/src/endpoint_auth.test.ts

@@ -1,9 +1,9 @@
 import 'source-map-support/register.js';
 
 export { speakerinfo } from "./speakerinfo";
-export { metadata, transcript } from "./transcript";
+export { transcript } from "./transcript";
 export { video_queue, vast } from "./video_queue";
 
 import { initializeFirebase } from "./utils/firebase";
 
-initializeFirebase({});
+initializeFirebase();

functions/src/index.ts

@@ -9,16 +9,14 @@
 //  /transcripts/public/[category]/sentences/[vid].eng.json
 
 import * as Constants from 'config/constants';
-import process from 'node:process';
-import { CloudStorageAccessor } from 'common/storage';
+import { getLzmaStorageAccessor } from 'utils/storage';
 import { DiarizedTranscript, makeSentenceTablePath, makeTranscriptDataPath } from 'common/transcript';
 import { basename } from 'node:path';
 import { makePublicPath } from 'common/paths';
 
-
 import type { Iso6393Code, VideoId } from "common/params";
 
-const accessor = new CloudStorageAccessor({keyfile:process.env.TRANSCRIPT_STORAGE_KEYFILE}); 
+const accessor = getLzmaStorageAccessor();
 
 for (const category of Constants.ALL_CATEGORIES) {
   const [allFiles] = await accessor.bucket.getFiles(