Merge pull request #24619 from overleaf/rh-team-invites-index

Add migration for subscriptions.teamInvites.email index

GitOrigin-RevId: 5f4bca578ae0dcf92c422596aa7834c42dc63bee

README.md

@@ -73,4 +73,4 @@ Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contribut
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
 
-Copyright (c) Overleaf, 2014-2024.
+Copyright (c) Overleaf, 2014-2025.

develop/README.md

@@ -11,12 +11,6 @@ bin/build
 > [!NOTE]
 > If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
 
-Next, initialize the database:
-
-```shell
-bin/init
-```
-
 Then start the services:
 
 ```shell

develop/bin/init

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-docker compose up --detach mongo
-curl --max-time 10 --retry 5 --retry-delay 5 --retry-all-errors --silent --output /dev/null localhost:27017
-docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })"
-docker compose down mongo

develop/docker-compose.yml

@@ -94,6 +94,14 @@ services:
       - "127.0.0.1:27017:27017" # for debugging
     volumes:
       - mongo-data:/data/db
+      - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 
   notifications:
     build:

docker-compose.yml

@@ -103,7 +103,7 @@ services:
         command: '--replSet overleaf'
         volumes:
             - ~/mongo_data:/data/db
-            - ./server-ce/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+            - ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
         environment:
           MONGO_INITDB_DATABASE: sharelatex
         extra_hosts:

libraries/fetch-utils/index.js

@@ -95,6 +95,19 @@ async function fetchNothing(url, opts = {}) {
  * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
  */
 async function fetchRedirect(url, opts = {}) {
+  const { location } = await fetchRedirectWithResponse(url, opts)
+  return location
+}
+
+/**
+ * Make a request and extract the redirect from the response.
+ *
+ * @param {string | URL} url - request URL
+ * @param {object} opts - fetch options
+ * @return {Promise<{location: string, response: Response}>}
+ * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
+ */
+async function fetchRedirectWithResponse(url, opts = {}) {
   const { fetchOpts } = parseOpts(opts)
   fetchOpts.redirect = 'manual'
   const response = await performRequest(url, fetchOpts)
@@ -112,7 +125,7 @@ async function fetchRedirect(url, opts = {}) {
     )
   }
   await discardResponseBody(response)
-  return location
+  return { location, response }
 }
 
 /**
@@ -297,6 +310,7 @@ module.exports = {
   fetchStreamWithResponse,
   fetchNothing,
   fetchRedirect,
+  fetchRedirectWithResponse,
   fetchString,
   fetchStringWithResponse,
   RequestFailedError,

libraries/mongo-utils/batchedUpdate.js

@@ -16,6 +16,7 @@ let VERBOSE_LOGGING
 let BATCH_RANGE_START
 let BATCH_RANGE_END
 let BATCH_MAX_TIME_SPAN_IN_MS
+let BATCHED_UPDATE_RUNNING = false
 
 /**
  * @typedef {import("mongodb").Collection} Collection
@@ -211,57 +212,66 @@ async function batchedUpdate(
   findOptions,
   batchedUpdateOptions
 ) {
-  ID_EDGE_PAST = await getIdEdgePast(collection)
-  if (!ID_EDGE_PAST) {
-    console.warn(
-      `The collection ${collection.collectionName} appears to be empty.`
-    )
-    return 0
+  // only a single batchedUpdate can run at a time due to global variables
+  if (BATCHED_UPDATE_RUNNING) {
+    throw new Error('batchedUpdate is already running')
   }
-  refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
-
-  findOptions = findOptions || {}
-  findOptions.readPreference = READ_PREFERENCE_SECONDARY
-
-  projection = projection || { _id: 1 }
-  let nextBatch
-  let updated = 0
-  let start = BATCH_RANGE_START
-
-  while (start !== BATCH_RANGE_END) {
-    let end = getNextEnd(start)
-    nextBatch = await getNextBatch(
-      collection,
-      query,
-      start,
-      end,
-      projection,
-      findOptions
-    )
-    if (nextBatch.length > 0) {
-      end = nextBatch[nextBatch.length - 1]._id
-      updated += nextBatch.length
-
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `Running update on batch with ids ${JSON.stringify(
-            nextBatch.map(entry => entry._id)
-          )}`
-        )
-      } else {
-        console.error(`Running update on batch ending ${renderObjectId(end)}`)
-      }
-
-      if (typeof update === 'function') {
-        await update(nextBatch)
-      } else {
-        await performUpdate(collection, nextBatch, update)
-      }
+  try {
+    BATCHED_UPDATE_RUNNING = true
+    ID_EDGE_PAST = await getIdEdgePast(collection)
+    if (!ID_EDGE_PAST) {
+      console.warn(
+        `The collection ${collection.collectionName} appears to be empty.`
+      )
+      return 0
     }
-    console.error(`Completed batch ending ${renderObjectId(end)}`)
-    start = end
+    refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
+
+    findOptions = findOptions || {}
+    findOptions.readPreference = READ_PREFERENCE_SECONDARY
+
+    projection = projection || { _id: 1 }
+    let nextBatch
+    let updated = 0
+    let start = BATCH_RANGE_START
+
+    while (start !== BATCH_RANGE_END) {
+      let end = getNextEnd(start)
+      nextBatch = await getNextBatch(
+        collection,
+        query,
+        start,
+        end,
+        projection,
+        findOptions
+      )
+      if (nextBatch.length > 0) {
+        end = nextBatch[nextBatch.length - 1]._id
+        updated += nextBatch.length
+
+        if (VERBOSE_LOGGING) {
+          console.log(
+            `Running update on batch with ids ${JSON.stringify(
+              nextBatch.map(entry => entry._id)
+            )}`
+          )
+        } else {
+          console.error(`Running update on batch ending ${renderObjectId(end)}`)
+        }
+
+        if (typeof update === 'function') {
+          await update(nextBatch)
+        } else {
+          await performUpdate(collection, nextBatch, update)
+        }
+      }
+      console.error(`Completed batch ending ${renderObjectId(end)}`)
+      start = end
+    }
+    return updated
+  } finally {
+    BATCHED_UPDATE_RUNNING = false
   }
-  return updated
 }
 
 /**

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -414,6 +414,16 @@ class CachedPerProjectEncryptedS3Persistor {
     return await this.sendStream(bucketName, path, fs.createReadStream(fsPath))
   }
 
+  /**
+   *
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<number>}
+   */
+  async getObjectSize(bucketName, path) {
+    return await this.#parent.getObjectSize(bucketName, path)
+  }
+
   /**
    * @param {string} bucketName
    * @param {string} path

libraries/promise-utils/index.js

@@ -13,6 +13,7 @@ module.exports = {
   expressify,
   expressifyErrorHandler,
   promiseMapWithLimit,
+  promiseMapSettledWithLimit,
 }
 
 /**
@@ -264,3 +265,19 @@ async function promiseMapWithLimit(concurrency, array, fn) {
   const limit = pLimit(concurrency)
   return await Promise.all(array.map(x => limit(() => fn(x))))
 }
+
+/**
+ * Map values in `array` with the async function `fn`
+ *
+ * Limit the number of unresolved promises to `concurrency`.
+ *
+ * @template T, U
+ * @param {number} concurrency
+ * @param {Array<T>} array
+ * @param {(T) => Promise<U>} fn
+ * @return {Promise<Array<PromiseSettledResult<U>>>}
+ */
+function promiseMapSettledWithLimit(concurrency, array, fn) {
+  const limit = pLimit(concurrency)
+  return Promise.allSettled(array.map(x => limit(() => fn(x))))
+}

package.json

@@ -73,6 +73,7 @@
     "services/third-party-datastore",
     "services/third-party-references",
     "services/tpdsworker",
-    "services/web"
+    "services/web",
+    "tools/saas-e2e"
   ]
 }

server-ce/Dockerfile-base

@@ -10,7 +10,7 @@ ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
 
 # Update to ensure dependencies are updated
 # ------------------------------------------
-ENV REBUILT_AFTER="2024-15-10"
+ENV REBUILT_AFTER="2025-03-27"
 
 # Install dependencies
 # --------------------

server-ce/bin/shared

@@ -0,0 +1 @@
+../../bin/shared/

server-ce/hotfix/4.2.9/Dockerfile

@@ -0,0 +1 @@
+FROM sharelatex/sharelatex:4.2.8

server-ce/hotfix/5.3.3/Dockerfile

@@ -0,0 +1 @@
+FROM sharelatex/sharelatex:5.3.2

server-ce/mongodb-init-replica-set.js

@@ -1 +0,0 @@
-rs.initiate({ _id: "overleaf", members: [ { _id: 0, host: "mongo:27017" } ] })

server-ce/nginx/clsi-nginx.conf

@@ -30,7 +30,7 @@ server {
       application/pdf pdf;
   }
   # handle output files for specific users
-  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
     alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/output.$4;
   }
   # handle .blg files for specific users
@@ -38,7 +38,7 @@ server {
     alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/$4.blg;
   }
   # handle output files for anonymous users
-  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
     alias /var/lib/overleaf/data/output/$1/generated-files/$2/output.$3;
   }
   # handle .blg files for anonymous users

server-ce/nginx/overleaf.conf

@@ -47,12 +47,12 @@ server {
 	}
 
   # handle output files for specific users
-  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
 		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }
   # handle output files for anonymous users
-  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
 		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }

server-ce/server-ce

@@ -1 +0,0 @@
-.

server-ce/test/create-and-compile-project.spec.ts

@@ -95,7 +95,9 @@ describe('Project creation and compilation', function () {
 
     cy.findByText('Share').click()
     cy.findByRole('dialog').within(() => {
-      cy.get('input').type('collaborator@example.com,')
+      cy.findByTestId('collaborator-email-input').type(
+        'collaborator@example.com,'
+      )
       cy.findByText('Invite').click({ force: true })
       cy.findByText('Invite not yet accepted.')
     })

server-ce/test/docker-compose.yml

@@ -38,7 +38,7 @@ services:
     image: mongo:6.0
     command: '--replSet overleaf'
     volumes:
-      - ../mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+      - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
     environment:
       MONGO_INITDB_DATABASE: sharelatex
     extra_hosts:

server-ce/test/editor.spec.ts

@@ -149,10 +149,10 @@ describe('editor', () => {
       openFile(fileName, 'static')
 
       cy.log('reject changes')
-      cy.findByText('Review').click()
+      cy.contains('.toolbar-item', 'Review').click()
       cy.get('.cm-content').should('not.contain.text', oldContent)
-      cy.findByText('Reject').click({ force: true })
-      cy.findByText('Review').click()
+      cy.findByText('Reject change').click({ force: true })
+      cy.contains('.toolbar-item', 'Review').click()
 
       cy.log('recompile to force flush')
       recompile()
@@ -205,10 +205,10 @@ describe('editor', () => {
       openFile(fileName, 'static')
 
       cy.log('reject changes')
-      cy.findByText('Review').click()
+      cy.contains('.toolbar-item', 'Review').click()
       cy.get('.cm-content').should('not.contain.text', oldContent)
-      cy.findAllByText('Reject').first().click({ force: true })
-      cy.findByText('Review').click()
+      cy.findAllByText('Reject change').first().click({ force: true })
+      cy.contains('.toolbar-item', 'Review').click()
 
       cy.log('recompile to force flush')
       recompile()

server-ce/test/external-auth.spec.ts

@@ -59,7 +59,7 @@ describe('LDAP', () => {
 
   it('login', () => {
     cy.visit('/')
-    cy.findByText('Login LDAP')
+    cy.findByText('Log in LDAP')
 
     cy.get('input[name="login"]').type('fry')
     cy.get('input[name="password"]').type('fry')

server-ce/test/git-bridge.spec.ts

@@ -136,7 +136,7 @@ describe('git-bridge', function () {
         shareProjectByEmailAndAcceptInviteViaDash(
           projectName,
           'collaborator-rw@example.com',
-          'Can edit'
+          'Editor'
         )
         maybeClearAllTokens()
         openProjectByName(projectName)
@@ -147,7 +147,7 @@ describe('git-bridge', function () {
         shareProjectByEmailAndAcceptInviteViaDash(
           projectName,
           'collaborator-ro@example.com',
-          'Can view'
+          'Viewer'
         )
         maybeClearAllTokens()
         openProjectByName(projectName)

server-ce/test/helpers/compile.ts

@@ -24,7 +24,7 @@ export function prepareWaitForNextCompileSlot() {
       queueReset()
       triggerCompile()
       cy.log('Wait for compile to finish')
-      cy.findByText('Recompile')
+      cy.findByText('Recompile').should('be.visible')
     })
   }
   function recompile() {

server-ce/test/helpers/project.ts

@@ -100,7 +100,7 @@ export function openProjectViaInviteNotification(projectName: string) {
 function shareProjectByEmail(
   projectName: string,
   email: string,
-  level: 'Can view' | 'Can edit'
+  level: 'Viewer' | 'Editor'
 ) {
   openProjectByName(projectName)
   cy.findByText('Share').click()
@@ -108,7 +108,13 @@ function shareProjectByEmail(
     cy.findByLabelText('Add people', { selector: 'input' }).type(`${email},`)
     cy.findByLabelText('Add people', { selector: 'input' })
       .parents('form')
-      .within(() => cy.findByText('Can edit').parent().select(level))
+      .within(() => {
+        cy.findByTestId('add-collaborator-select')
+          .click()
+          .then(() => {
+            cy.findByText(level).click()
+          })
+      })
     cy.findByText('Invite').click({ force: true })
     cy.findByText('Invite not yet accepted.')
   })
@@ -117,7 +123,7 @@ function shareProjectByEmail(
 export function shareProjectByEmailAndAcceptInviteViaDash(
   projectName: string,
   email: string,
-  level: 'Can view' | 'Can edit'
+  level: 'Viewer' | 'Editor'
 ) {
   shareProjectByEmail(projectName, email, level)
 
@@ -128,7 +134,7 @@ export function shareProjectByEmailAndAcceptInviteViaDash(
 export function shareProjectByEmailAndAcceptInviteViaEmail(
   projectName: string,
   email: string,
-  level: 'Can view' | 'Can edit'
+  level: 'Viewer' | 'Editor'
 ) {
   shareProjectByEmail(projectName, email, level)
 
@@ -212,11 +218,11 @@ export function createNewFile() {
 
 export function toggleTrackChanges(state: boolean) {
   cy.findByText('Review').click()
-  cy.get('.rp-tc-state-collapse').then(el => {
-    // TODO: simplify this in the frontend?
-    if (el.hasClass('rp-tc-state-collapse-on')) {
-      // make track-changes switches visible
-      cy.get('.rp-tc-state-collapse').click()
+  cy.get('.track-changes-menu-button').then(el => {
+    // when the menu is expanded renders the `expand_more` icon,
+    // and the `chevron_right` icon when it's collapsed
+    if (!el.text().includes('expand_more')) {
+      el.click()
     }
   })
 
@@ -241,5 +247,5 @@ export function toggleTrackChanges(state: boolean) {
         cy.wait(alias)
       })
     })
-  cy.findByText('Review').click()
+  cy.contains('.toolbar-item', 'Review').click()
 }

server-ce/test/project-sharing.spec.ts

@@ -154,7 +154,7 @@ describe('Project Sharing', function () {
 
     beforeEach(function () {
       login('user@example.com')
-      shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Can view')
+      shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Viewer')
     })
 
     it('should grant the collaborator read access', () => {
@@ -169,7 +169,7 @@ describe('Project Sharing', function () {
 
     beforeWithReRunOnTestRetry(function () {
       login('user@example.com')
-      shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can view')
+      shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Viewer')
     })
 
     it('should grant the collaborator read access', () => {
@@ -186,7 +186,7 @@ describe('Project Sharing', function () {
 
     beforeWithReRunOnTestRetry(function () {
       login('user@example.com')
-      shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can edit')
+      shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Editor')
     })
 
     it('should grant the collaborator write access', () => {

server-ce/test/sandboxed-compiles.spec.ts

@@ -204,9 +204,9 @@ describe('SandboxedCompiles', function () {
       cy.log('wait for compile')
       cy.get('.pdf-viewer').should('contain.text', 'sandboxed')
 
-      cy.log('Check which compiler version was used, expect 2024')
+      cy.log('Check which compiler version was used, expect 2025')
       cy.get('[aria-label="View logs"]').click()
-      cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2024\) /)
+      cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2025\) /)
 
       cy.log('Check that there is no TeX Live version toggle')
       cy.get('header').findByText('Menu').click()

services/chat/Makefile

@@ -116,13 +116,6 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
-		while ! mongosh --eval "db.version()" > /dev/null; do \
-			echo "Waiting for Mongo..."; \
-			sleep 1; \
-		done; \
-		mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/chat/docker-compose.ci.yml

@@ -26,7 +26,7 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
     user: node
     command: npm run test:acceptance
 
@@ -41,7 +41,12 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1

services/chat/docker-compose.yml

@@ -38,14 +38,19 @@ services:
     user: node
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
     command: npm run --silent test:acceptance
 
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 

services/clsi/app.js

@@ -309,6 +309,10 @@ const loadTcpServer = net.createServer(function (socket) {
     } else {
       // Ready will cancel the maint state.
       socket.write(`up, ready, ${Math.max(freeLoadPercentage, 1)}%\n`, 'ASCII')
+      if (freeLoadPercentage <= 0) {
+        // This metric records how often we would have gone into maintenance mode.
+        Metrics.inc('clsi-prevented-maint')
+      }
     }
     socket.end()
   } else {

services/clsi/app/js/OutputCacheManager.js

@@ -98,12 +98,11 @@ module.exports = OutputCacheManager = {
   CONTENT_SUBDIR: 'content',
   CACHE_SUBDIR: 'generated-files',
   ARCHIVE_SUBDIR: 'archived-logs',
-  // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
-  // for backwards compatibility, make the randombytes part optional
-  BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
-  CONTENT_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
+  // build id is HEXDATE-HEXRANDOM from Date.now() and RandomBytes
+  BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
+  CONTENT_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
   CACHE_LIMIT: 2, // maximum number of cache directories
-  CACHE_AGE: 60 * 60 * 1000, // up to one hour old
+  CACHE_AGE: 90 * 60 * 1000, // up to 90 minutes old
 
   init,
   queueDirOperation: callbackify(queueDirOperation),
@@ -137,7 +136,11 @@ module.exports = OutputCacheManager = {
     outputDir,
     callback
   ) {
-    OutputCacheManager.generateBuildId(function (err, buildId) {
+    const getBuildId = cb => {
+      if (request.buildId) return cb(null, request.buildId)
+      OutputCacheManager.generateBuildId(cb)
+    }
+    getBuildId(function (err, buildId) {
       if (err) {
         return callback(err)
       }

services/clsi/app/js/OutputFileArchiveManager.js

@@ -7,7 +7,7 @@ const { NotFoundError } = require('./Errors')
 const logger = require('@overleaf/logger')
 
 // NOTE: Updating this list requires a corresponding change in
-// * services/web/frontend/js/features/pdf-preview/util/file-list.js
+// * services/web/frontend/js/features/pdf-preview/util/file-list.ts
 const ignoreFiles = ['output.fls', 'output.fdb_latexmk']
 
 function getContentDir(projectId, userId) {

services/clsi/app/js/ProjectPersistenceManager.js

@@ -13,6 +13,7 @@ const CompileManager = require('./CompileManager')
 const async = require('async')
 const logger = require('@overleaf/logger')
 const oneDay = 24 * 60 * 60 * 1000
+const Metrics = require('@overleaf/metrics')
 const Settings = require('@overleaf/settings')
 const diskusage = require('diskusage')
 const { callbackify } = require('node:util')
@@ -22,33 +23,48 @@ const fs = require('node:fs')
 // projectId -> timestamp mapping.
 const LAST_ACCESS = new Map()
 
-async function refreshExpiryTimeout() {
+async function collectDiskStats() {
   const paths = [
     Settings.path.compilesDir,
     Settings.path.outputDir,
     Settings.path.clsiCacheDir,
   ]
+
+  const diskStats = {}
   for (const path of paths) {
     try {
       const stats = await diskusage.check(path)
-      const lowDisk = stats.available / stats.total < 0.1
-
-      const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
-      if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
-        logger.warn(
-          {
-            stats,
-            newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
-          },
-          'disk running low on space, modifying EXPIRY_TIMEOUT'
-        )
-        ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
-        break
-      }
+      const diskAvailablePercent = (stats.available / stats.total) * 100
+      Metrics.gauge('disk_available_percent', diskAvailablePercent, 1, {
+        path,
+      })
+      const lowDisk = diskAvailablePercent < 10
+      diskStats[path] = { stats, lowDisk }
     } catch (err) {
       logger.err({ err, path }, 'error getting disk usage')
     }
   }
+  return diskStats
+}
+
+async function refreshExpiryTimeout() {
+  for (const [path, { stats, lowDisk }] of Object.entries(
+    await collectDiskStats()
+  )) {
+    const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+    if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+      logger.warn(
+        {
+          path,
+          stats,
+          newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
+        },
+        'disk running low on space, modifying EXPIRY_TIMEOUT'
+      )
+      ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+      break
+    }
+  }
 }
 
 module.exports = ProjectPersistenceManager = {
@@ -103,6 +119,13 @@ module.exports = ProjectPersistenceManager = {
         }
       )
     })
+
+    // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter).
+    setInterval(() => {
+      collectDiskStats().catch(err => {
+        logger.err({ err }, 'low level error collecting disk stats')
+      })
+    }, 50_000)
   },
 
   markProjectAsJustAccessed(projectId, callback) {

services/clsi/app/js/RequestParser.js

@@ -1,4 +1,5 @@
 const settings = require('@overleaf/settings')
+const OutputCacheManager = require('./OutputCacheManager')
 
 const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
 const MAX_TIMEOUT = 600
@@ -135,6 +136,11 @@ function parse(body, callback) {
       }
     )
     response.rootResourcePath = _checkPath(rootResourcePath)
+
+    response.buildId = _parseAttribute('buildId', compile.options.buildId, {
+      type: 'string',
+      regex: OutputCacheManager.BUILD_REGEX,
+    })
   } catch (error1) {
     const error = error1
     return callback(error)
@@ -199,6 +205,13 @@ function _parseAttribute(name, attribute, options) {
         throw new Error(`${name} attribute should be a ${options.type}`)
       }
     }
+    if (options.type === 'string' && options.regex instanceof RegExp) {
+      if (!options.regex.test(attribute)) {
+        throw new Error(
+          `${name} attribute does not match regex ${options.regex}`
+        )
+      }
+    }
   } else {
     if (options.default != null) {
       return options.default

services/clsi/app/js/ResourceWriter.js

@@ -200,73 +200,22 @@ module.exports = ResourceWriter = {
     return OutputFileFinder.findOutputFiles(
       resources,
       basePath,
-      function (error, outputFiles, allFiles) {
+      (error, outputFiles, allFiles) => {
         if (error != null) {
           return callback(error)
         }
 
         const jobs = []
-        for (const file of Array.from(outputFiles || [])) {
-          ;(function (file) {
-            const { path } = file
-            let shouldDelete = true
-            if (
-              path.match(/^output\./) ||
-              path.match(/\.aux$/) ||
-              path.match(/^cache\//)
-            ) {
-              // knitr cache
-              shouldDelete = false
-            }
-            if (path.match(/^output-.*/)) {
-              // Tikz cached figures (default case)
-              shouldDelete = false
-            }
-            if (path.match(/\.(pdf|dpth|md5)$/)) {
-              // Tikz cached figures (by extension)
-              shouldDelete = false
-            }
-            if (
-              path.match(/\.(pygtex|pygstyle)$/) ||
-              path.match(/(^|\/)_minted-[^\/]+\//)
-            ) {
-              // minted files/directory
-              shouldDelete = false
-            }
-            if (
-              path.match(/\.md\.tex$/) ||
-              path.match(/(^|\/)_markdown_[^\/]+\//)
-            ) {
-              // markdown files/directory
-              shouldDelete = false
-            }
-            if (path.match(/-eps-converted-to\.pdf$/)) {
-              // Epstopdf generated files
-              shouldDelete = false
-            }
-            if (
-              path === 'output.pdf' ||
-              path === 'output.dvi' ||
-              path === 'output.log' ||
-              path === 'output.xdv' ||
-              path === 'output.stdout' ||
-              path === 'output.stderr'
-            ) {
-              shouldDelete = true
-            }
-            if (path === 'output.tex') {
-              // created by TikzManager if present in output files
-              shouldDelete = true
-            }
-            if (shouldDelete) {
-              return jobs.push(callback =>
-                ResourceWriter._deleteFileIfNotDirectory(
-                  Path.join(basePath, path),
-                  callback
-                )
+        for (const { path } of outputFiles || []) {
+          const shouldDelete = ResourceWriter.isExtraneousFile(path)
+          if (shouldDelete) {
+            jobs.push(callback =>
+              ResourceWriter._deleteFileIfNotDirectory(
+                Path.join(basePath, path),
+                callback
               )
-            }
-          })(file)
+            )
+          }
         }
 
         return async.series(jobs, function (error) {
@@ -279,6 +228,58 @@ module.exports = ResourceWriter = {
     )
   },
 
+  isExtraneousFile(path) {
+    let shouldDelete = true
+    if (
+      path.match(/^output\./) ||
+      path.match(/\.aux$/) ||
+      path.match(/^cache\//)
+    ) {
+      // knitr cache
+      shouldDelete = false
+    }
+    if (path.match(/^output-.*/)) {
+      // Tikz cached figures (default case)
+      shouldDelete = false
+    }
+    if (path.match(/\.(pdf|dpth|md5)$/)) {
+      // Tikz cached figures (by extension)
+      shouldDelete = false
+    }
+    if (
+      path.match(/\.(pygtex|pygstyle)$/) ||
+      path.match(/(^|\/)_minted-[^\/]+\//)
+    ) {
+      // minted files/directory
+      shouldDelete = false
+    }
+    if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) {
+      // markdown files/directory
+      shouldDelete = false
+    }
+    if (path.match(/-eps-converted-to\.pdf$/)) {
+      // Epstopdf generated files
+      shouldDelete = false
+    }
+    if (
+      path === 'output.synctex.gz' ||
+      path === 'output.pdfxref' ||
+      path === 'output.pdf' ||
+      path === 'output.dvi' ||
+      path === 'output.log' ||
+      path === 'output.xdv' ||
+      path === 'output.stdout' ||
+      path === 'output.stderr'
+    ) {
+      shouldDelete = true
+    }
+    if (path === 'output.tex') {
+      // created by TikzManager if present in output files
+      shouldDelete = true
+    }
+    return shouldDelete
+  },
+
   _deleteFileIfNotDirectory(path, callback) {
     if (callback == null) {
       callback = function () {}

services/clsi/nginx.conf

@@ -46,7 +46,7 @@ server {
   }
 
   # handle output files for specific users
-  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
     if ($request_method = 'OPTIONS') {
       # handle OPTIONS method for CORS requests
       add_header 'Allow' 'GET,HEAD';
@@ -64,7 +64,7 @@ server {
     alias /output/$1-$2/generated-files/$3/$4.blg;
   }
   # handle output files for anonymous users
-  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
     if ($request_method = 'OPTIONS') {
       # handle OPTIONS method for CORS requests
       add_header 'Allow' 'GET,HEAD';

services/clsi/package.json

@@ -28,7 +28,7 @@
     "body-parser": "^1.20.3",
     "bunyan": "^1.8.15",
     "diskusage": "^1.1.3",
-    "dockerode": "^3.1.0",
+    "dockerode": "^4.0.5",
     "express": "^4.21.2",
     "lodash": "^4.17.21",
     "p-limit": "^3.1.0",

services/clsi/test/acceptance/js/BrokenLatexFileTests.js

@@ -107,7 +107,6 @@ Hello world
           'output.fdb_latexmk',
           'output.fls',
           'output.log',
-          'output.pdfxref',
           'output.stderr',
           'output.stdout',
         ])

services/clsi/test/unit/js/DockerLockManagerTests.js

@@ -16,7 +16,7 @@ const modulePath = require('node:path').join(
   '../../../app/js/DockerLockManager'
 )
 
-describe('LockManager', function () {
+describe('DockerLockManager', function () {
   beforeEach(function () {
     return (this.LockManager = SandboxedModule.require(modulePath, {
       requires: {

services/clsi/test/unit/js/LockManagerTests.js

@@ -21,6 +21,7 @@ describe('LockManager', function () {
           compileConcurrencyLimit: 5,
         }),
         './Errors': (this.Erros = Errors),
+        './RequestParser': { MAX_TIMEOUT: 600 },
       },
     })
   })

services/clsi/test/unit/js/ProjectPersistenceManagerTests.js

@@ -23,6 +23,7 @@ describe('ProjectPersistenceManager', function () {
   beforeEach(function () {
     this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
       requires: {
+        '@overleaf/metrics': (this.Metrics = { gauge: sinon.stub() }),
         './UrlCache': (this.UrlCache = {}),
         './CompileManager': (this.CompileManager = {}),
         diskusage: (this.diskusage = { check: sinon.stub() }),
@@ -49,6 +50,10 @@ describe('ProjectPersistenceManager', function () {
       })
 
       this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.Metrics.gauge.should.have.been.calledWith(
+          'disk_available_percent',
+          40
+        )
         this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
           this.settings.project_cache_length_ms
         )
@@ -63,6 +68,10 @@ describe('ProjectPersistenceManager', function () {
       })
 
       this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.Metrics.gauge.should.have.been.calledWith(
+          'disk_available_percent',
+          5
+        )
         this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
         done()
       })
@@ -75,6 +84,10 @@ describe('ProjectPersistenceManager', function () {
       })
       this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
       this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.Metrics.gauge.should.have.been.calledWith(
+          'disk_available_percent',
+          5
+        )
         this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
         done()
       })

services/clsi/test/unit/js/RequestParserTests.js

@@ -30,6 +30,7 @@ describe('RequestParser', function () {
     this.RequestParser = SandboxedModule.require(modulePath, {
       requires: {
         '@overleaf/settings': (this.settings = {}),
+        './OutputCacheManager': { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ },
       },
     })
   })
@@ -274,6 +275,37 @@ describe('RequestParser', function () {
     })
   })
 
+  describe('with a valid buildId', function () {
+    beforeEach(function (done) {
+      this.validRequest.compile.options.buildId = '195a4869176-a4ad60bee7bf35e4'
+      this.RequestParser.parse(this.validRequest, (error, data) => {
+        if (error) return done(error)
+        this.data = data
+        done()
+      })
+    })
+
+    it('should return an error', function () {
+      this.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4')
+    })
+  })
+
+  describe('with a bad buildId', function () {
+    beforeEach(function () {
+      this.validRequest.compile.options.buildId = 'foo/bar'
+      this.RequestParser.parse(this.validRequest, this.callback)
+    })
+
+    it('should return an error', function () {
+      this.callback
+        .calledWithMatch({
+          message:
+            'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/',
+        })
+        .should.equal(true)
+    })
+  })
+
   describe('with a resource with a valid date', function () {
     beforeEach(function () {
       this.date = '12:00 01/02/03'

services/contacts/Makefile

@@ -116,13 +116,6 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
-		while ! mongosh --eval "db.version()" > /dev/null; do \
-			echo "Waiting for Mongo..."; \
-			sleep 1; \
-		done; \
-		mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/contacts/docker-compose.ci.yml

@@ -26,7 +26,7 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
     user: node
     command: npm run test:acceptance
 
@@ -41,7 +41,12 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1

services/contacts/docker-compose.yml

@@ -38,14 +38,19 @@ services:
     user: node
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
     command: npm run --silent test:acceptance
 
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 

services/docstore/Makefile

@@ -116,13 +116,6 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
-		while ! mongosh --eval "db.version()" > /dev/null; do \
-			echo "Waiting for Mongo..."; \
-			sleep 1; \
-		done; \
-		mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/docstore/app.js

@@ -88,14 +88,17 @@ app.get('/status', (req, res) => res.send('docstore is alive'))
 
 app.use(handleValidationErrors())
 app.use(function (error, req, res, next) {
-  logger.error({ err: error, req }, 'request errored')
   if (error instanceof Errors.NotFoundError) {
+    logger.warn({ req }, 'not found')
     res.sendStatus(404)
   } else if (error instanceof Errors.DocModifiedError) {
+    logger.warn({ req }, 'conflict: doc modified')
     res.sendStatus(409)
   } else if (error instanceof Errors.DocVersionDecrementedError) {
+    logger.warn({ req }, 'conflict: doc version decremented')
     res.sendStatus(409)
   } else {
+    logger.error({ err: error, req }, 'request errored')
     res.status(500).send('Oops, something went wrong')
   }
 })

services/docstore/docker-compose.ci.yml

@@ -29,7 +29,7 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       gcs:
         condition: service_healthy
     user: node
@@ -46,10 +46,15 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
   gcs:
     image: fsouza/fake-gcs-server:1.45.2
     command: ["--port=9090", "--scheme=http"]

services/docstore/docker-compose.yml

@@ -41,7 +41,7 @@ services:
     user: node
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       gcs:
         condition: service_healthy
     command: npm run --silent test:acceptance
@@ -49,10 +49,15 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 
   gcs:
     image: fsouza/fake-gcs-server:1.45.2

services/document-updater/Makefile

@@ -116,13 +116,6 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
-		while ! mongosh --eval "db.version()" > /dev/null; do \
-			echo "Waiting for Mongo..."; \
-			sleep 1; \
-		done; \
-		mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/document-updater/app.js

@@ -147,6 +147,10 @@ app.post(
   '/project/:project_id/get_and_flush_if_old',
   HttpController.getProjectDocsAndFlushIfOld
 )
+app.get(
+  '/project/:project_id/last_updated_at',
+  HttpController.getProjectLastUpdatedAt
+)
 app.post('/project/:project_id/clearState', HttpController.clearProjectState)
 app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
 app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)

services/document-updater/app/js/HttpController.js

@@ -129,6 +129,22 @@ function getProjectDocsAndFlushIfOld(req, res, next) {
   )
 }
 
+function getProjectLastUpdatedAt(req, res, next) {
+  const projectId = req.params.project_id
+  ProjectManager.getProjectDocsTimestamps(projectId, (err, timestamps) => {
+    if (err) return next(err)
+
+    // Filter out nulls. This can happen when
+    // - docs get flushed between the listing and getting the individual docs ts
+    // - a doc flush failed half way (doc keys removed, project tracking not updated)
+    timestamps = timestamps.filter(ts => !!ts)
+
+    timestamps = timestamps.map(ts => parseInt(ts, 10))
+    timestamps.sort((a, b) => (a > b ? 1 : -1))
+    res.json({ lastUpdatedAt: timestamps.pop() })
+  })
+}
+
 function clearProjectState(req, res, next) {
   const projectId = req.params.project_id
   const timer = new Metrics.Timer('http.clearProjectState')
@@ -521,6 +537,7 @@ module.exports = {
   getDoc,
   peekDoc,
   getProjectDocsAndFlushIfOld,
+  getProjectLastUpdatedAt,
   clearProjectState,
   appendToDoc,
   setDoc,

services/document-updater/docker-compose.ci.yml

@@ -29,7 +29,7 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       redis:
         condition: service_healthy
     user: node
@@ -53,7 +53,12 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1

services/document-updater/docker-compose.yml

@@ -41,7 +41,7 @@ services:
     user: node
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       redis:
         condition: service_healthy
     command: npm run --silent test:acceptance
@@ -56,8 +56,13 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 

services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js

@@ -109,11 +109,40 @@ describe('Applying updates to a doc', function () {
       )
     })
 
+    it('should yield last updated time', function (done) {
+      DocUpdaterClient.getProjectLastUpdatedAt(
+        this.project_id,
+        (error, res, body) => {
+          if (error != null) {
+            throw error
+          }
+          res.statusCode.should.equal(200)
+          body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
+          done()
+        }
+      )
+    })
+
+    it('should yield no last updated time for another project', function (done) {
+      DocUpdaterClient.getProjectLastUpdatedAt(
+        DocUpdaterClient.randomId(),
+        (error, res, body) => {
+          if (error != null) {
+            throw error
+          }
+          res.statusCode.should.equal(200)
+          body.should.deep.equal({})
+          done()
+        }
+      )
+    })
+
     describe('when sending another update', function () {
       before(function (done) {
-        this.timeout = 10000
-        this.second_update = Object.create(this.update)
+        this.timeout(10000)
+        this.second_update = Object.assign({}, this.update)
         this.second_update.v = this.version + 1
+        this.secondStartTime = Date.now()
         DocUpdaterClient.sendUpdate(
           this.project_id,
           this.doc_id,
@@ -127,6 +156,24 @@ describe('Applying updates to a doc', function () {
         )
       })
 
+      it('should update the doc', function (done) {
+        DocUpdaterClient.getDoc(
+          this.project_id,
+          this.doc_id,
+          (error, res, doc) => {
+            if (error) done(error)
+            doc.lines.should.deep.equal([
+              'one',
+              'one and a half',
+              'one and a half',
+              'two',
+              'three',
+            ])
+            done()
+          }
+        )
+      })
+
       it('should not change the first op timestamp', function (done) {
         rclientProjectHistory.get(
           ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
@@ -142,6 +189,23 @@ describe('Applying updates to a doc', function () {
           }
         )
       })
+
+      it('should yield last updated time', function (done) {
+        DocUpdaterClient.getProjectLastUpdatedAt(
+          this.project_id,
+          (error, res, body) => {
+            if (error != null) {
+              throw error
+            }
+            res.statusCode.should.equal(200)
+            body.lastUpdatedAt.should.be.within(
+              this.secondStartTime,
+              Date.now()
+            )
+            done()
+          }
+        )
+      })
     })
   })
 

services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js

@@ -119,6 +119,18 @@ module.exports = DocUpdaterClient = {
     )
   },
 
+  getProjectLastUpdatedAt(projectId, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
   preloadDoc(projectId, docId, callback) {
     DocUpdaterClient.getDoc(projectId, docId, callback)
   },

services/git-bridge/README.md

@@ -76,12 +76,10 @@ The configuration file is in `.json` format.
         "postbackBaseUrl" (string): the postback url,
         "serviceName" (string): current name of writeLaTeX
                                 in case it ever changes,
-        "oauth2" (object): { null or missing if oauth2 shouldn't be used
-            "oauth2ClientID" (string): oauth2 client ID,
-            "oauth2ClientSecret" (string): oauth2 client secret,
-            "oauth2Server" (string): oauth2 server,
-                                     with protocol and
-                                     without trailing slash
+        "oauth2Server" (string): oauth2 server,
+                                 with protocol and
+                                 without trailing slash,
+                                 null or missing if oauth2 shouldn't be used
         },
         "repoStore" (object, optional): { configure the repo store
             "maxFileSize" (long, optional): maximum size of a file, inclusive

services/git-bridge/conf/envsubst_template.json

@@ -7,11 +7,7 @@
   "apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
   "postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
   "serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
-  "oauth2": {
-    "oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
-    "oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
-    "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
-  },
+  "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}",
   "userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
   "repoStore": {
     "maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},

services/git-bridge/conf/example_config.json

@@ -7,11 +7,7 @@
     "apiBaseUrl": "https://localhost/api/v0",
     "postbackBaseUrl": "https://localhost",
     "serviceName": "Overleaf",
-    "oauth2": {
-        "oauth2ClientID": "asdf",
-        "oauth2ClientSecret": "asdf",
-        "oauth2Server": "https://localhost"
-    },
+    "oauth2Server": "https://localhost",
     "repoStore": {
         "maxFileNum": 2000,
         "maxFileSize": 52428800

services/git-bridge/conf/local.json

@@ -7,11 +7,7 @@
   "apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
   "postbackBaseUrl": "http://git-bridge:8000",
   "serviceName": "Overleaf",
-  "oauth2": {
-    "oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
-    "oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
-    "oauth2Server": "http://v2.overleaf.test:3000"
-  },
+  "oauth2Server": "http://v2.overleaf.test:3000",
   "repoStore": {
     "maxFileNum": 2000,
     "maxFileSize": 52428800

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java

@@ -30,7 +30,7 @@ public class Config implements JSONSource {
         config.apiBaseURL,
         config.postbackURL,
         config.serviceName,
-        Oauth2.asSanitised(config.oauth2),
+        config.oauth2Server,
         config.userPasswordEnabled,
         config.repoStore,
         SwapStoreConfig.sanitisedCopy(config.swapStore),
@@ -46,7 +46,7 @@ public class Config implements JSONSource {
   private String apiBaseURL;
   private String postbackURL;
   private String serviceName;
-  @Nullable private Oauth2 oauth2;
+  @Nullable private String oauth2Server;
   private boolean userPasswordEnabled;
   @Nullable private RepoStoreConfig repoStore;
   @Nullable private SwapStoreConfig swapStore;
@@ -70,7 +70,7 @@ public class Config implements JSONSource {
       String apiBaseURL,
       String postbackURL,
       String serviceName,
-      Oauth2 oauth2,
+      String oauth2Server,
       boolean userPasswordEnabled,
       RepoStoreConfig repoStore,
       SwapStoreConfig swapStore,
@@ -84,7 +84,7 @@ public class Config implements JSONSource {
     this.apiBaseURL = apiBaseURL;
     this.postbackURL = postbackURL;
     this.serviceName = serviceName;
-    this.oauth2 = oauth2;
+    this.oauth2Server = oauth2Server;
     this.userPasswordEnabled = userPasswordEnabled;
     this.repoStore = repoStore;
     this.swapStore = swapStore;
@@ -116,7 +116,7 @@ public class Config implements JSONSource {
     if (!postbackURL.endsWith("/")) {
       postbackURL += "/";
     }
-    oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
+    oauth2Server = getOptionalString(configObject, "oauth2Server");
     userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
     repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
     swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
@@ -166,19 +166,12 @@ public class Config implements JSONSource {
     return postbackURL;
   }
 
-  public boolean isUsingOauth2() {
-    return oauth2 != null;
-  }
-
   public boolean isUserPasswordEnabled() {
     return userPasswordEnabled;
   }
 
-  public Oauth2 getOauth2() {
-    if (!isUsingOauth2()) {
-      throw new AssertionError("Getting oauth2 when not using it");
-    }
-    return oauth2;
+  public String getOauth2Server() {
+    return oauth2Server;
   }
 
   public Optional<RepoStoreConfig> getRepoStore() {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java

@@ -1,33 +0,0 @@
-package uk.ac.ic.wlgitbridge.application.config;
-
-/*
- * Created by winston on 25/10/15.
- */
-public class Oauth2 {
-
-  private final String oauth2ClientID;
-  private final String oauth2ClientSecret;
-  private final String oauth2Server;
-
-  public Oauth2(String oauth2ClientID, String oauth2ClientSecret, String oauth2Server) {
-    this.oauth2ClientID = oauth2ClientID;
-    this.oauth2ClientSecret = oauth2ClientSecret;
-    this.oauth2Server = oauth2Server;
-  }
-
-  public String getOauth2ClientID() {
-    return oauth2ClientID;
-  }
-
-  public String getOauth2ClientSecret() {
-    return oauth2ClientSecret;
-  }
-
-  public String getOauth2Server() {
-    return oauth2Server;
-  }
-
-  public static Oauth2 asSanitised(Oauth2 oauth2) {
-    return new Oauth2("<oauth2ClientID>", "<oauth2ClientSecret>", oauth2.oauth2Server);
-  }
-}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java

@@ -151,9 +151,9 @@ public class GitBridgeServer {
       throws ServletException {
     final ServletContextHandler servletContextHandler =
         new ServletContextHandler(ServletContextHandler.SESSIONS);
-    if (config.isUsingOauth2()) {
+    if (config.getOauth2Server() != null) {
       Filter filter =
-          new Oauth2Filter(snapshotApi, config.getOauth2(), config.isUserPasswordEnabled());
+          new Oauth2Filter(snapshotApi, config.getOauth2Server(), config.isUserPasswordEnabled());
       servletContextHandler.addFilter(
           new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java

@@ -13,7 +13,6 @@ import javax.servlet.*;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.commons.codec.binary.Base64;
-import uk.ac.ic.wlgitbridge.application.config.Oauth2;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
 import uk.ac.ic.wlgitbridge.util.Instance;
 import uk.ac.ic.wlgitbridge.util.Log;
@@ -28,13 +27,13 @@ public class Oauth2Filter implements Filter {
 
   private final SnapshotApi snapshotApi;
 
-  private final Oauth2 oauth2;
+  private final String oauth2Server;
 
   private final boolean isUserPasswordEnabled;
 
-  public Oauth2Filter(SnapshotApi snapshotApi, Oauth2 oauth2, boolean isUserPasswordEnabled) {
+  public Oauth2Filter(SnapshotApi snapshotApi, String oauth2Server, boolean isUserPasswordEnabled) {
     this.snapshotApi = snapshotApi;
-    this.oauth2 = oauth2;
+    this.oauth2Server = oauth2Server;
     this.isUserPasswordEnabled = isUserPasswordEnabled;
   }
 
@@ -108,7 +107,7 @@ public class Oauth2Filter implements Filter {
       // fail later (for example, in the unlikely event that the token
       // expired between the two requests). In that case, JGit will
       // return a 401 without a custom error message.
-      int statusCode = checkAccessToken(oauth2, password, getClientIp(request));
+      int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
       if (statusCode == 429) {
         handleRateLimit(projectId, username, request, response);
         return;
@@ -238,10 +237,9 @@ public class Oauth2Filter implements Filter {
             "your Overleaf Account Settings."));
   }
 
-  private int checkAccessToken(Oauth2 oauth2, String accessToken, String clientIp)
+  private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
       throws IOException {
-    GenericUrl url =
-        new GenericUrl(oauth2.getOauth2Server() + "/oauth/token/info?client_ip=" + clientIp);
+    GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
     HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
     HttpHeaders headers = new HttpHeaders();
     headers.setAuthorization("Bearer " + accessToken);

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java

@@ -1495,13 +1495,9 @@ public class WLGitBridgeIntegrationTest {
             + port
             + "\",\n"
             + "    \"serviceName\": \"Overleaf\",\n"
-            + "    \"oauth2\": {\n"
-            + "        \"oauth2ClientID\": \"clientID\",\n"
-            + "        \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
-            + "        \"oauth2Server\": \"http://127.0.0.1:"
+            + "    \"oauth2Server\": \"http://127.0.0.1:"
             + apiPort
-            + "\"\n"
-            + "    }";
+            + "\"";
     if (swapCfg != null) {
       cfgStr +=
           ",\n"
@@ -1524,7 +1520,6 @@ public class WLGitBridgeIntegrationTest {
               + ",\n"
               + "        \"intervalMillis\": "
               + swapCfg.getIntervalMillis()
-              + "\n"
               + "    }\n";
     }
     cfgStr += "}\n";

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java

@@ -23,11 +23,7 @@ public class ConfigTest {
                 + "    \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
                 + "    \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
                 + "    \"serviceName\": \"Overleaf\",\n"
-                + "    \"oauth2\": {\n"
-                + "        \"oauth2ClientID\": \"clientID\",\n"
-                + "        \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
-                + "        \"oauth2Server\": \"https://www.overleaf.com\"\n"
-                + "    }\n"
+                + "    \"oauth2Server\": \"https://www.overleaf.com\"\n"
                 + "}\n");
     Config config = new Config(reader);
     assertEquals(80, config.getPort());
@@ -35,10 +31,7 @@ public class ConfigTest {
     assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
     assertEquals("http://127.0.0.1/", config.getPostbackURL());
     assertEquals("Overleaf", config.getServiceName());
-    assertTrue(config.isUsingOauth2());
-    assertEquals("clientID", config.getOauth2().getOauth2ClientID());
-    assertEquals("oauth2 client secret", config.getOauth2().getOauth2ClientSecret());
-    assertEquals("https://www.overleaf.com", config.getOauth2().getOauth2Server());
+    assertEquals("https://www.overleaf.com", config.getOauth2Server());
   }
 
   @Test(expected = AssertionError.class)
@@ -53,7 +46,7 @@ public class ConfigTest {
                 + "    \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
                 + "    \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
                 + "    \"serviceName\": \"Overleaf\",\n"
-                + "    \"oauth2\": null\n"
+                + "    \"oauth2Server\": null\n"
                 + "}\n");
     Config config = new Config(reader);
     assertEquals(80, config.getPort());
@@ -61,8 +54,7 @@ public class ConfigTest {
     assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
     assertEquals("http://127.0.0.1/", config.getPostbackURL());
     assertEquals("Overleaf", config.getServiceName());
-    assertFalse(config.isUsingOauth2());
-    config.getOauth2();
+    assertNull(config.getOauth2Server());
   }
 
   @Test
@@ -77,11 +69,7 @@ public class ConfigTest {
                 + "    \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
                 + "    \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
                 + "    \"serviceName\": \"Overleaf\",\n"
-                + "    \"oauth2\": {\n"
-                + "        \"oauth2ClientID\": \"my oauth2 client id\",\n"
-                + "        \"oauth2ClientSecret\": \"my oauth2 client secret\",\n"
-                + "        \"oauth2Server\": \"https://www.overleaf.com\"\n"
-                + "    }\n"
+                + "    \"oauth2Server\": \"https://www.overleaf.com\"\n"
                 + "}\n");
     Config config = new Config(reader);
     String expected =
@@ -94,11 +82,7 @@ public class ConfigTest {
             + "  \"apiBaseURL\": \"http://127.0.0.1:60000/api/v0/\",\n"
             + "  \"postbackURL\": \"http://127.0.0.1/\",\n"
             + "  \"serviceName\": \"Overleaf\",\n"
-            + "  \"oauth2\": {\n"
-            + "    \"oauth2ClientID\": \"<oauth2ClientID>\",\n"
-            + "    \"oauth2ClientSecret\": \"<oauth2ClientSecret>\",\n"
-            + "    \"oauth2Server\": \"https://www.overleaf.com\"\n"
-            + "  },\n"
+            + "  \"oauth2Server\": \"https://www.overleaf.com\",\n"
             + "  \"userPasswordEnabled\": false,\n"
             + "  \"repoStore\": null,\n"
             + "  \"swapStore\": null,\n"

services/history-v1/Makefile

@@ -116,13 +116,6 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
-	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
-		while ! mongosh --eval "db.version()" > /dev/null; do \
-			echo "Waiting for Mongo..."; \
-			sleep 1; \
-		done; \
-		mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/history-v1/api/controllers/project_import.js

@@ -22,6 +22,7 @@ const BlobStore = storage.BlobStore
 const chunkStore = storage.chunkStore
 const HashCheckBlobStore = storage.HashCheckBlobStore
 const persistChanges = storage.persistChanges
+const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
 
@@ -113,7 +114,8 @@ async function importChanges(req, res, next) {
       err instanceof File.NotEditableError ||
       err instanceof FileMap.PathnameError ||
       err instanceof Snapshot.EditMissingFileError ||
-      err instanceof chunkStore.ChunkVersionConflictError
+      err instanceof chunkStore.ChunkVersionConflictError ||
+      err instanceof InvalidChangeError
     ) {
       // If we failed to apply operations, that's probably because they were
       // invalid.

services/history-v1/backup-verifier-app.mjs

@@ -4,17 +4,24 @@ import '@overleaf/metrics/initialize.js'
 import http from 'node:http'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
+import { setTimeout } from 'node:timers/promises'
 import express from 'express'
 import logger from '@overleaf/logger'
 import Metrics from '@overleaf/metrics'
+import { healthCheck } from './backupVerifier/healthCheck.mjs'
 import {
   BackupCorruptedError,
-  healthCheck,
   verifyBlob,
 } from './storage/lib/backupVerifier.mjs'
 import { mongodb } from './storage/index.js'
 import { expressify } from '@overleaf/promise-utils'
 import { Blob } from 'overleaf-editor-core'
+import { loadGlobalBlobs } from './storage/lib/blob_store/index.js'
+import { EventEmitter } from 'node:events'
+import {
+  loopRandomProjects,
+  setWriteMetrics,
+} from './backupVerifier/ProjectVerifier.mjs'
 
 const app = express()
 
@@ -64,20 +71,46 @@ app.use((err, req, res, next) => {
   next(err)
 })
 
+const shutdownEmitter = new EventEmitter()
+
+shutdownEmitter.once('shutdown', async code => {
+  logger.info({ code }, 'shutting down')
+  await mongodb.client.close()
+  await setTimeout(100)
+  process.exit(code)
+})
+
+process.on('SIGTERM', () => {
+  shutdownEmitter.emit('shutdown', 0)
+})
+
+process.on('SIGINT', () => {
+  shutdownEmitter.emit('shutdown', 0)
+})
+
 /**
  * @param {number} port
  * @return {Promise<http.Server>}
  */
 export async function startApp(port) {
   await mongodb.client.connect()
+  await loadGlobalBlobs()
   await healthCheck()
   const server = http.createServer(app)
   await promisify(server.listen.bind(server, port))()
+  loopRandomProjects(shutdownEmitter)
   return server
 }
 
+setWriteMetrics(true)
+
 // Run this if we're called directly
 if (process.argv[1] === fileURLToPath(import.meta.url)) {
   const PORT = parseInt(process.env.PORT || '3102', 10)
-  await startApp(PORT)
+  try {
+    await startApp(PORT)
+  } catch (error) {
+    shutdownEmitter.emit('shutdown', 1)
+    logger.error({ error }, 'error starting app')
+  }
 }

services/history-v1/backup-worker-app.mjs

@@ -38,10 +38,10 @@ app.use((err, req, res, next) => {
 })
 
 async function triggerGracefulShutdown(server, signal) {
-  logger.warn({ signal }, 'graceful shutdown: started shutdown sequence')
+  logger.info({ signal }, 'graceful shutdown: started shutdown sequence')
   await drainQueue()
   server.close(function () {
-    logger.warn({ signal }, 'graceful shutdown: closed server')
+    logger.info({ signal }, 'graceful shutdown: closed server')
     setTimeout(() => {
       process.exit(0)
     }, 1000)

services/history-v1/backupVerifier/ProjectMetrics.mjs

@@ -0,0 +1,33 @@
+import Metrics from '@overleaf/metrics'
+import { objectIdFromDate } from './utils.mjs'
+import { db } from '../storage/lib/mongodb.js'
+
+const projectsCollection = db.collection('projects')
+
+/**
+ *
+ * @param {Date} beforeTime
+ * @return {Promise<void>}
+ */
+export async function measurePendingChangesBeforeTime(beforeTime) {
+  const pendingChangeCount = await projectsCollection.countDocuments({
+    'overleaf.backup.pendingChangeAt': {
+      $lt: beforeTime,
+    },
+  })
+
+  Metrics.gauge('backup_verification_pending_changes', pendingChangeCount)
+}
+
+/**
+ *
+ * @param {Date} graceTime
+ * @return {Promise<void>}
+ */
+export async function measureNeverBackedUpProjects(graceTime) {
+  const neverBackedUpCount = await projectsCollection.countDocuments({
+    'overleaf.backup.lastBackedUpVersion': null,
+    _id: { $lt: objectIdFromDate(graceTime) },
+  })
+  Metrics.gauge('backup_verification_never_backed_up', neverBackedUpCount)
+}

services/history-v1/backupVerifier/ProjectSampler.mjs

@@ -0,0 +1,79 @@
+// @ts-check
+import { objectIdFromDate } from './utils.mjs'
+import { db } from '../storage/lib/mongodb.js'
+import config from 'config'
+
+const projectsCollection = db.collection('projects')
+
+const HAS_PROJECTS_WITHOUT_HISTORY =
+  config.get('hasProjectsWithoutHistory') === 'true'
+
+/**
+ * @param {Date} start
+ * @param {Date} end
+ * @param {number} N
+ * @yields {string}
+ */
+export async function* getProjectsCreatedInDateRangeCursor(start, end, N) {
+  yield* getSampleProjectsCursor(N, [
+    {
+      $match: {
+        _id: {
+          $gt: objectIdFromDate(start),
+          $lte: objectIdFromDate(end),
+        },
+      },
+    },
+  ])
+}
+
+export async function* getProjectsUpdatedInDateRangeCursor(start, end, N) {
+  yield* getSampleProjectsCursor(N, [
+    {
+      $match: {
+        'overleaf.history.updatedAt': {
+          $gt: start,
+          $lte: end,
+        },
+      },
+    },
+  ])
+}
+
+/**
+ * @typedef {import('mongodb').Document} Document
+ */
+
+/**
+ *
+ * @generator
+ * @param {number} N
+ * @param {Array<Document>} preSampleAggregationStages
+ * @yields {string}
+ */
+export async function* getSampleProjectsCursor(
+  N,
+  preSampleAggregationStages = []
+) {
+  const cursor = projectsCollection.aggregate([
+    ...preSampleAggregationStages,
+    { $sample: { size: N } },
+    { $project: { 'overleaf.history.id': 1 } },
+  ])
+
+  let validProjects = 0
+  let hasInvalidProject = false
+
+  for await (const project of cursor) {
+    if (HAS_PROJECTS_WITHOUT_HISTORY && !project.overleaf?.history?.id) {
+      hasInvalidProject = true
+      continue
+    }
+    validProjects++
+    yield project.overleaf.history.id.toString()
+  }
+
+  if (validProjects === 0 && hasInvalidProject) {
+    yield* getSampleProjectsCursor(N, preSampleAggregationStages)
+  }
+}

services/history-v1/backupVerifier/ProjectVerifier.mjs

@@ -0,0 +1,320 @@
+// @ts-check
+import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
+import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
+import logger from '@overleaf/logger'
+import metrics from '@overleaf/metrics'
+import {
+  getSampleProjectsCursor,
+  getProjectsCreatedInDateRangeCursor,
+  getProjectsUpdatedInDateRangeCursor,
+} from './ProjectSampler.mjs'
+import OError from '@overleaf/o-error'
+import { setTimeout } from 'node:timers/promises'
+
+const MS_PER_30_DAYS = 30 * 24 * 60 * 60 * 1000
+
+const failureCounter = new metrics.prom.Counter({
+  name: 'backup_project_verification_failed',
+  help: 'Number of projects that failed verification',
+  labelNames: ['name'],
+})
+
+const successCounter = new metrics.prom.Counter({
+  name: 'backup_project_verification_succeeded',
+  help: 'Number of projects that succeeded verification',
+})
+
+let WRITE_METRICS = false
+
+/**
+ * @typedef {import('node:events').EventEmitter} EventEmitter
+ */
+
+/**
+ * Allows writing metrics to be enabled or disabled.
+ * @param {Boolean} writeMetrics
+ */
+export function setWriteMetrics(writeMetrics) {
+  WRITE_METRICS = writeMetrics
+}
+
+/**
+ *
+ * @param {Error|unknown} error
+ * @param {string} historyId
+ */
+function handleVerificationError(error, historyId) {
+  const name = error instanceof Error ? error.name : 'UnknownError'
+  logger.error({ historyId, error, name }, 'error verifying project backup')
+
+  WRITE_METRICS && failureCounter.inc({ name })
+
+  return name
+}
+
+/**
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} interval
+ * @returns {Array<VerificationJobSpecification>}
+ */
+function splitJobs(startDate, endDate, interval) {
+  /** @type {Array<VerificationJobSpecification>} */
+  const jobs = []
+  while (startDate < endDate) {
+    const nextStart = new Date(
+      Math.min(startDate.getTime() + interval, endDate.getTime())
+    )
+    jobs.push({ startDate, endDate: nextStart })
+    startDate = nextStart
+  }
+  return jobs
+}
+
+/**
+ *
+ * @param {AsyncGenerator<string>} historyIdCursor
+ * @param {EventEmitter} [eventEmitter]
+ * @param {number} [delay] - Allows a delay between each verification
+ * @return {Promise<{verified: number, total: number, errorTypes: *[], hasFailure: boolean}>}
+ */
+async function verifyProjectsFromCursor(
+  historyIdCursor,
+  eventEmitter,
+  delay = 0
+) {
+  const errorTypes = []
+  let verified = 0
+  let total = 0
+  let receivedShutdownSignal = false
+  if (eventEmitter) {
+    eventEmitter.once('shutdown', () => {
+      receivedShutdownSignal = true
+    })
+  }
+  for await (const historyId of historyIdCursor) {
+    if (receivedShutdownSignal) {
+      break
+    }
+    total++
+    try {
+      await verifyProjectWithErrorContext(historyId)
+      logger.debug({ historyId }, 'verified project backup successfully')
+      WRITE_METRICS && successCounter.inc()
+      verified++
+    } catch (error) {
+      const errorType = handleVerificationError(error, historyId)
+      errorTypes.push(errorType)
+    }
+    if (delay > 0) {
+      await setTimeout(delay)
+    }
+  }
+  return {
+    verified,
+    total,
+    errorTypes,
+    hasFailure: errorTypes.length > 0,
+  }
+}
+
+/**
+ *
+ * @param {number} nProjectsToSample
+ * @param {EventEmitter} [signal]
+ * @param {number} [delay]
+ * @return {Promise<VerificationJobStatus>}
+ */
+export async function verifyRandomProjectSample(
+  nProjectsToSample,
+  signal,
+  delay = 0
+) {
+  const historyIds = await getSampleProjectsCursor(nProjectsToSample)
+  return await verifyProjectsFromCursor(historyIds, signal, delay)
+}
+
+/**
+ * Samples projects with history IDs between the specified dates and verifies them.
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} projectsPerRange
+ * @param {EventEmitter} [signal]
+ * @return {Promise<VerificationJobStatus>}
+ */
+async function verifyRange(startDate, endDate, projectsPerRange, signal) {
+  logger.info({ startDate, endDate }, 'verifying range')
+
+  const results = await verifyProjectsFromCursor(
+    getProjectsCreatedInDateRangeCursor(startDate, endDate, projectsPerRange),
+    signal
+  )
+
+  if (results.total === 0) {
+    logger.debug(
+      { start: startDate, end: endDate },
+      'No projects found in range'
+    )
+  }
+
+  const jobStatus = {
+    ...results,
+    startDate,
+    endDate,
+  }
+
+  logger.debug(
+    { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
+    'Verified range'
+  )
+  return jobStatus
+}
+
+/**
+ * @typedef {Object} VerificationJobSpecification
+ * @property {Date} startDate
+ * @property {Date} endDate
+ */
+
+/**
+ * @typedef {import('./types.d.ts').VerificationJobStatus} VerificationJobStatus
+ */
+
+/**
+ * @typedef {Object} VerifyDateRangeOptions
+ * @property {Date} startDate
+ * @property {Date} endDate
+ * @property {number} [interval]
+ * @property {number} [projectsPerRange]
+ * @property {number} [concurrency]
+ * @property {EventEmitter} [signal]
+ */
+
+/**
+ *
+ * @param {VerifyDateRangeOptions} options
+ * @return {Promise<VerificationJobStatus>}
+ */
+export async function verifyProjectsCreatedInDateRange({
+  concurrency = 0,
+  projectsPerRange = 10,
+  startDate,
+  endDate,
+  interval = MS_PER_30_DAYS,
+  signal,
+}) {
+  const jobs = splitJobs(startDate, endDate, interval)
+  if (jobs.length === 0) {
+    throw new OError('Time range could not be split into jobs', {
+      start: startDate,
+      end: endDate,
+      interval,
+    })
+  }
+  const settlements = await promiseMapSettledWithLimit(
+    concurrency,
+    jobs,
+    ({ startDate, endDate }) =>
+      verifyRange(startDate, endDate, projectsPerRange, signal)
+  )
+  return settlements.reduce(
+    /**
+     *
+     * @param {VerificationJobStatus} acc
+     * @param settlement
+     * @return {VerificationJobStatus}
+     */
+    (acc, settlement) => {
+      if (settlement.status !== 'rejected') {
+        if (settlement.value.hasFailure) {
+          acc.hasFailure = true
+        }
+        acc.total += settlement.value.total
+        acc.verified += settlement.value.verified
+        acc.errorTypes = acc.errorTypes.concat(settlement.value.errorTypes)
+      } else {
+        logger.error({ ...settlement.reason }, 'Error processing range')
+      }
+      return acc
+    },
+    /** @type {VerificationJobStatus} */
+    {
+      startDate,
+      endDate,
+      verified: 0,
+      total: 0,
+      hasFailure: false,
+      errorTypes: [],
+    }
+  )
+}
+
+/**
+ * Verifies that projects that have recently gone out of RPO have been updated.
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} nProjects
+ * @param {EventEmitter} [signal]
+ * @return {Promise<VerificationJobStatus>}
+ */
+export async function verifyProjectsUpdatedInDateRange(
+  startDate,
+  endDate,
+  nProjects,
+  signal
+) {
+  logger.debug(
+    { startDate, endDate, nProjects },
+    'Sampling projects updated in date range'
+  )
+  const results = await verifyProjectsFromCursor(
+    getProjectsUpdatedInDateRangeCursor(startDate, endDate, nProjects),
+    signal
+  )
+
+  if (results.total === 0) {
+    logger.debug(
+      { start: startDate, end: endDate },
+      'No projects updated recently'
+    )
+  }
+
+  const jobStatus = {
+    ...results,
+    startDate,
+    endDate,
+  }
+
+  logger.debug(
+    { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
+    'Verified recently updated projects'
+  )
+  return jobStatus
+}
+
+/**
+ *
+ * @param {EventEmitter} signal
+ * @return {void}
+ */
+export function loopRandomProjects(signal) {
+  let shutdown = false
+  signal.on('shutdown', function () {
+    shutdown = true
+  })
+  async function loop() {
+    do {
+      try {
+        const result = await verifyRandomProjectSample(100, signal, 2_000)
+        logger.debug({ result }, 'verified random project sample')
+      } catch (error) {
+        logger.error({ error }, 'error verifying random project sample')
+      }
+      // eslint-disable-next-line no-unmodified-loop-condition
+    } while (!shutdown)
+  }
+  loop()
+}

services/history-v1/backupVerifier/healthCheck.mjs

@@ -0,0 +1,32 @@
+import config from 'config'
+import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
+import {
+  measureNeverBackedUpProjects,
+  measurePendingChangesBeforeTime,
+} from './ProjectMetrics.mjs'
+import { getEndDateForRPO, RPO } from './utils.mjs'
+
+/** @type {Array<string>} */
+const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
+
+export async function healthCheck() {
+  if (!Array.isArray(HEALTH_CHECK_PROJECTS)) {
+    throw new Error('expected healthCheckProjects to be an array')
+  }
+  if (HEALTH_CHECK_PROJECTS.length !== 2) {
+    throw new Error('expected 2 healthCheckProjects')
+  }
+  if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
+    throw new Error('expected mongo id in healthCheckProjects')
+  }
+  if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
+    throw new Error('expected postgres id in healthCheckProjects')
+  }
+
+  for (const historyId of HEALTH_CHECK_PROJECTS) {
+    await verifyProjectWithErrorContext(historyId)
+  }
+
+  await measurePendingChangesBeforeTime(getEndDateForRPO(2))
+  await measureNeverBackedUpProjects(getEndDateForRPO(2))
+}

services/history-v1/backupVerifier/types.d.ts

@@ -0,0 +1,8 @@
+export type VerificationJobStatus = {
+  verified: number
+  total: number
+  startDate?: Date
+  endDate?: Date
+  hasFailure: boolean
+  errorTypes: Array<string>
+}

services/history-v1/backupVerifier/utils.mjs

@@ -0,0 +1,35 @@
+import { ObjectId } from 'mongodb'
+import config from 'config'
+
+export const RPO = parseInt(config.get('backupRPOInMS'), 10)
+
+/**
+ * @param {Date} time
+ * @return {ObjectId}
+ */
+export function objectIdFromDate(time) {
+  return ObjectId.createFromTime(time.getTime() / 1000)
+}
+
+/**
+ * @param {number} [factor] - Multiply RPO by this factor, default is 1
+ * @return {Date}
+ */
+export function getEndDateForRPO(factor = 1) {
+  return new Date(Date.now() - RPO * factor)
+}
+
+/**
+ * Creates a startDate, endDate pair that checks a period of time before the RPO horizon
+ *
+ * @param {number} offset - How many seconds we should check
+ * @return {{endDate: Date, startDate: Date}}
+ */
+export function getDatesBeforeRPO(offset) {
+  const now = new Date()
+  const endDate = new Date(now.getTime() - RPO)
+  return {
+    endDate,
+    startDate: new Date(endDate.getTime() - offset * 1000),
+  }
+}

services/history-v1/buildscript.txt

@@ -7,4 +7,4 @@ history-v1
 --node-version=20.18.2
 --public-repo=False
 --script-version=4.5.0
---tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,api/**/*,migrations/**/*,storage/**/*
+--tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,backup-worker-app.mjs,api/**/*,migrations/**/*,storage/**/*

services/history-v1/config/custom-environment-variables.json

@@ -66,6 +66,7 @@
   },
   "healthCheckBlobs": "HEALTH_CHECK_BLOBS",
   "healthCheckProjects": "HEALTH_CHECK_PROJECTS",
+  "backupRPOInMS": "BACKUP_RPO_IN_MS",
   "minSoftDeletionPeriodDays": "MIN_SOFT_DELETION_PERIOD_DAYS",
   "mongo": {
     "uri": "MONGO_CONNECTION_STRING"

services/history-v1/config/default.json

@@ -23,12 +23,14 @@
       }
     }
   },
+  "backupRPOInMS": "3600000",
   "chunkStore": {
     "historyStoreConcurrency": "4"
   },
   "zipStore": {
     "zipTimeoutMs": "360000"
   },
+  "hasProjectsWithoutHistory": false,
   "minSoftDeletionPeriodDays": "90",
   "maxDeleteKeys": "1000",
   "useDeleteObjects": "true",

services/history-v1/config/test.json

@@ -36,6 +36,7 @@
   },
   "healthCheckBlobs": "[\"42/f70d7bba4ae1f07682e0358bd7a2068094fc023b\",\"000000000000000000000042/98d5521fe746bc2d11761edab5d0829bee286009\"]",
   "healthCheckProjects": "[\"42\",\"000000000000000000000042\"]",
+  "backupRPOInMS": "360000",
   "maxDeleteKeys": "3",
   "useDeleteObjects": "false",
   "mongo": {

services/history-v1/docker-compose.ci.yml

@@ -40,7 +40,7 @@ services:
       - ./test/acceptance/certs:/certs
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       redis:
         condition: service_healthy
       postgres:
@@ -74,10 +74,15 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
   postgres:
     image: postgres:10
     environment:

services/history-v1/docker-compose.yml

@@ -57,7 +57,7 @@ services:
     user: node
     depends_on:
       mongo:
-        condition: service_healthy
+        condition: service_started
       redis:
         condition: service_healthy
       postgres:
@@ -82,10 +82,15 @@ services:
   mongo:
     image: mongo:6.0.13
     command: --replSet overleaf
-    healthcheck:
-      test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
-      interval: 1s
-      retries: 20
+    volumes:
+      - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+    environment:
+      MONGO_INITDB_DATABASE: sharelatex
+    extra_hosts:
+      # Required when using the automatic database setup for initializing the
+      # replica set. This override is not needed when running the setup after
+      # starting up mongo.
+      - mongo:127.0.0.1
 
   postgres:
     image: postgres:10

services/history-v1/storage/index.js

@@ -15,3 +15,6 @@ exports.zipStore = require('./lib/zip_store')
 const { BlobStore, loadGlobalBlobs } = require('./lib/blob_store')
 exports.BlobStore = BlobStore
 exports.loadGlobalBlobs = loadGlobalBlobs
+
+const { InvalidChangeError } = require('./lib/errors')
+exports.InvalidChangeError = InvalidChangeError

services/history-v1/storage/lib/backupVerifier.mjs

@@ -1,14 +1,24 @@
 // @ts-check
-import config from 'config'
 import OError from '@overleaf/o-error'
-import { backupPersistor, projectBlobsBucket } from './backupPersistor.mjs'
-import { Blob } from 'overleaf-editor-core'
-import { BlobStore, makeProjectKey } from './blob_store/index.js'
+import chunkStore from '../lib/chunk_store/index.js'
+import {
+  backupPersistor,
+  chunksBucket,
+  projectBlobsBucket,
+} from './backupPersistor.mjs'
+import { Blob, Chunk, History } from 'overleaf-editor-core'
+import { BlobStore, GLOBAL_BLOBS, makeProjectKey } from './blob_store/index.js'
 import blobHash from './blob_hash.js'
 import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
+import logger from '@overleaf/logger'
+import path from 'node:path'
+import projectKey from './project_key.js'
+import streams from './streams.js'
+import objectPersistor from '@overleaf/object-persistor'
+import { getEndDateForRPO } from '../../backupVerifier/utils.mjs'
 
 /**
- * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
+ * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
  */
 
 /**
@@ -20,13 +30,13 @@ export async function verifyBlob(historyId, hash) {
 }
 
 /**
+ *
  * @param {string} historyId
- * @param {Array<string>} hashes
+ * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
  */
-export async function verifyBlobs(historyId, hashes) {
-  let projectCache
+async function getProjectPersistor(historyId) {
   try {
-    projectCache = await backupPersistor.forProjectRO(
+    return await backupPersistor.forProjectRO(
       projectBlobsBucket,
       makeProjectKey(historyId, '')
     )
@@ -36,16 +46,19 @@ export async function verifyBlobs(historyId, hashes) {
     }
     throw err
   }
-  await verifyBlobsWithCache(historyId, projectCache, hashes)
 }
 
 /**
  * @param {string} historyId
- * @param {CachedPerProjectEncryptedS3Persistor} projectCache
  * @param {Array<string>} hashes
+ * @param {CachedPerProjectEncryptedS3Persistor} [projectCache]
  */
-export async function verifyBlobsWithCache(historyId, projectCache, hashes) {
+export async function verifyBlobs(historyId, hashes, projectCache) {
   if (hashes.length === 0) throw new Error('bug: empty hashes')
+
+  if (!projectCache) {
+    projectCache = await getProjectPersistor(historyId)
+  }
   const blobStore = new BlobStore(historyId)
   for (const hash of hashes) {
     const path = makeProjectKey(historyId, hash)
@@ -58,41 +71,146 @@ export async function verifyBlobsWithCache(historyId, projectCache, hashes) {
       })
     } catch (err) {
       if (err instanceof NotFoundError) {
-        throw new BackupCorruptedError('missing blob')
+        throw new BackupCorruptedMissingBlobError('missing blob', {
+          path,
+          hash,
+        })
       }
       throw err
     }
     const backupHash = await blobHash.fromStream(blob.getByteLength(), stream)
     if (backupHash !== hash) {
-      throw new BackupCorruptedError('hash mismatch for backed up blob', {
-        path,
-        hash,
-        backupHash,
-      })
+      throw new BackupCorruptedInvalidBlobError(
+        'hash mismatch for backed up blob',
+        {
+          path,
+          hash,
+          backupHash,
+        }
+      )
     }
   }
 }
 
-export class BackupCorruptedError extends OError {}
-
-export async function healthCheck() {
-  /** @type {Array<string>} */
-  const HEALTH_CHECK_BLOBS = JSON.parse(config.get('healthCheckBlobs'))
-  if (HEALTH_CHECK_BLOBS.length !== 2) {
-    throw new Error('expected 2 healthCheckBlobs')
-  }
-  if (!HEALTH_CHECK_BLOBS.some(path => path.split('/')[0].length === 24)) {
-    throw new Error('expected mongo id in healthCheckBlobs')
-  }
-  if (!HEALTH_CHECK_BLOBS.some(path => path.split('/')[0].length < 24)) {
-    throw new Error('expected postgres id in healthCheckBlobs')
-  }
-  if (HEALTH_CHECK_BLOBS.some(path => path.split('/')[1]?.length !== 40)) {
-    throw new Error('expected hash in healthCheckBlobs')
-  }
-
-  for (const path of HEALTH_CHECK_BLOBS) {
-    const [historyId, hash] = path.split('/')
-    await verifyBlob(historyId, hash)
+/**
+ * @param {string} historyId
+ * @param {Date} [endTimestamp]
+ */
+export async function verifyProjectWithErrorContext(
+  historyId,
+  endTimestamp = getEndDateForRPO()
+) {
+  try {
+    await verifyProject(historyId, endTimestamp)
+  } catch (err) {
+    // @ts-ignore err is Error instance
+    throw OError.tag(err, 'verifyProject', { historyId, endTimestamp })
   }
 }
+
+/**
+ *
+ * @param {string} historyId
+ * @param {number} startVersion
+ * @param {CachedPerProjectEncryptedS3Persistor} backupPersistorForProject
+ * @return {Promise<any>}
+ */
+async function loadChunk(historyId, startVersion, backupPersistorForProject) {
+  const key = path.join(
+    projectKey.format(historyId),
+    projectKey.pad(startVersion)
+  )
+  try {
+    const buf = await streams.gunzipStreamToBuffer(
+      await backupPersistorForProject.getObjectStream(chunksBucket, key)
+    )
+    return JSON.parse(buf.toString('utf-8'))
+  } catch (err) {
+    if (err instanceof objectPersistor.Errors.NotFoundError) {
+      throw new Chunk.NotPersistedError(historyId)
+    }
+    if (err instanceof Error) {
+      throw OError.tag(err, 'Failed to load chunk', { historyId, startVersion })
+    }
+    throw err
+  }
+}
+
+/**
+ * @param {string} historyId
+ * @param {Date} endTimestamp
+ */
+export async function verifyProject(historyId, endTimestamp) {
+  const backend = chunkStore.getBackend(historyId)
+  const [first, last] = await Promise.all([
+    backend.getFirstChunkBeforeTimestamp(historyId, endTimestamp),
+    backend.getLastActiveChunkBeforeTimestamp(historyId, endTimestamp),
+  ])
+
+  const chunksRecordsToVerify = [
+    {
+      chunkId: first.id,
+      chunkLabel: 'first',
+    },
+  ]
+  if (first.startVersion !== last.startVersion) {
+    chunksRecordsToVerify.push({
+      chunkId: last.id,
+      chunkLabel: 'last before RPO',
+    })
+  }
+
+  const projectCache = await getProjectPersistor(historyId)
+
+  const chunks = await Promise.all(
+    chunksRecordsToVerify.map(async chunk => {
+      try {
+        return History.fromRaw(
+          await loadChunk(historyId, chunk.startVersion, projectCache)
+        )
+      } catch (err) {
+        if (err instanceof Chunk.NotPersistedError) {
+          throw new BackupRPOViolationChunkNotBackedUpError(
+            'BackupRPOviolation: chunk not backed up',
+            chunk
+          )
+        }
+        throw err
+      }
+    })
+  )
+  const seenBlobs = new Set()
+  const blobsToVerify = []
+  for (const chunk of chunks) {
+    /** @type {Set<string>} */
+    const chunkBlobs = new Set()
+    chunk.findBlobHashes(chunkBlobs)
+    let hasAddedBlobFromThisChunk = false
+    for (const blobHash of chunkBlobs) {
+      if (seenBlobs.has(blobHash)) continue // old blob
+      if (GLOBAL_BLOBS.has(blobHash)) continue // global blob
+      seenBlobs.add(blobHash)
+      if (!hasAddedBlobFromThisChunk) {
+        blobsToVerify.push(blobHash)
+        hasAddedBlobFromThisChunk = true
+      }
+    }
+  }
+  if (blobsToVerify.length === 0) {
+    logger.debug(
+      {
+        historyId,
+        chunksRecordsToVerify: chunksRecordsToVerify.map(c => c.chunkId),
+      },
+      'chunks contain no blobs to verify'
+    )
+    return
+  }
+  await verifyBlobs(historyId, blobsToVerify, projectCache)
+}
+
+export class BackupCorruptedError extends OError {}
+export class BackupRPOViolationError extends OError {}
+export class BackupCorruptedMissingBlobError extends BackupCorruptedError {}
+export class BackupCorruptedInvalidBlobError extends BackupCorruptedError {}
+export class BackupRPOViolationChunkNotBackedUpError extends OError {}

services/history-v1/storage/lib/backup_store/index.js

@@ -3,8 +3,18 @@ const { projects, backedUpBlobs } = require('../mongodb')
 const OError = require('@overleaf/o-error')
 
 // List projects with pending backups older than the specified interval
-function listPendingBackups(timeIntervalMs = 0) {
+function listPendingBackups(timeIntervalMs = 0, limit = null) {
   const cutoffTime = new Date(Date.now() - timeIntervalMs)
+  const options = {
+    projection: { 'overleaf.backup.pendingChangeAt': 1 },
+    sort: { 'overleaf.backup.pendingChangeAt': 1 },
+  }
+
+  // Apply limit if provided
+  if (limit) {
+    options.limit = limit
+  }
+
   const cursor = projects.find(
     {
       'overleaf.backup.pendingChangeAt': {
@@ -12,10 +22,30 @@ function listPendingBackups(timeIntervalMs = 0) {
         $lt: cutoffTime,
       },
     },
+    options
+  )
+  return cursor
+}
+
+// List projects that have never been backed up and are older than the specified interval
+function listUninitializedBackups(timeIntervalMs = 0, limit = null) {
+  const cutoffTimeInSeconds = (Date.now() - timeIntervalMs) / 1000
+  const options = {
+    projection: { _id: 1 },
+    sort: { _id: 1 },
+  }
+  // Apply limit if provided
+  if (limit) {
+    options.limit = limit
+  }
+  const cursor = projects.find(
     {
-      projection: { 'overleaf.backup': 1, 'overleaf.history': 1 },
-      sort: { 'overleaf.backup.pendingChangeAt': 1 },
-    }
+      'overleaf.backup.lastBackedUpVersion': null,
+      _id: {
+        $lt: ObjectId.createFromTime(cutoffTimeInSeconds),
+      },
+    },
+    options
   )
   return cursor
 }
@@ -176,6 +206,7 @@ module.exports = {
   updateCurrentMetadataIfNotSet,
   updatePendingChangeTimestamp,
   listPendingBackups,
+  listUninitializedBackups,
   getBackedUpBlobHashes,
   unsetBackedUpBlobHashes,
 }

services/history-v1/storage/lib/chunk_store/index.js

@@ -155,15 +155,22 @@ async function loadAtTimestamp(projectId, timestamp) {
  *
  * @param {string} projectId
  * @param {Chunk} chunk
+ * @param {Date} [earliestChangeTimestamp]
  * @return {Promise.<number>} for the chunkId of the inserted chunk
  */
-async function create(projectId, chunk) {
+async function create(projectId, chunk, earliestChangeTimestamp) {
   assert.projectId(projectId, 'bad projectId')
   assert.instance(chunk, Chunk, 'bad chunk')
+  assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
 
   const backend = getBackend(projectId)
   const chunkId = await uploadChunk(projectId, chunk)
-  await backend.confirmCreate(projectId, chunk, chunkId)
+  await backend.confirmCreate(
+    projectId,
+    chunk,
+    chunkId,
+    earliestChangeTimestamp
+  )
 }
 
 /**
@@ -195,18 +202,31 @@ async function uploadChunk(projectId, chunk) {
  * @param {string} projectId
  * @param {number} oldEndVersion
  * @param {Chunk} newChunk
+ * @param {Date} [earliestChangeTimestamp]
  * @return {Promise}
  */
-async function update(projectId, oldEndVersion, newChunk) {
+async function update(
+  projectId,
+  oldEndVersion,
+  newChunk,
+  earliestChangeTimestamp
+) {
   assert.projectId(projectId, 'bad projectId')
   assert.integer(oldEndVersion, 'bad oldEndVersion')
   assert.instance(newChunk, Chunk, 'bad newChunk')
+  assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
 
   const backend = getBackend(projectId)
   const oldChunkId = await getChunkIdForVersion(projectId, oldEndVersion)
   const newChunkId = await uploadChunk(projectId, newChunk)
 
-  await backend.confirmUpdate(projectId, oldChunkId, newChunk, newChunkId)
+  await backend.confirmUpdate(
+    projectId,
+    oldChunkId,
+    newChunk,
+    newChunkId,
+    earliestChangeTimestamp
+  )
 }
 
 /**

services/history-v1/storage/lib/chunk_store/mongo.js

@@ -54,6 +54,35 @@ async function getChunkForVersion(projectId, version) {
   return chunkFromRecord(record)
 }
 
+/**
+ * Get the metadata for the chunk that contains the given version before the endTime.
+ */
+async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
+  assert.mongoId(projectId, 'bad projectId')
+  assert.date(timestamp, 'bad timestamp')
+
+  const recordActive = await getChunkForVersion(projectId, 0)
+  if (recordActive && recordActive.endTimestamp <= timestamp) {
+    return recordActive
+  }
+
+  // fallback to deleted chunk
+  const recordDeleted = await mongodb.chunks.findOne(
+    {
+      projectId: new ObjectId(projectId),
+      state: 'deleted',
+      startVersion: 0,
+      updatedAt: { $lte: timestamp }, // indexed for state=deleted
+      endTimestamp: { $lte: timestamp },
+    },
+    { sort: { updatedAt: -1 } }
+  )
+  if (recordDeleted) {
+    return chunkFromRecord(recordDeleted)
+  }
+  throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+}
+
 /**
  * Get the metadata for the chunk that contains the version that was current at
  * the given timestamp.
@@ -86,6 +115,39 @@ async function getChunkForTimestamp(projectId, timestamp) {
   return chunkFromRecord(record)
 }
 
+/**
+ * Get the metadata for the chunk that contains the version that was current before
+ * the given timestamp.
+ */
+async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
+  assert.mongoId(projectId, 'bad projectId')
+  assert.date(timestamp, 'bad timestamp')
+
+  const record = await mongodb.chunks.findOne(
+    {
+      projectId: new ObjectId(projectId),
+      state: 'active',
+      $or: [
+        {
+          endTimestamp: {
+            $lte: timestamp,
+          },
+        },
+        {
+          endTimestamp: null,
+        },
+      ],
+    },
+    // We use the index on the startVersion for sorting records. This assumes
+    // that timestamps go up with each version.
+    { sort: { startVersion: -1 } }
+  )
+  if (record == null) {
+    throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+  }
+  return chunkFromRecord(record)
+}
+
 /**
  * Get all of a project's chunk ids
  */
@@ -137,7 +199,13 @@ async function insertPendingChunk(projectId, chunk) {
 /**
  * Record that a new chunk was created.
  */
-async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
+async function confirmCreate(
+  projectId,
+  chunk,
+  chunkId,
+  earliestChangeTimestamp,
+  mongoOpts = {}
+) {
   assert.mongoId(projectId, 'bad projectId')
   assert.instance(chunk, Chunk, 'bad chunk')
   assert.mongoId(chunkId, 'bad chunkId')
@@ -166,13 +234,23 @@ async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
   if (result.matchedCount === 0) {
     throw new OError('pending chunk not found', { projectId, chunkId })
   }
-  await updateProjectRecord(projectId, chunk, mongoOpts)
+  await updateProjectRecord(
+    projectId,
+    chunk,
+    earliestChangeTimestamp,
+    mongoOpts
+  )
 }
 
 /**
  * Write the metadata to the project record
  */
-async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
+async function updateProjectRecord(
+  projectId,
+  chunk,
+  earliestChangeTimestamp,
+  mongoOpts = {}
+) {
   // record the end version against the project
   await mongodb.projects.updateOne(
     {
@@ -189,7 +267,7 @@ async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
       // be cleared every time a backup is completed.
       $min: {
         'overleaf.backup.pendingChangeAt':
-          chunk.getEndTimestamp() || new Date(),
+          earliestChangeTimestamp || chunk.getEndTimestamp() || new Date(),
       },
     },
     mongoOpts
@@ -199,7 +277,13 @@ async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
 /**
  * Record that a chunk was replaced by a new one.
  */
-async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
+async function confirmUpdate(
+  projectId,
+  oldChunkId,
+  newChunk,
+  newChunkId,
+  earliestChangeTimestamp
+) {
   assert.mongoId(projectId, 'bad projectId')
   assert.mongoId(oldChunkId, 'bad oldChunkId')
   assert.instance(newChunk, Chunk, 'bad newChunk')
@@ -209,7 +293,13 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
   try {
     await session.withTransaction(async () => {
       await deleteChunk(projectId, oldChunkId, { session })
-      await confirmCreate(projectId, newChunk, newChunkId, { session })
+      await confirmCreate(
+        projectId,
+        newChunk,
+        newChunkId,
+        earliestChangeTimestamp,
+        { session }
+      )
     })
   } finally {
     await session.endSession()
@@ -310,6 +400,8 @@ function chunkFromRecord(record) {
 
 module.exports = {
   getLatestChunk,
+  getFirstChunkBeforeTimestamp,
+  getLastActiveChunkBeforeTimestamp,
   getChunkForVersion,
   getChunkForTimestamp,
   getProjectChunkIds,

services/history-v1/storage/lib/chunk_store/postgres.js

@@ -46,6 +46,59 @@ async function getChunkForVersion(projectId, version) {
   return chunkFromRecord(record)
 }
 
+/**
+ * Get the metadata for the chunk that contains the given version.
+ */
+async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
+  assert.date(timestamp, 'bad timestamp')
+
+  const recordActive = await getChunkForVersion(projectId, 0)
+  // projectId must be valid if getChunkForVersion did not throw
+  projectId = parseInt(projectId, 10)
+  if (recordActive && recordActive.endTimestamp <= timestamp) {
+    return recordActive
+  }
+
+  // fallback to deleted chunk
+  const recordDeleted = await knex('old_chunks')
+    .where('doc_id', projectId)
+    .where('start_version', '=', 0)
+    .where('end_timestamp', '<=', timestamp)
+    .orderBy('end_version', 'desc')
+    .first()
+  if (recordDeleted) {
+    return chunkFromRecord(recordDeleted)
+  }
+  throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+}
+
+/**
+ * Get the metadata for the chunk that contains the version that was current at
+ * the given timestamp.
+ */
+async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
+  assert.date(timestamp, 'bad timestamp')
+  assert.postgresId(projectId, 'bad projectId')
+  projectId = parseInt(projectId, 10)
+
+  const query = knex('chunks')
+    .where('doc_id', projectId)
+    .where(function () {
+      this.where('end_timestamp', '<=', timestamp).orWhere(
+        'end_timestamp',
+        null
+      )
+    })
+    .orderBy('end_version', 'desc', 'last')
+
+  const record = await query.first()
+
+  if (!record) {
+    throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+  }
+  return chunkFromRecord(record)
+}
+
 /**
  * Get the metadata for the chunk that contains the version that was current at
  * the given timestamp.
@@ -140,7 +193,12 @@ async function insertPendingChunk(projectId, chunk) {
 /**
  * Record that a new chunk was created.
  */
-async function confirmCreate(projectId, chunk, chunkId) {
+async function confirmCreate(
+  projectId,
+  chunk,
+  chunkId,
+  earliestChangeTimestamp
+) {
   assert.postgresId(projectId, `bad projectId ${projectId}`)
   projectId = parseInt(projectId, 10)
 
@@ -149,14 +207,20 @@ async function confirmCreate(projectId, chunk, chunkId) {
       _deletePendingChunk(tx, projectId, chunkId),
       _insertChunk(tx, projectId, chunk, chunkId),
     ])
-    await updateProjectRecord(projectId, chunk)
+    await updateProjectRecord(projectId, chunk, earliestChangeTimestamp)
   })
 }
 
 /**
  * Record that a chunk was replaced by a new one.
  */
-async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
+async function confirmUpdate(
+  projectId,
+  oldChunkId,
+  newChunk,
+  newChunkId,
+  earliestChangeTimestamp
+) {
   assert.postgresId(projectId, `bad projectId ${projectId}`)
   projectId = parseInt(projectId, 10)
 
@@ -166,7 +230,7 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
       _deletePendingChunk(tx, projectId, newChunkId),
       _insertChunk(tx, projectId, newChunk, newChunkId),
     ])
-    await updateProjectRecord(projectId, newChunk)
+    await updateProjectRecord(projectId, newChunk, earliestChangeTimestamp)
   })
 }
 
@@ -280,6 +344,8 @@ async function generateProjectId() {
 
 module.exports = {
   getLatestChunk,
+  getFirstChunkBeforeTimestamp,
+  getLastActiveChunkBeforeTimestamp,
   getChunkForVersion,
   getChunkForTimestamp,
   getProjectChunkIds,

services/history-v1/storage/lib/persist_changes.js

@@ -65,6 +65,9 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
 
   const blobStore = new BlobStore(projectId)
 
+  const earliestChangeTimestamp =
+    allChanges.length > 0 ? allChanges[0].getTimestamp() : null
+
   let currentChunk
 
   /**
@@ -78,12 +81,6 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
   let originalEndVersion
   let changesToPersist
 
-  /**
-   * It's only useful to log validation errors once per flush. When we enforce
-   * content hash validation, it will stop the flush right away anyway.
-   */
-  let validationErrorLogged = false
-
   limits = limits || {}
   _.defaults(limits, {
     changeBucketMinutes: 60,
@@ -128,22 +125,7 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
       for (const operation of change.iterativelyApplyTo(currentSnapshot, {
         strict: true,
       })) {
-        try {
-          await validateContentHash(operation)
-        } catch (err) {
-          // Temporary: skip validation errors
-          if (err instanceof InvalidChangeError) {
-            if (!validationErrorLogged) {
-              logger.warn(
-                { err, projectId },
-                'content snapshot mismatch (ignored)'
-              )
-              validationErrorLogged = true
-            }
-          } else {
-            throw err
-          }
-        }
+        await validateContentHash(operation)
       }
 
       chunk.pushChanges([change])
@@ -220,7 +202,12 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
 
     checkElapsedTime(timer)
 
-    await chunkStore.update(projectId, originalEndVersion, currentChunk)
+    await chunkStore.update(
+      projectId,
+      originalEndVersion,
+      currentChunk,
+      earliestChangeTimestamp
+    )
   }
 
   async function createNewChunksAsNeeded() {
@@ -234,7 +221,7 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
       if (changesPushed) {
         checkElapsedTime(timer)
         currentChunk = chunk
-        await chunkStore.create(projectId, chunk)
+        await chunkStore.create(projectId, chunk, earliestChangeTimestamp)
       } else {
         throw new Error('failed to fill empty chunk')
       }

services/history-v1/storage/scripts/backup.mjs

@@ -2,8 +2,12 @@
 
 import logger from '@overleaf/logger'
 import commandLineArgs from 'command-line-args'
-import { History } from 'overleaf-editor-core'
-import { getProjectChunks, loadLatestRaw } from '../lib/chunk_store/index.js'
+import { Chunk, History, Snapshot } from 'overleaf-editor-core'
+import {
+  getProjectChunks,
+  loadLatestRaw,
+  create,
+} from '../lib/chunk_store/index.js'
 import { client } from '../lib/mongodb.js'
 import knex from '../lib/knex.js'
 import { historyStore } from '../lib/history_store.js'
@@ -30,7 +34,7 @@ import {
   projectBlobsBucket,
 } from '../lib/backupPersistor.mjs'
 import { backupGenerator } from '../lib/backupGenerator.mjs'
-import { promises as fs } from 'node:fs'
+import { promises as fs, createWriteStream } from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
 import projectKey from '../lib/project_key.js'
@@ -89,7 +93,7 @@ process.on('SIGTERM', handleSignal)
 
 function handleSignal() {
   gracefulShutdownInitiated = true
-  console.warn('graceful shutdown initiated, draining queue')
+  logger.info({}, 'graceful shutdown initiated, draining queue')
 }
 
 async function retry(fn, times, delayMs) {
@@ -321,12 +325,18 @@ const optionDefinitions = [
     description: 'Time interval in seconds for pending backups (default: 3600)',
     defaultValue: 3600,
   },
+  {
+    name: 'fix',
+    type: Number,
+    description: 'Fix projects without chunks',
+  },
   {
     name: 'init',
     alias: 'I',
     type: Boolean,
     description: 'Initialize backups for all projects.',
   },
+  { name: 'output', alias: 'o', type: String, description: 'Output file' },
   {
     name: 'start-date',
     type: String,
@@ -366,6 +376,7 @@ function handleOptions() {
     !options.list &&
     !options.pending &&
     !options.init &&
+    !(options.fix >= 0) &&
     !(options.compare && options['start-date'] && options['end-date'])
 
   if (projectIdRequired && !options.projectId) {
@@ -680,19 +691,68 @@ function convertToISODate(dateStr) {
   return new Date(dateStr + 'T00:00:00.000Z').toISOString()
 }
 
+export async function fixProjectsWithoutChunks(options) {
+  const limit = options.fix || 1
+  const query = {
+    'overleaf.history.id': { $exists: true },
+    'overleaf.backup.lastBackedUpVersion': { $in: [null] },
+  }
+  const cursor = client
+    .db()
+    .collection('projects')
+    .find(query, {
+      projection: { _id: 1, 'overleaf.history.id': 1 },
+      readPreference: READ_PREFERENCE_SECONDARY,
+    })
+    .limit(limit)
+  for await (const project of cursor) {
+    const historyId = project.overleaf.history.id.toString()
+    const chunks = await getProjectChunks(historyId)
+    if (chunks.length > 0) {
+      continue
+    }
+    if (DRY_RUN) {
+      console.log(
+        'Would create new chunk for Project ID:',
+        project._id.toHexString(),
+        'History ID:',
+        historyId,
+        'Chunks:',
+        chunks
+      )
+    } else {
+      console.log(
+        'Creating new chunk for Project ID:',
+        project._id.toHexString(),
+        'History ID:',
+        historyId,
+        'Chunks:',
+        chunks
+      )
+      const snapshot = new Snapshot()
+      const history = new History(snapshot, [])
+      const chunk = new Chunk(history, 0)
+      await create(historyId, chunk)
+      const newChunks = await getProjectChunks(historyId)
+      console.log('New chunk:', newChunks)
+    }
+  }
+}
+
 export async function initializeProjects(options) {
   await ensureGlobalBlobsLoaded()
   let totalErrors = 0
   let totalProjects = 0
 
   const query = {
-    'overleaf.history.id': { $exists: true },
-    'overleaf.backup.lastBackedUpVersion': { $exists: false },
-    'overleaf.backup.pendingChangeAt': { $exists: false },
-    _id: {
+    'overleaf.backup.lastBackedUpVersion': { $in: [null] },
+  }
+
+  if (options['start-date'] && options['end-date']) {
+    query._id = {
       $gte: objectIdFromInput(convertToISODate(options['start-date'])),
       $lt: objectIdFromInput(convertToISODate(options['end-date'])),
-    },
+    }
   }
 
   const cursor = client
@@ -703,6 +763,18 @@ export async function initializeProjects(options) {
       readPreference: READ_PREFERENCE_SECONDARY,
     })
 
+  if (options.output) {
+    console.log("Writing project IDs to file: '" + options.output + "'")
+    const output = createWriteStream(options.output)
+    for await (const project of cursor) {
+      output.write(project._id.toHexString() + '\n')
+      totalProjects++
+    }
+    output.end()
+    console.log('Wrote ' + totalProjects + ' project IDs to file')
+    return
+  }
+
   for await (const project of cursor) {
     if (gracefulShutdownInitiated) {
       console.warn('graceful shutdown: stopping project initialization')
@@ -969,11 +1041,12 @@ async function main() {
   const options = handleOptions()
   await ensureGlobalBlobsLoaded()
   const projectId = options.projectId
-
   if (options.status) {
     await displayBackupStatus(projectId)
   } else if (options.list) {
     await displayPendingBackups(options)
+  } else if (options.fix !== undefined) {
+    await fixProjectsWithoutChunks(options)
   } else if (options.pending) {
     await backupPendingProjects(options)
   } else if (options.init) {

services/history-v1/storage/scripts/backup_blob.mjs

@@ -0,0 +1,171 @@
+// @ts-check
+import commandLineArgs from 'command-line-args'
+import { backupBlob, downloadBlobToDir } from '../lib/backupBlob.mjs'
+import withTmpDir from '../../api/controllers/with_tmp_dir.js'
+import {
+  BlobStore,
+  GLOBAL_BLOBS,
+  loadGlobalBlobs,
+} from '../lib/blob_store/index.js'
+import assert from '../lib/assert.js'
+import knex from '../lib/knex.js'
+import { client } from '../lib/mongodb.js'
+import { setTimeout } from 'node:timers/promises'
+import fs from 'node:fs'
+
+await loadGlobalBlobs()
+
+/**
+ * Gracefully shutdown the process
+ * @return {Promise<void>}
+ */
+async function gracefulShutdown() {
+  console.log('Gracefully shutting down')
+  await knex.destroy()
+  await client.close()
+  await setTimeout(100)
+  process.exit()
+}
+
+/**
+ *
+ * @param {string} row
+ * @return {BackupBlobJob}
+ */
+function parseCSVRow(row) {
+  const [historyId, hash] = row.split(',')
+  validateBackedUpBlobJob({ historyId, hash })
+  return { historyId, hash }
+}
+
+/**
+ *
+ * @param {BackupBlobJob} job
+ */
+function validateBackedUpBlobJob(job) {
+  assert.projectId(job.historyId)
+  assert.blobHash(job.hash)
+}
+
+/**
+ *
+ * @param {string} path
+ * @return {Promise<Array<BackupBlobJob>>}
+ */
+async function readCSV(path) {
+  let fh
+  /** @type {Array<BackupBlobJob>} */
+  const rows = []
+  try {
+    fh = await fs.promises.open(path, 'r')
+  } catch (error) {
+    console.error(`Could not open file: ${error}`)
+    throw error
+  }
+  for await (const line of fh.readLines()) {
+    try {
+      const row = parseCSVRow(line)
+      if (GLOBAL_BLOBS.has(row.hash)) {
+        console.log(`Skipping global blob: ${line}`)
+        continue
+      }
+      rows.push(row)
+    } catch (error) {
+      console.error(error instanceof Error ? error.message : error)
+      console.log(`Skipping invalid row: ${line}`)
+    }
+  }
+  return rows
+}
+
+/**
+ * @typedef {Object} BackupBlobJob
+ * @property {string} hash
+ * @property {string} historyId
+ */
+
+/**
+ * @param {Object} options
+ * @property {string} [options.historyId]
+ * @property {string} [options.hash]
+ * @property {string} [options.input]
+ * @return {Promise<Array<BackupBlobJob>>}
+ */
+async function initialiseJobs({ historyId, hash, input }) {
+  if (input) {
+    return await readCSV(input)
+  }
+
+  if (!historyId) {
+    console.error('historyId is required')
+    process.exitCode = 1
+    await gracefulShutdown()
+  }
+
+  if (!hash) {
+    console.error('hash is required')
+    process.exitCode = 1
+    await gracefulShutdown()
+  }
+
+  validateBackedUpBlobJob({ historyId, hash })
+
+  if (GLOBAL_BLOBS.has(hash)) {
+    console.error(`Blob ${hash} is a global blob; not backing up`)
+    process.exitCode = 1
+    await gracefulShutdown()
+  }
+  return [{ hash, historyId }]
+}
+
+/**
+ *
+ * @param {string} historyId
+ * @param {string} hash
+ * @return {Promise<void>}
+ */
+export async function downloadAndBackupBlob(historyId, hash) {
+  const blobStore = new BlobStore(historyId)
+  const blob = await blobStore.getBlob(hash)
+  if (!blob) {
+    throw new Error(`Blob ${hash} could not be loaded`)
+  }
+  await withTmpDir(`blob-${hash}`, async tmpDir => {
+    const filePath = await downloadBlobToDir(historyId, blob, tmpDir)
+    console.log(`Downloaded blob ${hash} to ${filePath}`)
+    await backupBlob(historyId, blob, filePath)
+    console.log('Backed up blob')
+  })
+}
+
+let jobs
+
+const options = commandLineArgs([
+  { name: 'historyId', type: String },
+  { name: 'hash', type: String },
+  { name: 'input', type: String },
+])
+
+try {
+  jobs = await initialiseJobs(options)
+} catch (error) {
+  console.error(error)
+  await gracefulShutdown()
+}
+
+if (!Array.isArray(jobs)) {
+  // This is mostly to satisfy typescript
+  process.exitCode = 1
+  await gracefulShutdown()
+  process.exit(1)
+}
+
+for (const { historyId, hash } of jobs) {
+  try {
+    await downloadAndBackupBlob(historyId, hash)
+  } catch (error) {
+    console.error(error)
+    process.exitCode = 1
+  }
+}
+await gracefulShutdown()

services/history-v1/storage/scripts/backup_sample.mjs

@@ -32,34 +32,18 @@ async function takeSample(sampleSize) {
       [
         { $sample: { size: sampleSize } },
         {
-          $project: {
-            _id: 0,
-            hasBackup: {
-              $ifNull: ['$overleaf.backup.lastBackedUpVersion', false],
-            },
-          },
+          $match: { 'overleaf.backup.lastBackedUpVersion': { $exists: true } },
         },
         {
-          $group: {
-            _id: null,
-            totalSampled: { $sum: 1 },
-            backedUp: {
-              $sum: {
-                $cond: ['$hasBackup', 1, 0],
-              },
-            },
-          },
+          $count: 'total',
         },
       ],
       { readPreference: READ_PREFERENCE_SECONDARY }
     )
     .toArray()
 
-  if (results.length === 0) {
-    return { totalSampled: 0, backedUp: 0 }
-  }
-
-  return results[0]
+  const count = results[0]?.total || 0
+  return { totalSampled: sampleSize, backedUp: count }
 }
 
 function calculateStatistics(
@@ -67,7 +51,7 @@ function calculateStatistics(
   cumulativeBackedUp,
   totalPopulation
 ) {
-  const proportion = cumulativeBackedUp / cumulativeSampled
+  const proportion = Math.max(1, cumulativeBackedUp) / cumulativeSampled
 
   // Standard error with finite population correction
   const fpc = Math.sqrt(

services/history-v1/storage/scripts/backup_scheduler.mjs

@@ -2,6 +2,11 @@ import Queue from 'bull'
 import config from 'config'
 import commandLineArgs from 'command-line-args'
 import logger from '@overleaf/logger'
+import {
+  listPendingBackups,
+  listUninitializedBackups,
+  getBackupStatus,
+} from '../lib/backup_store/index.js'
 
 logger.initialize('backup-queue')
 
@@ -28,16 +33,100 @@ const optionDefinitions = [
     description: 'Project IDs or date range in YYYY-MM-DD:YYYY-MM-DD format',
   },
   { name: 'monitor', type: Boolean },
+  {
+    name: 'queue-pending',
+    type: Number,
+    description:
+      'Find projects with pending changes older than N seconds and add them to the queue',
+  },
+  {
+    name: 'show-pending',
+    type: Number,
+    description:
+      'Show count of pending projects older than N seconds without adding to queue',
+  },
+  {
+    name: 'limit',
+    type: Number,
+    description: 'Limit the number of jobs to be added',
+  },
+  {
+    name: 'interval',
+    type: Number,
+    description: 'Time in seconds to spread jobs over (default: 300)',
+    defaultValue: 300,
+  },
+  {
+    name: 'backoff-delay',
+    type: Number,
+    description:
+      'Backoff delay in milliseconds for failed jobs (default: 1000)',
+    defaultValue: 1000,
+  },
+  {
+    name: 'attempts',
+    type: Number,
+    description: 'Number of retry attempts for failed jobs (default: 3)',
+    defaultValue: 3,
+  },
+  {
+    name: 'warn-threshold',
+    type: Number,
+    description: 'Warn about any project exceeding this pending age',
+    defaultValue: 2 * 3600, // 2 hours
+  },
+  {
+    name: 'verbose',
+    alias: 'v',
+    type: Boolean,
+    description: 'Show detailed information when used with --show-pending',
+  },
 ]
 
 // Parse command line arguments
 const options = commandLineArgs(optionDefinitions)
+const WARN_THRESHOLD = options['warn-threshold']
 
 // Helper to validate date format
 function isValidDateFormat(dateStr) {
   return /^\d{4}-\d{2}-\d{2}$/.test(dateStr)
 }
 
+// Helper to validate the pending time parameter
+function validatePendingTime(option, value) {
+  if (typeof value !== 'number' || value <= 0) {
+    console.error(
+      `Error: --${option} requires a positive numeric TIME argument in seconds`
+    )
+    console.error(`Example: --${option} 3600`)
+    process.exit(1)
+  }
+  return value
+}
+
+// Helper to format the pending time display
+function formatPendingTime(timestamp) {
+  const now = new Date()
+  const diffMs = now - timestamp
+  const seconds = Math.floor(diffMs / 1000)
+  return `${timestamp.toISOString()} (${seconds} seconds ago)`
+}
+
+// Helper to add a job to the queue, checking for duplicates
+async function addJobWithCheck(queue, data, options) {
+  const jobId = options.jobId
+
+  // Check if the job already exists
+  const existingJob = await queue.getJob(jobId)
+
+  if (existingJob) {
+    return { job: existingJob, added: false }
+  } else {
+    const job = await queue.add(data, options)
+    return { job, added: true }
+  }
+}
+
 // Setup queue event listeners
 function setupMonitoring() {
   console.log('Starting queue monitoring. Press Ctrl+C to exit.')
@@ -99,15 +188,125 @@ async function addDateRangeJob(input) {
     )
     return
   }
-  const job = await backupQueue.add(
+
+  const jobId = `backup-${startDate}-to-${endDate}`
+  const { job, added } = await addJobWithCheck(
+    backupQueue,
     { startDate, endDate },
-    { jobId: `backup-${startDate}-to-${endDate}` }
+    { jobId }
   )
+
   console.log(
-    `Added date range backup job: ${startDate} to ${endDate}, job ID: ${job.id}`
+    `${added ? 'Added' : 'Already exists'}: date range backup job: ${startDate} to ${endDate}, job ID: ${job.id}`
   )
 }
 
+// Helper to list pending and uninitialized backups
+// This function combines the two cursors into a single generator
+// to yield projects from both lists
+async function* pendingCursor(timeIntervalMs, limit) {
+  for await (const project of listPendingBackups(timeIntervalMs, limit)) {
+    yield project
+  }
+  for await (const project of listUninitializedBackups(timeIntervalMs, limit)) {
+    yield project
+  }
+}
+
+// Process pending projects with changes older than the specified seconds
+async function processPendingProjects(
+  age,
+  showOnly,
+  limit,
+  verbose,
+  jobInterval,
+  jobOpts = {}
+) {
+  const timeIntervalMs = age * 1000
+  console.log(
+    `Finding projects with pending changes older than ${age} seconds${showOnly ? ' (count only)' : ''}`
+  )
+
+  let count = 0
+  let addedCount = 0
+  let existingCount = 0
+  // Pass the limit directly to MongoDB query for better performance
+  const changeTimes = []
+  for await (const project of pendingCursor(timeIntervalMs, limit)) {
+    const projectId = project._id.toHexString()
+    const pendingAt =
+      project.overleaf?.backup?.pendingChangeAt || project._id.getTimestamp()
+    if (pendingAt) {
+      changeTimes.push(pendingAt)
+      const pendingAge = Math.floor((Date.now() - pendingAt.getTime()) / 1000)
+      if (pendingAge > WARN_THRESHOLD) {
+        const backupStatus = await getBackupStatus(projectId)
+        logger.warn(
+          {
+            projectId,
+            pendingAt,
+            pendingAge,
+            backupStatus,
+            warnThreshold: WARN_THRESHOLD,
+          },
+          `pending change exceeds rpo warning threshold`
+        )
+      }
+    }
+    if (showOnly && verbose) {
+      console.log(
+        `Project: ${projectId} (pending since: ${formatPendingTime(pendingAt)})`
+      )
+    } else if (!showOnly) {
+      const delay = Math.floor(Math.random() * jobInterval * 1000) // add random delay to avoid all jobs running simultaneously
+      const { job, added } = await addJobWithCheck(
+        backupQueue,
+        { projectId, pendingChangeAt: pendingAt.getTime() },
+        { ...jobOpts, delay, jobId: projectId }
+      )
+
+      if (added) {
+        if (verbose) {
+          console.log(
+            `Added job for project: ${projectId}, job ID: ${job.id} (pending since: ${formatPendingTime(pendingAt)})`
+          )
+        }
+        addedCount++
+      } else {
+        if (verbose) {
+          console.log(
+            `Job already exists for project: ${projectId}, job ID: ${job.id} (pending since: ${formatPendingTime(pendingAt)})`
+          )
+        }
+        existingCount++
+      }
+    }
+
+    count++
+    if (count % 1000 === 0) {
+      console.log(
+        `Processed ${count} projects`,
+        showOnly ? '' : `(${addedCount} added, ${existingCount} existing)`
+      )
+    }
+  }
+
+  const oldestChange = changeTimes.reduce((min, time) =>
+    time < min ? time : min
+  )
+
+  if (showOnly) {
+    console.log(
+      `Found ${count} projects with pending changes (not added to queue)`
+    )
+  } else {
+    console.log(`Found ${count} projects with pending changes:`)
+    console.log(`  ${addedCount} jobs added to queue`)
+    console.log(`  ${existingCount} jobs already existed in queue`)
+    console.log(`  Oldest pending change: ${formatPendingTime(oldestChange)}`)
+  }
+}
+
 // Main execution block
 async function run() {
   const optionCount = [
@@ -115,6 +314,8 @@ async function run() {
     options.status,
     options.add,
     options.monitor,
+    options['queue-pending'] !== undefined,
+    options['show-pending'] !== undefined,
   ].filter(Boolean).length
   if (optionCount > 1) {
     console.error('Only one option can be specified')
@@ -141,24 +342,65 @@ async function run() {
         await addDateRangeJob(input)
       } else {
         // Handle project ID format
-        const job = await backupQueue.add(
+        const { job, added } = await addJobWithCheck(
+          backupQueue,
           { projectId: input },
           { jobId: input }
         )
-        console.log(`Added job for project: ${input}, job ID: ${job.id}`)
+        console.log(
+          `${added ? 'Added' : 'Already exists'}: job for project: ${input}, job ID: ${job.id}`
+        )
       }
     }
   } else if (options.monitor) {
     setupMonitoring()
+  } else if (options['queue-pending'] !== undefined) {
+    const age = validatePendingTime('queue-pending', options['queue-pending'])
+    await processPendingProjects(
+      age,
+      false,
+      options.limit,
+      options.verbose,
+      options.interval,
+      {
+        attempts: options.attempts,
+        backoff: {
+          type: 'exponential',
+          delay: options['backoff-delay'],
+        },
+      }
+    )
+  } else if (options['show-pending'] !== undefined) {
+    const age = validatePendingTime('show-pending', options['show-pending'])
+    await processPendingProjects(age, true, options.limit, options.verbose)
   } else {
     console.log('Usage:')
-    console.log('  --clean   Clean up completed and failed jobs')
-    console.log('  --status  Show current job counts')
-    console.log('  --add [projectId] Add a job for the specified projectId')
+    console.log('  --clean               Clean up completed and failed jobs')
+    console.log('  --status              Show current job counts')
+    console.log('  --add [projectId]     Add a job for the specified projectId')
     console.log(
       '  --add [YYYY-MM-DD:YYYY-MM-DD] Add a job for the specified date range'
     )
-    console.log('  --monitor Monitor queue events')
+    console.log('  --monitor             Monitor queue events')
+    console.log(
+      '  --queue-pending TIME  Find projects with changes older than TIME seconds and add them to the queue'
+    )
+    console.log(
+      '  --show-pending TIME   Show count of pending projects older than TIME seconds'
+    )
+    console.log('  --limit N             Limit the number of jobs to be added')
+    console.log(
+      '  --interval TIME       Time interval in seconds to spread jobs over'
+    )
+    console.log(
+      '  --backoff-delay TIME  Backoff delay in milliseconds for failed jobs (default: 1000)'
+    )
+    console.log(
+      '  --attempts N          Number of retry attempts for failed jobs (default: 3)'
+    )
+    console.log(
+      '  --verbose, -v         Show detailed information when used with --show-pending'
+    )
   }
 }
 

services/history-v1/storage/scripts/backup_worker.mjs

@@ -9,8 +9,12 @@ import {
 } from './backup.mjs'
 
 const CONCURRENCY = 15
+const WARN_THRESHOLD = 2 * 60 * 60 * 1000 // warn if projects are older than this
 const redisOptions = config.get('redis.queue')
-const TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000]
+const JOB_TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000] // milliseconds
+const LAG_TIME_BUCKETS_HRS = [
+  0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.75, 2, 3, 4, 5, 6,
+] // hours
 
 // Configure backup settings to match worker concurrency
 configureBackup({ concurrency: 50, useSecondary: true })
@@ -27,12 +31,12 @@ const backupQueue = new Queue('backup', {
 
 // Log queue events
 backupQueue.on('active', job => {
-  logger.info({ job }, 'job  is now active')
+  logger.debug({ job }, 'job is now active')
 })
 
 backupQueue.on('completed', (job, result) => {
   metrics.inc('backup_worker_job', 1, { status: 'completed' })
-  logger.info({ job, result }, 'job completed')
+  logger.debug({ job, result }, 'job completed')
 })
 
 backupQueue.on('failed', (job, err) => {
@@ -41,7 +45,7 @@ backupQueue.on('failed', (job, err) => {
 })
 
 backupQueue.on('waiting', jobId => {
-  logger.info({ jobId }, 'job is waiting')
+  logger.debug({ jobId }, 'job is waiting')
 })
 
 backupQueue.on('error', error => {
@@ -69,7 +73,7 @@ backupQueue.process(CONCURRENCY, async job => {
   const { projectId, startDate, endDate } = job.data
 
   if (projectId) {
-    return await runBackup(projectId)
+    return await runBackup(projectId, job.data, job)
   } else if (startDate && endDate) {
     return await runInit(startDate, endDate)
   } else {
@@ -77,23 +81,40 @@ backupQueue.process(CONCURRENCY, async job => {
   }
 })
 
-async function runBackup(projectId) {
+async function runBackup(projectId, data, job) {
+  const { pendingChangeAt } = data
+  // record the time it takes to run the backup job
   const timer = new metrics.Timer(
     'backup_worker_job_duration',
     1,
     {},
-    TIME_BUCKETS
+    JOB_TIME_BUCKETS
   )
+  const pendingAge = Date.now() - pendingChangeAt
+  if (pendingAge > WARN_THRESHOLD) {
+    logger.warn(
+      { projectId, pendingAge, job },
+      'project has been pending for a long time'
+    )
+  }
   try {
-    logger.info({ projectId }, 'processing backup for project')
-    const { errors, completed } = await backupProject(projectId, {})
-    metrics.inc('backup_worker_project', completed - errors, {
+    logger.debug({ projectId }, 'processing backup for project')
+    await backupProject(projectId, {})
+    metrics.inc('backup_worker_project', 1, {
       status: 'success',
     })
-    metrics.inc('backup_worker_project', errors, { status: 'failed' })
     timer.done()
-    return `backup completed ${projectId} (${errors} failed in ${completed} projects)`
+    // record the replication lag (time from change to backup)
+    if (pendingChangeAt) {
+      metrics.histogram(
+        'backup_worker_replication_lag_in_hours',
+        (Date.now() - pendingChangeAt) / (3600 * 1000),
+        LAG_TIME_BUCKETS_HRS
+      )
+    }
+    return `backup completed ${projectId}`
   } catch (err) {
+    metrics.inc('backup_worker_project', 1, { status: 'failed' })
     logger.error({ projectId, err }, 'backup failed')
     throw err // Re-throw to mark job as failed
   }

services/history-v1/storage/scripts/verify_blob_backed_up_by_path_bulk.mjs

@@ -0,0 +1,177 @@
+import fs from 'node:fs'
+import { makeProjectKey } from '../lib/blob_store/index.js'
+import { backupPersistor, projectBlobsBucket } from '../lib/backupPersistor.mjs'
+import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
+import commandLineArgs from 'command-line-args'
+import OError from '@overleaf/o-error'
+import assert from '../lib/assert.js'
+import { client, projects } from '../lib/mongodb.js'
+import { ObjectId } from 'mongodb'
+import { setTimeout } from 'node:timers/promises'
+
+const { input, verbose } = commandLineArgs([
+  { name: 'input', type: String },
+  { name: 'verbose', type: Boolean, defaultValue: false },
+])
+
+function parseCSVRow(row) {
+  const [path] = row.split(',')
+  const pathSegments = path.split('/')
+  const historyId = `${pathSegments[0]}${pathSegments[1]}${pathSegments[2]}`
+    .split('')
+    .reverse()
+    .join('')
+
+  return { historyId, path, hash: `${pathSegments[3]}${pathSegments[4]}` }
+}
+
+async function* readCSV(path) {
+  let fh
+  try {
+    fh = await fs.promises.open(path, 'r')
+  } catch (error) {
+    console.error(`Could not open file: ${error}`)
+    throw error
+  }
+  for await (const line of fh.readLines()) {
+    try {
+      const row = parseCSVRow(line)
+      yield row
+    } catch (error) {
+      console.error(error instanceof Error ? error.message : error)
+      console.log(`Skipping invalid row: ${line}`)
+    }
+  }
+}
+
+class MissingDEKError extends OError {}
+class InvalidHistoryIdError extends OError {}
+class MissingProjectError extends OError {}
+class MissingBlobError extends OError {}
+
+async function getProjectPersistor(historyId) {
+  try {
+    return await backupPersistor.forProjectRO(
+      projectBlobsBucket,
+      makeProjectKey(historyId, '')
+    )
+  } catch (err) {
+    if (err instanceof NotFoundError) {
+      throw new MissingDEKError('dek does not exist', { historyId }, err)
+    }
+    throw err
+  }
+}
+
+async function checkBlobExists(path, historyId) {
+  const persistor = await getProjectPersistor(historyId)
+  return await persistor.getObjectSize(projectBlobsBucket, path)
+}
+
+let total = 0
+const errors = {
+  invalidProjectId: 0,
+  notBackedUpProjectId: 0,
+  missingBlob: 0,
+  notInMongo: 0,
+  unknown: 0,
+}
+
+const notInMongoProjectIds = new Set()
+const notBackedUpProjectIds = new Set()
+
+let stopping = false
+
+process.on('SIGTERM', () => {
+  console.log('SIGTERM received')
+  stopping = true
+})
+
+process.on('SIGINT', () => {
+  console.log('SIGINT received')
+  stopping = true
+})
+
+/**
+ *
+ * @param {string} historyId
+ * @param {string} path
+ * @param {string} hash
+ * @return {Promise<void>}
+ */
+async function checkPath(historyId, path, hash) {
+  try {
+    assert.mongoId(historyId)
+  } catch (error) {
+    throw InvalidHistoryIdError('invalid history id', { historyId })
+  }
+  if (notInMongoProjectIds.has(historyId)) {
+    throw new MissingProjectError('project not in mongo', { historyId })
+  }
+  if (notBackedUpProjectIds.has(historyId)) {
+    throw new MissingDEKError('project not backed up', { historyId })
+  }
+
+  const project = await projects.findOne({ _id: new ObjectId(historyId) })
+  if (!project) {
+    notInMongoProjectIds.add(historyId)
+    throw new MissingProjectError('project not in mongo', { historyId })
+  }
+  try {
+    await checkBlobExists(path, historyId)
+  } catch (error) {
+    if (error instanceof NotFoundError) {
+      throw new MissingBlobError('missing blob', { historyId, hash })
+    }
+    if (error instanceof MissingDEKError) {
+      notBackedUpProjectIds.add(historyId)
+    }
+    throw error
+  }
+}
+
+for await (const line of readCSV(input)) {
+  if (stopping) break
+  total++
+  if (total % 10_000 === 0) {
+    console.log(`checked ${total}`)
+  }
+  const { historyId, path, hash } = line
+  try {
+    await checkPath(historyId, path, hash)
+    if (verbose) {
+      console.log(`✓ Project ${historyId} has ${hash} backed up`)
+    }
+  } catch (error) {
+    if (error instanceof InvalidHistoryIdError) {
+      errors.invalidProjectId++
+      console.warn(`invalid historyId ${historyId}`)
+      continue
+    } else if (error instanceof MissingProjectError) {
+      errors.notInMongo++
+      console.warn(`✗ project ${historyId} not in mongo`)
+      continue
+    } else if (error instanceof MissingDEKError) {
+      errors.notBackedUpProjectId++
+      console.error(`✗ Project DEK ${historyId} not found`)
+      continue
+    } else if (error instanceof MissingBlobError) {
+      errors.missingBlob++
+      console.error(`✗ missing blob ${hash} from project ${historyId}`)
+      continue
+    }
+    errors.unknown++
+    console.error(error)
+  }
+}
+
+console.log(`total checked: ${total}`)
+console.log(`invalid project id: ${errors.invalidProjectId}`)
+console.log(`not found in mongo: ${errors.notInMongo}`)
+console.log(`missing blob: ${errors.missingBlob}`)
+console.log(`project not backed up: ${errors.notBackedUpProjectId}`)
+console.log(`unknown errors: ${errors.unknown}`)
+
+await client.close()
+await setTimeout(100)
+process.exit()

services/history-v1/storage/scripts/verify_project.mjs

@@ -0,0 +1,33 @@
+import commandLineArgs from 'command-line-args'
+import { verifyProjectWithErrorContext } from '../lib/backupVerifier.mjs'
+import knex from '../lib/knex.js'
+import { client } from '../lib/mongodb.js'
+import { setTimeout } from 'node:timers/promises'
+import { loadGlobalBlobs } from '../lib/blob_store/index.js'
+
+const { historyId } = commandLineArgs([{ name: 'historyId', type: String }])
+
+async function gracefulShutdown(code = process.exitCode) {
+  await knex.destroy()
+  await client.close()
+  await setTimeout(1_000)
+  process.exit(code)
+}
+
+if (!historyId) {
+  console.error('missing --historyId')
+  process.exitCode = 1
+  await gracefulShutdown()
+}
+
+await loadGlobalBlobs()
+
+try {
+  await verifyProjectWithErrorContext(historyId)
+  console.log('OK')
+} catch (error) {
+  console.error('error verifying', error)
+  process.exitCode = 1
+} finally {
+  await gracefulShutdown()
+}

services/history-v1/storage/scripts/verify_sampled_projects.mjs

@@ -0,0 +1,215 @@
+// @ts-check
+import commandLineArgs from 'command-line-args'
+import {
+  setWriteMetrics,
+  verifyProjectsCreatedInDateRange,
+  verifyRandomProjectSample,
+  verifyProjectsUpdatedInDateRange,
+} from '../../backupVerifier/ProjectVerifier.mjs'
+import knex from '../lib/knex.js'
+import { client } from '../lib/mongodb.js'
+import { setTimeout } from 'node:timers/promises'
+import logger from '@overleaf/logger'
+import { loadGlobalBlobs } from '../lib/blob_store/index.js'
+import { getDatesBeforeRPO } from '../../backupVerifier/utils.mjs'
+import { EventEmitter } from 'node:events'
+import { mongodb } from '../index.js'
+
+logger.logger.level('fatal')
+
+const usageMessage = [
+  'Usage: node verify_sampled_projects.mjs [--startDate <start>] [--endDate <end>] [--nProjects <n>] [--verbose] [--usage] [--writeMetrics] [--concurrency <n>] [--strategy <range|random>]',
+  'strategy: defaults to "range"; startDate and endDate are required for "range" strategy',
+].join('\n')
+
+/**
+ * Gracefully shutdown the process
+ * @param code
+ * @return {Promise<void>}
+ */
+async function gracefulShutdown(code = process.exitCode) {
+  await knex.destroy()
+  await client.close()
+  await setTimeout(1_000)
+  process.exit(code)
+}
+
+const STATS = {
+  verifiable: 0,
+  unverifiable: 0,
+}
+
+/**
+ * @typedef {Object} CLIOptions
+ * @property {(signal: EventEmitter) => Promise<VerificationJobStatus>} projectVerifier
+ * @property {boolean} verbose
+ */
+
+/**
+ * @typedef {import('../../backupVerifier/types.d.ts').VerificationJobStatus} VerificationJobStatus
+ */
+
+/**
+ *
+ * @return {CLIOptions}
+ */
+function getOptions() {
+  const {
+    startDate,
+    endDate,
+    concurrency,
+    writeMetrics,
+    verbose,
+    nProjects,
+    strategy,
+    usage,
+  } = commandLineArgs([
+    { name: 'startDate', type: String },
+    { name: 'endDate', type: String },
+    { name: 'concurrency', type: Number, defaultValue: 1 },
+    { name: 'verbose', type: Boolean, defaultValue: false },
+    { name: 'nProjects', type: Number, defaultValue: 10 },
+    { name: 'usage', type: Boolean, defaultValue: false },
+    { name: 'writeMetrics', type: Boolean, defaultValue: false },
+    { name: 'strategy', type: String, defaultValue: 'range' },
+  ])
+
+  if (usage) {
+    console.log(usageMessage)
+    process.exit(0)
+  }
+
+  if (!['range', 'random', 'recent'].includes(strategy)) {
+    throw new Error(`Invalid strategy: ${strategy}`)
+  }
+
+  setWriteMetrics(writeMetrics)
+
+  switch (strategy) {
+    case 'random':
+      console.log('Verifying random projects')
+      return {
+        verbose,
+        projectVerifier: signal => verifyRandomProjectSample(nProjects, signal),
+      }
+    case 'recent':
+      return {
+        verbose,
+        projectVerifier: async signal => {
+          const { startDate, endDate } = getDatesBeforeRPO(3 * 3600)
+          return await verifyProjectsUpdatedInDateRange(
+            startDate,
+            endDate,
+            nProjects,
+            signal
+          )
+        },
+      }
+    case 'range':
+    default: {
+      if (!startDate || !endDate) {
+        throw new Error(usageMessage)
+      }
+      const start = Date.parse(startDate)
+      const end = Date.parse(endDate)
+      if (Number.isNaN(start)) {
+        throw new Error(`Invalid start date: ${startDate}`)
+      }
+
+      if (Number.isNaN(end)) {
+        throw new Error(`Invalid end date: ${endDate}`)
+      }
+      if (verbose) {
+        console.log(`Verifying from ${startDate} to ${endDate}`)
+        console.log(`Concurrency: ${concurrency}`)
+      }
+      STATS.ranges = 0
+      return {
+        projectVerifier: signal =>
+          verifyProjectsCreatedInDateRange({
+            startDate: new Date(start),
+            endDate: new Date(end),
+            projectsPerRange: nProjects,
+            concurrency,
+            signal,
+          }),
+        verbose,
+      }
+    }
+  }
+}
+
+/**
+ * @type {CLIOptions}
+ */
+let options
+try {
+  options = getOptions()
+} catch (error) {
+  console.error(error)
+  process.exitCode = 1
+  await gracefulShutdown(1)
+  process.exit() // just here so the type checker knows that the process will exit
+}
+
+const { projectVerifier, verbose } = options
+
+if (verbose) {
+  logger.logger.level('debug')
+}
+
+/**
+ *
+ * @param {Array<string>} array
+ * @param {string} matchString
+ * @return {*}
+ */
+function sumStringInstances(array, matchString) {
+  return array.reduce((total, string) => {
+    return string === matchString ? total + 1 : total
+  }, 0)
+}
+
+/**
+ *
+ * @param {VerificationJobStatus} stats
+ */
+function displayStats(stats) {
+  console.log(`Verified projects: ${stats.verified}`)
+  console.log(`Total projects sampled: ${stats.total}`)
+  if (stats.errorTypes.length > 0) {
+    console.log('Errors:')
+    for (const error of new Set(stats.errorTypes)) {
+      console.log(`${error}: ${sumStringInstances(stats.errorTypes, error)}`)
+    }
+  }
+}
+
+const shutdownEmitter = new EventEmitter()
+
+shutdownEmitter.on('shutdown', async () => {
+  await gracefulShutdown()
+})
+
+process.on('SIGTERM', () => {
+  shutdownEmitter.emit('shutdown')
+})
+
+process.on('SIGINT', () => {
+  shutdownEmitter.emit('shutdown')
+})
+
+await loadGlobalBlobs()
+
+try {
+  const stats = await projectVerifier(shutdownEmitter)
+  displayStats(stats)
+  console.log(`completed`)
+} catch (error) {
+  console.error(error)
+  console.log('completed with errors')
+  process.exitCode = 1
+} finally {
+  console.log('shutting down')
+  await gracefulShutdown()
+}