Fix casing in E2E tests (Example project, Blank project, ...) (#25812)

* Fix casing in Tests (Example project, Blank project, Trashed projects, ...)

* Check "Blank Project" case insentively

* Fix git-bridge tests

GitOrigin-RevId: 52339258016c3a923c6207a65d058cb0d6e9d3a2

develop/docker-compose.yml

@@ -1,6 +1,5 @@
 volumes:
   clsi-cache:
-  clsi-output:
   filestore-public-files:
   filestore-template-files:
   filestore-uploads:
@@ -33,9 +32,9 @@ services:
     user: root
     volumes:
       - ${PWD}/compiles:/overleaf/services/clsi/compiles
+      - ${PWD}/output:/overleaf/services/clsi/output
       - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
       - clsi-cache:/overleaf/services/clsi/cache
-      - clsi-output:/overleaf/services/clsi/output
 
   contacts:
     build:

docker-compose.yml

@@ -73,7 +73,11 @@ services:
             ## Server Pro ##
             ################
 
-            ## Sandboxed Compiles: https://github.com/overleaf/overleaf/wiki/Server-Pro:-Sandboxed-Compiles
+            ## The Community Edition is intended for use in environments where all users are trusted and is not appropriate for
+            ## scenarios where isolation of users is required. Sandboxed Compiles are not available in the Community Edition,
+            ## so the following environment variables must be commented out to avoid compile issues.
+            ##
+            ## Sandboxed Compiles: https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles
             SANDBOXED_COMPILES: 'true'
             ### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
             SANDBOXED_COMPILES_HOST_DIR_COMPILES: '/home/user/sharelatex_data/data/compiles'

libraries/access-token-encryptor/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/access-token-encryptor/.gitignore

@@ -1,46 +0,0 @@
-compileFolder
-
-Compiled source #
-###################
-*.com
-*.class
-*.dll
-*.exe
-*.o
-*.so
-
-# Packages #
-############
-# it's better to unpack these files and commit the raw source
-# git has its own built in compression methods
-*.7z
-*.dmg
-*.gz
-*.iso
-*.jar
-*.rar
-*.tar
-*.zip
-
-# Logs and databases #
-######################
-*.log
-*.sql
-*.sqlite
-
-# OS generated files #
-######################
-.DS_Store?
-ehthumbs.db
-Icon?
-Thumbs.db
-
-/node_modules/*
-data/*/*
-
-**.swp
-
-/log.json
-hash_folder
-
-.npmrc

libraries/access-token-encryptor/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/access-token-encryptor/buildscript.txt

@@ -5,6 +5,6 @@ access-token-encryptor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/fetch-utils/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/fetch-utils/.gitignore

@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/fetch-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/fetch-utils/buildscript.txt

@@ -5,6 +5,6 @@ fetch-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/logger/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/logger/.gitignore

@@ -1,3 +0,0 @@
-node_modules
-
-.npmrc

libraries/logger/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/logger/buildscript.txt

@@ -5,6 +5,6 @@ logger
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/metrics/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/metrics/.gitignore

@@ -1,3 +0,0 @@
-node_modules
-
-.npmrc

libraries/metrics/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/metrics/buildscript.txt

@@ -5,6 +5,6 @@ metrics
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/metrics/initialize.js

@@ -5,6 +5,8 @@
  * before any other module to support code instrumentation.
  */
 
+const metricsModuleImportStartTime = performance.now()
+
 const APP_NAME = process.env.METRICS_APP_NAME || 'unknown'
 const BUILD_VERSION = process.env.BUILD_VERSION
 const ENABLE_PROFILE_AGENT = process.env.ENABLE_PROFILE_AGENT === 'true'
@@ -103,3 +105,5 @@ function recordProcessStart() {
   const metrics = require('.')
   metrics.inc('process_startup')
 }
+
+module.exports = { metricsModuleImportStartTime }

libraries/metrics/package.json

@@ -9,7 +9,7 @@
   "main": "index.js",
   "dependencies": {
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
-    "@google-cloud/profiler": "^6.0.0",
+    "@google-cloud/profiler": "^6.0.3",
     "@opentelemetry/api": "^1.4.1",
     "@opentelemetry/auto-instrumentations-node": "^0.39.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.41.2",

libraries/mongo-utils/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/mongo-utils/.gitignore

@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/mongo-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/mongo-utils/buildscript.txt

@@ -5,6 +5,6 @@ mongo-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/o-error/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/o-error/.gitignore

@@ -1,5 +0,0 @@
-.nyc_output
-coverage
-node_modules/
-
-.npmrc

libraries/o-error/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/o-error/buildscript.txt

@@ -5,6 +5,6 @@ o-error
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/o-error/index.cjs

@@ -1,20 +1,34 @@
+// @ts-check
+
 /**
  * Light-weight helpers for handling JavaScript Errors in node.js and the
  * browser.
  */
 class OError extends Error {
+  /**
+   * The error that is the underlying cause of this error
+   *
+   * @type {unknown}
+   */
+  cause
+
+  /**
+   * List of errors encountered as the callback chain is unwound
+   *
+   * @type {TaggedError[] | undefined}
+   */
+  _oErrorTags
+
   /**
    * @param {string} message as for built-in Error
    * @param {Object} [info] extra data to attach to the error
-   * @param {Error} [cause] the internal error that caused this error
+   * @param {unknown} [cause] the internal error that caused this error
    */
   constructor(message, info, cause) {
     super(message)
     this.name = this.constructor.name
     if (info) this.info = info
     if (cause) this.cause = cause
-    /** @private @type {Array<TaggedError> | undefined} */
-    this._oErrorTags // eslint-disable-line
   }
 
   /**
@@ -31,7 +45,7 @@ class OError extends Error {
   /**
    * Wrap the given error, which caused this error.
    *
-   * @param {Error} cause the internal error that caused this error
+   * @param {unknown} cause the internal error that caused this error
    * @return {this}
    */
   withCause(cause) {
@@ -65,13 +79,16 @@ class OError extends Error {
    *   }
    * }
    *
-   * @param {Error} error the error to tag
+   * @template {unknown} E
+   * @param {E} error the error to tag
    * @param {string} [message] message with which to tag `error`
    * @param {Object} [info] extra data with wich to tag `error`
-   * @return {Error} the modified `error` argument
+   * @return {E} the modified `error` argument
    */
   static tag(error, message, info) {
-    const oError = /** @type{OError} */ (error)
+    const oError = /** @type {{ _oErrorTags: TaggedError[] | undefined }} */ (
+      error
+    )
 
     if (!oError._oErrorTags) oError._oErrorTags = []
 
@@ -102,7 +119,7 @@ class OError extends Error {
    *
    * If an info property is repeated, the last one wins.
    *
-   * @param {Error | null | undefined} error any error (may or may not be an `OError`)
+   * @param {unknown} error any error (may or may not be an `OError`)
    * @return {Object}
    */
   static getFullInfo(error) {
@@ -129,7 +146,7 @@ class OError extends Error {
    * Return the `stack` property from `error`, including the `stack`s for any
    * tagged errors added with `OError.tag` and for any `cause`s.
    *
-   * @param {Error | null | undefined} error any error (may or may not be an `OError`)
+   * @param {unknown} error any error (may or may not be an `OError`)
    * @return {string}
    */
   static getFullStack(error) {
@@ -143,7 +160,7 @@ class OError extends Error {
       stack += `\n${oError._oErrorTags.map(tag => tag.stack).join('\n')}`
     }
 
-    const causeStack = oError.cause && OError.getFullStack(oError.cause)
+    const causeStack = OError.getFullStack(oError.cause)
     if (causeStack) {
       stack += '\ncaused by:\n' + indent(causeStack)
     }

libraries/o-error/test/o-error-util.test.js

@@ -268,6 +268,11 @@ describe('utils', function () {
       expect(OError.getFullInfo(null)).to.deep.equal({})
     })
 
+    it('works when given a string', function () {
+      const err = 'not an error instance'
+      expect(OError.getFullInfo(err)).to.deep.equal({})
+    })
+
     it('works on a normal error', function () {
       const err = new Error('foo')
       expect(OError.getFullInfo(err)).to.deep.equal({})

libraries/o-error/test/o-error.test.js

@@ -35,6 +35,14 @@ describe('OError', function () {
     expect(err2.cause.message).to.equal('cause 2')
   })
 
+  it('accepts non-Error causes', function () {
+    const err1 = new OError('foo', {}, 'not-an-error')
+    expect(err1.cause).to.equal('not-an-error')
+
+    const err2 = new OError('foo').withCause('not-an-error')
+    expect(err2.cause).to.equal('not-an-error')
+  })
+
   it('handles a custom error type with a cause', function () {
     function doSomethingBadInternally() {
       throw new Error('internal error')

libraries/object-persistor/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/object-persistor/.gitignore

@@ -1,4 +0,0 @@
-/node_modules
-*.swp
-
-.npmrc

libraries/object-persistor/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/object-persistor/buildscript.txt

@@ -5,6 +5,6 @@ object-persistor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/object-persistor/src/FSPersistor.js

@@ -305,8 +305,10 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   async _listDirectory(path) {
     if (this.useSubdirectories) {
+      // eslint-disable-next-line @typescript-eslint/return-await
       return await glob(Path.join(path, '**'))
     } else {
+      // eslint-disable-next-line @typescript-eslint/return-await
       return await glob(`${path}_*`)
     }
   }

libraries/overleaf-editor-core/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/overleaf-editor-core/.gitignore

@@ -1,5 +0,0 @@
-/coverage
-/node_modules
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/overleaf-editor-core/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/overleaf-editor-core/buildscript.txt

@@ -5,6 +5,6 @@ overleaf-editor-core
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/overleaf-editor-core/index.js

@@ -18,6 +18,7 @@ const MoveFileOperation = require('./lib/operation/move_file_operation')
 const SetCommentStateOperation = require('./lib/operation/set_comment_state_operation')
 const EditFileOperation = require('./lib/operation/edit_file_operation')
 const EditNoOperation = require('./lib/operation/edit_no_operation')
+const EditOperationTransformer = require('./lib/operation/edit_operation_transformer')
 const SetFileMetadataOperation = require('./lib/operation/set_file_metadata_operation')
 const NoOperation = require('./lib/operation/no_operation')
 const Operation = require('./lib/operation')
@@ -43,6 +44,8 @@ const TrackingProps = require('./lib/file_data/tracking_props')
 const Range = require('./lib/range')
 const CommentList = require('./lib/file_data/comment_list')
 const LazyStringFileData = require('./lib/file_data/lazy_string_file_data')
+const StringFileData = require('./lib/file_data/string_file_data')
+const EditOperationBuilder = require('./lib/operation/edit_operation_builder')
 
 exports.AddCommentOperation = AddCommentOperation
 exports.Author = Author
@@ -58,6 +61,7 @@ exports.DeleteCommentOperation = DeleteCommentOperation
 exports.File = File
 exports.FileMap = FileMap
 exports.LazyStringFileData = LazyStringFileData
+exports.StringFileData = StringFileData
 exports.History = History
 exports.Label = Label
 exports.AddFileOperation = AddFileOperation
@@ -65,6 +69,8 @@ exports.MoveFileOperation = MoveFileOperation
 exports.SetCommentStateOperation = SetCommentStateOperation
 exports.EditFileOperation = EditFileOperation
 exports.EditNoOperation = EditNoOperation
+exports.EditOperationBuilder = EditOperationBuilder
+exports.EditOperationTransformer = EditOperationTransformer
 exports.SetFileMetadataOperation = SetFileMetadataOperation
 exports.NoOperation = NoOperation
 exports.Operation = Operation

libraries/overleaf-editor-core/lib/change.js

@@ -13,7 +13,7 @@ const V2DocVersions = require('./v2_doc_versions')
 
 /**
  * @import Author from "./author"
- * @import { BlobStore } from "./types"
+ * @import { BlobStore, RawChange } from "./types"
  */
 
 /**
@@ -54,7 +54,7 @@ class Change {
   /**
    * For serialization.
    *
-   * @return {Object}
+   * @return {RawChange}
    */
   toRaw() {
     function toRaw(object) {

libraries/overleaf-editor-core/lib/file_data/string_file_data.js

@@ -88,6 +88,14 @@ class StringFileData extends FileData {
     return content
   }
 
+  /**
+   * Return docstore view of a doc: each line separated
+   * @return {string[]}
+   */
+  getLines() {
+    return this.getContent({ filterTrackedDeletes: true }).split('\n')
+  }
+
   /** @inheritdoc */
   getByteLength() {
     return Buffer.byteLength(this.content)

libraries/overleaf-editor-core/lib/operation/edit_operation_builder.js

@@ -36,6 +36,20 @@ class EditOperationBuilder {
     }
     throw new Error('Unsupported operation in EditOperationBuilder.fromJSON')
   }
+
+  /**
+   * @param {unknown} raw
+   * @return {raw is RawEditOperation}
+   */
+  static isValid(raw) {
+    return (
+      isTextOperation(raw) ||
+      isRawAddCommentOperation(raw) ||
+      isRawDeleteCommentOperation(raw) ||
+      isRawSetCommentStateOperation(raw) ||
+      isRawEditNoOperation(raw)
+    )
+  }
 }
 
 /**

libraries/promise-utils/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/promise-utils/.gitignore

@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/promise-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/promise-utils/buildscript.txt

@@ -5,6 +5,6 @@ promise-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/ranges-tracker/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/ranges-tracker/.gitignore

@@ -1,13 +0,0 @@
-**.swp
-
-app.js
-app/js/
-test/unit/js/
-public/build/
-
-node_modules/
-
-/public/js/chat.js
-plato/
-
-.npmrc

libraries/ranges-tracker/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/ranges-tracker/buildscript.txt

@@ -5,6 +5,6 @@ ranges-tracker
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/redis-wrapper/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/redis-wrapper/.gitignore

@@ -1,13 +0,0 @@
-**.swp
-
-app.js
-app/js/
-test/unit/js/
-public/build/
-
-node_modules/
-
-/public/js/chat.js
-plato/
-
-.npmrc

libraries/redis-wrapper/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/redis-wrapper/RedisLocker.js

@@ -97,7 +97,8 @@ module.exports = class RedisLocker {
   }
 
   /**
-   * @param {Callback} callback
+   * @param {string} id
+   * @param {function(Error, boolean, string): void} callback
    */
   tryLock(id, callback) {
     if (callback == null) {
@@ -106,7 +107,7 @@ module.exports = class RedisLocker {
     const lockValue = this.randomLock()
     const key = this.getKey(id)
     const startTime = Date.now()
-    return this.rclient.set(
+    this.rclient.set(
       key,
       lockValue,
       'EX',
@@ -121,7 +122,7 @@ module.exports = class RedisLocker {
           const timeTaken = Date.now() - startTime
           if (timeTaken > MAX_REDIS_REQUEST_LENGTH) {
             // took too long, so try to free the lock
-            return this.releaseLock(id, lockValue, function (err, result) {
+            this.releaseLock(id, lockValue, function (err, result) {
               if (err != null) {
                 return callback(err)
               } // error freeing lock
@@ -139,7 +140,8 @@ module.exports = class RedisLocker {
   }
 
   /**
-   * @param {Callback} callback
+   * @param {string} id
+   * @param {function(Error, string): void} callback
    */
   getLock(id, callback) {
     if (callback == null) {
@@ -153,7 +155,7 @@ module.exports = class RedisLocker {
         return callback(e)
       }
 
-      return this.tryLock(id, (error, gotLock, lockValue) => {
+      this.tryLock(id, (error, gotLock, lockValue) => {
         if (error != null) {
           return callback(error)
         }
@@ -173,14 +175,15 @@ module.exports = class RedisLocker {
   }
 
   /**
-   * @param {Callback} callback
+   * @param {string} id
+   * @param {function(Error, boolean): void} callback
    */
   checkLock(id, callback) {
     if (callback == null) {
       callback = function () {}
     }
     const key = this.getKey(id)
-    return this.rclient.exists(key, (err, exists) => {
+    this.rclient.exists(key, (err, exists) => {
       if (err != null) {
         return callback(err)
       }
@@ -196,30 +199,26 @@ module.exports = class RedisLocker {
   }
 
   /**
-   * @param {Callback} callback
+   * @param {string} id
+   * @param {string} lockValue
+   * @param {function(Error, boolean): void} callback
    */
   releaseLock(id, lockValue, callback) {
     const key = this.getKey(id)
-    return this.rclient.eval(
+    this.rclient.eval(UNLOCK_SCRIPT, 1, key, lockValue, (err, result) => {
-      UNLOCK_SCRIPT,
+      if (err != null) {
-      1,
+        return callback(err)
-      key,
+      } else if (result != null && result !== 1) {
-      lockValue,
+        // successful unlock should release exactly one key
-      (err, result) => {
+        logger.error(
-        if (err != null) {
+          { id, key, lockValue, redis_err: err, redis_result: result },
-          return callback(err)
+          'unlocking error'
-        } else if (result != null && result !== 1) {
+        )
-          // successful unlock should release exactly one key
+        metrics.inc(this.metricsPrefix + '-unlock-error')
-          logger.error(
+        return callback(new Error('tried to release timed out lock'))
-            { id, key, lockValue, redis_err: err, redis_result: result },
+      } else {
-            'unlocking error'
+        return callback(null, result)
-          )
-          metrics.inc(this.metricsPrefix + '-unlock-error')
-          return callback(new Error('tried to release timed out lock'))
-        } else {
-          return callback(null, result)
-        }
       }
-    )
+    })
   }
 }

libraries/redis-wrapper/buildscript.txt

@@ -5,6 +5,6 @@ redis-wrapper
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/settings/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/settings/.gitignore

@@ -1,5 +0,0 @@
-/.npmrc
-/node_modules
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/settings/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/settings/buildscript.txt

@@ -5,6 +5,6 @@ settings
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

libraries/stream-utils/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/stream-utils/.gitignore

@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/stream-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

libraries/stream-utils/buildscript.txt

@@ -5,6 +5,6 @@ stream-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

package.json

@@ -37,7 +37,7 @@
     },
     "swagger-tools": {
       "body-parser": "1.20.3",
-      "multer": "1.4.5-lts.1",
+      "multer": "2.0.0",
       "path-to-regexp": "3.3.0",
       "qs": "6.13.0"
     }

server-ce/Dockerfile-base

@@ -2,7 +2,7 @@
 # Overleaf Base Image (sharelatex/sharelatex-base)
 # --------------------------------------------------
 
-FROM phusion/baseimage:noble-1.0.0
+FROM phusion/baseimage:noble-1.0.2
 
 # Makes sure LuaTex cache is writable
 # -----------------------------------
@@ -10,7 +10,7 @@ ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
 
 # Update to ensure dependencies are updated
 # ------------------------------------------
-ENV REBUILT_AFTER="2025-03-27"
+ENV REBUILT_AFTER="2025-05-19"
 
 # Install dependencies
 # --------------------
@@ -30,7 +30,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 # install Node.js https://github.com/nodesource/distributions#nodejs
 &&  mkdir -p /etc/apt/keyrings \
 &&  curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-&&  echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
+&&  echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
 &&  apt-get update \
 &&  apt-get install -y nodejs \
     \

server-ce/config/crontab-history

@@ -1,3 +1,4 @@
 */20 * * * *  root  /overleaf/cron/project-history-periodic-flush.sh >> /var/log/overleaf/cron-project-history-periodic-flush.log 2>&1
 30 * * * *    root  /overleaf/cron/project-history-retry-soft.sh >> /var/log/overleaf/project-history-retry-soft.log 2>&1
 45 * * * *    root  /overleaf/cron/project-history-retry-hard.sh >> /var/log/overleaf/project-history-retry-hard.log 2>&1
+0 3 * * *     root  /overleaf/cron/project-history-flush-all.sh >> /var/log/overleaf/project-history-flush-all.log 2>&1

server-ce/config/settings.js

@@ -79,6 +79,7 @@ const settings = {
       host: process.env.OVERLEAF_REDIS_HOST || 'dockerhost',
       port: process.env.OVERLEAF_REDIS_PORT || '6379',
       password: process.env.OVERLEAF_REDIS_PASS || undefined,
+      tls: process.env.OVERLEAF_REDIS_TLS === 'true' ? {} : undefined,
       key_schema: {
         // document-updater
         blockingKey({ doc_id }) {

server-ce/cron/project-history-flush-all.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eux
+
+echo "---------------------------------"
+echo "Flush all project-history changes"
+echo "---------------------------------"
+date
+
+source /etc/container_environment.sh
+source /etc/overleaf/env.sh
+cd /overleaf/services/project-history && node scripts/flush_all.js
+
+echo "Done flushing all project-history changes"

server-ce/hotfix/5.4.1/Dockerfile

@@ -0,0 +1,14 @@
+FROM sharelatex/sharelatex:5.4.0
+
+RUN apt update && apt install -y linux-libc-dev \
+    && unattended-upgrade --verbose --no-minimal-upgrade-steps \
+    && rm -rf /var/lib/apt/lists/*
+COPY package-lock.json.diff .
+RUN patch package-lock.json < package-lock.json.diff
+RUN npm install --omit=dev
+
+
+# fix tls configuration in redis
+COPY issue_24996.patch .
+RUN patch -p0 /etc/overleaf/settings.js < issue_24996.patch \
+    && rm issue_24996.patch

server-ce/hotfix/5.4.1/issue_24996.patch

@@ -0,0 +1,10 @@
+--- settings.js
++++ settings.js
+@@ -79,6 +79,7 @@ const settings = {
+       host: process.env.OVERLEAF_REDIS_HOST || 'dockerhost',
+       port: process.env.OVERLEAF_REDIS_PORT || '6379',
+       password: process.env.OVERLEAF_REDIS_PASS || undefined,
++      tls: process.env.OVERLEAF_REDIS_TLS === 'true' ? {} : undefined,
+       key_schema: {
+         // document-updater
+         blockingKey({ doc_id }) {

server-ce/test/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.18.2
+FROM node:22.15.1
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
 &&  echo \
       "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \

server-ce/test/admin.spec.ts

@@ -294,7 +294,7 @@ describe('admin panel', function () {
 
       cy.log('navigate to thrashed projects and delete the project')
       cy.get('.project-list-sidebar-scroll').within(() => {
-        cy.findByText('Trashed Projects').click()
+        cy.findByText('Trashed projects').click()
       })
       findProjectRow(deletedProjectName).within(() =>
         cy.findByRole('button', { name: 'Delete' }).click()
@@ -319,7 +319,7 @@ describe('admin panel', function () {
       login(user1)
       cy.visit('/project')
       cy.get('.project-list-sidebar-scroll').within(() => {
-        cy.findByText('Trashed Projects').click()
+        cy.findByText('Trashed projects').click()
       })
       cy.findByText(`${deletedProjectName} (Restored)`)
     })

server-ce/test/create-and-compile-project.spec.ts

@@ -51,7 +51,7 @@ describe('Project creation and compilation', function () {
     login('user@example.com')
 
     createProject(sourceProjectName, {
-      type: 'Example Project',
+      type: 'Example project',
       open: false,
     }).as('sourceProjectId')
     createProject(targetProjectName)
@@ -79,7 +79,7 @@ describe('Project creation and compilation', function () {
     const targetProjectName = `${sourceProjectName}-target`
     login('user@example.com')
     createProject(sourceProjectName, {
-      type: 'Example Project',
+      type: 'Example project',
       open: false,
     }).as('sourceProjectId')
     createProject(targetProjectName).as('targetProjectId')

server-ce/test/editor.spec.ts

@@ -1,11 +1,7 @@
 import {
   createNewFile,
   createProject,
-  enableLinkSharing,
-  openFile,
   openProjectById,
-  openProjectViaLinkSharingAsUser,
-  toggleTrackChanges,
 } from './helpers/project'
 import { isExcludedBySharding, startWith } from './helpers/config'
 import { ensureUserExists, login } from './helpers/login'
@@ -26,7 +22,7 @@ describe('editor', () => {
   beforeWithReRunOnTestRetry(function () {
     projectName = `project-${uuid()}`
     login('user@example.com')
-    createProject(projectName, { type: 'Example Project', open: false }).then(
+    createProject(projectName, { type: 'Example project', open: false }).then(
       id => (projectId = id)
     )
     ;({ recompile, waitForCompileRateLimitCoolOff } =
@@ -66,12 +62,12 @@ describe('editor', () => {
 
       cy.log('add word to dictionary')
       cy.get('.ol-cm-spelling-error').contains(word).rightclick()
-      cy.findByText('Add to Dictionary').click()
+      cy.findByText('Add to dictionary').click()
       cy.get('.ol-cm-spelling-error').should('not.exist')
 
       cy.log('remove word from dictionary')
       cy.get('button').contains('Menu').click()
-      cy.get('button').contains('Edit').click()
+      cy.get('button#dictionary-settings').contains('Edit').click()
       cy.get('[id="dictionary-modal"]').within(() => {
         cy.findByText(word)
           .parent()
@@ -92,136 +88,6 @@ describe('editor', () => {
     })
   })
 
-  describe('collaboration', () => {
-    beforeWithReRunOnTestRetry(function () {
-      enableLinkSharing().then(({ linkSharingReadAndWrite }) => {
-        const email = 'collaborator@example.com'
-        login(email)
-        openProjectViaLinkSharingAsUser(
-          linkSharingReadAndWrite,
-          projectName,
-          email
-        )
-      })
-
-      login('user@example.com')
-      waitForCompileRateLimitCoolOff(() => {
-        openProjectById(projectId)
-      })
-    })
-
-    it('track-changes', () => {
-      cy.log('disable track-changes before populating doc')
-      toggleTrackChanges(false)
-
-      const fileName = createNewFile()
-      const oldContent = 'oldContent'
-      cy.get('.cm-line').type(`${oldContent}\n\nstatic`)
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      cy.log('enable track-changes for everyone')
-      toggleTrackChanges(true)
-
-      login('collaborator@example.com')
-      waitForCompileRateLimitCoolOff(() => {
-        openProjectById(projectId)
-      })
-      openFile(fileName, 'static')
-
-      cy.log('make changes in main file')
-      // cy.type() "clicks" in the center of the selected element before typing. This "click" discards the text as selected by the dblclick.
-      // Go down to the lower level event based typing, the frontend tests in web use similar events.
-      cy.get('.cm-editor').as('editor')
-      cy.get('@editor').findByText(oldContent).dblclick()
-      cy.get('@editor').trigger('keydown', { key: 'Delete' })
-      cy.get('@editor').trigger('keydown', { key: 'Enter' })
-      cy.get('@editor').trigger('keydown', { key: 'Enter' })
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      login('user@example.com')
-      waitForCompileRateLimitCoolOff(() => {
-        openProjectById(projectId)
-      })
-      openFile(fileName, 'static')
-
-      cy.log('reject changes')
-      cy.contains('.toolbar-item', 'Review').click()
-      cy.get('.cm-content').should('not.contain.text', oldContent)
-      cy.findByText('Reject change').click({ force: true })
-      cy.contains('.toolbar-item', 'Review').click()
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      cy.log('verify the changes are applied')
-      cy.get('.cm-content').should('contain.text', oldContent)
-
-      cy.log('disable track-changes for everyone again')
-      toggleTrackChanges(false)
-    })
-
-    it('track-changes rich text', () => {
-      cy.log('disable track-changes before populating doc')
-      toggleTrackChanges(false)
-
-      const fileName = createNewFile()
-      const oldContent = 'oldContent'
-      cy.get('.cm-line').type(`\\section{{}${oldContent}}\n\nstatic`)
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      cy.log('enable track-changes for everyone')
-      toggleTrackChanges(true)
-
-      login('collaborator@example.com')
-      waitForCompileRateLimitCoolOff(() => {
-        openProjectById(projectId)
-      })
-      cy.log('enable visual editor and make changes in main file')
-      cy.findByText('Visual Editor').click()
-
-      openFile(fileName, 'static')
-
-      // cy.type() "clicks" in the center of the selected element before typing. This "click" discards the text as selected by the dblclick.
-      // Go down to the lower level event based typing, the frontend tests in web use similar events.
-      cy.get('.cm-editor').as('editor')
-      cy.get('@editor').findByText(oldContent).dblclick()
-      cy.get('@editor').trigger('keydown', { key: 'Delete' })
-      cy.get('@editor').trigger('keydown', { key: 'Enter' })
-      cy.get('@editor').trigger('keydown', { key: 'Enter' })
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      login('user@example.com')
-      waitForCompileRateLimitCoolOff(() => {
-        openProjectById(projectId)
-      })
-      openFile(fileName, 'static')
-
-      cy.log('reject changes')
-      cy.contains('.toolbar-item', 'Review').click()
-      cy.get('.cm-content').should('not.contain.text', oldContent)
-      cy.findAllByText('Reject change').first().click({ force: true })
-      cy.contains('.toolbar-item', 'Review').click()
-
-      cy.log('recompile to force flush')
-      recompile()
-
-      cy.log('verify the changes are applied in the visual editor')
-      cy.findByText('Visual Editor').click()
-      cy.get('.cm-content').should('contain.text', oldContent)
-
-      cy.log('disable track-changes for everyone again')
-      toggleTrackChanges(false)
-    })
-  })
-
   describe('editor', () => {
     it('renders jpg', () => {
       cy.findByTestId('file-tree').findByText('frog.jpg').click()

server-ce/test/git-bridge.spec.ts

@@ -46,7 +46,7 @@ describe('git-bridge', function () {
 
     function maybeClearAllTokens() {
       cy.visit('/user/settings')
-      cy.findByText('Git Integration')
+      cy.findByText('Git integration')
       cy.get('button')
         .contains(/Generate token|Add another token/)
         .then(btn => {
@@ -63,7 +63,7 @@ describe('git-bridge', function () {
     it('should render the git-bridge UI in the settings', () => {
       maybeClearAllTokens()
       cy.visit('/user/settings')
-      cy.findByText('Git Integration')
+      cy.findByText('Git integration')
       cy.get('button').contains('Generate token').click()
       cy.get('code')
         .contains(/olp_[a-zA-Z0-9]{16}/)
@@ -93,7 +93,7 @@ describe('git-bridge', function () {
           cy.get('code').contains(`git clone ${gitURL(id.toString())}`)
         })
         cy.findByRole('button', {
-          name: 'Generate token',
+          name: /generate token/i,
         }).click()
         cy.get('code').contains(/olp_[a-zA-Z0-9]{16}/)
       })
@@ -107,7 +107,7 @@ describe('git-bridge', function () {
           cy.get('code').contains(`git clone ${gitURL(id.toString())}`)
         })
         cy.findByText('Generate token').should('not.exist')
-        cy.findByText(/generate a new one in Account Settings/)
+        cy.findByText(/generate a new one in Account settings/i)
         cy.findByText('Go to settings')
           .should('have.attr', 'target', '_blank')
           .and('have.attr', 'href', '/user/settings')
@@ -196,7 +196,7 @@ describe('git-bridge', function () {
           cy.get('code').contains(`git clone ${gitURL(projectId.toString())}`)
         })
         cy.findByRole('button', {
-          name: 'Generate token',
+          name: /generate token/i,
         }).click()
         cy.get('code')
           .contains(/olp_[a-zA-Z0-9]{16}/)
@@ -365,7 +365,7 @@ Hello world
     it('should not render the git-bridge UI in the settings', () => {
       login('user@example.com')
       cy.visit('/user/settings')
-      cy.findByText('Git Integration').should('not.exist')
+      cy.findByText('Git integration').should('not.exist')
     })
     it('should not render the git-bridge UI in the editor', function () {
       login('user@example.com')

server-ce/test/helpers/project.ts

@@ -5,11 +5,11 @@ import { v4 as uuid } from 'uuid'
 export function createProject(
   name: string,
   {
-    type = 'Blank Project',
+    type = 'Blank project',
     newProjectButtonMatcher = /new project/i,
     open = true,
   }: {
-    type?: 'Blank Project' | 'Example Project'
+    type?: 'Blank project' | 'Example project'
     newProjectButtonMatcher?: RegExp
     open?: boolean
   } = {}
@@ -37,7 +37,7 @@ export function createProject(
   }
   cy.findAllByRole('button').contains(newProjectButtonMatcher).click()
   // FIXME: This should only look in the left menu
-  cy.findAllByText(type).first().click()
+  cy.findAllByText(new RegExp(type, 'i')).first().click()
   cy.findByRole('dialog').within(() => {
     cy.get('input').type(name)
     cy.findByText('Create').click()
@@ -215,37 +215,3 @@ export function createNewFile() {
 
   return fileName
 }
-
-export function toggleTrackChanges(state: boolean) {
-  cy.findByText('Review').click()
-  cy.get('.track-changes-menu-button').then(el => {
-    // when the menu is expanded renders the `expand_more` icon,
-    // and the `chevron_right` icon when it's collapsed
-    if (!el.text().includes('expand_more')) {
-      el.click()
-    }
-  })
-
-  cy.findByText('Everyone')
-    .parent()
-    .within(() => {
-      cy.get('.form-check-input').then(el => {
-        if (el.prop('checked') === state) return
-
-        const id = uuid()
-        const alias = `@${id}`
-        cy.intercept({
-          method: 'POST',
-          url: '**/track_changes',
-          times: 1,
-        }).as(id)
-        if (state) {
-          cy.get('.form-check-input').check()
-        } else {
-          cy.get('.form-check-input').uncheck()
-        }
-        cy.wait(alias)
-      })
-    })
-  cy.contains('.toolbar-item', 'Review').click()
-}

server-ce/test/project-list.spec.ts

@@ -32,7 +32,7 @@ describe('Project List', () => {
 
     before(() => {
       login(REGULAR_USER)
-      createProject(projectName, { type: 'Example Project', open: false })
+      createProject(projectName, { type: 'Example project', open: false })
     })
     beforeEach(function () {
       login(REGULAR_USER)

server-ce/test/sandboxed-compiles.spec.ts

@@ -59,7 +59,9 @@ describe('SandboxedCompiles', function () {
   })
 
   function checkSyncTeX() {
-    describe('SyncTeX', function () {
+    // TODO(25342): re-enable
+    // eslint-disable-next-line mocha/no-skipped-tests
+    describe.skip('SyncTeX', function () {
       let projectName: string
       beforeEach(function () {
         projectName = `Project ${uuid()}`

server-ce/test/templates.spec.ts

@@ -47,7 +47,9 @@ describe('Templates', () => {
       cy.url().should('match', /\/templates$/)
     })
 
-    it('should have templates feature', () => {
+    // TODO(25342): re-enable
+    // eslint-disable-next-line mocha/no-skipped-tests
+    it.skip('should have templates feature', () => {
       login(TEMPLATES_USER)
       const name = `Template ${Date.now()}`
       const description = `Template Description ${Date.now()}`

services/chat/.gitignore

@@ -1,12 +0,0 @@
-**.swp
-
-public/build/
-
-node_modules/
-
-plato/
-
-**/*.map
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc

services/chat/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

services/chat/Dockerfile

@@ -2,7 +2,7 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-FROM node:20.18.2 AS base
+FROM node:22.15.1 AS base
 
 WORKDIR /overleaf/services/chat
 

services/chat/Makefile

@@ -32,12 +32,12 @@ HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)
 # Run the linting commands in the scope of the monorepo.
 # Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:20.18.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.15.1 npm run --silent
 
 RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
 
 # Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:20.18.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.15.1 npm run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/chat/buildscript.txt

@@ -4,6 +4,6 @@ chat
 --env-add=
 --env-pass-through=
 --esmock-loader=False
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=False
 --script-version=4.7.0

services/chat/docker-compose.ci.yml

@@ -39,7 +39,7 @@ services:
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
   mongo:
-    image: mongo:6.0.13
+    image: mongo:7.0.20
     command: --replSet overleaf
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js

services/chat/docker-compose.yml

@@ -6,7 +6,7 @@ version: "2.3"
 
 services:
   test_unit:
-    image: node:20.18.2
+    image: node:22.15.1
     volumes:
       - .:/overleaf/services/chat
       - ../../node_modules:/overleaf/node_modules
@@ -21,7 +21,7 @@ services:
     user: node
 
   test_acceptance:
-    image: node:20.18.2
+    image: node:22.15.1
     volumes:
       - .:/overleaf/services/chat
       - ../../node_modules:/overleaf/node_modules
@@ -42,7 +42,7 @@ services:
     command: npm run --silent test:acceptance
 
   mongo:
-    image: mongo:6.0.13
+    image: mongo:7.0.20
     command: --replSet overleaf
     volumes:
       - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js

services/clsi/.gitignore

@@ -1,14 +1,3 @@
-**.swp
-node_modules
-test/acceptance/fixtures/tmp
 compiles
 output
-.DS_Store
-*~
 cache
-.vagrant
-config/*
-npm-debug.log
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc

services/clsi/.nvmrc

@@ -1 +1 @@
-20.18.2
+22.15.1

services/clsi/Dockerfile

@@ -2,7 +2,7 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-FROM node:20.18.2 AS base
+FROM node:22.15.1 AS base
 
 WORKDIR /overleaf/services/clsi
 COPY services/clsi/install_deps.sh /overleaf/services/clsi/

services/clsi/Makefile

@@ -24,7 +24,6 @@ DOCKER_COMPOSE_TEST_UNIT = \
 
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
 	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
@@ -33,12 +32,12 @@ HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)
 # Run the linting commands in the scope of the monorepo.
 # Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:20.18.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.15.1 npm run --silent
 
 RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
 
 # Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:20.18.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.15.1 npm run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \
@@ -129,11 +128,10 @@ build:
 		--pull \
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
 		--tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
-		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
-		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
-		--cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
-		--cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):main \
 		--tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
+		--cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
+		--cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \
 		--file Dockerfile \
 		../..
 

services/clsi/app.js

@@ -249,6 +249,9 @@ app.get('/health_check', function (req, res) {
   if (Settings.processTooOld) {
     return res.status(500).json({ processTooOld: true })
   }
+  if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
+    return res.status(500).json({ diskCritical: true })
+  }
   smokeTest.sendLastResult(res)
 })
 
@@ -296,9 +299,14 @@ const loadTcpServer = net.createServer(function (socket) {
     }
 
     const freeLoad = availableWorkingCpus - currentLoad
-    const freeLoadPercentage = Math.round(
+    let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
-      (freeLoad / availableWorkingCpus) * 100
+    if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
-    )
+      freeLoadPercentage = 0
+    }
+    if (ProjectPersistenceManager.isAnyDiskLow()) {
+      freeLoadPercentage = freeLoadPercentage / 2
+    }
+
     if (
       Settings.internal.load_balancer_agent.allow_maintenance &&
       freeLoadPercentage <= 0

services/clsi/app/js/CLSICacheHandler.js

@@ -20,6 +20,19 @@ const TIMING_BUCKETS = [
   0, 10, 100, 1000, 2000, 5000, 10000, 15000, 20000, 30000,
 ]
 const MAX_ENTRIES_IN_OUTPUT_TAR = 100
+const OBJECT_ID_REGEX = /^[0-9a-f]{24}$/
+
+/**
+ * @param {string} projectId
+ * @return {{shard: string, url: string}}
+ */
+function getShard(projectId) {
+  // [timestamp 4bytes][random per machine 5bytes][counter 3bytes]
+  //                                          [32bit       4bytes]
+  const last4Bytes = Buffer.from(projectId, 'hex').subarray(8, 12)
+  const idx = last4Bytes.readUInt32BE() % Settings.apis.clsiCache.shards.length
+  return Settings.apis.clsiCache.shards[idx]
+}
 
 /**
  * @param {string} projectId
@@ -29,6 +42,7 @@ const MAX_ENTRIES_IN_OUTPUT_TAR = 100
  * @param {[{path: string}]} outputFiles
  * @param {string} compileGroup
  * @param {Record<string, any>} options
+ * @return {string | undefined}
  */
 function notifyCLSICacheAboutBuild({
   projectId,
@@ -39,14 +53,16 @@ function notifyCLSICacheAboutBuild({
   compileGroup,
   options,
 }) {
-  if (!Settings.apis.clsiCache.enabled) return
+  if (!Settings.apis.clsiCache.enabled) return undefined
+  if (!OBJECT_ID_REGEX.test(projectId)) return undefined
+  const { url, shard } = getShard(projectId)
 
   /**
    * @param {[{path: string}]} files
    */
   const enqueue = files => {
     Metrics.count('clsi_cache_enqueue_files', files.length)
-    fetchNothing(`${Settings.apis.clsiCache.url}/enqueue`, {
+    fetchNothing(`${url}/enqueue`, {
       method: 'POST',
       json: {
         projectId,
@@ -97,6 +113,8 @@ function notifyCLSICacheAboutBuild({
         'build output.tar.gz for clsi cache failed'
       )
     })
+
+  return shard
 }
 
 /**
@@ -155,6 +173,7 @@ async function downloadOutputDotSynctexFromCompileCache(
   outputDir
 ) {
   if (!Settings.apis.clsiCache.enabled) return false
+  if (!OBJECT_ID_REGEX.test(projectId)) return false
 
   const timer = new Metrics.Timer(
     'clsi_cache_download',
@@ -165,7 +184,7 @@ async function downloadOutputDotSynctexFromCompileCache(
   let stream
   try {
     stream = await fetchStream(
-      `${Settings.apis.clsiCache.url}/project/${projectId}/${
+      `${getShard(projectId).url}/project/${projectId}/${
         userId ? `user/${userId}/` : ''
       }build/${editorId}-${buildId}/search/output/output.synctex.gz`,
       {
@@ -205,8 +224,9 @@ async function downloadOutputDotSynctexFromCompileCache(
  */
 async function downloadLatestCompileCache(projectId, userId, compileDir) {
   if (!Settings.apis.clsiCache.enabled) return false
+  if (!OBJECT_ID_REGEX.test(projectId)) return false
 
-  const url = `${Settings.apis.clsiCache.url}/project/${projectId}/${
+  const url = `${getShard(projectId).url}/project/${projectId}/${
     userId ? `user/${userId}/` : ''
   }latest/output/output.tar.gz`
   const timer = new Metrics.Timer(

services/clsi/app/js/CompileController.js

@@ -112,12 +112,13 @@ function compile(req, res, next) {
               buildId = error.buildId
             }
 
+            let clsiCacheShard
             if (
               status === 'success' &&
               request.editorId &&
               request.populateClsiCache
             ) {
-              notifyCLSICacheAboutBuild({
+              clsiCacheShard = notifyCLSICacheAboutBuild({
                 projectId: request.project_id,
                 userId: request.user_id,
                 buildId: outputFiles[0].build,
@@ -144,6 +145,7 @@ function compile(req, res, next) {
                 stats,
                 timings,
                 buildId,
+                clsiCacheShard,
                 outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
                 outputFiles: outputFiles.map(file => ({
                   url:
@@ -188,7 +190,8 @@ function clearCache(req, res, next) {
 }
 
 function syncFromCode(req, res, next) {
-  const { file, editorId, buildId, compileFromClsiCache } = req.query
+  const { file, editorId, buildId } = req.query
+  const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
   const line = parseInt(req.query.line, 10)
   const column = parseInt(req.query.column, 10)
   const { imageName } = req.query
@@ -201,12 +204,13 @@ function syncFromCode(req, res, next) {
     line,
     column,
     { imageName, editorId, buildId, compileFromClsiCache },
-    function (error, pdfPositions) {
+    function (error, pdfPositions, downloadedFromCache) {
       if (error) {
         return next(error)
       }
       res.json({
         pdf: pdfPositions,
+        downloadedFromCache,
       })
     }
   )
@@ -216,7 +220,8 @@ function syncFromPdf(req, res, next) {
   const page = parseInt(req.query.page, 10)
   const h = parseFloat(req.query.h)
   const v = parseFloat(req.query.v)
-  const { imageName, editorId, buildId, compileFromClsiCache } = req.query
+  const { imageName, editorId, buildId } = req.query
+  const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
   const projectId = req.params.project_id
   const userId = req.params.user_id
   CompileManager.syncFromPdf(
@@ -226,12 +231,13 @@ function syncFromPdf(req, res, next) {
     h,
     v,
     { imageName, editorId, buildId, compileFromClsiCache },
-    function (error, codePositions) {
+    function (error, codePositions, downloadedFromCache) {
       if (error) {
         return next(error)
       }
       res.json({
         code: codePositions,
+        downloadedFromCache,
       })
     }
   )

services/clsi/app/js/CompileManager.js

@@ -23,6 +23,7 @@ const {
   downloadLatestCompileCache,
   downloadOutputDotSynctexFromCompileCache,
 } = require('./CLSICacheHandler')
+const { callbackifyMultiResult } = require('@overleaf/promise-utils')
 
 const COMPILE_TIME_BUCKETS = [
   // NOTE: These buckets are locked in per metric name.
@@ -447,12 +448,20 @@ async function syncFromCode(projectId, userId, filename, line, column, opts) {
     '-o',
     outputFilePath,
   ]
-  const stdout = await _runSynctex(projectId, userId, command, opts)
+  const { stdout, downloadedFromCache } = await _runSynctex(
+    projectId,
+    userId,
+    command,
+    opts
+  )
   logger.debug(
     { projectId, userId, filename, line, column, command, stdout },
     'synctex code output'
   )
-  return SynctexOutputParser.parseViewOutput(stdout)
+  return {
+    codePositions: SynctexOutputParser.parseViewOutput(stdout),
+    downloadedFromCache,
+  }
 }
 
 async function syncFromPdf(projectId, userId, page, h, v, opts) {
@@ -465,9 +474,17 @@ async function syncFromPdf(projectId, userId, page, h, v, opts) {
     '-o',
     `${page}:${h}:${v}:${outputFilePath}`,
   ]
-  const stdout = await _runSynctex(projectId, userId, command, opts)
+  const { stdout, downloadedFromCache } = await _runSynctex(
+    projectId,
+    userId,
+    command,
+    opts
+  )
   logger.debug({ projectId, userId, page, h, v, stdout }, 'synctex pdf output')
-  return SynctexOutputParser.parseEditOutput(stdout, baseDir)
+  return {
+    pdfPositions: SynctexOutputParser.parseEditOutput(stdout, baseDir),
+    downloadedFromCache,
+  }
 }
 
 async function _checkFileExists(dir, filename) {
@@ -522,9 +539,10 @@ async function _runSynctex(projectId, userId, command, opts) {
   return await OutputCacheManager.promises.queueDirOperation(
     outputDir,
     /**
-     * @return {Promise<string>}
+     * @return {Promise<{stdout: string, downloadedFromCache: boolean}>}
      */
     async () => {
+      let downloadedFromCache = false
       try {
         await _checkFileExists(directory, 'output.synctex.gz')
       } catch (err) {
@@ -535,13 +553,14 @@ async function _runSynctex(projectId, userId, command, opts) {
           buildId
         ) {
           try {
-            await downloadOutputDotSynctexFromCompileCache(
+            downloadedFromCache =
-              projectId,
+              await downloadOutputDotSynctexFromCompileCache(
-              userId,
+                projectId,
-              editorId,
+                userId,
-              buildId,
+                editorId,
-              directory
+                buildId,
-            )
+                directory
+              )
           } catch (err) {
             logger.warn(
               { err, projectId, userId, editorId, buildId },
@@ -554,7 +573,7 @@ async function _runSynctex(projectId, userId, command, opts) {
         }
       }
       try {
-        const output = await CommandRunner.promises.run(
+        const { stdout } = await CommandRunner.promises.run(
           compileName,
           command,
           directory,
@@ -563,7 +582,10 @@ async function _runSynctex(projectId, userId, command, opts) {
           {},
           compileGroup
         )
-        return output.stdout
+        return {
+          stdout,
+          downloadedFromCache,
+        }
       } catch (error) {
         throw OError.tag(error, 'error running synctex', {
           command,
@@ -686,8 +708,14 @@ module.exports = {
   stopCompile: callbackify(stopCompile),
   clearProject: callbackify(clearProject),
   clearExpiredProjects: callbackify(clearExpiredProjects),
-  syncFromCode: callbackify(syncFromCode),
+  syncFromCode: callbackifyMultiResult(syncFromCode, [
-  syncFromPdf: callbackify(syncFromPdf),
+    'codePositions',
+    'downloadedFromCache',
+  ]),
+  syncFromPdf: callbackifyMultiResult(syncFromPdf, [
+    'pdfPositions',
+    'downloadedFromCache',
+  ]),
   wordcount: callbackify(wordcount),
   promises: {
     doCompileWithLock,

services/clsi/app/js/ProjectPersistenceManager.js

@@ -22,6 +22,9 @@ const fs = require('node:fs')
 // projectId -> timestamp mapping.
 const LAST_ACCESS = new Map()
 
+let ANY_DISK_LOW = false
+let ANY_DISK_CRITICAL_LOW = false
+
 async function collectDiskStats() {
   const paths = [
     Settings.path.compilesDir,
@@ -30,6 +33,8 @@ async function collectDiskStats() {
   ]
 
   const diskStats = {}
+  let anyDiskLow = false
+  let anyDiskCriticalLow = false
   for (const path of paths) {
     try {
       const { blocks, bavail, bsize } = await fs.promises.statfs(path)
@@ -45,10 +50,16 @@ async function collectDiskStats() {
       })
       const lowDisk = diskAvailablePercent < 10
       diskStats[path] = { stats, lowDisk }
+
+      const criticalLowDisk = diskAvailablePercent < 3
+      anyDiskLow = anyDiskLow || lowDisk
+      anyDiskCriticalLow = anyDiskCriticalLow || criticalLowDisk
     } catch (err) {
       logger.err({ err, path }, 'error getting disk usage')
     }
   }
+  ANY_DISK_LOW = anyDiskLow
+  ANY_DISK_CRITICAL_LOW = anyDiskCriticalLow
   return diskStats
 }
 
@@ -70,11 +81,22 @@ async function refreshExpiryTimeout() {
       break
     }
   }
+  Metrics.gauge(
+    'project_persistence_expiry_timeout',
+    ProjectPersistenceManager.EXPIRY_TIMEOUT
+  )
 }
 
 module.exports = ProjectPersistenceManager = {
   EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
 
+  isAnyDiskLow() {
+    return ANY_DISK_LOW
+  },
+  isAnyDiskCriticalLow() {
+    return ANY_DISK_CRITICAL_LOW
+  },
+
   promises: {
     refreshExpiryTimeout,
   },
@@ -125,12 +147,12 @@ module.exports = ProjectPersistenceManager = {
       )
     })
 
-    // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter).
+    // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter) or every 5th scrape of the load agent (3s +- jitter).
     setInterval(() => {
       collectDiskStats().catch(err => {
         logger.err({ err }, 'low level error collecting disk stats')
       })
-    }, 50_000)
+    }, 15_000)
   },
 
   markProjectAsJustAccessed(projectId, callback) {

services/clsi/buildscript.txt

@@ -1,11 +1,11 @@
 clsi
 --data-dirs=cache,compiles,output
 --dependencies=
---docker-repos=gcr.io/overleaf-ops,us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles,OUTPUT_HOST_DIR=$PWD/output
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles,OUTPUT_HOST_DIR=$PWD/output
 --env-pass-through=
 --esmock-loader=False
---node-version=20.18.2
+--node-version=22.15.1
 --public-repo=True
 --script-version=4.7.0
 --use-large-ci-runner=True