+diff --git a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
+index 2480b7f061f..8e5429dbde6 100644
+--- a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
++++ b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
+@@ -1,4 +1,4 @@
+-import { useState } from 'react'
++import React, { useState } from 'react'
+ import { sendMB } from '@/infrastructure/event-tracking'
+ import { useTranslation } from 'react-i18next'
+ import { Button, Container, Nav, Navbar } from 'react-bootstrap'
+@@ -13,9 +13,15 @@ import MaterialIcon from '@/shared/components/material-icon'
+ import { useContactUsModal } from '@/shared/hooks/use-contact-us-modal'
+ import { UserProvider } from '@/shared/context/user-context'
+ import { X } from '@phosphor-icons/react'
++import overleafWhiteLogo from '@/shared/svgs/overleaf-white.svg'
++import overleafBlackLogo from '@/shared/svgs/overleaf-black.svg'
++import type { CSSPropertiesWithVariables } from '../../../../../../../types/css-properties-with-variables'
+
+-function DefaultNavbar(props: DefaultNavbarMetadata) {
++function DefaultNavbar(
++ props: DefaultNavbarMetadata & { overleafLogo?: string }
++) {
+ const {
++ overleafLogo,
+ customLogo,
+ title,
+ canDisplayAdminMenu,
+@@ -49,10 +55,20 @@ function DefaultNavbar(props: DefaultNavbarMetadata) {
+ className="navbar-default navbar-main"
+ expand="lg"
+ onToggle={expanded => setExpanded(expanded)}
++ style={
++ {
++ '--navbar-brand-image-default-url': `url("${overleafWhiteLogo}")`,
++ '--navbar-brand-image-redesign-url': `url("${overleafBlackLogo}")`,
++ } as CSSPropertiesWithVariables
++ }
+ >
+
+
+-
++
+ {enableUpgradeButton ? (
+
) {
++}: Pick & {
++ overleafLogo?: string
++}) {
+ const { appName } = getMeta('ol-ExposedSettings')
+-
+ if (customLogo) {
+ return (
+ // eslint-disable-next-line jsx-a11y/anchor-has-content
+@@ -24,9 +26,16 @@ export default function HeaderLogoOrTitle({
+
+ )
+ } else {
++ const style = overleafLogo
++ ? {
++ style: {
++ backgroundImage: `url("${overleafLogo}")`,
++ },
++ }
++ : null
+ return (
+ // eslint-disable-next-line jsx-a11y/anchor-has-content
+-
++
++
++
++
++
+diff --git a/services/web/frontend/js/shared/svgs/overleaf-white.svg b/services/web/frontend/js/shared/svgs/overleaf-white.svg
+new file mode 100644
+index 00000000000..2ced81aa46d
+--- /dev/null
++++ b/services/web/frontend/js/shared/svgs/overleaf-white.svg
+@@ -0,0 +1 @@
++ {
+ cy.url().then(url => {
+ if (!url.endsWith('/project')) {
+ cy.visit('/project')
+ }
+ })
+ const interceptId = uuid()
+ let projectId = ''
+ if (!open) {
+ cy.then(() => {
+ // Register intercept just before creating the project, otherwise we might
+ // intercept a request from a prior createProject invocation.
+ cy.intercept(
+ { method: 'GET', url: /\/project\/[a-fA-F0-9]{24}$/, times: 1 },
+ req => {
+ projectId = req.url.split('/').pop()!
+ // Redirect back to the project dashboard, effectively reload the page.
+ req.redirect('/project')
+ }
+ ).as(interceptId)
+ })
+ }
cy.findAllByRole('button').contains(newProjectButtonMatcher).click()
// FIXME: This should only look in the left menu
- cy.findAllByText(type).first().click()
+ // The upgrading tests create projects in older versions of Server Pro which used different casing of the project type. Use case-insensitive match.
+ cy.findAllByText(type, { exact: false }).first().click()
cy.findByRole('dialog').within(() => {
cy.get('input').type(name)
cy.findByText('Create').click()
})
- return cy
- .url()
- .should('match', /\/project\/[a-fA-F0-9]{24}/)
- .then(url => url.split('/').pop())
+ if (open) {
+ cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
+ waitForMainDocToLoad()
+ return cy
+ .url()
+ .should('match', /\/project\/[a-fA-F0-9]{24}/)
+ .then(url => url.split('/').pop())
+ } else {
+ const alias = `@${interceptId}` // IDEs do not like computed values in cy.wait().
+ cy.wait(alias)
+ return cy.then(() => projectId)
+ }
+}
+
+export function openProjectByName(projectName: string) {
+ cy.visit('/project')
+ cy.findByText(projectName).click()
+ waitForMainDocToLoad()
+}
+
+export function openProjectById(projectId: string) {
+ cy.visit(`/project/${projectId}`)
+ waitForMainDocToLoad()
+}
+
+export function openProjectViaLinkSharingAsAnon(url: string) {
+ cy.visit(url)
+ waitForMainDocToLoad()
+}
+
+export function openProjectViaLinkSharingAsUser(
+ url: string,
+ projectName: string,
+ email: string
+) {
+ cy.visit(url)
+ cy.findByText(projectName) // wait for lazy loading
+ cy.contains(`as ${email}`)
+ cy.findByText('OK, join project').click()
+ waitForMainDocToLoad()
+}
+
+export function openProjectViaInviteNotification(projectName: string) {
+ cy.visit('/project')
+ cy.findByText(projectName)
+ .parent()
+ .parent()
+ .within(() => {
+ cy.findByText('Join Project').click()
+ })
+ cy.findByText('Open Project').click()
+ cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
+ waitForMainDocToLoad()
}
function shareProjectByEmail(
projectName: string,
email: string,
- level: 'Read only' | 'Can edit'
+ level: 'Viewer' | 'Editor'
) {
- cy.visit('/project')
- cy.findByText(projectName).click()
+ openProjectByName(projectName)
cy.findByText('Share').click()
cy.findByRole('dialog').within(() => {
- cy.get('input').type(`${email},`)
- cy.get('input')
+ cy.findByLabelText('Add people', { selector: 'input' }).type(`${email},`)
+ cy.findByLabelText('Add people', { selector: 'input' })
.parents('form')
- .within(() => cy.findByText('Can edit').parent().select(level))
- cy.findByText('Share').click({ force: true })
+ .within(() => {
+ cy.findByTestId('add-collaborator-select')
+ .click()
+ .then(() => {
+ cy.findByText(level).click()
+ })
+ })
+ cy.findByText('Invite').click({ force: true })
+ cy.findByText('Invite not yet accepted.')
})
}
export function shareProjectByEmailAndAcceptInviteViaDash(
projectName: string,
email: string,
- level: 'Read only' | 'Can edit'
+ level: 'Viewer' | 'Editor'
) {
shareProjectByEmail(projectName, email, level)
login(email)
- cy.visit('/project')
- cy.findByText(new RegExp(projectName))
- .parent()
- .parent()
- .within(() => {
- cy.findByText('Join Project').click()
- })
+ openProjectViaInviteNotification(projectName)
}
export function shareProjectByEmailAndAcceptInviteViaEmail(
projectName: string,
email: string,
- level: 'Read only' | 'Can edit'
+ level: 'Viewer' | 'Editor'
) {
shareProjectByEmail(projectName, email, level)
@@ -79,23 +153,27 @@ export function shareProjectByEmailAndAcceptInviteViaEmail(
cy.findByText(/user would like you to join/)
cy.contains(new RegExp(`You are accepting this invite as ${email}`))
cy.findByText('Join Project').click()
+ waitForMainDocToLoad()
}
export function enableLinkSharing() {
let linkSharingReadOnly: string
let linkSharingReadAndWrite: string
+ const origin = new URL(Cypress.config().baseUrl!).origin
+
+ waitForMainDocToLoad()
cy.findByText('Share').click()
cy.findByText('Turn on link sharing').click()
cy.findByText('Anyone with this link can view this project')
.next()
- .should('contain.text', 'http://sharelatex/')
+ .should('contain.text', origin + '/read')
.then(el => {
linkSharingReadOnly = el.text()
})
cy.findByText('Anyone with this link can edit this project')
.next()
- .should('contain.text', 'http://sharelatex/')
+ .should('contain.text', origin + '/')
.then(el => {
linkSharingReadAndWrite = el.text()
})
@@ -104,3 +182,77 @@ export function enableLinkSharing() {
return { linkSharingReadOnly, linkSharingReadAndWrite }
})
}
+
+export function waitForMainDocToLoad() {
+ cy.log('Wait for main doc to load; it will steal the focus after loading')
+ cy.get('.cm-content').should('contain.text', 'Introduction')
+}
+
+export function openFile(fileName: string, waitFor: string) {
+ // force: The file-tree pane is too narrow to display the full name.
+ cy.findByTestId('file-tree').findByText(fileName).click({ force: true })
+
+ // wait until we've switched to the selected file
+ cy.findByText('Loading…').should('not.exist')
+ cy.findByText(waitFor)
+}
+
+export function createNewFile() {
+ const fileName = `${uuid()}.tex`
+
+ cy.log('create new project file')
+ cy.get('button').contains('New file').click({ force: true })
+ cy.findByRole('dialog').within(() => {
+ cy.get('input').clear()
+ cy.get('input').type(fileName)
+ cy.findByText('Create').click()
+ })
+ // force: The file-tree pane is too narrow to display the full name.
+ cy.findByTestId('file-tree').findByText(fileName).click({ force: true })
+
+ // wait until we've switched to the newly created empty file
+ cy.findByText('Loading…').should('not.exist')
+ cy.get('.cm-line').should('have.length', 1)
+
+ return fileName
+}
+
+export function prepareFileUploadTest(binary = false) {
+ const name = `${uuid()}.txt`
+ const content = `Test File Content ${name}${binary ? ' \x00' : ''}`
+ cy.get('button').contains('Upload').click({ force: true })
+ cy.get('input[type=file]')
+ .first()
+ .selectFile(
+ {
+ contents: Cypress.Buffer.from(content),
+ fileName: name,
+ lastModified: Date.now(),
+ },
+ { force: true }
+ )
+
+ // wait for the upload to finish
+ cy.findByRole('treeitem', { name })
+
+ return function check() {
+ cy.findByRole('treeitem', { name }).click()
+ if (binary) {
+ cy.findByText(content).should('not.have.class', 'cm-line')
+ } else {
+ cy.findByText(content).should('have.class', 'cm-line')
+ }
+ }
+}
+
+export function testNewFileUpload() {
+ it('can upload text file', () => {
+ const check = prepareFileUploadTest(false)
+ check()
+ })
+
+ it('can upload binary file', () => {
+ const check = prepareFileUploadTest(true)
+ check()
+ })
+}
diff --git a/server-ce/test/history.spec.ts b/server-ce/test/history.spec.ts
index cc1950f240..f0d7e74fb3 100644
--- a/server-ce/test/history.spec.ts
+++ b/server-ce/test/history.spec.ts
@@ -40,7 +40,6 @@ describe('History', function () {
const CLASS_DELETION = 'ol-cm-deletion-marker'
it('should support labels, comparison and download', () => {
- cy.visit('/project')
createProject('labels')
const recompile = throttledRecompile()
diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js
index 9e4cd5d360..b3dcd72b1f 100644
--- a/server-ce/test/host-admin.js
+++ b/server-ce/test/host-admin.js
@@ -29,6 +29,17 @@ const IMAGES = {
PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
}
+function defaultDockerComposeOverride() {
+ return {
+ services: {
+ sharelatex: {
+ environment: {},
+ },
+ 'git-bridge': {},
+ },
+ }
+}
+
let previousConfig = ''
function readDockerComposeOverride() {
@@ -38,14 +49,7 @@ function readDockerComposeOverride() {
if (error.code !== 'ENOENT') {
throw error
}
- return {
- services: {
- sharelatex: {
- environment: {},
- },
- 'git-bridge': {},
- },
- }
+ return defaultDockerComposeOverride
}
}
@@ -77,12 +81,21 @@ app.use(bodyParser.json())
app.use((req, res, next) => {
// Basic access logs
console.log(req.method, req.url, req.body)
+ const json = res.json
+ res.json = body => {
+ console.log(req.method, req.url, req.body, '->', body)
+ json.call(res, body)
+ }
+ next()
+})
+app.use((req, res, next) => {
// Add CORS headers
const accessControlAllowOrigin =
process.env.ACCESS_CONTROL_ALLOW_ORIGIN || 'http://sharelatex'
res.setHeader('Access-Control-Allow-Origin', accessControlAllowOrigin)
res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
res.setHeader('Access-Control-Max-Age', '3600')
+ res.setHeader('Access-Control-Allow-Methods', 'DELETE, GET, HEAD, POST, PUT')
next()
})
@@ -131,10 +144,9 @@ const allowedVars = Joi.object(
'GIT_BRIDGE_HOST',
'GIT_BRIDGE_PORT',
'V1_HISTORY_URL',
- 'DOCKER_RUNNER',
'SANDBOXED_COMPILES',
- 'SANDBOXED_COMPILES_SIBLING_CONTAINERS',
'ALL_TEX_LIVE_DOCKER_IMAGE_NAMES',
+ 'OVERLEAF_FILESTORE_MIGRATION_LEVEL',
'OVERLEAF_TEMPLATES_USER_ID',
'OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS',
'OVERLEAF_ALLOW_PUBLIC_ACCESS',
@@ -196,10 +208,7 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
)
}
- if (
- cfg.services.sharelatex.environment
- .SANDBOXED_COMPILES_SIBLING_CONTAINERS === 'true'
- ) {
+ if (cfg.services.sharelatex.environment.SANDBOXED_COMPILES === 'true') {
cfg.services.sharelatex.environment.SANDBOXED_COMPILES_HOST_DIR =
PATHS.SANDBOXED_COMPILES_HOST_DIR
cfg.services.sharelatex.environment.TEX_LIVE_DOCKER_IMAGE =
@@ -324,8 +333,19 @@ app.get('/redis/keys', (req, res) => {
)
})
+app.delete('/data/user_files', (req, res) => {
+ runDockerCompose(
+ 'exec',
+ ['sharelatex', 'rm', '-rf', '/var/lib/overleaf/data/user_files'],
+ (error, stdout, stderr) => {
+ res.json({ error, stdout, stderr })
+ }
+ )
+})
+
app.use(handleValidationErrors())
purgeDataDir()
+writeDockerComposeOverride(defaultDockerComposeOverride())
app.listen(80)
diff --git a/server-ce/test/package-lock.json b/server-ce/test/package-lock.json
index 6e84524ae0..05870284e8 100644
--- a/server-ce/test/package-lock.json
+++ b/server-ce/test/package-lock.json
@@ -12,10 +12,10 @@
"@types/pdf-parse": "^1.1.4",
"@types/uuid": "^9.0.8",
"adm-zip": "^0.5.12",
- "body-parser": "^1.20.2",
+ "body-parser": "^1.20.3",
"celebrate": "^15.0.3",
"cypress": "13.13.2",
- "express": "^4.19.2",
+ "express": "^4.21.2",
"isomorphic-git": "^1.25.10",
"js-yaml": "^4.1.0",
"pdf-parse": "^1.1.1",
@@ -609,9 +609,9 @@
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"node_modules/body-parser": {
- "version": "1.20.2",
- "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz",
- "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==",
+ "version": "1.20.3",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
+ "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -621,7 +621,7 @@
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
- "qs": "6.11.0",
+ "qs": "6.13.0",
"raw-body": "2.5.2",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
@@ -645,11 +645,11 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/body-parser/node_modules/qs": {
- "version": "6.11.0",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
- "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
+ "version": "6.13.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
+ "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"dependencies": {
- "side-channel": "^1.0.4"
+ "side-channel": "^1.0.6"
},
"engines": {
"node": ">=0.6"
@@ -718,6 +718,33 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/call-bind-apply-helpers": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+ "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+ "dependencies": {
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/call-bound": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz",
+ "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==",
+ "dependencies": {
+ "call-bind-apply-helpers": "^1.0.1",
+ "get-intrinsic": "^1.2.6"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
@@ -891,9 +918,9 @@
}
},
"node_modules/cookie": {
- "version": "0.6.0",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
- "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
+ "version": "0.7.1",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
+ "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
"engines": {
"node": ">= 0.6"
}
@@ -920,9 +947,9 @@
}
},
"node_modules/cross-spawn": {
- "version": "7.0.3",
- "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
- "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "version": "7.0.6",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+ "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -1140,6 +1167,19 @@
"resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz",
"integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg=="
},
+ "node_modules/dunder-proto": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+ "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+ "dependencies": {
+ "call-bind-apply-helpers": "^1.0.1",
+ "es-errors": "^1.3.0",
+ "gopd": "^1.2.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1160,9 +1200,9 @@
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/encodeurl": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
- "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+ "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"engines": {
"node": ">= 0.8"
}
@@ -1187,6 +1227,22 @@
"node": ">=8.6"
}
},
+ "node_modules/es-define-property": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+ "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/es-errors": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+ "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/es-get-iterator": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz",
@@ -1206,6 +1262,17 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/es-object-atoms": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+ "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+ "dependencies": {
+ "es-errors": "^1.3.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -1266,36 +1333,36 @@
}
},
"node_modules/express": {
- "version": "4.19.2",
- "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
- "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
+ "version": "4.21.2",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
+ "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
- "body-parser": "1.20.2",
+ "body-parser": "1.20.3",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
- "cookie": "0.6.0",
+ "cookie": "0.7.1",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
- "finalhandler": "1.2.0",
+ "finalhandler": "1.3.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
- "merge-descriptors": "1.0.1",
+ "merge-descriptors": "1.0.3",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
- "path-to-regexp": "0.1.7",
+ "path-to-regexp": "0.1.12",
"proxy-addr": "~2.0.7",
- "qs": "6.11.0",
+ "qs": "6.13.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
- "send": "0.18.0",
- "serve-static": "1.15.0",
+ "send": "0.19.0",
+ "serve-static": "1.16.2",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"type-is": "~1.6.18",
@@ -1304,6 +1371,10 @@
},
"engines": {
"node": ">= 0.10.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/express"
}
},
"node_modules/express/node_modules/debug": {
@@ -1320,11 +1391,11 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/express/node_modules/qs": {
- "version": "6.11.0",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
- "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
+ "version": "6.13.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
+ "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"dependencies": {
- "side-channel": "^1.0.4"
+ "side-channel": "^1.0.6"
},
"engines": {
"node": ">=0.6"
@@ -1393,12 +1464,12 @@
}
},
"node_modules/finalhandler": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
- "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
+ "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"dependencies": {
"debug": "2.6.9",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
@@ -1498,19 +1569,40 @@
}
},
"node_modules/get-intrinsic": {
- "version": "1.2.2",
- "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
- "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+ "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"dependencies": {
+ "call-bind-apply-helpers": "^1.0.2",
+ "es-define-property": "^1.0.1",
+ "es-errors": "^1.3.0",
+ "es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
- "has-proto": "^1.0.1",
- "has-symbols": "^1.0.3",
- "hasown": "^2.0.0"
+ "get-proto": "^1.0.1",
+ "gopd": "^1.2.0",
+ "has-symbols": "^1.1.0",
+ "hasown": "^2.0.2",
+ "math-intrinsics": "^1.1.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/get-proto": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+ "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+ "dependencies": {
+ "dunder-proto": "^1.0.1",
+ "es-object-atoms": "^1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/get-stream": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
@@ -1556,11 +1648,11 @@
}
},
"node_modules/gopd": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
- "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
- "dependencies": {
- "get-intrinsic": "^1.1.3"
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+ "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -1598,21 +1690,10 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/has-proto": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
- "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
"node_modules/has-symbols": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
- "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+ "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"engines": {
"node": ">= 0.4"
},
@@ -1635,9 +1716,9 @@
}
},
"node_modules/hasown": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
- "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+ "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"dependencies": {
"function-bind": "^1.1.2"
},
@@ -2284,6 +2365,14 @@
"lz-string": "bin/bin.js"
}
},
+ "node_modules/math-intrinsics": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+ "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -2293,9 +2382,12 @@
}
},
"node_modules/merge-descriptors": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
- "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
+ "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
},
"node_modules/merge-stream": {
"version": "2.0.0",
@@ -2405,9 +2497,12 @@
}
},
"node_modules/object-inspect": {
- "version": "1.13.1",
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
- "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
+ "version": "1.13.4",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
+ "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
+ "engines": {
+ "node": ">= 0.4"
+ },
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -2526,9 +2621,9 @@
}
},
"node_modules/path-to-regexp": {
- "version": "0.1.7",
- "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
- "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
+ "version": "0.1.12",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
+ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
},
"node_modules/pdf-parse": {
"version": "1.1.1",
@@ -2807,9 +2902,9 @@
}
},
"node_modules/send": {
- "version": "0.18.0",
- "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
- "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
+ "version": "0.19.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
+ "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -2842,20 +2937,28 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
+ "node_modules/send/node_modules/encodeurl": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
"node_modules/send/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/serve-static": {
- "version": "1.15.0",
- "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
- "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
+ "version": "1.16.2",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
+ "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"dependencies": {
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
- "send": "0.18.0"
+ "send": "0.19.0"
},
"engines": {
"node": ">= 0.8.0"
@@ -2925,13 +3028,68 @@
}
},
"node_modules/side-channel": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
- "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
+ "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==",
"dependencies": {
- "call-bind": "^1.0.0",
- "get-intrinsic": "^1.0.2",
- "object-inspect": "^1.9.0"
+ "es-errors": "^1.3.0",
+ "object-inspect": "^1.13.3",
+ "side-channel-list": "^1.0.0",
+ "side-channel-map": "^1.0.1",
+ "side-channel-weakmap": "^1.0.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/side-channel-list": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz",
+ "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==",
+ "dependencies": {
+ "es-errors": "^1.3.0",
+ "object-inspect": "^1.13.3"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/side-channel-map": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
+ "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
+ "dependencies": {
+ "call-bound": "^1.0.2",
+ "es-errors": "^1.3.0",
+ "get-intrinsic": "^1.2.5",
+ "object-inspect": "^1.13.3"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/side-channel-weakmap": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
+ "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
+ "dependencies": {
+ "call-bound": "^1.0.2",
+ "es-errors": "^1.3.0",
+ "get-intrinsic": "^1.2.5",
+ "object-inspect": "^1.13.3",
+ "side-channel-map": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
diff --git a/server-ce/test/package.json b/server-ce/test/package.json
index 674154dd39..36ba3df2dd 100644
--- a/server-ce/test/package.json
+++ b/server-ce/test/package.json
@@ -15,10 +15,10 @@
"@types/pdf-parse": "^1.1.4",
"@types/uuid": "^9.0.8",
"adm-zip": "^0.5.12",
- "body-parser": "^1.20.2",
+ "body-parser": "^1.20.3",
"celebrate": "^15.0.3",
"cypress": "13.13.2",
- "express": "^4.19.2",
+ "express": "^4.21.2",
"isomorphic-git": "^1.25.10",
"js-yaml": "^4.1.0",
"pdf-parse": "^1.1.1",
diff --git a/server-ce/test/project-list.spec.ts b/server-ce/test/project-list.spec.ts
index 9ee9ac9ca0..998fcf9ffb 100644
--- a/server-ce/test/project-list.spec.ts
+++ b/server-ce/test/project-list.spec.ts
@@ -18,11 +18,11 @@ describe('Project List', () => {
describe('user with no projects', () => {
ensureUserExists({ email: WITHOUT_PROJECTS_USER })
- it("'Import from Github' is not displayed in the welcome page", () => {
+ it("'Import from GitHub' is not displayed in the welcome page", () => {
login(WITHOUT_PROJECTS_USER)
cy.visit('/project')
cy.findByText('Create a new project').click()
- cy.findByText(/Import from Github/i).should('not.exist')
+ cy.findByText(/Import from GitHub/i).should('not.exist')
})
})
@@ -31,29 +31,27 @@ describe('Project List', () => {
ensureUserExists({ email: REGULAR_USER })
before(() => {
+ login(REGULAR_USER)
+ createProject(projectName, { type: 'Example project', open: false })
+ })
+ beforeEach(function () {
login(REGULAR_USER)
cy.visit('/project')
- createProject(projectName, { type: 'Example Project' })
})
it('Can download project sources', () => {
- login(REGULAR_USER)
- cy.visit('/project')
-
findProjectRow(projectName).within(() =>
cy.findByRole('button', { name: 'Download .zip file' }).click()
)
+ const zipName = projectName.replaceAll('-', '_')
cy.task('readFileInZip', {
- pathToZip: `cypress/downloads/${projectName}.zip`,
+ pathToZip: `cypress/downloads/${zipName}.zip`,
fileToRead: 'main.tex',
}).should('contain', 'Your introduction goes here')
})
it('Can download project PDF', () => {
- login(REGULAR_USER)
- cy.visit('/project')
-
findProjectRow(projectName).within(() =>
cy.findByRole('button', { name: 'Download PDF' }).click()
)
@@ -67,9 +65,6 @@ describe('Project List', () => {
it('can assign and remove tags to projects', () => {
const tagName = uuid().slice(0, 7) // long tag names are truncated in the UI, which affects selectors
- login(REGULAR_USER)
- cy.visit('/project')
-
cy.log('select project')
cy.get(`[aria-label="Select ${projectName}"]`).click()
@@ -90,9 +85,7 @@ describe('Project List', () => {
cy.log('create a separate project to filter')
const nonTaggedProjectName = `project-${uuid()}`
login(REGULAR_USER)
- cy.visit('/project')
- createProject(nonTaggedProjectName)
- cy.visit('/project')
+ createProject(nonTaggedProjectName, { open: false })
cy.log('select project')
cy.get(`[aria-label="Select ${projectName}"]`).click()
diff --git a/server-ce/test/project-sharing.spec.ts b/server-ce/test/project-sharing.spec.ts
index e14f36d778..4da2209332 100644
--- a/server-ce/test/project-sharing.spec.ts
+++ b/server-ce/test/project-sharing.spec.ts
@@ -4,6 +4,9 @@ import { ensureUserExists, login } from './helpers/login'
import {
createProject,
enableLinkSharing,
+ openProjectByName,
+ openProjectViaLinkSharingAsAnon,
+ openProjectViaLinkSharingAsUser,
shareProjectByEmailAndAcceptInviteViaDash,
shareProjectByEmailAndAcceptInviteViaEmail,
} from './helpers/project'
@@ -31,7 +34,6 @@ describe('Project Sharing', function () {
function setupTestProject() {
login('user@example.com')
- cy.visit('/project')
createProject(projectName)
// Add chat message
@@ -53,8 +55,15 @@ describe('Project Sharing', function () {
function expectContentReadOnlyAccess() {
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
- cy.get('.cm-content').should('contain.text', '\\maketitle')
- cy.get('.cm-content').should('have.attr', 'contenteditable', 'false')
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
+ 'contain.text',
+ '\\maketitle'
+ )
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
+ 'have.attr',
+ 'contenteditable',
+ 'false'
+ )
}
function expectContentWriteAccess() {
@@ -62,13 +71,23 @@ describe('Project Sharing', function () {
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
const recompile = throttledRecompile()
// wait for the editor to finish loading
- cy.get('.cm-content').should('contain.text', '\\maketitle')
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
+ 'contain.text',
+ '\\maketitle'
+ )
// the editor should be writable
- cy.get('.cm-content').should('have.attr', 'contenteditable', 'true')
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
+ 'have.attr',
+ 'contenteditable',
+ 'true'
+ )
cy.findByText('\\maketitle').parent().click()
cy.findByText('\\maketitle').parent().type(`\n\\section{{}${section}}`)
// should have written
- cy.get('.cm-content').should('contain.text', `\\section{${section}}`)
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
+ 'contain.text',
+ `\\section{${section}}`
+ )
// check PDF
recompile()
cy.get('.pdf-viewer').should('contain.text', projectName)
@@ -152,16 +171,10 @@ describe('Project Sharing', function () {
beforeEach(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaEmail(
- projectName,
- email,
- 'Read only'
- )
+ shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Viewer')
})
it('should grant the collaborator read access', () => {
- cy.visit('/project')
- cy.findByText(projectName).click()
expectFullReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -173,13 +186,12 @@ describe('Project Sharing', function () {
beforeWithReRunOnTestRetry(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Read only')
+ shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Viewer')
})
it('should grant the collaborator read access', () => {
login(email)
- cy.visit('/project')
- cy.findByText(projectName).click()
+ openProjectByName(projectName)
expectFullReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -191,13 +203,12 @@ describe('Project Sharing', function () {
beforeWithReRunOnTestRetry(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can edit')
+ shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Editor')
})
it('should grant the collaborator write access', () => {
login(email)
- cy.visit('/project')
- cy.findByText(projectName).click()
+ openProjectByName(projectName)
expectReadAndWriteAccess()
expectEditAuthoredAs('You')
expectProjectDashboardEntry()
@@ -212,9 +223,11 @@ describe('Project Sharing', function () {
it('should grant restricted read access', () => {
login(email)
- cy.visit(linkSharingReadOnly)
- cy.findByText(projectName) // wait for lazy loading
- cy.findByText('Join Project').click()
+ openProjectViaLinkSharingAsUser(
+ linkSharingReadOnly,
+ projectName,
+ email
+ )
expectRestrictedReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -226,9 +239,11 @@ describe('Project Sharing', function () {
it('should grant full write access', () => {
login(email)
- cy.visit(linkSharingReadAndWrite)
- cy.findByText(projectName) // wait for lazy loading
- cy.findByText('Join Project').click()
+ openProjectViaLinkSharingAsUser(
+ linkSharingReadAndWrite,
+ projectName,
+ email
+ )
expectReadAndWriteAccess()
expectEditAuthoredAs('You')
expectProjectDashboardEntry()
@@ -272,7 +287,7 @@ describe('Project Sharing', function () {
withDataDir: true,
})
it('should grant read access with read link', () => {
- cy.visit(linkSharingReadOnly)
+ openProjectViaLinkSharingAsAnon(linkSharingReadOnly)
expectRestrictedReadOnlyAccess()
})
@@ -292,12 +307,12 @@ describe('Project Sharing', function () {
})
it('should grant read access with read link', () => {
- cy.visit(linkSharingReadOnly)
+ openProjectViaLinkSharingAsAnon(linkSharingReadOnly)
expectRestrictedReadOnlyAccess()
})
it('should grant write access with write link', () => {
- cy.visit(linkSharingReadAndWrite)
+ openProjectViaLinkSharingAsAnon(linkSharingReadAndWrite)
expectReadAndWriteAccess()
expectEditAuthoredAs('Anonymous')
})
diff --git a/server-ce/test/sandboxed-compiles.spec.ts b/server-ce/test/sandboxed-compiles.spec.ts
index e50aa36283..71f5b43392 100644
--- a/server-ce/test/sandboxed-compiles.spec.ts
+++ b/server-ce/test/sandboxed-compiles.spec.ts
@@ -1,7 +1,7 @@
import { ensureUserExists, login } from './helpers/login'
import { createProject } from './helpers/project'
import { isExcludedBySharding, startWith } from './helpers/config'
-import { throttledRecompile } from './helpers/compile'
+import { throttledRecompile, stopCompile } from './helpers/compile'
import { v4 as uuid } from 'uuid'
import { waitUntilScrollingFinished } from './helpers/waitUntilScrollingFinished'
import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
@@ -10,9 +10,7 @@ const LABEL_TEX_LIVE_VERSION = 'TeX Live version'
describe('SandboxedCompiles', function () {
const enabledVars = {
- DOCKER_RUNNER: 'true',
SANDBOXED_COMPILES: 'true',
- SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true',
ALL_TEX_LIVE_DOCKER_IMAGE_NAMES: '2023,2022',
}
@@ -29,7 +27,6 @@ describe('SandboxedCompiles', function () {
})
it('should offer TexLive images and switch the compiler', function () {
- cy.visit('/project')
createProject('sandboxed')
const recompile = throttledRecompile()
cy.log('wait for compile')
@@ -46,7 +43,7 @@ describe('SandboxedCompiles', function () {
.findByText('2023')
.parent()
.select('2022')
- cy.get('#left-menu-modal').click()
+ cy.get('.left-menu-modal-backdrop').click()
cy.log('Trigger compile with other TeX Live version')
recompile()
@@ -59,14 +56,47 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
+ checkStopCompile()
})
+ function checkStopCompile() {
+ it('users can stop a running compile', function () {
+ login('user@example.com')
+ createProject('test-project')
+ // create an infinite loop in the main document
+ // this will cause the compile to run indefinitely
+ cy.findByText('\\maketitle').parent().click()
+ cy.findByText('\\maketitle')
+ .parent()
+ .type('\n\\def\\x{{}Hello!\\par\\x}\\x')
+ cy.log('Start compile')
+ // We need to start the compile manually because we do not want to wait for it to finish
+ cy.findByText('Recompile').click()
+ // Now stop the compile and kill the latex process
+ stopCompile({ delay: 1000 })
+ cy.get('.logs-pane')
+ .invoke('text')
+ .should('match', /PDF Rendering Error|Compilation cancelled/)
+ // Check that the previous compile is not running in the background by
+ // disabling the infinite loop and recompiling
+ cy.findByText('\\def').parent().click()
+ cy.findByText('\\def').parent().type('{home}disabled loop% ')
+ cy.findByText('Recompile').click()
+ cy.get('.pdf-viewer').should('contain.text', 'disabled loop')
+ cy.get('.logs-pane').should(
+ 'not.contain.text',
+ 'A previous compile is still running'
+ )
+ })
+ }
+
function checkSyncTeX() {
- describe('SyncTeX', function () {
+ // TODO(25342): re-enable
+ // eslint-disable-next-line mocha/no-skipped-tests
+ describe.skip('SyncTeX', function () {
let projectName: string
beforeEach(function () {
projectName = `Project ${uuid()}`
- cy.visit('/project')
createProject(projectName)
const recompile = throttledRecompile()
cy.findByText('\\maketitle').parent().click()
@@ -131,7 +161,9 @@ describe('SandboxedCompiles', function () {
})
cy.log('navigate to Section A')
- cy.get('.cm-content').within(() => cy.findByText('Section A').click())
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).within(
+ () => cy.findByText('Section A').click()
+ )
cy.get('[aria-label="Go to code location in PDF"]').click()
cy.get('@title').then((title: any) => {
waitUntilScrollingFinished('.pdfjs-viewer-inner', title)
@@ -140,7 +172,9 @@ describe('SandboxedCompiles', function () {
})
cy.log('navigate to Section B')
- cy.get('.cm-content').within(() => cy.findByText('Section B').click())
+ cy.findByRole('textbox', { name: /Source Editor editing/i }).within(
+ () => cy.findByText('Section B').click()
+ )
cy.get('[aria-label="Go to code location in PDF"]').click()
cy.get('@sectionA').then((title: any) => {
waitUntilScrollingFinished('.pdfjs-viewer-inner', title)
@@ -154,7 +188,6 @@ describe('SandboxedCompiles', function () {
function checkRecompilesAfterErrors() {
it('recompiles even if there are Latex errors', function () {
login('user@example.com')
- cy.visit('/project')
createProject('test-project')
const recompile = throttledRecompile()
cy.findByText('\\maketitle').parent().click()
@@ -170,7 +203,6 @@ describe('SandboxedCompiles', function () {
function checkXeTeX() {
it('should be able to use XeLaTeX', function () {
- cy.visit('/project')
createProject('XeLaTeX')
const recompile = throttledRecompile()
cy.log('wait for compile')
@@ -187,7 +219,7 @@ describe('SandboxedCompiles', function () {
.findByText('pdfLaTeX')
.parent()
.select('XeLaTeX')
- cy.get('#left-menu-modal').click()
+ cy.get('.left-menu-modal-backdrop').click()
cy.log('Trigger compile with other compiler')
recompile()
@@ -204,14 +236,13 @@ describe('SandboxedCompiles', function () {
})
it('should not offer TexLive images and use default compiler', function () {
- cy.visit('/project')
createProject('sandboxed')
cy.log('wait for compile')
cy.get('.pdf-viewer').should('contain.text', 'sandboxed')
- cy.log('Check which compiler version was used, expect 2024')
+ cy.log('Check which compiler version was used, expect 2025')
cy.get('[aria-label="View logs"]').click()
- cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2024\) /)
+ cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2025\) /)
cy.log('Check that there is no TeX Live version toggle')
cy.get('header').findByText('Menu').click()
@@ -232,6 +263,7 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
+ checkStopCompile()
})
describe.skip('unavailable in CE', function () {
@@ -246,5 +278,6 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
+ checkStopCompile()
})
})
diff --git a/server-ce/test/templates.spec.ts b/server-ce/test/templates.spec.ts
index b9cc3f87eb..4959e149fc 100644
--- a/server-ce/test/templates.spec.ts
+++ b/server-ce/test/templates.spec.ts
@@ -47,7 +47,9 @@ describe('Templates', () => {
cy.url().should('match', /\/templates$/)
})
- it('should have templates feature', () => {
+ // TODO(25342): re-enable
+ // eslint-disable-next-line mocha/no-skipped-tests
+ it.skip('should have templates feature', () => {
login(TEMPLATES_USER)
const name = `Template ${Date.now()}`
const description = `Template Description ${Date.now()}`
@@ -64,7 +66,7 @@ describe('Templates', () => {
.get('textarea')
.type(description)
cy.findByText('Publish').click()
- cy.findByText('Publishing…').should('be.disabled')
+ cy.findByText('Publishing…').parent().should('be.disabled')
cy.findByText('Publish').should('not.exist')
cy.findByText('Unpublish', { timeout: 10_000 })
cy.findByText('Republish')
@@ -96,12 +98,12 @@ describe('Templates', () => {
.parent()
.parent()
.within(() => cy.get('input[type="checkbox"]').first().check())
- cy.get('.project-list-sidebar-react').within(() => {
+ cy.get('.project-list-sidebar-scroll').within(() => {
cy.findAllByText('New Tag').first().click()
})
cy.focused().type(tagName)
cy.findByText('Create').click()
- cy.get('.project-list-sidebar-react').within(() => {
+ cy.get('.project-list-sidebar-scroll').within(() => {
cy.findByText(tagName)
.parent()
.within(() => cy.get('.name').should('have.text', `${tagName} (1)`))
@@ -203,6 +205,7 @@ describe('Templates', () => {
.click()
cy.findAllByText('All Templates')
.first()
+ .parent()
.should('have.attr', 'href', '/templates/all')
})
})
diff --git a/server-ce/test/upgrading.spec.ts b/server-ce/test/upgrading.spec.ts
index 86a3ea0cad..16e0320dcc 100644
--- a/server-ce/test/upgrading.spec.ts
+++ b/server-ce/test/upgrading.spec.ts
@@ -1,7 +1,7 @@
import { ensureUserExists, login } from './helpers/login'
import { isExcludedBySharding, startWith } from './helpers/config'
import { dockerCompose, runScript } from './helpers/hostAdminClient'
-import { createProject } from './helpers/project'
+import { createProject, openProjectByName } from './helpers/project'
import { throttledRecompile } from './helpers/compile'
import { v4 as uuid } from 'uuid'
@@ -38,8 +38,6 @@ describe('Upgrading', function () {
before(() => {
cy.log('Populate old instance')
login(USER)
-
- cy.visit('/project')
createProject(PROJECT_NAME, {
newProjectButtonMatcher: startOptions.newProjectButtonMatcher,
})
@@ -59,7 +57,7 @@ describe('Upgrading', function () {
recompile()
cy.get('header').findByText('Menu').click()
cy.findByText('Source').click()
- cy.get('#left-menu-modal').click()
+ cy.get('.left-menu-modal-backdrop').click({ force: true })
}
cy.log('Check compile and history')
@@ -115,8 +113,7 @@ describe('Upgrading', function () {
})
it('should open the old project', () => {
- cy.visit('/project')
- cy.findByText(PROJECT_NAME).click()
+ openProjectByName(PROJECT_NAME)
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
cy.findByRole('navigation').within(() => {
diff --git a/services/chat/.gitignore b/services/chat/.gitignore
deleted file mode 100644
index f0cf94b147..0000000000
--- a/services/chat/.gitignore
+++ /dev/null
@@ -1,12 +0,0 @@
-**.swp
-
-public/build/
-
-node_modules/
-
-plato/
-
-**/*.map
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc
diff --git a/services/chat/.nvmrc b/services/chat/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/chat/.nvmrc
+++ b/services/chat/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/chat/Dockerfile b/services/chat/Dockerfile
index 242daa44c0..66a8bc3ded 100644
--- a/services/chat/Dockerfile
+++ b/services/chat/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/chat
diff --git a/services/chat/Makefile b/services/chat/Makefile
index cbe59db223..792f5d2cd6 100644
--- a/services/chat/Makefile
+++ b/services/chat/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -98,13 +116,6 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
- while ! mongosh --eval "db.version()" > /dev/null; do \
- echo "Waiting for Mongo..."; \
- sleep 1; \
- done; \
- mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -137,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/chat/app/js/Features/Messages/MessageHttpController.js b/services/chat/app/js/Features/Messages/MessageHttpController.js
index a20d005864..45208e2c03 100644
--- a/services/chat/app/js/Features/Messages/MessageHttpController.js
+++ b/services/chat/app/js/Features/Messages/MessageHttpController.js
@@ -74,6 +74,10 @@ export async function deleteMessage(context) {
return await callMessageHttpController(context, _deleteMessage)
}
+export async function deleteUserMessage(context) {
+ return await callMessageHttpController(context, _deleteUserMessage)
+}
+
export async function getResolvedThreadIds(context) {
return await callMessageHttpController(context, _getResolvedThreadIds)
}
@@ -190,6 +194,13 @@ const _deleteMessage = async (req, res) => {
res.status(204)
}
+const _deleteUserMessage = async (req, res) => {
+ const { projectId, threadId, userId, messageId } = req.params
+ const room = await ThreadManager.findOrCreateThread(projectId, threadId)
+ await MessageManager.deleteUserMessage(userId, room._id, messageId)
+ res.status(204)
+}
+
const _getResolvedThreadIds = async (req, res) => {
const { projectId } = req.params
const resolvedThreadIds = await ThreadManager.getResolvedThreadIds(projectId)
diff --git a/services/chat/app/js/Features/Messages/MessageManager.js b/services/chat/app/js/Features/Messages/MessageManager.js
index cb8818e3b6..efff22a2a4 100644
--- a/services/chat/app/js/Features/Messages/MessageManager.js
+++ b/services/chat/app/js/Features/Messages/MessageManager.js
@@ -77,6 +77,14 @@ export async function deleteMessage(roomId, messageId) {
await db.messages.deleteOne(query)
}
+export async function deleteUserMessage(userId, roomId, messageId) {
+ await db.messages.deleteOne({
+ _id: new ObjectId(messageId),
+ user_id: new ObjectId(userId),
+ room_id: new ObjectId(roomId),
+ })
+}
+
function _ensureIdsAreObjectIds(query) {
if (query.user_id && !(query.user_id instanceof ObjectId)) {
query.user_id = new ObjectId(query.user_id)
diff --git a/services/chat/app/js/server.js b/services/chat/app/js/server.js
index aa4ea42482..80970fc22e 100644
--- a/services/chat/app/js/server.js
+++ b/services/chat/app/js/server.js
@@ -1,10 +1,10 @@
-import http from 'http'
+import http from 'node:http'
import metrics from '@overleaf/metrics'
import logger from '@overleaf/logger'
import express from 'express'
import exegesisExpress from 'exegesis-express'
-import path from 'path'
-import { fileURLToPath } from 'url'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
import * as messagesController from './Features/Messages/MessageHttpController.js'
const __dirname = fileURLToPath(new URL('.', import.meta.url))
diff --git a/services/chat/buildscript.txt b/services/chat/buildscript.txt
index 8470e45e93..35114bd2a4 100644
--- a/services/chat/buildscript.txt
+++ b/services/chat/buildscript.txt
@@ -4,6 +4,6 @@ chat
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=False
---script-version=4.5.0
+--script-version=4.7.0
diff --git a/services/chat/chat.yaml b/services/chat/chat.yaml
index 3ccdf9bc30..35ed3d378d 100644
--- a/services/chat/chat.yaml
+++ b/services/chat/chat.yaml
@@ -177,6 +177,34 @@ paths:
'204':
description: No Content
description: 'Delete message with Message ID provided, from the Thread with ThreadID and ProjectID provided'
+ '/project/{projectId}/thread/{threadId}/user/{userId}/messages/{messageId}':
+ parameters:
+ - schema:
+ type: string
+ name: projectId
+ in: path
+ required: true
+ - schema:
+ type: string
+ name: threadId
+ in: path
+ required: true
+ - schema:
+ type: string
+ name: userId
+ in: path
+ required: true
+ - schema:
+ type: string
+ name: messageId
+ in: path
+ required: true
+ delete:
+ summary: Delete message written by a given user
+ operationId: deleteUserMessage
+ responses:
+ '204':
+ description: No Content
'/project/{projectId}/thread/{threadId}/resolve':
parameters:
- schema:
diff --git a/services/chat/config/settings.defaults.cjs b/services/chat/config/settings.defaults.cjs
index 48734ff11e..4b7dc293a9 100644
--- a/services/chat/config/settings.defaults.cjs
+++ b/services/chat/config/settings.defaults.cjs
@@ -1,3 +1,9 @@
+const http = require('node:http')
+const https = require('node:https')
+
+http.globalAgent.keepAlive = false
+https.globalAgent.keepAlive = false
+
module.exports = {
internal: {
chat: {
diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml
index 6f1a608534..ca3303a079 100644
--- a/services/chat/docker-compose.ci.yml
+++ b/services/chat/docker-compose.ci.yml
@@ -24,10 +24,13 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
+ volumes:
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
user: node
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -39,9 +42,14 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml
index 5474cc3574..e7b8ce7385 100644
--- a/services/chat/docker-compose.yml
+++ b/services/chat/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/chat
- ../../node_modules:/overleaf/node_modules
@@ -14,37 +14,45 @@ services:
working_dir: /overleaf/services/chat
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/chat
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/chat
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/chat/package.json b/services/chat/package.json
index f6132b969c..f3d37eb6d3 100644
--- a/services/chat/package.json
+++ b/services/chat/package.json
@@ -12,8 +12,8 @@
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -24,15 +24,15 @@
"async": "^3.2.5",
"body-parser": "^1.20.3",
"exegesis-express": "^4.0.0",
- "express": "^4.21.0",
- "mongodb": "^6.1.0"
+ "express": "^4.21.2",
+ "mongodb": "6.12.0"
},
"devDependencies": {
"acorn": "^7.1.1",
"ajv": "^6.12.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^10.2.0",
+ "mocha": "^11.1.0",
"request": "^2.88.2",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
diff --git a/services/chat/test/acceptance/js/helpers/ChatApp.js b/services/chat/test/acceptance/js/helpers/ChatApp.js
index 7116ea36f6..3a0baf5ce0 100644
--- a/services/chat/test/acceptance/js/helpers/ChatApp.js
+++ b/services/chat/test/acceptance/js/helpers/ChatApp.js
@@ -1,5 +1,5 @@
import { createServer } from '../../../../app/js/server.js'
-import { promisify } from 'util'
+import { promisify } from 'node:util'
export { db } from '../../../../app/js/mongodb.js'
diff --git a/services/clsi/.gitignore b/services/clsi/.gitignore
index 360466227e..a85e6b757a 100644
--- a/services/clsi/.gitignore
+++ b/services/clsi/.gitignore
@@ -1,14 +1,3 @@
-**.swp
-node_modules
-test/acceptance/fixtures/tmp
compiles
output
-.DS_Store
-*~
cache
-.vagrant
-config/*
-npm-debug.log
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc
diff --git a/services/clsi/.nvmrc b/services/clsi/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/clsi/.nvmrc
+++ b/services/clsi/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile
index dc776e4cea..77c26fab23 100644
--- a/services/clsi/Dockerfile
+++ b/services/clsi/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/clsi
COPY services/clsi/install_deps.sh /overleaf/services/clsi/
diff --git a/services/clsi/Makefile b/services/clsi/Makefile
index 069c3c9426..e02697f4e9 100644
--- a/services/clsi/Makefile
+++ b/services/clsi/Makefile
@@ -24,7 +24,6 @@ DOCKER_COMPOSE_TEST_UNIT = \
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
- -docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
@@ -33,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -64,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -111,11 +128,10 @@ build:
--pull \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
- --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
- --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
- --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
- --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):main \
--tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+ --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
+ --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
+ --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \
--file Dockerfile \
../..
@@ -132,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/clsi/README.md b/services/clsi/README.md
index 0626bb4cfe..f1cf927d3d 100644
--- a/services/clsi/README.md
+++ b/services/clsi/README.md
@@ -19,18 +19,18 @@ The CLSI can be configured through the following environment variables:
* `ALLOWED_IMAGES` - Space separated list of allowed Docker TeX Live images
* `CATCH_ERRORS` - Set to `true` to log uncaught exceptions
* `COMPILE_GROUP_DOCKER_CONFIGS` - JSON string of Docker configs for compile groups
-* `COMPILES_HOST_DIR` - Working directory for LaTeX compiles
+* `SANDBOXED_COMPILES` - Set to true to use sibling containers
+* `SANDBOXED_COMPILES_HOST_DIR_COMPILES` - Working directory for LaTeX compiles
+* `SANDBOXED_COMPILES_HOST_DIR_OUTPUT` - Output directory for LaTeX compiles
* `COMPILE_SIZE_LIMIT` - Sets the body-parser [limit](https://github.com/expressjs/body-parser#limit)
-* `DOCKER_RUNNER` - Set to true to use sibling containers
* `DOCKER_RUNTIME` -
* `FILESTORE_DOMAIN_OVERRIDE` - The url for the filestore service e.g.`http://$FILESTORE_HOST:3009`
* `FILESTORE_PARALLEL_FILE_DOWNLOADS` - Number of parallel file downloads
* `LISTEN_ADDRESS` - The address for the RESTful service to listen on. Set to `0.0.0.0` to listen on all network interfaces
* `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
-* `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
* `SMOKE_TEST` - Whether to run smoke tests
-* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
-* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops`
+* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2017.1`
+* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker`
* `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`
* `TEXLIVE_OPENOUT_ANY` - Sets the `openout_any` environment variable for TeX Live (see the `\openout` primitive [documentation](http://tug.org/texinfohtml/web2c.html#tex-invocation))
@@ -63,10 +63,10 @@ Then start the Docker container:
docker run --rm \
-p 127.0.0.1:3013:3013 \
-e LISTEN_ADDRESS=0.0.0.0 \
- -e DOCKER_RUNNER=true \
+ -e SANDBOXED_COMPILES=true \
-e TEXLIVE_IMAGE=texlive/texlive \
-e TEXLIVE_IMAGE_USER=root \
- -e COMPILES_HOST_DIR="$PWD/compiles" \
+ -e SANDBOXED_COMPILES_HOST_DIR_COMPILES="$PWD/compiles" \
-v "$PWD/compiles:/overleaf/services/clsi/compiles" \
-v "$PWD/cache:/overleaf/services/clsi/cache" \
-v /var/run/docker.sock:/var/run/docker.sock \
diff --git a/services/clsi/app.js b/services/clsi/app.js
index bcf89aaaf9..872f612d9c 100644
--- a/services/clsi/app.js
+++ b/services/clsi/app.js
@@ -6,9 +6,6 @@ const ContentController = require('./app/js/ContentController')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
logger.initialize('clsi')
-if (Settings.sentry.dsn != null) {
- logger.initializeErrorReporting(Settings.sentry.dsn)
-}
const Metrics = require('@overleaf/metrics')
const smokeTest = require('./test/smoke/js/SmokeTests')
@@ -16,7 +13,7 @@ const ContentTypeMapper = require('./app/js/ContentTypeMapper')
const Errors = require('./app/js/Errors')
const { createOutputZip } = require('./app/js/OutputController')
-const Path = require('path')
+const Path = require('node:path')
Metrics.open_sockets.monitor(true)
Metrics.memory.monitor(logger)
@@ -131,26 +128,6 @@ const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks')
// create a static server which does not allow access to any symlinks
// avoids possible mismatch of root directory between middleware check
// and serving the files
-const staticCompileServer = ForbidSymlinks(
- express.static,
- Settings.path.compilesDir,
- {
- setHeaders(res, path, stat) {
- if (Path.basename(path) === 'output.pdf') {
- // Calculate an etag in the same way as nginx
- // https://github.com/tj/send/issues/65
- const etag = (path, stat) =>
- `"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
- '-' +
- Number(stat.size).toString(16) +
- '"'
- res.set('Etag', etag(path, stat))
- }
- res.set('Content-Type', ContentTypeMapper.map(path))
- },
- }
-)
-
const staticOutputServer = ForbidSymlinks(
express.static,
Settings.path.outputDir,
@@ -216,32 +193,6 @@ app.get(
}
)
-app.get(
- '/project/:project_id/user/:user_id/output/*',
- function (req, res, next) {
- // for specific user get the path to the top level file
- logger.warn(
- { url: req.url },
- 'direct request for file in compile directory'
- )
- req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}`
- staticCompileServer(req, res, next)
- }
-)
-
-app.get('/project/:project_id/output/*', function (req, res, next) {
- logger.warn({ url: req.url }, 'direct request for file in compile directory')
- if (req.query?.build?.match(OutputCacheManager.BUILD_REGEX)) {
- // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
- req.url =
- `/${req.params.project_id}/` +
- OutputCacheManager.path(req.query.build, `/${req.params[0]}`)
- } else {
- req.url = `/${req.params.project_id}/${req.params[0]}`
- }
- staticCompileServer(req, res, next)
-})
-
app.get('/oops', function (req, res, next) {
logger.error({ err: 'hello' }, 'test error')
res.send('error\n')
@@ -298,6 +249,9 @@ app.get('/health_check', function (req, res) {
if (Settings.processTooOld) {
return res.status(500).json({ processTooOld: true })
}
+ if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
+ return res.status(500).json({ diskCritical: true })
+ }
smokeTest.sendLastResult(res)
})
@@ -307,6 +261,8 @@ app.use(function (error, req, res, next) {
if (error instanceof Errors.NotFoundError) {
logger.debug({ err: error, url: req.url }, 'not found error')
res.sendStatus(404)
+ } else if (error instanceof Errors.InvalidParameter) {
+ res.status(400).send(error.message)
} else if (error.code === 'EPIPE') {
// inspect container returns EPIPE when shutting down
res.sendStatus(503) // send 503 Unavailable response
@@ -316,8 +272,8 @@ app.use(function (error, req, res, next) {
}
})
-const net = require('net')
-const os = require('os')
+const net = require('node:net')
+const os = require('node:os')
let STATE = 'up'
@@ -343,10 +299,18 @@ const loadTcpServer = net.createServer(function (socket) {
}
const freeLoad = availableWorkingCpus - currentLoad
- const freeLoadPercentage = Math.round(
- (freeLoad / availableWorkingCpus) * 100
- )
- if (freeLoadPercentage <= 0) {
+ let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
+ if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
+ freeLoadPercentage = 0
+ }
+ if (ProjectPersistenceManager.isAnyDiskLow()) {
+ freeLoadPercentage = freeLoadPercentage / 2
+ }
+
+ if (
+ Settings.internal.load_balancer_agent.allow_maintenance &&
+ freeLoadPercentage <= 0
+ ) {
// When its 0 the server is set to drain implicitly.
// Drain will move new projects to different servers.
// Drain will keep existing projects assigned to the same server.
@@ -354,7 +318,11 @@ const loadTcpServer = net.createServer(function (socket) {
socket.write(`maint, 0%\n`, 'ASCII')
} else {
// Ready will cancel the maint state.
- socket.write(`up, ready, ${freeLoadPercentage}%\n`, 'ASCII')
+ socket.write(`up, ready, ${Math.max(freeLoadPercentage, 1)}%\n`, 'ASCII')
+ if (freeLoadPercentage <= 0) {
+ // This metric records how often we would have gone into maintenance mode.
+ Metrics.inc('clsi-prevented-maint')
+ }
}
socket.end()
} else {
diff --git a/services/clsi/app/js/CLSICacheHandler.js b/services/clsi/app/js/CLSICacheHandler.js
new file mode 100644
index 0000000000..26acd221f9
--- /dev/null
+++ b/services/clsi/app/js/CLSICacheHandler.js
@@ -0,0 +1,304 @@
+const crypto = require('node:crypto')
+const fs = require('node:fs')
+const Path = require('node:path')
+const { pipeline } = require('node:stream/promises')
+const { createGzip, createGunzip } = require('node:zlib')
+const tarFs = require('tar-fs')
+const _ = require('lodash')
+const {
+ fetchNothing,
+ fetchStream,
+ RequestFailedError,
+} = require('@overleaf/fetch-utils')
+const logger = require('@overleaf/logger')
+const Metrics = require('@overleaf/metrics')
+const Settings = require('@overleaf/settings')
+const { MeteredStream } = require('@overleaf/stream-utils')
+const { CACHE_SUBDIR } = require('./OutputCacheManager')
+const { isExtraneousFile } = require('./ResourceWriter')
+
+const TIMING_BUCKETS = [
+ 0, 10, 100, 1000, 2000, 5000, 10000, 15000, 20000, 30000,
+]
+const MAX_ENTRIES_IN_OUTPUT_TAR = 100
+const OBJECT_ID_REGEX = /^[0-9a-f]{24}$/
+
+/**
+ * @param {string} projectId
+ * @return {{shard: string, url: string}}
+ */
+function getShard(projectId) {
+ // [timestamp 4bytes][random per machine 5bytes][counter 3bytes]
+ // [32bit 4bytes]
+ const last4Bytes = Buffer.from(projectId, 'hex').subarray(8, 12)
+ const idx = last4Bytes.readUInt32BE() % Settings.apis.clsiCache.shards.length
+ return Settings.apis.clsiCache.shards[idx]
+}
+
+/**
+ * @param {string} projectId
+ * @param {string} userId
+ * @param {string} buildId
+ * @param {string} editorId
+ * @param {[{path: string}]} outputFiles
+ * @param {string} compileGroup
+ * @param {Record} options
+ * @return {string | undefined}
+ */
+function notifyCLSICacheAboutBuild({
+ projectId,
+ userId,
+ buildId,
+ editorId,
+ outputFiles,
+ compileGroup,
+ options,
+}) {
+ if (!Settings.apis.clsiCache.enabled) return undefined
+ if (!OBJECT_ID_REGEX.test(projectId)) return undefined
+ const { url, shard } = getShard(projectId)
+
+ /**
+ * @param {[{path: string}]} files
+ */
+ const enqueue = files => {
+ Metrics.count('clsi_cache_enqueue_files', files.length)
+ fetchNothing(`${url}/enqueue`, {
+ method: 'POST',
+ json: {
+ projectId,
+ userId,
+ buildId,
+ editorId,
+ files,
+ downloadHost: Settings.apis.clsi.downloadHost,
+ clsiServerId: Settings.apis.clsi.clsiServerId,
+ compileGroup,
+ options,
+ },
+ signal: AbortSignal.timeout(15_000),
+ }).catch(err => {
+ logger.warn(
+ { err, projectId, userId, buildId },
+ 'enqueue for clsi cache failed'
+ )
+ })
+ }
+
+ // PDF preview
+ enqueue(
+ outputFiles
+ .filter(
+ f =>
+ f.path === 'output.pdf' ||
+ f.path === 'output.log' ||
+ f.path === 'output.synctex.gz' ||
+ f.path.endsWith('.blg')
+ )
+ .map(f => {
+ if (f.path === 'output.pdf') {
+ return _.pick(f, 'path', 'size', 'contentId', 'ranges')
+ }
+ return _.pick(f, 'path')
+ })
+ )
+
+ // Compile Cache
+ buildTarball({ projectId, userId, buildId, outputFiles })
+ .then(() => {
+ enqueue([{ path: 'output.tar.gz' }])
+ })
+ .catch(err => {
+ logger.warn(
+ { err, projectId, userId, buildId },
+ 'build output.tar.gz for clsi cache failed'
+ )
+ })
+
+ return shard
+}
+
+/**
+ * @param {string} projectId
+ * @param {string} userId
+ * @param {string} buildId
+ * @param {[{path: string}]} outputFiles
+ * @return {Promise}
+ */
+async function buildTarball({ projectId, userId, buildId, outputFiles }) {
+ const timer = new Metrics.Timer('clsi_cache_build', 1, {}, TIMING_BUCKETS)
+ const outputDir = Path.join(
+ Settings.path.outputDir,
+ userId ? `${projectId}-${userId}` : projectId,
+ CACHE_SUBDIR,
+ buildId
+ )
+
+ const files = outputFiles.filter(f => !isExtraneousFile(f.path))
+ if (files.length > MAX_ENTRIES_IN_OUTPUT_TAR) {
+ Metrics.inc('clsi_cache_build_too_many_entries')
+ throw new Error('too many output files for output.tar.gz')
+ }
+ Metrics.count('clsi_cache_build_files', files.length)
+
+ const path = Path.join(outputDir, 'output.tar.gz')
+ try {
+ await pipeline(
+ tarFs.pack(outputDir, { entries: files.map(f => f.path) }),
+ createGzip(),
+ fs.createWriteStream(path)
+ )
+ } catch (err) {
+ try {
+ await fs.promises.unlink(path)
+ } catch (e) {}
+ throw err
+ } finally {
+ timer.done()
+ }
+}
+
+/**
+ * @param {string} projectId
+ * @param {string} userId
+ * @param {string} editorId
+ * @param {string} buildId
+ * @param {string} outputDir
+ * @return {Promise}
+ */
+async function downloadOutputDotSynctexFromCompileCache(
+ projectId,
+ userId,
+ editorId,
+ buildId,
+ outputDir
+) {
+ if (!Settings.apis.clsiCache.enabled) return false
+ if (!OBJECT_ID_REGEX.test(projectId)) return false
+
+ const timer = new Metrics.Timer(
+ 'clsi_cache_download',
+ 1,
+ { method: 'synctex' },
+ TIMING_BUCKETS
+ )
+ let stream
+ try {
+ stream = await fetchStream(
+ `${getShard(projectId).url}/project/${projectId}/${
+ userId ? `user/${userId}/` : ''
+ }build/${editorId}-${buildId}/search/output/output.synctex.gz`,
+ {
+ method: 'GET',
+ signal: AbortSignal.timeout(10_000),
+ }
+ )
+ } catch (err) {
+ if (err instanceof RequestFailedError && err.response.status === 404) {
+ timer.done({ status: 'not-found' })
+ return false
+ }
+ timer.done({ status: 'error' })
+ throw err
+ }
+ await fs.promises.mkdir(outputDir, { recursive: true })
+ const dst = Path.join(outputDir, 'output.synctex.gz')
+ const tmp = dst + crypto.randomUUID()
+ try {
+ await pipeline(
+ stream,
+ new MeteredStream(Metrics, 'clsi_cache_egress', {
+ path: 'output.synctex.gz',
+ }),
+ fs.createWriteStream(tmp)
+ )
+ await fs.promises.rename(tmp, dst)
+ } catch (err) {
+ try {
+ await fs.promises.unlink(tmp)
+ } catch {}
+ throw err
+ }
+ timer.done({ status: 'success' })
+ return true
+}
+
+/**
+ * @param {string} projectId
+ * @param {string} userId
+ * @param {string} compileDir
+ * @return {Promise}
+ */
+async function downloadLatestCompileCache(projectId, userId, compileDir) {
+ if (!Settings.apis.clsiCache.enabled) return false
+ if (!OBJECT_ID_REGEX.test(projectId)) return false
+
+ const url = `${getShard(projectId).url}/project/${projectId}/${
+ userId ? `user/${userId}/` : ''
+ }latest/output/output.tar.gz`
+ const timer = new Metrics.Timer(
+ 'clsi_cache_download',
+ 1,
+ { method: 'tar' },
+ TIMING_BUCKETS
+ )
+ let stream
+ try {
+ stream = await fetchStream(url, {
+ method: 'GET',
+ signal: AbortSignal.timeout(10_000),
+ })
+ } catch (err) {
+ if (err instanceof RequestFailedError && err.response.status === 404) {
+ timer.done({ status: 'not-found' })
+ return false
+ }
+ timer.done({ status: 'error' })
+ throw err
+ }
+ let n = 0
+ let abort = false
+ await pipeline(
+ stream,
+ new MeteredStream(Metrics, 'clsi_cache_egress', { path: 'output.tar.gz' }),
+ createGunzip(),
+ tarFs.extract(compileDir, {
+ // use ignore hook for counting entries (files+folders) and validation.
+ // Include folders as they incur mkdir calls.
+ ignore(_, header) {
+ if (abort) return true // log once
+ n++
+ if (n > MAX_ENTRIES_IN_OUTPUT_TAR) {
+ abort = true
+ logger.warn(
+ {
+ url,
+ compileDir,
+ },
+ 'too many entries in tar-ball from clsi-cache'
+ )
+ } else if (header.type !== 'file' && header.type !== 'directory') {
+ abort = true
+ logger.warn(
+ {
+ url,
+ compileDir,
+ entryType: header.type,
+ },
+ 'unexpected entry in tar-ball from clsi-cache'
+ )
+ }
+ return abort
+ },
+ })
+ )
+ Metrics.count('clsi_cache_download_entries', n)
+ timer.done({ status: 'success' })
+ return !abort
+}
+
+module.exports = {
+ notifyCLSICacheAboutBuild,
+ downloadLatestCompileCache,
+ downloadOutputDotSynctexFromCompileCache,
+}
diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js
index 3da884eed7..b3343ee233 100644
--- a/services/clsi/app/js/CompileController.js
+++ b/services/clsi/app/js/CompileController.js
@@ -1,3 +1,4 @@
+const Path = require('node:path')
const RequestParser = require('./RequestParser')
const CompileManager = require('./CompileManager')
const Settings = require('@overleaf/settings')
@@ -5,6 +6,7 @@ const Metrics = require('./Metrics')
const ProjectPersistenceManager = require('./ProjectPersistenceManager')
const logger = require('@overleaf/logger')
const Errors = require('./Errors')
+const { notifyCLSICacheAboutBuild } = require('./CLSICacheHandler')
let lastSuccessfulCompileTimestamp = 0
@@ -29,101 +31,135 @@ function compile(req, res, next) {
if (error) {
return next(error)
}
- CompileManager.doCompileWithLock(request, (error, result) => {
- let { buildId, outputFiles, stats, timings } = result || {}
- let code, status
- if (outputFiles == null) {
- outputFiles = []
- }
- if (error instanceof Errors.AlreadyCompilingError) {
- code = 423 // Http 423 Locked
- status = 'compile-in-progress'
- } else if (error instanceof Errors.FilesOutOfSyncError) {
- code = 409 // Http 409 Conflict
- status = 'retry'
- logger.warn(
- {
+ const stats = {}
+ const timings = {}
+ CompileManager.doCompileWithLock(
+ request,
+ stats,
+ timings,
+ (error, result) => {
+ let { buildId, outputFiles } = result || {}
+ let code, status
+ if (outputFiles == null) {
+ outputFiles = []
+ }
+ if (error instanceof Errors.AlreadyCompilingError) {
+ code = 423 // Http 423 Locked
+ status = 'compile-in-progress'
+ } else if (error instanceof Errors.FilesOutOfSyncError) {
+ code = 409 // Http 409 Conflict
+ status = 'retry'
+ logger.warn(
+ {
+ projectId: request.project_id,
+ userId: request.user_id,
+ },
+ 'files out of sync, please retry'
+ )
+ } else if (
+ error?.code === 'EPIPE' ||
+ error instanceof Errors.TooManyCompileRequestsError
+ ) {
+ // docker returns EPIPE when shutting down
+ code = 503 // send 503 Unavailable response
+ status = 'unavailable'
+ } else if (error?.terminated) {
+ status = 'terminated'
+ } else if (error?.validate) {
+ status = `validation-${error.validate}`
+ } else if (error?.timedout) {
+ status = 'timedout'
+ logger.debug(
+ { err: error, projectId: request.project_id },
+ 'timeout running compile'
+ )
+ } else if (error) {
+ status = 'error'
+ code = 500
+ logger.error(
+ { err: error, projectId: request.project_id },
+ 'error running compile'
+ )
+ } else {
+ if (
+ outputFiles.some(
+ file => file.path === 'output.pdf' && file.size > 0
+ )
+ ) {
+ status = 'success'
+ lastSuccessfulCompileTimestamp = Date.now()
+ } else if (request.stopOnFirstError) {
+ status = 'stopped-on-first-error'
+ } else {
+ status = 'failure'
+ logger.warn(
+ { projectId: request.project_id, outputFiles },
+ 'project failed to compile successfully, no output.pdf generated'
+ )
+ }
+
+ // log an error if any core files are found
+ if (outputFiles.some(file => file.path === 'core')) {
+ logger.error(
+ { projectId: request.project_id, req, outputFiles },
+ 'core file found in output'
+ )
+ }
+ }
+
+ if (error) {
+ outputFiles = error.outputFiles || []
+ buildId = error.buildId
+ }
+
+ let clsiCacheShard
+ if (
+ status === 'success' &&
+ request.editorId &&
+ request.populateClsiCache
+ ) {
+ clsiCacheShard = notifyCLSICacheAboutBuild({
projectId: request.project_id,
userId: request.user_id,
+ buildId: outputFiles[0].build,
+ editorId: request.editorId,
+ outputFiles,
+ compileGroup: request.compileGroup,
+ options: {
+ compiler: request.compiler,
+ draft: request.draft,
+ imageName: request.imageName
+ ? request.imageName
+ : undefined,
+ rootResourcePath: request.rootResourcePath,
+ stopOnFirstError: request.stopOnFirstError,
+ },
+ })
+ }
+
+ timer.done()
+ res.status(code || 200).send({
+ compile: {
+ status,
+ error: error?.message || error,
+ stats,
+ timings,
+ buildId,
+ clsiCacheShard,
+ outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
+ outputFiles: outputFiles.map(file => ({
+ url:
+ `${Settings.apis.clsi.url}/project/${request.project_id}` +
+ (request.user_id != null
+ ? `/user/${request.user_id}`
+ : '') +
+ `/build/${file.build}/output/${file.path}`,
+ ...file,
+ })),
},
- 'files out of sync, please retry'
- )
- } else if (
- error?.code === 'EPIPE' ||
- error instanceof Errors.TooManyCompileRequestsError
- ) {
- // docker returns EPIPE when shutting down
- code = 503 // send 503 Unavailable response
- status = 'unavailable'
- } else if (error?.terminated) {
- status = 'terminated'
- } else if (error?.validate) {
- status = `validation-${error.validate}`
- } else if (error?.timedout) {
- status = 'timedout'
- logger.debug(
- { err: error, projectId: request.project_id },
- 'timeout running compile'
- )
- } else if (error) {
- status = 'error'
- code = 500
- logger.error(
- { err: error, projectId: request.project_id },
- 'error running compile'
- )
- } else {
- if (
- outputFiles.some(
- file => file.path === 'output.pdf' && file.size > 0
- )
- ) {
- status = 'success'
- lastSuccessfulCompileTimestamp = Date.now()
- } else if (request.stopOnFirstError) {
- status = 'stopped-on-first-error'
- } else {
- status = 'failure'
- logger.warn(
- { projectId: request.project_id, outputFiles },
- 'project failed to compile successfully, no output.pdf generated'
- )
- }
-
- // log an error if any core files are found
- if (outputFiles.some(file => file.path === 'core')) {
- logger.error(
- { projectId: request.project_id, req, outputFiles },
- 'core file found in output'
- )
- }
+ })
}
-
- if (error) {
- outputFiles = error.outputFiles || []
- buildId = error.buildId
- }
-
- timer.done()
- res.status(code || 200).send({
- compile: {
- status,
- error: error?.message || error,
- stats,
- timings,
- buildId,
- outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
- outputFiles: outputFiles.map(file => ({
- url:
- `${Settings.apis.clsi.url}/project/${request.project_id}` +
- (request.user_id != null ? `/user/${request.user_id}` : '') +
- (file.build != null ? `/build/${file.build}` : '') +
- `/output/${file.path}`,
- ...file,
- })),
- },
- })
- })
+ )
}
)
})
@@ -154,30 +190,27 @@ function clearCache(req, res, next) {
}
function syncFromCode(req, res, next) {
- const { file } = req.query
+ const { file, editorId, buildId } = req.query
+ const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
const line = parseInt(req.query.line, 10)
const column = parseInt(req.query.column, 10)
const { imageName } = req.query
const projectId = req.params.project_id
const userId = req.params.user_id
-
- if (imageName && !_isImageNameAllowed(imageName)) {
- return res.status(400).send('invalid image')
- }
-
CompileManager.syncFromCode(
projectId,
userId,
file,
line,
column,
- imageName,
- function (error, pdfPositions) {
+ { imageName, editorId, buildId, compileFromClsiCache },
+ function (error, pdfPositions, downloadedFromCache) {
if (error) {
return next(error)
}
res.json({
pdf: pdfPositions,
+ downloadedFromCache,
})
}
)
@@ -187,26 +220,24 @@ function syncFromPdf(req, res, next) {
const page = parseInt(req.query.page, 10)
const h = parseFloat(req.query.h)
const v = parseFloat(req.query.v)
- const { imageName } = req.query
+ const { imageName, editorId, buildId } = req.query
+ const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
const projectId = req.params.project_id
const userId = req.params.user_id
-
- if (imageName && !_isImageNameAllowed(imageName)) {
- return res.status(400).send('invalid image')
- }
CompileManager.syncFromPdf(
projectId,
userId,
page,
h,
v,
- imageName,
- function (error, codePositions) {
+ { imageName, editorId, buildId, compileFromClsiCache },
+ function (error, codePositions, downloadedFromCache) {
if (error) {
return next(error)
}
res.json({
code: codePositions,
+ downloadedFromCache,
})
}
)
@@ -217,9 +248,6 @@ function wordcount(req, res, next) {
const projectId = req.params.project_id
const userId = req.params.user_id
const { image } = req.query
- if (image && !_isImageNameAllowed(image)) {
- return res.status(400).send('invalid image')
- }
logger.debug({ image, file, projectId }, 'word count request')
CompileManager.wordcount(
@@ -242,12 +270,6 @@ function status(req, res, next) {
res.send('OK')
}
-function _isImageNameAllowed(imageName) {
- const ALLOWED_IMAGES =
- Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages
- return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName)
-}
-
module.exports = {
compile,
stopCompile,
diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js
index 9d1981263d..1b66927412 100644
--- a/services/clsi/app/js/CompileManager.js
+++ b/services/clsi/app/js/CompileManager.js
@@ -1,7 +1,7 @@
-const fsPromises = require('fs/promises')
-const os = require('os')
-const Path = require('path')
-const { callbackify } = require('util')
+const fsPromises = require('node:fs/promises')
+const os = require('node:os')
+const Path = require('node:path')
+const { callbackify } = require('node:util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
@@ -19,6 +19,11 @@ const Errors = require('./Errors')
const CommandRunner = require('./CommandRunner')
const { emitPdfStats } = require('./ContentCacheMetrics')
const SynctexOutputParser = require('./SynctexOutputParser')
+const {
+ downloadLatestCompileCache,
+ downloadOutputDotSynctexFromCompileCache,
+} = require('./CLSICacheHandler')
+const { callbackifyMultiResult } = require('@overleaf/promise-utils')
const COMPILE_TIME_BUCKETS = [
// NOTE: These buckets are locked in per metric name.
@@ -42,22 +47,22 @@ function getOutputDir(projectId, userId) {
return Path.join(Settings.path.outputDir, getCompileName(projectId, userId))
}
-async function doCompileWithLock(request) {
+async function doCompileWithLock(request, stats, timings) {
const compileDir = getCompileDir(request.project_id, request.user_id)
- await fsPromises.mkdir(compileDir, { recursive: true })
+ request.isInitialCompile =
+ (await fsPromises.mkdir(compileDir, { recursive: true })) === compileDir
// prevent simultaneous compiles
const lock = LockManager.acquire(compileDir)
try {
- return await doCompile(request)
+ return await doCompile(request, stats, timings)
} finally {
lock.release()
}
}
-async function doCompile(request) {
+async function doCompile(request, stats, timings) {
+ const { project_id: projectId, user_id: userId } = request
const compileDir = getCompileDir(request.project_id, request.user_id)
- const stats = {}
- const timings = {}
const timerE2E = new Metrics.Timer(
'compile-e2e-v2',
@@ -65,6 +70,25 @@ async function doCompile(request) {
request.metricsOpts,
COMPILE_TIME_BUCKETS
)
+ if (request.isInitialCompile) {
+ stats.isInitialCompile = 1
+ request.metricsOpts.compile = 'initial'
+ if (request.compileFromClsiCache) {
+ try {
+ if (await downloadLatestCompileCache(projectId, userId, compileDir)) {
+ stats.restoredClsiCache = 1
+ request.metricsOpts.compile = 'from-clsi-cache'
+ }
+ } catch (err) {
+ logger.warn(
+ { err, projectId, userId },
+ 'failed to populate compile dir from cache'
+ )
+ }
+ }
+ } else {
+ request.metricsOpts.compile = 'recompile'
+ }
const writeToDiskTimer = new Metrics.Timer(
'write-to-disk',
1,
@@ -296,7 +320,7 @@ async function doCompile(request) {
emitPdfStats(stats, timings, request)
}
- return { outputFiles, stats, timings, buildId }
+ return { outputFiles, buildId }
}
async function _saveOutputFiles({
@@ -313,24 +337,16 @@ async function _saveOutputFiles({
)
const outputDir = getOutputDir(request.project_id, request.user_id)
- let { outputFiles, allEntries } =
+ const { outputFiles: rawOutputFiles, allEntries } =
await OutputFileFinder.promises.findOutputFiles(resourceList, compileDir)
- let buildId
-
- try {
- const saveResult = await OutputCacheManager.promises.saveOutputFiles(
+ const { buildId, outputFiles } =
+ await OutputCacheManager.promises.saveOutputFiles(
{ request, stats, timings },
- outputFiles,
+ rawOutputFiles,
compileDir,
outputDir
)
- buildId = saveResult.buildId
- outputFiles = saveResult.outputFiles
- } catch (err) {
- const { project_id: projectId, user_id: userId } = request
- logger.err({ projectId, userId, err }, 'failed to save output files')
- }
timings.output = timer.done()
return { outputFiles, allEntries, buildId }
@@ -416,14 +432,7 @@ async function _checkDirectory(compileDir) {
return true
}
-async function syncFromCode(
- projectId,
- userId,
- filename,
- line,
- column,
- imageName
-) {
+async function syncFromCode(projectId, userId, filename, line, column, opts) {
// If LaTeX was run in a virtual environment, the file path that synctex expects
// might not match the file path on the host. The .synctex.gz file however, will be accessed
// wherever it is on the host.
@@ -439,15 +448,23 @@ async function syncFromCode(
'-o',
outputFilePath,
]
- const stdout = await _runSynctex(projectId, userId, command, imageName)
+ const { stdout, downloadedFromCache } = await _runSynctex(
+ projectId,
+ userId,
+ command,
+ opts
+ )
logger.debug(
{ projectId, userId, filename, line, column, command, stdout },
'synctex code output'
)
- return SynctexOutputParser.parseViewOutput(stdout)
+ return {
+ codePositions: SynctexOutputParser.parseViewOutput(stdout),
+ downloadedFromCache,
+ }
}
-async function syncFromPdf(projectId, userId, page, h, v, imageName) {
+async function syncFromPdf(projectId, userId, page, h, v, opts) {
const compileName = getCompileName(projectId, userId)
const baseDir = Settings.path.synctexBaseDir(compileName)
const outputFilePath = `${baseDir}/output.pdf`
@@ -457,9 +474,17 @@ async function syncFromPdf(projectId, userId, page, h, v, imageName) {
'-o',
`${page}:${h}:${v}:${outputFilePath}`,
]
- const stdout = await _runSynctex(projectId, userId, command, imageName)
+ const { stdout, downloadedFromCache } = await _runSynctex(
+ projectId,
+ userId,
+ command,
+ opts
+ )
logger.debug({ projectId, userId, page, h, v, stdout }, 'synctex pdf output')
- return SynctexOutputParser.parseEditOutput(stdout, baseDir)
+ return {
+ pdfPositions: SynctexOutputParser.parseEditOutput(stdout, baseDir),
+ downloadedFromCache,
+ }
}
async function _checkFileExists(dir, filename) {
@@ -486,32 +511,90 @@ async function _checkFileExists(dir, filename) {
}
}
-async function _runSynctex(projectId, userId, command, imageName) {
- const directory = getCompileDir(projectId, userId)
+async function _runSynctex(projectId, userId, command, opts) {
+ const { imageName, editorId, buildId, compileFromClsiCache } = opts
+
+ if (imageName && !_isImageNameAllowed(imageName)) {
+ throw new Errors.InvalidParameter('invalid image')
+ }
+ if (editorId && !/^[a-f0-9-]+$/.test(editorId)) {
+ throw new Errors.InvalidParameter('invalid editorId')
+ }
+ if (buildId && !OutputCacheManager.BUILD_REGEX.test(buildId)) {
+ throw new Errors.InvalidParameter('invalid buildId')
+ }
+
+ const outputDir = getOutputDir(projectId, userId)
+ const runInOutputDir = buildId && CommandRunner.canRunSyncTeXInOutputDir()
+
+ const directory = runInOutputDir
+ ? Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR, buildId)
+ : getCompileDir(projectId, userId)
const timeout = 60 * 1000 // increased to allow for large projects
const compileName = getCompileName(projectId, userId)
- const compileGroup = 'synctex'
+ const compileGroup = runInOutputDir ? 'synctex-output' : 'synctex'
const defaultImageName =
Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.image
- await _checkFileExists(directory, 'output.synctex.gz')
- try {
- const output = await CommandRunner.promises.run(
- compileName,
- command,
- directory,
- imageName || defaultImageName,
- timeout,
- {},
- compileGroup
- )
- return output.stdout
- } catch (error) {
- throw OError.tag(error, 'error running synctex', {
- command,
- projectId,
- userId,
- })
- }
+ // eslint-disable-next-line @typescript-eslint/return-await
+ return await OutputCacheManager.promises.queueDirOperation(
+ outputDir,
+ /**
+ * @return {Promise<{stdout: string, downloadedFromCache: boolean}>}
+ */
+ async () => {
+ let downloadedFromCache = false
+ try {
+ await _checkFileExists(directory, 'output.synctex.gz')
+ } catch (err) {
+ if (
+ err instanceof Errors.NotFoundError &&
+ compileFromClsiCache &&
+ editorId &&
+ buildId
+ ) {
+ try {
+ downloadedFromCache =
+ await downloadOutputDotSynctexFromCompileCache(
+ projectId,
+ userId,
+ editorId,
+ buildId,
+ directory
+ )
+ } catch (err) {
+ logger.warn(
+ { err, projectId, userId, editorId, buildId },
+ 'failed to download output.synctex.gz from clsi-cache'
+ )
+ }
+ await _checkFileExists(directory, 'output.synctex.gz')
+ } else {
+ throw err
+ }
+ }
+ try {
+ const { stdout } = await CommandRunner.promises.run(
+ compileName,
+ command,
+ directory,
+ imageName || defaultImageName,
+ timeout,
+ {},
+ compileGroup
+ )
+ return {
+ stdout,
+ downloadedFromCache,
+ }
+ } catch (error) {
+ throw OError.tag(error, 'error running synctex', {
+ command,
+ projectId,
+ userId,
+ })
+ }
+ }
+ )
}
async function wordcount(projectId, userId, filename, image) {
@@ -523,6 +606,10 @@ async function wordcount(projectId, userId, filename, image) {
const compileName = getCompileName(projectId, userId)
const compileGroup = 'wordcount'
+ if (image && !_isImageNameAllowed(image)) {
+ throw new Errors.InvalidParameter('invalid image')
+ }
+
try {
await fsPromises.mkdir(compileDir, { recursive: true })
} catch (err) {
@@ -610,13 +697,25 @@ function _parseWordcountFromOutput(output) {
return results
}
+function _isImageNameAllowed(imageName) {
+ const ALLOWED_IMAGES =
+ Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages
+ return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName)
+}
+
module.exports = {
doCompileWithLock: callbackify(doCompileWithLock),
stopCompile: callbackify(stopCompile),
clearProject: callbackify(clearProject),
clearExpiredProjects: callbackify(clearExpiredProjects),
- syncFromCode: callbackify(syncFromCode),
- syncFromPdf: callbackify(syncFromPdf),
+ syncFromCode: callbackifyMultiResult(syncFromCode, [
+ 'codePositions',
+ 'downloadedFromCache',
+ ]),
+ syncFromPdf: callbackifyMultiResult(syncFromPdf, [
+ 'pdfPositions',
+ 'downloadedFromCache',
+ ]),
wordcount: callbackify(wordcount),
promises: {
doCompileWithLock,
diff --git a/services/clsi/app/js/ContentCacheManager.js b/services/clsi/app/js/ContentCacheManager.js
index 016b264b86..5457c0dce0 100644
--- a/services/clsi/app/js/ContentCacheManager.js
+++ b/services/clsi/app/js/ContentCacheManager.js
@@ -2,10 +2,10 @@
* ContentCacheManager - maintains a cache of stream hashes from a PDF file
*/
-const { callbackify } = require('util')
-const fs = require('fs')
-const crypto = require('crypto')
-const Path = require('path')
+const { callbackify } = require('node:util')
+const fs = require('node:fs')
+const crypto = require('node:crypto')
+const Path = require('node:path')
const Settings = require('@overleaf/settings')
const OError = require('@overleaf/o-error')
const pLimit = require('p-limit')
diff --git a/services/clsi/app/js/ContentCacheMetrics.js b/services/clsi/app/js/ContentCacheMetrics.js
index 0337876feb..1e2b598286 100644
--- a/services/clsi/app/js/ContentCacheMetrics.js
+++ b/services/clsi/app/js/ContentCacheMetrics.js
@@ -1,6 +1,6 @@
const logger = require('@overleaf/logger')
const Metrics = require('./Metrics')
-const os = require('os')
+const os = require('node:os')
let CACHED_LOAD = {
expires: -1,
diff --git a/services/clsi/app/js/ContentController.js b/services/clsi/app/js/ContentController.js
index b154bea175..96eba613ed 100644
--- a/services/clsi/app/js/ContentController.js
+++ b/services/clsi/app/js/ContentController.js
@@ -1,4 +1,4 @@
-const Path = require('path')
+const Path = require('node:path')
const send = require('send')
const Settings = require('@overleaf/settings')
const OutputCacheManager = require('./OutputCacheManager')
diff --git a/services/clsi/app/js/ContentTypeMapper.js b/services/clsi/app/js/ContentTypeMapper.js
index 6301dce489..5bf0c31423 100644
--- a/services/clsi/app/js/ContentTypeMapper.js
+++ b/services/clsi/app/js/ContentTypeMapper.js
@@ -4,7 +4,7 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
let ContentTypeMapper
-const Path = require('path')
+const Path = require('node:path')
// here we coerce html, css and js to text/plain,
// otherwise choose correct mime type based on file extension,
diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js
index c1f08c26e7..97053c1875 100644
--- a/services/clsi/app/js/DockerRunner.js
+++ b/services/clsi/app/js/DockerRunner.js
@@ -1,26 +1,17 @@
-const { promisify } = require('util')
+const { promisify } = require('node:util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const Docker = require('dockerode')
const dockerode = new Docker()
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const async = require('async')
const LockManager = require('./DockerLockManager')
-const fs = require('fs')
-const Path = require('path')
+const Path = require('node:path')
const _ = require('lodash')
const ONE_HOUR_IN_MS = 60 * 60 * 1000
logger.debug('using docker runner')
-function usingSiblingContainers() {
- return (
- Settings != null &&
- Settings.path != null &&
- Settings.path.sandboxedCompilesHostDir != null
- )
-}
-
let containerMonitorTimeout
let containerMonitorInterval
@@ -35,24 +26,6 @@ const DockerRunner = {
compileGroup,
callback
) {
- if (usingSiblingContainers()) {
- const _newPath = Settings.path.sandboxedCompilesHostDir
- logger.debug(
- { path: _newPath },
- 'altering bind path for sibling containers'
- )
- // Server Pro, example:
- // '/var/lib/overleaf/data/compiles/'
- // ... becomes ...
- // '/opt/overleaf_data/data/compiles/'
- directory = Path.join(
- Settings.path.sandboxedCompilesHostDir,
- Path.basename(directory)
- )
- }
-
- const volumes = { [directory]: '/compile' }
-
command = command.map(arg =>
arg.toString().replace('$COMPILE_DIR', '/compile')
)
@@ -72,7 +45,32 @@ const DockerRunner = {
image = `${Settings.texliveImageNameOveride}/${img[2]}`
}
- if (compileGroup === 'synctex' || compileGroup === 'wordcount') {
+ if (compileGroup === 'synctex-output') {
+ // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId'
+ // directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId'
+ // sandboxedCompilesHostDirOutput = '/host/output'
+ // Out: directory = '/host/output/projectId-userId/generated-files/buildId'
+ directory = Path.join(
+ Settings.path.sandboxedCompilesHostDirOutput,
+ ...directory.split('/').slice(-3)
+ )
+ } else {
+ // In: directory = '/overleaf/services/clsi/compiles/projectId-userId'
+ // Path.basename(directory) === 'projectId-userId'
+ // sandboxedCompilesHostDirCompiles = '/host/compiles'
+ // Out: directory = '/host/compiles/projectId-userId'
+ directory = Path.join(
+ Settings.path.sandboxedCompilesHostDirCompiles,
+ Path.basename(directory)
+ )
+ }
+
+ const volumes = { [directory]: '/compile' }
+ if (
+ compileGroup === 'synctex' ||
+ compileGroup === 'synctex-output' ||
+ compileGroup === 'wordcount'
+ ) {
volumes[directory] += ':ro'
}
@@ -234,8 +232,8 @@ const DockerRunner = {
}
}
// set the path based on the image year
- const match = image.match(/:([0-9]+)\.[0-9]+/)
- const year = match ? match[1] : '2014'
+ const match = image.match(/:([0-9]+)\.[0-9]+|:TL([0-9]+)/)
+ const year = match ? match[1] || match[2] : '2014'
env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
const options = {
Cmd: command,
@@ -309,50 +307,17 @@ const DockerRunner = {
LockManager.runWithLock(
options.name,
releaseLock =>
- // Check that volumes exist before starting the container.
- // When a container is started with volume pointing to a
- // non-existent directory then docker creates the directory but
- // with root ownership.
- DockerRunner._checkVolumes(options, volumes, err => {
- if (err != null) {
- return releaseLock(err)
- }
- DockerRunner._startContainer(
- options,
- volumes,
- attachStreamHandler,
- releaseLock
- )
- }),
-
+ DockerRunner._startContainer(
+ options,
+ volumes,
+ attachStreamHandler,
+ releaseLock
+ ),
callback
)
},
// Check that volumes exist and are directories
- _checkVolumes(options, volumes, callback) {
- if (usingSiblingContainers()) {
- // Server Pro, with sibling-containers active, skip checks
- return callback(null)
- }
-
- const checkVolume = (path, cb) =>
- fs.stat(path, (err, stats) => {
- if (err != null) {
- return cb(err)
- }
- if (!stats.isDirectory()) {
- return cb(new Error('not a directory'))
- }
- cb()
- })
- const jobs = []
- for (const vol in volumes) {
- jobs.push(cb => checkVolume(vol, cb))
- }
- async.series(jobs, callback)
- },
-
_startContainer(options, volumes, attachStreamHandler, callback) {
callback = _.once(callback)
const { name } = options
@@ -617,6 +582,10 @@ const DockerRunner = {
containerMonitorInterval = undefined
}
},
+
+ canRunSyncTeXInOutputDir() {
+ return Boolean(Settings.path.sandboxedCompilesHostDirOutput)
+ },
}
DockerRunner.startContainerMonitor()
diff --git a/services/clsi/app/js/DraftModeManager.js b/services/clsi/app/js/DraftModeManager.js
index a5b26348e6..cf8ababc47 100644
--- a/services/clsi/app/js/DraftModeManager.js
+++ b/services/clsi/app/js/DraftModeManager.js
@@ -1,5 +1,5 @@
-const fsPromises = require('fs/promises')
-const { callbackify } = require('util')
+const fsPromises = require('node:fs/promises')
+const { callbackify } = require('node:util')
const logger = require('@overleaf/logger')
async function injectDraftMode(filename) {
diff --git a/services/clsi/app/js/Errors.js b/services/clsi/app/js/Errors.js
index 5c5fd3745a..64c3c7b59a 100644
--- a/services/clsi/app/js/Errors.js
+++ b/services/clsi/app/js/Errors.js
@@ -35,6 +35,7 @@ class QueueLimitReachedError extends OError {}
class TimedOutError extends OError {}
class NoXrefTableError extends OError {}
class TooManyCompileRequestsError extends OError {}
+class InvalidParameter extends OError {}
module.exports = Errors = {
QueueLimitReachedError,
@@ -44,4 +45,5 @@ module.exports = Errors = {
AlreadyCompilingError,
NoXrefTableError,
TooManyCompileRequestsError,
+ InvalidParameter,
}
diff --git a/services/clsi/app/js/LatexRunner.js b/services/clsi/app/js/LatexRunner.js
index d956ee4949..beefa002ab 100644
--- a/services/clsi/app/js/LatexRunner.js
+++ b/services/clsi/app/js/LatexRunner.js
@@ -1,9 +1,9 @@
-const Path = require('path')
-const { promisify } = require('util')
+const Path = require('node:path')
+const { promisify } = require('node:util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const CommandRunner = require('./CommandRunner')
-const fs = require('fs')
+const fs = require('node:fs')
const ProcessTable = {} // table of currently running jobs (pids or docker container names)
diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js
index d909ec601c..aa62825443 100644
--- a/services/clsi/app/js/LocalCommandRunner.js
+++ b/services/clsi/app/js/LocalCommandRunner.js
@@ -12,8 +12,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CommandRunner
-const { spawn } = require('child_process')
-const { promisify } = require('util')
+const { spawn } = require('node:child_process')
+const { promisify } = require('node:util')
const _ = require('lodash')
const logger = require('@overleaf/logger')
@@ -54,6 +54,7 @@ module.exports = CommandRunner = {
cwd: directory,
env,
stdio: ['pipe', 'pipe', 'ignore'],
+ detached: true,
})
let stdout = ''
@@ -99,6 +100,10 @@ module.exports = CommandRunner = {
}
return callback()
},
+
+ canRunSyncTeXInOutputDir() {
+ return true
+ },
}
module.exports.promises = {
diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js
index 13aad1bc74..a1a0a89aa7 100644
--- a/services/clsi/app/js/OutputCacheManager.js
+++ b/services/clsi/app/js/OutputCacheManager.js
@@ -1,12 +1,12 @@
let OutputCacheManager
-const { callbackify, promisify } = require('util')
+const { callbackify, promisify } = require('node:util')
const async = require('async')
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
const logger = require('@overleaf/logger')
const _ = require('lodash')
const Settings = require('@overleaf/settings')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const Metrics = require('./Metrics')
const OutputFileOptimiser = require('./OutputFileOptimiser')
@@ -83,6 +83,13 @@ async function cleanupDirectory(dir, options) {
})
}
+/**
+ * @template T
+ *
+ * @param {string} dir
+ * @param {() => Promise} fn
+ * @return {Promise}
+ */
async function queueDirOperation(dir, fn) {
const pending = PENDING_PROJECT_ACTIONS.get(dir) || Promise.resolve()
const p = pending.then(fn, fn).finally(() => {
@@ -98,12 +105,11 @@ module.exports = OutputCacheManager = {
CONTENT_SUBDIR: 'content',
CACHE_SUBDIR: 'generated-files',
ARCHIVE_SUBDIR: 'archived-logs',
- // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
- // for backwards compatibility, make the randombytes part optional
- BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
- CONTENT_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
+ // build id is HEXDATE-HEXRANDOM from Date.now() and RandomBytes
+ BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
+ CONTENT_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
CACHE_LIMIT: 2, // maximum number of cache directories
- CACHE_AGE: 60 * 60 * 1000, // up to one hour old
+ CACHE_AGE: 90 * 60 * 1000, // up to 90 minutes old
init,
queueDirOperation: callbackify(queueDirOperation),
@@ -137,7 +143,11 @@ module.exports = OutputCacheManager = {
outputDir,
callback
) {
- OutputCacheManager.generateBuildId(function (err, buildId) {
+ const getBuildId = cb => {
+ if (request.buildId) return cb(null, request.buildId)
+ OutputCacheManager.generateBuildId(cb)
+ }
+ getBuildId(function (err, buildId) {
if (err) {
return callback(err)
}
@@ -245,7 +255,7 @@ module.exports = OutputCacheManager = {
{ err, directory: cacheDir },
'error creating cache directory'
)
- callback(err, outputFiles)
+ callback(err)
} else {
// copy all the output files into the new cache directory
const results = []
@@ -263,7 +273,6 @@ module.exports = OutputCacheManager = {
return cb()
}
// copy other files into cache directory if valid
- const newFile = _.clone(file)
const src = Path.join(compileDir, file.path)
const dst = Path.join(cacheDir, file.path)
OutputCacheManager._checkIfShouldCopy(
@@ -279,8 +288,8 @@ module.exports = OutputCacheManager = {
if (err) {
return cb(err)
}
- newFile.build = buildId // attach a build id if we cached the file
- results.push(newFile)
+ file.build = buildId
+ results.push(file)
cb()
})
}
@@ -288,8 +297,7 @@ module.exports = OutputCacheManager = {
},
function (err) {
if (err) {
- // pass back the original files if we encountered *any* error
- callback(err, outputFiles)
+ callback(err)
// clean up the directory we just created
fs.rm(cacheDir, { force: true, recursive: true }, function (err) {
if (err) {
@@ -301,7 +309,7 @@ module.exports = OutputCacheManager = {
})
} else {
// pass back the list of new files in the cache
- callback(err, results)
+ callback(null, results)
// let file expiry run in the background, expire all previous files if per-user
cleanupDirectory(outputDir, {
keep: buildId,
@@ -676,4 +684,5 @@ OutputCacheManager.promises = {
saveOutputFilesInBuildDir: promisify(
OutputCacheManager.saveOutputFilesInBuildDir
),
+ queueDirOperation,
}
diff --git a/services/clsi/app/js/OutputFileArchiveManager.js b/services/clsi/app/js/OutputFileArchiveManager.js
index 3c5a6c8197..64c5198392 100644
--- a/services/clsi/app/js/OutputFileArchiveManager.js
+++ b/services/clsi/app/js/OutputFileArchiveManager.js
@@ -7,7 +7,7 @@ const { NotFoundError } = require('./Errors')
const logger = require('@overleaf/logger')
// NOTE: Updating this list requires a corresponding change in
-// * services/web/frontend/js/features/pdf-preview/util/file-list.js
+// * services/web/frontend/js/features/pdf-preview/util/file-list.ts
const ignoreFiles = ['output.fls', 'output.fdb_latexmk']
function getContentDir(projectId, userId) {
@@ -93,8 +93,11 @@ module.exports = {
)
return outputFiles.filter(
- // Ignore the pdf and also ignore the files ignored by the frontend.
- ({ path }) => path !== 'output.pdf' && !ignoreFiles.includes(path)
+ // Ignore the pdf, clsi-cache tar-ball and also ignore the files ignored by the frontend.
+ ({ path }) =>
+ path !== 'output.pdf' &&
+ path !== 'output.tar.gz' &&
+ !ignoreFiles.includes(path)
)
} catch (error) {
if (
diff --git a/services/clsi/app/js/OutputFileFinder.js b/services/clsi/app/js/OutputFileFinder.js
index 8ca13183dc..e62038c614 100644
--- a/services/clsi/app/js/OutputFileFinder.js
+++ b/services/clsi/app/js/OutputFileFinder.js
@@ -1,5 +1,5 @@
-const Path = require('path')
-const fs = require('fs')
+const Path = require('node:path')
+const fs = require('node:fs')
const { callbackifyMultiResult } = require('@overleaf/promise-utils')
async function walkFolder(compileDir, d, files, allEntries) {
diff --git a/services/clsi/app/js/OutputFileOptimiser.js b/services/clsi/app/js/OutputFileOptimiser.js
index c97ef7ffdc..09ca98672d 100644
--- a/services/clsi/app/js/OutputFileOptimiser.js
+++ b/services/clsi/app/js/OutputFileOptimiser.js
@@ -13,9 +13,9 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputFileOptimiser
-const fs = require('fs')
-const Path = require('path')
-const { spawn } = require('child_process')
+const fs = require('node:fs')
+const Path = require('node:path')
+const { spawn } = require('node:child_process')
const logger = require('@overleaf/logger')
const Metrics = require('./Metrics')
const _ = require('lodash')
@@ -74,9 +74,7 @@ module.exports = OutputFileOptimiser = {
logger.debug({ args }, 'running qpdf command')
const timer = new Metrics.Timer('qpdf')
- const proc = spawn('qpdf', args)
- let stdout = ''
- proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
+ const proc = spawn('qpdf', args, { stdio: 'ignore' })
callback = _.once(callback) // avoid double call back for error and close event
proc.on('error', function (err) {
logger.warn({ err, args }, 'qpdf failed')
diff --git a/services/clsi/app/js/ProjectPersistenceManager.js b/services/clsi/app/js/ProjectPersistenceManager.js
index 4dbe8b636e..41cdd07f4d 100644
--- a/services/clsi/app/js/ProjectPersistenceManager.js
+++ b/services/clsi/app/js/ProjectPersistenceManager.js
@@ -13,47 +13,90 @@ const CompileManager = require('./CompileManager')
const async = require('async')
const logger = require('@overleaf/logger')
const oneDay = 24 * 60 * 60 * 1000
+const Metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
-const diskusage = require('diskusage')
-const { callbackify } = require('util')
-const Path = require('path')
-const fs = require('fs')
+const { callbackify } = require('node:util')
+const Path = require('node:path')
+const fs = require('node:fs')
// projectId -> timestamp mapping.
const LAST_ACCESS = new Map()
-async function refreshExpiryTimeout() {
+let ANY_DISK_LOW = false
+let ANY_DISK_CRITICAL_LOW = false
+
+async function collectDiskStats() {
const paths = [
Settings.path.compilesDir,
Settings.path.outputDir,
Settings.path.clsiCacheDir,
]
+
+ const diskStats = {}
+ let anyDiskLow = false
+ let anyDiskCriticalLow = false
for (const path of paths) {
try {
- const stats = await diskusage.check(path)
- const lowDisk = stats.available / stats.total < 0.1
-
- const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
- if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
- logger.warn(
- {
- stats,
- newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
- },
- 'disk running low on space, modifying EXPIRY_TIMEOUT'
- )
- ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
- break
+ const { blocks, bavail, bsize } = await fs.promises.statfs(path)
+ const stats = {
+ // Warning: these values will be wrong by a factor in Docker-for-Mac.
+ // See https://github.com/docker/for-mac/issues/2136
+ total: blocks * bsize, // Total size of the file system in bytes
+ available: bavail * bsize, // Free space available to unprivileged users.
}
+ const diskAvailablePercent = (stats.available / stats.total) * 100
+ Metrics.gauge('disk_available_percent', diskAvailablePercent, 1, {
+ path,
+ })
+ const lowDisk = diskAvailablePercent < 10
+ diskStats[path] = { stats, lowDisk }
+
+ const criticalLowDisk = diskAvailablePercent < 3
+ anyDiskLow = anyDiskLow || lowDisk
+ anyDiskCriticalLow = anyDiskCriticalLow || criticalLowDisk
} catch (err) {
logger.err({ err, path }, 'error getting disk usage')
}
}
+ ANY_DISK_LOW = anyDiskLow
+ ANY_DISK_CRITICAL_LOW = anyDiskCriticalLow
+ return diskStats
+}
+
+async function refreshExpiryTimeout() {
+ for (const [path, { stats, lowDisk }] of Object.entries(
+ await collectDiskStats()
+ )) {
+ const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+ if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+ logger.warn(
+ {
+ path,
+ stats,
+ newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
+ },
+ 'disk running low on space, modifying EXPIRY_TIMEOUT'
+ )
+ ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+ break
+ }
+ }
+ Metrics.gauge(
+ 'project_persistence_expiry_timeout',
+ ProjectPersistenceManager.EXPIRY_TIMEOUT
+ )
}
module.exports = ProjectPersistenceManager = {
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
+ isAnyDiskLow() {
+ return ANY_DISK_LOW
+ },
+ isAnyDiskCriticalLow() {
+ return ANY_DISK_CRITICAL_LOW
+ },
+
promises: {
refreshExpiryTimeout,
},
@@ -103,6 +146,13 @@ module.exports = ProjectPersistenceManager = {
}
)
})
+
+ // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter) or every 5th scrape of the load agent (3s +- jitter).
+ setInterval(() => {
+ collectDiskStats().catch(err => {
+ logger.err({ err }, 'low level error collecting disk stats')
+ })
+ }, 15_000)
},
markProjectAsJustAccessed(projectId, callback) {
diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js
index 61d3b9d229..4e9d722921 100644
--- a/services/clsi/app/js/RequestParser.js
+++ b/services/clsi/app/js/RequestParser.js
@@ -1,7 +1,9 @@
const settings = require('@overleaf/settings')
+const OutputCacheManager = require('./OutputCacheManager')
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
const MAX_TIMEOUT = 600
+const EDITOR_ID_REGEX = /^[a-f0-9-]{36}$/ // UUID
function parse(body, callback) {
const response = {}
@@ -27,12 +29,24 @@ function parse(body, callback) {
default: '',
type: 'string',
}),
+ // Will be populated later. Must always be populated for prom library.
+ compile: 'initial',
}
response.compiler = _parseAttribute('compiler', compile.options.compiler, {
validValues: VALID_COMPILERS,
default: 'pdflatex',
type: 'string',
})
+ response.compileFromClsiCache = _parseAttribute(
+ 'compileFromClsiCache',
+ compile.options.compileFromClsiCache,
+ { default: false, type: 'boolean' }
+ )
+ response.populateClsiCache = _parseAttribute(
+ 'populateClsiCache',
+ compile.options.populateClsiCache,
+ { default: false, type: 'boolean' }
+ )
response.enablePdfCaching = _parseAttribute(
'enablePdfCaching',
compile.options.enablePdfCaching,
@@ -135,6 +149,15 @@ function parse(body, callback) {
}
)
response.rootResourcePath = _checkPath(rootResourcePath)
+
+ response.editorId = _parseAttribute('editorId', compile.options.editorId, {
+ type: 'string',
+ regex: EDITOR_ID_REGEX,
+ })
+ response.buildId = _parseAttribute('buildId', compile.options.buildId, {
+ type: 'string',
+ regex: OutputCacheManager.BUILD_REGEX,
+ })
} catch (error1) {
const error = error1
return callback(error)
@@ -169,11 +192,15 @@ function _parseResource(resource) {
if (resource.url != null && typeof resource.url !== 'string') {
throw new Error('url attribute should be a string')
}
+ if (resource.fallbackURL && typeof resource.fallbackURL !== 'string') {
+ throw new Error('fallbackURL attribute should be a string')
+ }
return {
path: resource.path,
modified,
url: resource.url,
+ fallbackURL: resource.fallbackURL,
content: resource.content,
}
}
@@ -195,6 +222,13 @@ function _parseAttribute(name, attribute, options) {
throw new Error(`${name} attribute should be a ${options.type}`)
}
}
+ if (options.type === 'string' && options.regex instanceof RegExp) {
+ if (!options.regex.test(attribute)) {
+ throw new Error(
+ `${name} attribute does not match regex ${options.regex}`
+ )
+ }
+ }
} else {
if (options.default != null) {
return options.default
diff --git a/services/clsi/app/js/ResourceStateManager.js b/services/clsi/app/js/ResourceStateManager.js
index dbfb3c9fc4..a5f747e1cd 100644
--- a/services/clsi/app/js/ResourceStateManager.js
+++ b/services/clsi/app/js/ResourceStateManager.js
@@ -1,5 +1,5 @@
-const Path = require('path')
-const fs = require('fs')
+const Path = require('node:path')
+const fs = require('node:fs')
const logger = require('@overleaf/logger')
const Errors = require('./Errors')
const SafeReader = require('./SafeReader')
diff --git a/services/clsi/app/js/ResourceWriter.js b/services/clsi/app/js/ResourceWriter.js
index 11fb4500f4..bf88538746 100644
--- a/services/clsi/app/js/ResourceWriter.js
+++ b/services/clsi/app/js/ResourceWriter.js
@@ -13,10 +13,10 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ResourceWriter
-const { promisify } = require('util')
+const { promisify } = require('node:util')
const UrlCache = require('./UrlCache')
-const Path = require('path')
-const fs = require('fs')
+const Path = require('node:path')
+const fs = require('node:fs')
const async = require('async')
const OutputFileFinder = require('./OutputFileFinder')
const ResourceStateManager = require('./ResourceStateManager')
@@ -200,73 +200,22 @@ module.exports = ResourceWriter = {
return OutputFileFinder.findOutputFiles(
resources,
basePath,
- function (error, outputFiles, allFiles) {
+ (error, outputFiles, allFiles) => {
if (error != null) {
return callback(error)
}
const jobs = []
- for (const file of Array.from(outputFiles || [])) {
- ;(function (file) {
- const { path } = file
- let shouldDelete = true
- if (
- path.match(/^output\./) ||
- path.match(/\.aux$/) ||
- path.match(/^cache\//)
- ) {
- // knitr cache
- shouldDelete = false
- }
- if (path.match(/^output-.*/)) {
- // Tikz cached figures (default case)
- shouldDelete = false
- }
- if (path.match(/\.(pdf|dpth|md5)$/)) {
- // Tikz cached figures (by extension)
- shouldDelete = false
- }
- if (
- path.match(/\.(pygtex|pygstyle)$/) ||
- path.match(/(^|\/)_minted-[^\/]+\//)
- ) {
- // minted files/directory
- shouldDelete = false
- }
- if (
- path.match(/\.md\.tex$/) ||
- path.match(/(^|\/)_markdown_[^\/]+\//)
- ) {
- // markdown files/directory
- shouldDelete = false
- }
- if (path.match(/-eps-converted-to\.pdf$/)) {
- // Epstopdf generated files
- shouldDelete = false
- }
- if (
- path === 'output.pdf' ||
- path === 'output.dvi' ||
- path === 'output.log' ||
- path === 'output.xdv' ||
- path === 'output.stdout' ||
- path === 'output.stderr'
- ) {
- shouldDelete = true
- }
- if (path === 'output.tex') {
- // created by TikzManager if present in output files
- shouldDelete = true
- }
- if (shouldDelete) {
- return jobs.push(callback =>
- ResourceWriter._deleteFileIfNotDirectory(
- Path.join(basePath, path),
- callback
- )
+ for (const { path } of outputFiles || []) {
+ const shouldDelete = ResourceWriter.isExtraneousFile(path)
+ if (shouldDelete) {
+ jobs.push(callback =>
+ ResourceWriter._deleteFileIfNotDirectory(
+ Path.join(basePath, path),
+ callback
)
- }
- })(file)
+ )
+ }
}
return async.series(jobs, function (error) {
@@ -279,6 +228,59 @@ module.exports = ResourceWriter = {
)
},
+ isExtraneousFile(path) {
+ let shouldDelete = true
+ if (
+ path.match(/^output\./) ||
+ path.match(/\.aux$/) ||
+ path.match(/^cache\//)
+ ) {
+ // knitr cache
+ shouldDelete = false
+ }
+ if (path.match(/^output-.*/)) {
+ // Tikz cached figures (default case)
+ shouldDelete = false
+ }
+ if (path.match(/\.(pdf|dpth|md5)$/)) {
+ // Tikz cached figures (by extension)
+ shouldDelete = false
+ }
+ if (
+ path.match(/\.(pygtex|pygstyle)$/) ||
+ path.match(/(^|\/)_minted-[^\/]+\//)
+ ) {
+ // minted files/directory
+ shouldDelete = false
+ }
+ if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) {
+ // markdown files/directory
+ shouldDelete = false
+ }
+ if (path.match(/-eps-converted-to\.pdf$/)) {
+ // Epstopdf generated files
+ shouldDelete = false
+ }
+ if (
+ path === 'output.tar.gz' ||
+ path === 'output.synctex.gz' ||
+ path === 'output.pdfxref' ||
+ path === 'output.pdf' ||
+ path === 'output.dvi' ||
+ path === 'output.log' ||
+ path === 'output.xdv' ||
+ path === 'output.stdout' ||
+ path === 'output.stderr'
+ ) {
+ shouldDelete = true
+ }
+ if (path === 'output.tex') {
+ // created by TikzManager if present in output files
+ shouldDelete = true
+ }
+ return shouldDelete
+ },
+
_deleteFileIfNotDirectory(path, callback) {
if (callback == null) {
callback = function () {}
@@ -333,6 +335,7 @@ module.exports = ResourceWriter = {
return UrlCache.downloadUrlToFile(
projectId,
resource.url,
+ resource.fallbackURL,
path,
resource.modified,
function (err) {
diff --git a/services/clsi/app/js/SafeReader.js b/services/clsi/app/js/SafeReader.js
index 756747af47..8b1b5abb54 100644
--- a/services/clsi/app/js/SafeReader.js
+++ b/services/clsi/app/js/SafeReader.js
@@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let SafeReader
-const fs = require('fs')
+const fs = require('node:fs')
const logger = require('@overleaf/logger')
module.exports = SafeReader = {
diff --git a/services/clsi/app/js/StaticServerForbidSymlinks.js b/services/clsi/app/js/StaticServerForbidSymlinks.js
index 219408eb11..a5ec774396 100644
--- a/services/clsi/app/js/StaticServerForbidSymlinks.js
+++ b/services/clsi/app/js/StaticServerForbidSymlinks.js
@@ -13,8 +13,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ForbidSymlinks
-const Path = require('path')
-const fs = require('fs')
+const Path = require('node:path')
+const fs = require('node:fs')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
diff --git a/services/clsi/app/js/SynctexOutputParser.js b/services/clsi/app/js/SynctexOutputParser.js
index e54add65ae..5b2d237825 100644
--- a/services/clsi/app/js/SynctexOutputParser.js
+++ b/services/clsi/app/js/SynctexOutputParser.js
@@ -1,4 +1,4 @@
-const Path = require('path')
+const Path = require('node:path')
/**
* Parse output from the `synctex view` command
diff --git a/services/clsi/app/js/TikzManager.js b/services/clsi/app/js/TikzManager.js
index 7d5f6c1b81..ca9db6b005 100644
--- a/services/clsi/app/js/TikzManager.js
+++ b/services/clsi/app/js/TikzManager.js
@@ -11,9 +11,9 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let TikzManager
-const fs = require('fs')
-const Path = require('path')
-const { promisify } = require('util')
+const fs = require('node:fs')
+const Path = require('node:path')
+const { promisify } = require('node:util')
const ResourceWriter = require('./ResourceWriter')
const SafeReader = require('./SafeReader')
const logger = require('@overleaf/logger')
diff --git a/services/clsi/app/js/UrlCache.js b/services/clsi/app/js/UrlCache.js
index ae8eae6b2a..36703e7091 100644
--- a/services/clsi/app/js/UrlCache.js
+++ b/services/clsi/app/js/UrlCache.js
@@ -12,9 +12,9 @@
*/
const UrlFetcher = require('./UrlFetcher')
const Settings = require('@overleaf/settings')
-const fs = require('fs')
-const Path = require('path')
-const { callbackify } = require('util')
+const fs = require('node:fs')
+const Path = require('node:path')
+const { callbackify } = require('node:util')
const Metrics = require('./Metrics')
const PENDING_DOWNLOADS = new Map()
@@ -47,14 +47,29 @@ async function createProjectDir(projectId) {
await fs.promises.mkdir(getProjectDir(projectId), { recursive: true })
}
-async function downloadUrlToFile(projectId, url, destPath, lastModified) {
+async function downloadUrlToFile(
+ projectId,
+ url,
+ fallbackURL,
+ destPath,
+ lastModified
+) {
const cachePath = getCachePath(projectId, url, lastModified)
try {
const timer = new Metrics.Timer('url_cache', {
status: 'cache-hit',
path: 'copy',
})
- await fs.promises.copyFile(cachePath, destPath)
+ try {
+ await fs.promises.copyFile(cachePath, destPath)
+ } catch (err) {
+ if (err.code === 'ENOENT' && fallbackURL) {
+ const fallbackPath = getCachePath(projectId, fallbackURL, lastModified)
+ await fs.promises.copyFile(fallbackPath, destPath)
+ } else {
+ throw err
+ }
+ }
// the metric is only updated if the file is present in the cache
timer.done()
return
@@ -70,7 +85,7 @@ async function downloadUrlToFile(projectId, url, destPath, lastModified) {
path: 'download',
})
try {
- await download(url, cachePath)
+ await download(url, fallbackURL, cachePath)
} finally {
timer.done()
}
@@ -86,13 +101,17 @@ async function downloadUrlToFile(projectId, url, destPath, lastModified) {
}
}
-async function download(url, cachePath) {
+async function download(url, fallbackURL, cachePath) {
let pending = PENDING_DOWNLOADS.get(cachePath)
if (pending) {
return pending
}
- pending = UrlFetcher.promises.pipeUrlToFileWithRetry(url, cachePath)
+ pending = UrlFetcher.promises.pipeUrlToFileWithRetry(
+ url,
+ fallbackURL,
+ cachePath
+ )
PENDING_DOWNLOADS.set(cachePath, pending)
try {
await pending
diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js
index 8d604d267b..2c44f3a6dd 100644
--- a/services/clsi/app/js/UrlFetcher.js
+++ b/services/clsi/app/js/UrlFetcher.js
@@ -1,20 +1,21 @@
-const fs = require('fs')
+const fs = require('node:fs')
const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
const {
CustomHttpAgent,
CustomHttpsAgent,
fetchStream,
+ RequestFailedError,
} = require('@overleaf/fetch-utils')
-const { URL } = require('url')
-const { pipeline } = require('stream/promises')
+const { URL } = require('node:url')
+const { pipeline } = require('node:stream/promises')
const Metrics = require('./Metrics')
const MAX_CONNECT_TIME = 1000
const httpAgent = new CustomHttpAgent({ connectTimeout: MAX_CONNECT_TIME })
const httpsAgent = new CustomHttpsAgent({ connectTimeout: MAX_CONNECT_TIME })
-async function pipeUrlToFileWithRetry(url, filePath) {
+async function pipeUrlToFileWithRetry(url, fallbackURL, filePath) {
let remainingAttempts = 3
let lastErr
while (remainingAttempts-- > 0) {
@@ -22,7 +23,7 @@ async function pipeUrlToFileWithRetry(url, filePath) {
path: lastErr ? ' retry' : 'fetch',
})
try {
- await pipeUrlToFile(url, filePath)
+ await pipeUrlToFile(url, fallbackURL, filePath)
timer.done({ status: 'success' })
return
} catch (err) {
@@ -37,7 +38,7 @@ async function pipeUrlToFileWithRetry(url, filePath) {
throw lastErr
}
-async function pipeUrlToFile(url, filePath) {
+async function pipeUrlToFile(url, fallbackURL, filePath) {
const u = new URL(url)
if (
Settings.filestoreDomainOveride &&
@@ -45,21 +46,55 @@ async function pipeUrlToFile(url, filePath) {
) {
url = `${Settings.filestoreDomainOveride}${u.pathname}${u.search}`
}
+ if (fallbackURL) {
+ const u2 = new URL(fallbackURL)
+ if (
+ Settings.filestoreDomainOveride &&
+ u2.host !== Settings.apis.clsiPerf.host
+ ) {
+ fallbackURL = `${Settings.filestoreDomainOveride}${u2.pathname}${u2.search}`
+ }
+ }
- const stream = await fetchStream(url, {
- signal: AbortSignal.timeout(60 * 1000),
- // provide a function to get the agent for each request
- // as there may be multiple requests with different protocols
- // due to redirects.
- agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
- })
+ let stream
+ try {
+ stream = await fetchStream(url, {
+ signal: AbortSignal.timeout(60 * 1000),
+ // provide a function to get the agent for each request
+ // as there may be multiple requests with different protocols
+ // due to redirects.
+ agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
+ })
+ } catch (err) {
+ if (
+ fallbackURL &&
+ err instanceof RequestFailedError &&
+ err.response.status === 404
+ ) {
+ stream = await fetchStream(fallbackURL, {
+ signal: AbortSignal.timeout(60 * 1000),
+ // provide a function to get the agent for each request
+ // as there may be multiple requests with different protocols
+ // due to redirects.
+ agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
+ })
+ url = fallbackURL
+ } else {
+ throw err
+ }
+ }
+
+ const source = inferSource(url)
+ Metrics.inc('url_source', 1, { path: source })
const atomicWrite = filePath + '~'
try {
const output = fs.createWriteStream(atomicWrite)
await pipeline(stream, output)
await fs.promises.rename(atomicWrite, filePath)
- Metrics.count('UrlFetcher.downloaded_bytes', output.bytesWritten)
+ Metrics.count('UrlFetcher.downloaded_bytes', output.bytesWritten, {
+ path: source,
+ })
} catch (err) {
try {
await fs.promises.unlink(atomicWrite)
@@ -68,6 +103,20 @@ async function pipeUrlToFile(url, filePath) {
}
}
+const BUCKET_REGEX = /\/bucket\/([^/]+)\/key\//
+
+function inferSource(url) {
+ if (url.includes(Settings.apis.clsiPerf.host)) {
+ return 'clsi-perf'
+ } else if (url.includes('/project/') && url.includes('/file/')) {
+ return 'user-files'
+ } else if (url.includes('/key/')) {
+ const match = url.match(BUCKET_REGEX)
+ if (match) return match[1]
+ }
+ return 'unknown'
+}
+
module.exports.promises = {
pipeUrlToFileWithRetry,
}
diff --git a/services/clsi/app/js/XrefParser.js b/services/clsi/app/js/XrefParser.js
index 76636acfe9..5f2d154679 100644
--- a/services/clsi/app/js/XrefParser.js
+++ b/services/clsi/app/js/XrefParser.js
@@ -1,5 +1,5 @@
const { NoXrefTableError } = require('./Errors')
-const fs = require('fs')
+const fs = require('node:fs')
const { O_RDONLY, O_NOFOLLOW } = fs.constants
const MAX_XREF_FILE_SIZE = 1024 * 1024
diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt
index 52113b47e0..09c21888df 100644
--- a/services/clsi/buildscript.txt
+++ b/services/clsi/buildscript.txt
@@ -1,11 +1,11 @@
clsi
--data-dirs=cache,compiles,output
--dependencies=
---docker-repos=gcr.io/overleaf-ops,us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles
+--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=True
---script-version=4.5.0
+--script-version=4.7.0
--use-large-ci-runner=True
diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js
index 934ef6d8c8..bd5614eb98 100644
--- a/services/clsi/config/settings.defaults.js
+++ b/services/clsi/config/settings.defaults.js
@@ -1,7 +1,8 @@
-const Path = require('path')
-const os = require('os')
+const Path = require('node:path')
+const os = require('node:os')
-const isPreEmptible = os.hostname().includes('pre-emp')
+const isPreEmptible = process.env.PREEMPTIBLE === 'TRUE'
+const CLSI_SERVER_ID = os.hostname().replace('-ctr', '')
module.exports = {
compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
@@ -33,6 +34,10 @@ module.exports = {
report_load: process.env.LOAD_BALANCER_AGENT_REPORT_LOAD !== 'false',
load_port: 3048,
local_port: 3049,
+ allow_maintenance:
+ (
+ process.env.LOAD_BALANCER_AGENT_ALLOW_MAINTENANCE ?? ''
+ ).toLowerCase() !== 'false',
},
},
apis: {
@@ -41,12 +46,19 @@ module.exports = {
url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`,
// External url prefix for output files, e.g. for requests via load-balancers.
outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`,
+ clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID,
+
+ downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:3013',
},
clsiPerf: {
host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${
process.env.CLSI_PERF_PORT || '3043'
}`,
},
+ clsiCache: {
+ enabled: !!process.env.CLSI_CACHE_SHARDS,
+ shards: JSON.parse(process.env.CLSI_CACHE_SHARDS || '[]'),
+ },
},
smokeTest: process.env.SMOKE_TEST || false,
@@ -56,10 +68,6 @@ module.exports = {
texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
texliveMaxPrintLine: process.env.TEXLIVE_MAX_PRINT_LINE,
- sentry: {
- dsn: process.env.SENTRY_DSN,
- },
-
enablePdfCaching: process.env.ENABLE_PDF_CACHING === 'true',
enablePdfCachingDark: process.env.ENABLE_PDF_CACHING_DARK === 'true',
pdfCachingMinChunkSize:
@@ -85,20 +93,21 @@ if (process.env.ALLOWED_COMPILE_GROUPS) {
}
}
-if (process.env.DOCKER_RUNNER) {
- let seccompProfilePath
+if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
module.exports.clsi = {
- dockerRunner: process.env.DOCKER_RUNNER === 'true',
+ dockerRunner: true,
docker: {
runtime: process.env.DOCKER_RUNTIME,
image:
- process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
+ process.env.TEXLIVE_IMAGE ||
+ process.env.TEX_LIVE_DOCKER_IMAGE ||
+ 'quay.io/sharelatex/texlive-full:2017.1',
env: {
HOME: '/tmp',
CLSI: 1,
},
socketPath: '/var/run/docker.sock',
- user: process.env.TEXLIVE_IMAGE_USER || 'tex',
+ user: process.env.TEXLIVE_IMAGE_USER || 'www-data',
},
optimiseInDocker: true,
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
@@ -118,6 +127,7 @@ if (process.env.DOCKER_RUNNER) {
const defaultCompileGroupConfig = {
wordcount: { 'HostConfig.AutoRemove': true },
synctex: { 'HostConfig.AutoRemove': true },
+ 'synctex-output': { 'HostConfig.AutoRemove': true },
}
module.exports.clsi.docker.compileGroupConfig = Object.assign(
defaultCompileGroupConfig,
@@ -128,11 +138,14 @@ if (process.env.DOCKER_RUNNER) {
process.exit(1)
}
+ let seccompProfilePath
try {
seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
- module.exports.clsi.docker.seccomp_profile = JSON.stringify(
- JSON.parse(require('fs').readFileSync(seccompProfilePath))
- )
+ module.exports.clsi.docker.seccomp_profile =
+ process.env.SECCOMP_PROFILE ||
+ JSON.stringify(
+ JSON.parse(require('node:fs').readFileSync(seccompProfilePath))
+ )
} catch (error) {
console.error(
error,
@@ -162,5 +175,23 @@ if (process.env.DOCKER_RUNNER) {
module.exports.path.synctexBaseDir = () => '/compile'
- module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
+ module.exports.path.sandboxedCompilesHostDirCompiles =
+ process.env.SANDBOXED_COMPILES_HOST_DIR_COMPILES ||
+ process.env.SANDBOXED_COMPILES_HOST_DIR ||
+ process.env.COMPILES_HOST_DIR
+ if (!module.exports.path.sandboxedCompilesHostDirCompiles) {
+ throw new Error(
+ 'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_COMPILES not set'
+ )
+ }
+
+ module.exports.path.sandboxedCompilesHostDirOutput =
+ process.env.SANDBOXED_COMPILES_HOST_DIR_OUTPUT ||
+ process.env.OUTPUT_HOST_DIR
+ if (!module.exports.path.sandboxedCompilesHostDirOutput) {
+ // TODO(das7pad): Enforce in a future major version of Server Pro.
+ // throw new Error(
+ // 'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_OUTPUT not set'
+ // )
+ }
}
diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml
index 00f54c6e72..77a45615b7 100644
--- a/services/clsi/docker-compose.ci.yml
+++ b/services/clsi/docker-compose.ci.yml
@@ -27,10 +27,11 @@ services:
PDF_CACHING_ENABLE_WORKER_POOL: "true"
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
- TEX_LIVE_IMAGE_NAME_OVERRIDE: gcr.io/overleaf-ops
+ TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
TEXLIVE_IMAGE_USER: "tex"
- DOCKER_RUNNER: "true"
- COMPILES_HOST_DIR: $PWD/compiles
+ SANDBOXED_COMPILES: "true"
+ SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
+ SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
volumes:
- ./compiles:/overleaf/services/clsi/compiles
- /var/run/docker.sock:/var/run/docker.sock
diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml
index c72fb8b2c4..b8112a8e17 100644
--- a/services/clsi/docker-compose.yml
+++ b/services/clsi/docker-compose.yml
@@ -17,6 +17,7 @@ services:
working_dir: /overleaf/services/clsi
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
@@ -37,16 +38,17 @@ services:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_PDF_CACHING: "true"
PDF_CACHING_ENABLE_WORKER_POOL: "true"
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
- TEX_LIVE_IMAGE_NAME_OVERRIDE: gcr.io/overleaf-ops
+ TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
TEXLIVE_IMAGE_USER: "tex"
- DOCKER_RUNNER: "true"
- COMPILES_HOST_DIR: $PWD/compiles
+ SANDBOXED_COMPILES: "true"
+ SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
+ SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
command: npm run --silent test:acceptance
diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh
index 9446ab9e2d..b45899ab17 100755
--- a/services/clsi/entrypoint.sh
+++ b/services/clsi/entrypoint.sh
@@ -2,13 +2,12 @@
# add the node user to the docker group on the host
DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock)
-groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost
+groupadd --non-unique --gid "${DOCKER_GROUP}" dockeronhost
usermod -aG dockeronhost node
# compatibility: initial volume setup
mkdir -p /overleaf/services/clsi/cache && chown node:node /overleaf/services/clsi/cache
mkdir -p /overleaf/services/clsi/compiles && chown node:node /overleaf/services/clsi/compiles
-mkdir -p /overleaf/services/clsi/db && chown node:node /overleaf/services/clsi/db
mkdir -p /overleaf/services/clsi/output && chown node:node /overleaf/services/clsi/output
exec runuser -u node -- "$@"
diff --git a/services/clsi/kube.yaml b/services/clsi/kube.yaml
deleted file mode 100644
index d3fb04291e..0000000000
--- a/services/clsi/kube.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: clsi
- namespace: default
-spec:
- type: LoadBalancer
- ports:
- - port: 80
- protocol: TCP
- targetPort: 80
- selector:
- run: clsi
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: clsi
- namespace: default
-spec:
- replicas: 2
- template:
- metadata:
- labels:
- run: clsi
- spec:
- containers:
- - name: clsi
- image: gcr.io/henry-terraform-admin/clsi
- imagePullPolicy: Always
- readinessProbe:
- httpGet:
- path: status
- port: 80
- periodSeconds: 5
- initialDelaySeconds: 0
- failureThreshold: 3
- successThreshold: 1
-
-
-
diff --git a/services/clsi/nginx.conf b/services/clsi/nginx.conf
index 2290aeb444..604eb93fbf 100644
--- a/services/clsi/nginx.conf
+++ b/services/clsi/nginx.conf
@@ -46,7 +46,7 @@ server {
}
# handle output files for specific users
- location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+ location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD';
@@ -64,7 +64,7 @@ server {
alias /output/$1-$2/generated-files/$3/$4.blg;
}
# handle output files for anonymous users
- location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
+ location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD';
diff --git a/services/clsi/package.json b/services/clsi/package.json
index 8df44c7150..fe31c430bd 100644
--- a/services/clsi/package.json
+++ b/services/clsi/package.json
@@ -11,8 +11,8 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -23,24 +23,24 @@
"@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/settings": "*",
+ "@overleaf/stream-utils": "*",
"archiver": "5.3.2",
"async": "^3.2.5",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "diskusage": "^1.1.3",
- "dockerode": "^3.1.0",
- "express": "^4.21.0",
+ "dockerode": "^4.0.7",
+ "express": "^4.21.2",
"lodash": "^4.17.21",
"p-limit": "^3.1.0",
"request": "^2.88.2",
"send": "^0.19.0",
+ "tar-fs": "^3.0.9",
"workerpool": "^6.1.5"
},
"devDependencies": {
- "@types/workerpool": "^6.1.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^10.2.0",
+ "mocha": "^11.1.0",
"mock-fs": "^5.1.2",
"node-fetch": "^2.7.0",
"sandboxed-module": "^2.0.4",
diff --git a/services/clsi/scripts/demo-pdfjs-Xref.js b/services/clsi/scripts/demo-pdfjs-Xref.js
index 149e55ee9e..1f55c571a7 100644
--- a/services/clsi/scripts/demo-pdfjs-Xref.js
+++ b/services/clsi/scripts/demo-pdfjs-Xref.js
@@ -1,4 +1,4 @@
-const fs = require('fs')
+const fs = require('node:fs')
const { parseXrefTable } = require('../app/lib/pdfjs/parseXrefTable')
const pdfPath = process.argv[2]
diff --git a/services/clsi/seccomp/clsi-profile.json b/services/clsi/seccomp/clsi-profile.json
index 084354b15c..ad95130f76 100644
--- a/services/clsi/seccomp/clsi-profile.json
+++ b/services/clsi/seccomp/clsi-profile.json
@@ -829,13 +829,19 @@
"args": []
},
{
- "name": "gettimeofday",
- "action": "SCMP_ACT_ALLOW",
- "args": []
- }, {
- "name": "epoll_pwait",
- "action": "SCMP_ACT_ALLOW",
- "args": []
+ "name": "gettimeofday",
+ "action": "SCMP_ACT_ALLOW",
+ "args": []
+ },
+ {
+ "name": "epoll_pwait",
+ "action": "SCMP_ACT_ALLOW",
+ "args": []
+ },
+ {
+ "name": "poll",
+ "action": "SCMP_ACT_ALLOW",
+ "args": []
}
]
-}
\ No newline at end of file
+}
diff --git a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
index 897f5d9c85..9cd7a65930 100644
--- a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
+++ b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
@@ -109,6 +109,7 @@ Hello world
width: 343.71106,
},
],
+ downloadedFromCache: false,
})
done()
}
@@ -146,6 +147,7 @@ Hello world
expect(error).to.not.exist
expect(result).to.deep.equal({
code: [{ file: 'main.tex', line: 3, column: -1 }],
+ downloadedFromCache: false,
})
done()
}
diff --git a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
index 71e9956c0d..46d07da092 100644
--- a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
+++ b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
@@ -11,6 +11,7 @@
const Client = require('./helpers/Client')
const request = require('request')
const ClsiApp = require('./helpers/ClsiApp')
+const { expect } = require('chai')
describe('Broken LaTeX file', function () {
before(function (done) {
@@ -58,9 +59,27 @@ Hello world
)
})
- return it('should return a failure status', function () {
+ it('should return a failure status', function () {
return this.body.compile.status.should.equal('failure')
})
+
+ it('should return isInitialCompile flag', function () {
+ expect(this.body.compile.stats.isInitialCompile).to.equal(1)
+ })
+
+ it('should return output files', function () {
+ // NOTE: No output.pdf file.
+ this.body.compile.outputFiles
+ .map(f => f.path)
+ .should.deep.equal([
+ 'output.aux',
+ 'output.fdb_latexmk',
+ 'output.fls',
+ 'output.log',
+ 'output.stderr',
+ 'output.stdout',
+ ])
+ })
})
return describe('on second run', function () {
@@ -80,8 +99,26 @@ Hello world
})
})
- return it('should return a failure status', function () {
+ it('should return a failure status', function () {
return this.body.compile.status.should.equal('failure')
})
+
+ it('should not return isInitialCompile flag', function () {
+ expect(this.body.compile.stats.isInitialCompile).to.not.exist
+ })
+
+ it('should return output files', function () {
+ // NOTE: No output.pdf file.
+ this.body.compile.outputFiles
+ .map(f => f.path)
+ .should.deep.equal([
+ 'output.aux',
+ 'output.fdb_latexmk',
+ 'output.fls',
+ 'output.log',
+ 'output.stderr',
+ 'output.stdout',
+ ])
+ })
})
})
diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js
index 404f8c4e90..b463584501 100644
--- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js
+++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js
@@ -14,19 +14,19 @@
*/
const Client = require('./helpers/Client')
const fetch = require('node-fetch')
-const { pipeline } = require('stream')
-const fs = require('fs')
-const ChildProcess = require('child_process')
+const { pipeline } = require('node:stream')
+const fs = require('node:fs')
+const ChildProcess = require('node:child_process')
const ClsiApp = require('./helpers/ClsiApp')
const logger = require('@overleaf/logger')
-const Path = require('path')
+const Path = require('node:path')
const fixturePath = path => {
if (path.slice(0, 3) === 'tmp') {
return '/tmp/clsi_acceptance_tests' + path.slice(3)
}
return Path.join(__dirname, '../fixtures/', path)
}
-const process = require('process')
+const process = require('node:process')
console.log(
process.pid,
process.ppid,
diff --git a/services/clsi/test/acceptance/js/StopCompile.js b/services/clsi/test/acceptance/js/StopCompile.js
new file mode 100644
index 0000000000..103a70f37d
--- /dev/null
+++ b/services/clsi/test/acceptance/js/StopCompile.js
@@ -0,0 +1,47 @@
+const Client = require('./helpers/Client')
+const ClsiApp = require('./helpers/ClsiApp')
+const { expect } = require('chai')
+
+describe('Stop compile', function () {
+ before(function (done) {
+ this.request = {
+ options: {
+ timeout: 100,
+ }, // seconds
+ resources: [
+ {
+ path: 'main.tex',
+ content: `\
+\\documentclass{article}
+\\begin{document}
+\\def\\x{Hello!\\par\\x}
+\\x
+\\end{document}\
+`,
+ },
+ ],
+ }
+ this.project_id = Client.randomId()
+ ClsiApp.ensureRunning(() => {
+ // start the compile in the background
+ Client.compile(this.project_id, this.request, (error, res, body) => {
+ this.compileResult = { error, res, body }
+ })
+ // wait for 1 second before stopping the compile
+ setTimeout(() => {
+ Client.stopCompile(this.project_id, (error, res, body) => {
+ this.stopResult = { error, res, body }
+ setTimeout(done, 1000) // allow time for the compile request to terminate
+ })
+ }, 1000)
+ })
+ })
+
+ it('should force a compile response with an error status', function () {
+ expect(this.stopResult.error).to.be.null
+ expect(this.stopResult.res.statusCode).to.equal(204)
+ expect(this.compileResult.res.statusCode).to.equal(200)
+ expect(this.compileResult.body.compile.status).to.equal('terminated')
+ expect(this.compileResult.body.compile.error).to.equal('terminated')
+ })
+})
diff --git a/services/clsi/test/acceptance/js/SynctexTests.js b/services/clsi/test/acceptance/js/SynctexTests.js
index 899898b5b2..049f260259 100644
--- a/services/clsi/test/acceptance/js/SynctexTests.js
+++ b/services/clsi/test/acceptance/js/SynctexTests.js
@@ -13,7 +13,7 @@ const Client = require('./helpers/Client')
const request = require('request')
const { expect } = require('chai')
const ClsiApp = require('./helpers/ClsiApp')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
describe('Syncing', function () {
before(function (done) {
@@ -67,6 +67,7 @@ Hello world
width: 343.71106,
},
],
+ downloadedFromCache: false,
})
return done()
}
@@ -87,6 +88,7 @@ Hello world
}
expect(codePositions).to.deep.equal({
code: [{ file: 'main.tex', line: 3, column: -1 }],
+ downloadedFromCache: false,
})
return done()
}
diff --git a/services/clsi/test/acceptance/js/TimeoutTests.js b/services/clsi/test/acceptance/js/TimeoutTests.js
index bca8ae71d2..e9175d223c 100644
--- a/services/clsi/test/acceptance/js/TimeoutTests.js
+++ b/services/clsi/test/acceptance/js/TimeoutTests.js
@@ -11,6 +11,7 @@
const Client = require('./helpers/Client')
const request = require('request')
const ClsiApp = require('./helpers/ClsiApp')
+const { expect } = require('chai')
describe('Timed out compile', function () {
before(function (done) {
@@ -54,6 +55,10 @@ describe('Timed out compile', function () {
return this.body.compile.status.should.equal('timedout')
})
+ it('should return isInitialCompile flag', function () {
+ expect(this.body.compile.stats.isInitialCompile).to.equal(1)
+ })
+
return it('should return the log output file name', function () {
const outputFilePaths = this.body.compile.outputFiles.map(x => x.path)
return outputFilePaths.should.include('output.log')
diff --git a/services/clsi/test/acceptance/js/UrlCachingTests.js b/services/clsi/test/acceptance/js/UrlCachingTests.js
index 424f1ad1b7..9fc9608204 100644
--- a/services/clsi/test/acceptance/js/UrlCachingTests.js
+++ b/services/clsi/test/acceptance/js/UrlCachingTests.js
@@ -10,10 +10,12 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const express = require('express')
-const Path = require('path')
+const Path = require('node:path')
const Client = require('./helpers/Client')
const sinon = require('sinon')
const ClsiApp = require('./helpers/ClsiApp')
+const request = require('request')
+const Settings = require('@overleaf/settings')
const Server = {
run() {
@@ -35,6 +37,21 @@ const Server = {
}
})
+ app.get('/not-found', (req, res, next) => {
+ this.getFile(req.url)
+ res.status(404).end()
+ })
+
+ app.get('/project/:projectId/file/:fileId', (req, res, next) => {
+ this.getFile(req.url)
+ return res.send(`${req.params.projectId}:${req.params.fileId}`)
+ })
+
+ app.get('/bucket/:bucket/key/*', (req, res, next) => {
+ this.getFile(req.url)
+ return res.send(`${req.params.bucket}:${req.params[0]}`)
+ })
+
app.get('/:random_id/*', (req, res, next) => {
this.getFile(req.url)
req.url = `/${req.params[0]}`
@@ -218,9 +235,24 @@ describe('Url Caching', function () {
return Server.getFile.restore()
})
- return it('should not download the image again', function () {
+ it('should not download the image again', function () {
return Server.getFile.called.should.equal(false)
})
+
+ it('should gather metrics', function (done) {
+ request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
+ if (err) return done(err)
+ body
+ .split('\n')
+ .some(line => {
+ return (
+ line.startsWith('url_source') && line.includes('path="unknown"')
+ )
+ })
+ .should.equal(true)
+ done()
+ })
+ })
})
describe('When an image is in the cache and the last modified date is advanced', function () {
@@ -391,7 +423,7 @@ describe('Url Caching', function () {
})
})
- return describe('After clearing the cache', function () {
+ describe('After clearing the cache', function () {
before(function (done) {
this.project_id = Client.randomId()
this.file = `${Server.randomId()}/lion.png`
@@ -446,4 +478,140 @@ describe('Url Caching', function () {
return Server.getFile.called.should.equal(true)
})
})
+
+ describe('fallbackURL', function () {
+ describe('when the primary resource is available', function () {
+ before(function (done) {
+ this.project_id = Client.randomId()
+ this.file = `/project/${Server.randomId()}/file/${Server.randomId()}`
+ this.fallback = `/bucket/project-blobs/key/ab/cd/${Server.randomId()}`
+ this.request = {
+ resources: [
+ {
+ path: 'main.tex',
+ content: `\
+\\documentclass{article}
+\\usepackage{graphicx}
+\\begin{document}
+\\includegraphics{lion.png}
+\\end{document}\
+`,
+ },
+ {
+ path: 'lion.png',
+ url: `http://filestore${this.file}`,
+ fallbackURL: `http://filestore${this.fallback}`,
+ },
+ ],
+ }
+
+ sinon.spy(Server, 'getFile')
+ return ClsiApp.ensureRunning(() => {
+ return Client.compile(
+ this.project_id,
+ this.request,
+ (error, res, body) => {
+ this.error = error
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+ })
+
+ after(function () {
+ return Server.getFile.restore()
+ })
+
+ it('should download from the primary', function () {
+ Server.getFile.calledWith(this.file).should.equal(true)
+ })
+ it('should not download from the fallback', function () {
+ Server.getFile.calledWith(this.fallback).should.equal(false)
+ })
+
+ it('should gather metrics', function (done) {
+ request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
+ if (err) return done(err)
+ body
+ .split('\n')
+ .some(line => {
+ return (
+ line.startsWith('url_source') &&
+ line.includes('path="user-files"')
+ )
+ })
+ .should.equal(true)
+ done()
+ })
+ })
+ })
+
+ describe('when the primary resource is not available', function () {
+ before(function (done) {
+ this.project_id = Client.randomId()
+ this.file = `/project/${Server.randomId()}/file/${Server.randomId()}`
+ this.fallback = `/bucket/project-blobs/key/ab/cd/${Server.randomId()}`
+ this.request = {
+ resources: [
+ {
+ path: 'main.tex',
+ content: `\
+\\documentclass{article}
+\\usepackage{graphicx}
+\\begin{document}
+\\includegraphics{lion.png}
+\\end{document}\
+`,
+ },
+ {
+ path: 'lion.png',
+ url: `http://filestore/not-found`,
+ fallbackURL: `http://filestore${this.fallback}`,
+ },
+ ],
+ }
+
+ sinon.spy(Server, 'getFile')
+ return ClsiApp.ensureRunning(() => {
+ return Client.compile(
+ this.project_id,
+ this.request,
+ (error, res, body) => {
+ this.error = error
+ this.res = res
+ this.body = body
+ return done()
+ }
+ )
+ })
+ })
+
+ after(function () {
+ return Server.getFile.restore()
+ })
+
+ it('should download from the fallback', function () {
+ Server.getFile.calledWith(`/not-found`).should.equal(true)
+ Server.getFile.calledWith(this.fallback).should.equal(true)
+ })
+
+ it('should gather metrics', function (done) {
+ request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
+ if (err) return done(err)
+ body
+ .split('\n')
+ .some(line => {
+ return (
+ line.startsWith('url_source') &&
+ line.includes('path="project-blobs"')
+ )
+ })
+ .should.equal(true)
+ done()
+ })
+ })
+ })
+ })
})
diff --git a/services/clsi/test/acceptance/js/WordcountTests.js b/services/clsi/test/acceptance/js/WordcountTests.js
index d3fa7d2b94..626b5d7034 100644
--- a/services/clsi/test/acceptance/js/WordcountTests.js
+++ b/services/clsi/test/acceptance/js/WordcountTests.js
@@ -12,8 +12,8 @@
const Client = require('./helpers/Client')
const request = require('request')
const { expect } = require('chai')
-const path = require('path')
-const fs = require('fs')
+const path = require('node:path')
+const fs = require('node:fs')
const ClsiApp = require('./helpers/ClsiApp')
describe('Syncing', function () {
diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js
index 389a26201e..49bf7390c6 100644
--- a/services/clsi/test/acceptance/js/helpers/Client.js
+++ b/services/clsi/test/acceptance/js/helpers/Client.js
@@ -13,7 +13,7 @@
let Client
const express = require('express')
const request = require('request')
-const fs = require('fs')
+const fs = require('node:fs')
const Settings = require('@overleaf/settings')
module.exports = Client = {
@@ -42,6 +42,16 @@ module.exports = Client = {
)
},
+ stopCompile(projectId, callback) {
+ if (callback == null) {
+ callback = function () {}
+ }
+ return request.post(
+ { url: `${this.host}/project/${projectId}/compile/stop` },
+ callback
+ )
+ },
+
clearCache(projectId, callback) {
if (callback == null) {
callback = function () {}
diff --git a/services/clsi/test/acceptance/js/helpers/ClsiApp.js b/services/clsi/test/acceptance/js/helpers/ClsiApp.js
index 4736315df8..38308e9129 100644
--- a/services/clsi/test/acceptance/js/helpers/ClsiApp.js
+++ b/services/clsi/test/acceptance/js/helpers/ClsiApp.js
@@ -10,8 +10,6 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
-require('@overleaf/logger').logger.level('info')
-const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
module.exports = {
@@ -37,7 +35,6 @@ module.exports = {
throw error
}
this.running = true
- logger.info('clsi running in dev mode')
return (() => {
const result = []
diff --git a/services/clsi/test/bench/hashbench.js b/services/clsi/test/bench/hashbench.js
index 652ad39c7d..1e19af6a35 100644
--- a/services/clsi/test/bench/hashbench.js
+++ b/services/clsi/test/bench/hashbench.js
@@ -1,8 +1,8 @@
const ContentCacheManager = require('../../app/js/ContentCacheManager')
-const fs = require('fs')
-const crypto = require('crypto')
-const path = require('path')
-const os = require('os')
+const fs = require('node:fs')
+const crypto = require('node:crypto')
+const path = require('node:path')
+const os = require('node:os')
const async = require('async')
const _createHash = crypto.createHash
diff --git a/services/clsi/test/load/js/loadTest.js b/services/clsi/test/load/js/loadTest.js
index 196d8579ca..506b51bf6f 100644
--- a/services/clsi/test/load/js/loadTest.js
+++ b/services/clsi/test/load/js/loadTest.js
@@ -9,7 +9,7 @@
const request = require('request')
const Settings = require('@overleaf/settings')
const async = require('async')
-const fs = require('fs')
+const fs = require('node:fs')
const _ = require('lodash')
const concurentCompiles = 5
const totalCompiles = 50
diff --git a/services/clsi/test/setup.js b/services/clsi/test/setup.js
index 653a8a69b2..b17507bf92 100644
--- a/services/clsi/test/setup.js
+++ b/services/clsi/test/setup.js
@@ -20,5 +20,10 @@ SandboxedModule.configure({
err() {},
},
},
- globals: { Buffer, console, process, URL },
+ globals: { Buffer, console, process, URL, Math },
+ sourceTransformers: {
+ removeNodePrefix: function (source) {
+ return source.replace(/require\(['"]node:/g, "require('")
+ },
+ },
})
diff --git a/services/clsi/test/unit/js/CompileControllerTests.js b/services/clsi/test/unit/js/CompileControllerTests.js
index ac807d212e..2ac8d9c2d7 100644
--- a/services/clsi/test/unit/js/CompileControllerTests.js
+++ b/services/clsi/test/unit/js/CompileControllerTests.js
@@ -1,54 +1,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const { expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/CompileController'
)
const Errors = require('../../../app/js/Errors')
-function tryImageNameValidation(method, imageNameField) {
- describe('when allowedImages is set', function () {
- beforeEach(function () {
- this.Settings.clsi = { docker: {} }
- this.Settings.clsi.docker.allowedImages = [
- 'repo/image:tag1',
- 'repo/image:tag2',
- ]
- this.res.send = sinon.stub()
- this.res.status = sinon.stub().returns({ send: this.res.send })
-
- this.CompileManager[method].reset()
- })
-
- describe('with an invalid image', function () {
- beforeEach(function () {
- this.req.query[imageNameField] = 'something/evil:1337'
- this.CompileController[method](this.req, this.res, this.next)
- })
- it('should return a 400', function () {
- expect(this.res.status.calledWith(400)).to.equal(true)
- })
- it('should not run the query', function () {
- expect(this.CompileManager[method].called).to.equal(false)
- })
- })
-
- describe('with a valid image', function () {
- beforeEach(function () {
- this.req.query[imageNameField] = 'repo/image:tag1'
- this.CompileController[method](this.req, this.res, this.next)
- })
- it('should not return a 400', function () {
- expect(this.res.status.calledWith(400)).to.equal(false)
- })
- it('should run the query', function () {
- expect(this.CompileManager[method].called).to.equal(true)
- })
- })
- })
-}
-
describe('CompileController', function () {
beforeEach(function () {
this.buildId = 'build-id-123'
@@ -61,6 +18,11 @@ describe('CompileController', function () {
clsi: {
url: 'http://clsi.example.com',
outputUrlPrefix: '/zone/b',
+ downloadHost: 'http://localhost:3013',
+ },
+ clsiCache: {
+ enabled: false,
+ url: 'http://localhost:3044',
},
},
}),
@@ -68,6 +30,11 @@ describe('CompileController', function () {
Timer: sinon.stub().returns({ done: sinon.stub() }),
},
'./ProjectPersistenceManager': (this.ProjectPersistenceManager = {}),
+ './CLSICacheHandler': {
+ notifyCLSICacheAboutBuild: sinon.stub(),
+ downloadLatestCompileCache: sinon.stub().resolves(),
+ downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
+ },
'./Errors': (this.Erros = Errors),
},
})
@@ -113,16 +80,21 @@ describe('CompileController', function () {
this.timings = { bar: 2 }
this.res.status = sinon.stub().returnsThis()
this.res.send = sinon.stub()
+
+ this.CompileManager.doCompileWithLock = sinon
+ .stub()
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(null, {
+ outputFiles: this.output_files,
+ buildId: this.buildId,
+ })
+ })
})
describe('successfully', function () {
beforeEach(function () {
- this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
- outputFiles: this.output_files,
- stats: this.stats,
- timings: this.timings,
- buildId: this.buildId,
- })
this.CompileController.compile(this.req, this.res)
})
@@ -157,6 +129,7 @@ describe('CompileController', function () {
url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
...file,
})),
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -166,12 +139,6 @@ describe('CompileController', function () {
describe('without a outputUrlPrefix', function () {
beforeEach(function () {
this.Settings.apis.clsi.outputUrlPrefix = ''
- this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
- outputFiles: this.output_files,
- stats: this.stats,
- timings: this.timings,
- buildId: this.buildId,
- })
this.CompileController.compile(this.req, this.res)
})
@@ -190,6 +157,7 @@ describe('CompileController', function () {
url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
...file,
})),
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -210,33 +178,36 @@ describe('CompileController', function () {
build: 1234,
},
]
- this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
- outputFiles: this.output_files,
- stats: this.stats,
- timings: this.timings,
- buildId: this.buildId,
- })
+ this.CompileManager.doCompileWithLock = sinon
+ .stub()
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(null, {
+ outputFiles: this.output_files,
+ buildId: this.buildId,
+ })
+ })
this.CompileController.compile(this.req, this.res)
})
it('should return the JSON response with status failure', function () {
this.res.status.calledWith(200).should.equal(true)
- this.res.send
- .calledWith({
- compile: {
- status: 'failure',
- error: null,
- stats: this.stats,
- timings: this.timings,
- outputUrlPrefix: '/zone/b',
- buildId: this.buildId,
- outputFiles: this.output_files.map(file => ({
- url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
- ...file,
- })),
- },
- })
- .should.equal(true)
+ this.res.send.should.have.been.calledWith({
+ compile: {
+ status: 'failure',
+ error: null,
+ stats: this.stats,
+ timings: this.timings,
+ outputUrlPrefix: '/zone/b',
+ buildId: this.buildId,
+ outputFiles: this.output_files.map(file => ({
+ url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
+ ...file,
+ })),
+ clsiCacheShard: undefined,
+ },
+ })
})
})
@@ -255,33 +226,36 @@ describe('CompileController', function () {
build: 1234,
},
]
- this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
- outputFiles: this.output_files,
- stats: this.stats,
- timings: this.timings,
- buildId: this.buildId,
- })
+ this.CompileManager.doCompileWithLock = sinon
+ .stub()
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(null, {
+ outputFiles: this.output_files,
+ buildId: this.buildId,
+ })
+ })
this.CompileController.compile(this.req, this.res)
})
it('should return the JSON response with status failure', function () {
this.res.status.calledWith(200).should.equal(true)
- this.res.send
- .calledWith({
- compile: {
- status: 'failure',
- error: null,
- stats: this.stats,
- buildId: this.buildId,
- timings: this.timings,
- outputUrlPrefix: '/zone/b',
- outputFiles: this.output_files.map(file => ({
- url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
- ...file,
- })),
- },
- })
- .should.equal(true)
+ this.res.send.should.have.been.calledWith({
+ compile: {
+ status: 'failure',
+ error: null,
+ stats: this.stats,
+ buildId: this.buildId,
+ timings: this.timings,
+ outputUrlPrefix: '/zone/b',
+ outputFiles: this.output_files.map(file => ({
+ url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
+ ...file,
+ })),
+ clsiCacheShard: undefined,
+ },
+ })
})
})
@@ -291,7 +265,11 @@ describe('CompileController', function () {
error.buildId = this.buildId
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsArgWith(1, error, null)
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(error)
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -305,9 +283,9 @@ describe('CompileController', function () {
outputUrlPrefix: '/zone/b',
outputFiles: [],
buildId: this.buildId,
- // JSON.stringify will omit these
- stats: undefined,
- timings: undefined,
+ stats: this.stats,
+ timings: this.timings,
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -321,7 +299,11 @@ describe('CompileController', function () {
)
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsArgWith(1, error, null)
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(error)
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -334,9 +316,11 @@ describe('CompileController', function () {
error: 'too many concurrent compile requests',
outputUrlPrefix: '/zone/b',
outputFiles: [],
+ stats: this.stats,
+ timings: this.timings,
+ // JSON.stringify will omit these undefined values
buildId: undefined,
- stats: undefined,
- timings: undefined,
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -349,7 +333,11 @@ describe('CompileController', function () {
this.error.timedout = true
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsArgWith(1, this.error, null)
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(this.error)
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -362,10 +350,11 @@ describe('CompileController', function () {
error: this.message,
outputUrlPrefix: '/zone/b',
outputFiles: [],
- // JSON.stringify will omit these
+ stats: this.stats,
+ timings: this.timings,
+ // JSON.stringify will omit these undefined values
buildId: undefined,
- stats: undefined,
- timings: undefined,
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -376,7 +365,11 @@ describe('CompileController', function () {
beforeEach(function () {
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsArgWith(1, null, [])
+ .callsFake((_req, stats, timings, cb) => {
+ Object.assign(stats, this.stats)
+ Object.assign(timings, this.timings)
+ cb(null, {})
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -389,10 +382,11 @@ describe('CompileController', function () {
status: 'failure',
outputUrlPrefix: '/zone/b',
outputFiles: [],
- // JSON.stringify will omit these
+ stats: this.stats,
+ timings: this.timings,
+ // JSON.stringify will omit these undefined values
buildId: undefined,
- stats: undefined,
- timings: undefined,
+ clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -416,7 +410,7 @@ describe('CompileController', function () {
this.CompileManager.syncFromCode = sinon
.stub()
- .yields(null, (this.pdfPositions = ['mock-positions']))
+ .yields(null, (this.pdfPositions = ['mock-positions']), true)
this.CompileController.syncFromCode(this.req, this.res, this.next)
})
@@ -436,11 +430,10 @@ describe('CompileController', function () {
this.res.json
.calledWith({
pdf: this.pdfPositions,
+ downloadedFromCache: true,
})
.should.equal(true)
})
-
- tryImageNameValidation('syncFromCode', 'imageName')
})
describe('syncFromPdf', function () {
@@ -459,7 +452,7 @@ describe('CompileController', function () {
this.CompileManager.syncFromPdf = sinon
.stub()
- .yields(null, (this.codePositions = ['mock-positions']))
+ .yields(null, (this.codePositions = ['mock-positions']), true)
this.CompileController.syncFromPdf(this.req, this.res, this.next)
})
@@ -473,11 +466,10 @@ describe('CompileController', function () {
this.res.json
.calledWith({
code: this.codePositions,
+ downloadedFromCache: true,
})
.should.equal(true)
})
-
- tryImageNameValidation('syncFromPdf', 'imageName')
})
describe('wordcount', function () {
@@ -511,7 +503,5 @@ describe('CompileController', function () {
})
.should.equal(true)
})
-
- tryImageNameValidation('wordcount', 'image')
})
})
diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js
index 6f5b5baf64..30ef538ac3 100644
--- a/services/clsi/test/unit/js/CompileManagerTests.js
+++ b/services/clsi/test/unit/js/CompileManagerTests.js
@@ -1,9 +1,9 @@
-const Path = require('path')
+const Path = require('node:path')
const SandboxedModule = require('sandboxed-module')
const { expect } = require('chai')
const sinon = require('sinon')
-const MODULE_PATH = require('path').join(
+const MODULE_PATH = require('node:path').join(
__dirname,
'../../../app/js/CompileManager'
)
@@ -35,7 +35,7 @@ describe('CompileManager', function () {
build: 1234,
},
]
- this.buildId = 'build-id-123'
+ this.buildId = '00000000000-0000000000000000'
this.commandOutput = 'Dummy output'
this.compileBaseDir = '/compile/dir'
this.outputBaseDir = '/output/dir'
@@ -61,7 +61,10 @@ describe('CompileManager', function () {
},
}
this.OutputCacheManager = {
+ BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
+ CACHE_SUBDIR: 'generated-files',
promises: {
+ queueDirOperation: sinon.stub().callsArg(1),
saveOutputFiles: sinon
.stub()
.resolves({ outputFiles: this.buildFiles, buildId: this.buildId }),
@@ -87,9 +90,10 @@ describe('CompileManager', function () {
execFile: sinon.stub().yields(),
}
this.CommandRunner = {
+ canRunSyncTeXInOutputDir: sinon.stub().returns(false),
promises: {
run: sinon.stub().callsFake((_1, _2, _3, _4, _5, _6, compileGroup) => {
- if (compileGroup === 'synctex') {
+ if (compileGroup === 'synctex' || compileGroup === 'synctex-output') {
return Promise.resolve({ stdout: this.commandOutput })
} else {
return Promise.resolve({
@@ -140,6 +144,12 @@ describe('CompileManager', function () {
.withArgs(Path.join(this.compileDir, 'output.synctex.gz'))
.resolves(this.fileStats)
+ this.CLSICacheHandler = {
+ notifyCLSICacheAboutBuild: sinon.stub(),
+ downloadLatestCompileCache: sinon.stub().resolves(),
+ downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
+ }
+
this.CompileManager = SandboxedModule.require(MODULE_PATH, {
requires: {
'./LatexRunner': this.LatexRunner,
@@ -160,6 +170,7 @@ describe('CompileManager', function () {
'./LockManager': this.LockManager,
'./SynctexOutputParser': this.SynctexOutputParser,
'fs/promises': this.fsPromises,
+ './CLSICacheHandler': this.CLSICacheHandler,
},
})
})
@@ -177,6 +188,11 @@ describe('CompileManager', function () {
flags: (this.flags = ['-file-line-error']),
compileGroup: (this.compileGroup = 'compile-group'),
stopOnFirstError: false,
+ metricsOpts: {
+ path: 'clsi-perf',
+ method: 'minimal',
+ compile: 'initial',
+ },
}
this.env = {
OVERLEAF_PROJECT_ID: this.projectId,
@@ -188,7 +204,7 @@ describe('CompileManager', function () {
const error = new Error('locked')
this.LockManager.acquire.throws(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request)
+ this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
).to.be.rejectedWith(error)
})
@@ -206,7 +222,9 @@ describe('CompileManager', function () {
describe('normally', function () {
beforeEach(async function () {
this.result = await this.CompileManager.promises.doCompileWithLock(
- this.request
+ this.request,
+ {},
+ {}
)
})
@@ -260,7 +278,11 @@ describe('CompileManager', function () {
describe('with draft mode', function () {
beforeEach(async function () {
this.request.draft = true
- await this.CompileManager.promises.doCompileWithLock(this.request)
+ await this.CompileManager.promises.doCompileWithLock(
+ this.request,
+ {},
+ {}
+ )
})
it('should inject the draft mode header', function () {
@@ -273,7 +295,11 @@ describe('CompileManager', function () {
describe('with a check option', function () {
beforeEach(async function () {
this.request.check = 'error'
- await this.CompileManager.promises.doCompileWithLock(this.request)
+ await this.CompileManager.promises.doCompileWithLock(
+ this.request,
+ {},
+ {}
+ )
})
it('should run chktex', function () {
@@ -305,7 +331,11 @@ describe('CompileManager', function () {
beforeEach(async function () {
this.request.rootResourcePath = 'main.Rtex'
this.request.check = 'error'
- await this.CompileManager.promises.doCompileWithLock(this.request)
+ await this.CompileManager.promises.doCompileWithLock(
+ this.request,
+ {},
+ {}
+ )
})
it('should not run chktex', function () {
@@ -334,7 +364,7 @@ describe('CompileManager', function () {
error.timedout = true
this.LatexRunner.promises.runLatex.rejects(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request)
+ this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
).to.be.rejected
})
@@ -357,7 +387,7 @@ describe('CompileManager', function () {
error.terminated = true
this.LatexRunner.promises.runLatex.rejects(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request)
+ this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
).to.be.rejected
})
@@ -437,12 +467,83 @@ describe('CompileManager', function () {
this.compileDir,
this.Settings.clsi.docker.image,
60000,
- {}
+ {},
+ 'synctex'
)
})
it('should return the parsed output', function () {
- expect(this.result).to.deep.equal(this.records)
+ expect(this.result).to.deep.equal({
+ codePositions: this.records,
+ downloadedFromCache: false,
+ })
+ })
+ })
+
+ describe('from cache in docker', function () {
+ beforeEach(async function () {
+ this.CommandRunner.canRunSyncTeXInOutputDir.returns(true)
+ this.Settings.path.synctexBaseDir
+ .withArgs(`${this.projectId}-${this.userId}`)
+ .returns('/compile')
+
+ const errNotFound = new Error()
+ errNotFound.code = 'ENOENT'
+ this.outputDir = `${this.outputBaseDir}/${this.projectId}-${this.userId}/${this.OutputCacheManager.CACHE_SUBDIR}/${this.buildId}`
+ const filename = Path.join(this.outputDir, 'output.synctex.gz')
+ this.fsPromises.stat
+ .withArgs(this.outputDir)
+ .onFirstCall()
+ .rejects(errNotFound)
+ this.fsPromises.stat
+ .withArgs(this.outputDir)
+ .onSecondCall()
+ .resolves(this.dirStats)
+ this.fsPromises.stat.withArgs(filename).resolves(this.fileStats)
+ this.CLSICacheHandler.downloadOutputDotSynctexFromCompileCache.resolves(
+ true
+ )
+ this.result = await this.CompileManager.promises.syncFromCode(
+ this.projectId,
+ this.userId,
+ this.filename,
+ this.line,
+ this.column,
+ {
+ imageName: 'image',
+ editorId: '00000000-0000-0000-0000-000000000000',
+ buildId: this.buildId,
+ compileFromClsiCache: true,
+ }
+ )
+ })
+
+ it('should run in output dir', function () {
+ const outputFilePath = '/compile/output.pdf'
+ const inputFilePath = `/compile/${this.filename}`
+ expect(this.CommandRunner.promises.run).to.have.been.calledWith(
+ `${this.projectId}-${this.userId}`,
+ [
+ 'synctex',
+ 'view',
+ '-i',
+ `${this.line}:${this.column}:${inputFilePath}`,
+ '-o',
+ outputFilePath,
+ ],
+ this.outputDir,
+ 'image',
+ 60000,
+ {},
+ 'synctex-output'
+ )
+ })
+
+ it('should return the parsed output', function () {
+ expect(this.result).to.deep.equal({
+ codePositions: this.records,
+ downloadedFromCache: true,
+ })
})
})
@@ -455,7 +556,7 @@ describe('CompileManager', function () {
this.filename,
this.line,
this.column,
- customImageName
+ { imageName: customImageName }
)
})
@@ -475,7 +576,8 @@ describe('CompileManager', function () {
this.compileDir,
customImageName,
60000,
- {}
+ {},
+ 'synctex'
)
})
})
@@ -497,7 +599,7 @@ describe('CompileManager', function () {
this.page,
this.h,
this.v,
- ''
+ { imageName: '' }
)
})
@@ -519,7 +621,10 @@ describe('CompileManager', function () {
})
it('should return the parsed output', function () {
- expect(this.result).to.deep.equal(this.records)
+ expect(this.result).to.deep.equal({
+ pdfPositions: this.records,
+ downloadedFromCache: false,
+ })
})
})
@@ -532,7 +637,7 @@ describe('CompileManager', function () {
this.page,
this.h,
this.v,
- customImageName
+ { imageName: customImageName }
)
})
diff --git a/services/clsi/test/unit/js/ContentCacheManagerTests.js b/services/clsi/test/unit/js/ContentCacheManagerTests.js
index be2d17039d..df3bce212b 100644
--- a/services/clsi/test/unit/js/ContentCacheManagerTests.js
+++ b/services/clsi/test/unit/js/ContentCacheManagerTests.js
@@ -1,5 +1,5 @@
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
const { expect } = require('chai')
const MODULE_PATH = '../../../app/js/ContentCacheManager'
diff --git a/services/clsi/test/unit/js/ContentTypeMapperTests.js b/services/clsi/test/unit/js/ContentTypeMapperTests.js
index a80ef991ca..a413337153 100644
--- a/services/clsi/test/unit/js/ContentTypeMapperTests.js
+++ b/services/clsi/test/unit/js/ContentTypeMapperTests.js
@@ -11,7 +11,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/ContentTypeMapper'
)
diff --git a/services/clsi/test/unit/js/DockerLockManagerTests.js b/services/clsi/test/unit/js/DockerLockManagerTests.js
index 5708faf292..f69179443c 100644
--- a/services/clsi/test/unit/js/DockerLockManagerTests.js
+++ b/services/clsi/test/unit/js/DockerLockManagerTests.js
@@ -11,12 +11,12 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/DockerLockManager'
)
-describe('LockManager', function () {
+describe('DockerLockManager', function () {
beforeEach(function () {
return (this.LockManager = SandboxedModule.require(modulePath, {
requires: {
diff --git a/services/clsi/test/unit/js/DockerRunnerTests.js b/services/clsi/test/unit/js/DockerRunnerTests.js
index 8ff9c6107b..d70aab52c7 100644
--- a/services/clsi/test/unit/js/DockerRunnerTests.js
+++ b/services/clsi/test/unit/js/DockerRunnerTests.js
@@ -15,11 +15,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/DockerRunner'
)
-const Path = require('path')
+const Path = require('node:path')
describe('DockerRunner', function () {
beforeEach(function () {
@@ -76,8 +76,11 @@ describe('DockerRunner', function () {
this.env = {}
this.callback = sinon.stub()
this.project_id = 'project-id-123'
- this.volumes = { '/local/compile/directory': '/compile' }
+ this.volumes = { '/some/host/dir/compiles/directory': '/compile' }
this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
+ this.Settings.path.sandboxedCompilesHostDirCompiles =
+ '/some/host/dir/compiles'
+ this.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output'
this.compileGroup = 'compile-group'
return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
})
@@ -151,9 +154,8 @@ describe('DockerRunner', function () {
})
})
- describe('when path.sandboxedCompilesHostDir is set', function () {
+ describe('standard compile', function () {
beforeEach(function () {
- this.Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
this.directory = '/var/lib/overleaf/data/compiles/xyz'
this.DockerRunner._runAndWaitForContainer = sinon
.stub()
@@ -183,6 +185,99 @@ describe('DockerRunner', function () {
})
})
+ describe('synctex-output', function () {
+ beforeEach(function () {
+ this.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id'
+ this.DockerRunner._runAndWaitForContainer = sinon
+ .stub()
+ .callsArgWith(3, null, (this.output = 'mock-output'))
+ this.DockerRunner.run(
+ this.project_id,
+ this.command,
+ this.directory,
+ this.image,
+ this.timeout,
+ this.env,
+ 'synctex-output',
+ this.callback
+ )
+ })
+
+ it('should re-write the bind directory and set ro flag', function () {
+ const volumes =
+ this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+ expect(volumes).to.deep.equal({
+ '/some/host/dir/output/xyz/generated-files/id': '/compile:ro',
+ })
+ })
+
+ it('should call the callback', function () {
+ this.callback.calledWith(null, this.output).should.equal(true)
+ })
+ })
+
+ describe('synctex', function () {
+ beforeEach(function () {
+ this.directory = '/var/lib/overleaf/data/compile/xyz'
+ this.DockerRunner._runAndWaitForContainer = sinon
+ .stub()
+ .callsArgWith(3, null, (this.output = 'mock-output'))
+ this.DockerRunner.run(
+ this.project_id,
+ this.command,
+ this.directory,
+ this.image,
+ this.timeout,
+ this.env,
+ 'synctex',
+ this.callback
+ )
+ })
+
+ it('should re-write the bind directory', function () {
+ const volumes =
+ this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+ expect(volumes).to.deep.equal({
+ '/some/host/dir/compiles/xyz': '/compile:ro',
+ })
+ })
+
+ it('should call the callback', function () {
+ this.callback.calledWith(null, this.output).should.equal(true)
+ })
+ })
+
+ describe('wordcount', function () {
+ beforeEach(function () {
+ this.directory = '/var/lib/overleaf/data/compile/xyz'
+ this.DockerRunner._runAndWaitForContainer = sinon
+ .stub()
+ .callsArgWith(3, null, (this.output = 'mock-output'))
+ this.DockerRunner.run(
+ this.project_id,
+ this.command,
+ this.directory,
+ this.image,
+ this.timeout,
+ this.env,
+ 'wordcount',
+ this.callback
+ )
+ })
+
+ it('should re-write the bind directory', function () {
+ const volumes =
+ this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+ expect(volumes).to.deep.equal({
+ '/some/host/dir/compiles/xyz': '/compile:ro',
+ })
+ })
+
+ it('should call the callback', function () {
+ this.callback.calledWith(null, this.output).should.equal(true)
+ })
+ })
+
describe('when the run throws an error', function () {
beforeEach(function () {
let firstTime = true
@@ -390,7 +485,7 @@ describe('DockerRunner', function () {
const options =
this.DockerRunner._runAndWaitForContainer.lastCall.args[0]
return expect(options.HostConfig).to.deep.include({
- Binds: ['/local/compile/directory:/compile:rw'],
+ Binds: ['/some/host/dir/compiles/directory:/compile:rw'],
LogConfig: { Type: 'none', Config: {} },
CapDrop: 'ALL',
SecurityOpt: ['no-new-privileges'],
@@ -562,82 +657,6 @@ describe('DockerRunner', function () {
})
})
- describe('when a volume does not exist', function () {
- beforeEach(function () {
- this.fs.stat = sinon.stub().yields(new Error('no such path'))
- return this.DockerRunner.startContainer(
- this.options,
- this.volumes,
- this.attachStreamHandler,
- this.callback
- )
- })
-
- it('should not try to create the container', function () {
- return this.createContainer.called.should.equal(false)
- })
-
- it('should call the callback with an error', function () {
- this.callback.calledWith(sinon.match(Error)).should.equal(true)
- })
- })
-
- describe('when a volume exists but is not a directory', function () {
- beforeEach(function () {
- this.fs.stat = sinon.stub().yields(null, {
- isDirectory() {
- return false
- },
- })
- return this.DockerRunner.startContainer(
- this.options,
- this.volumes,
- this.attachStreamHandler,
- this.callback
- )
- })
-
- it('should not try to create the container', function () {
- return this.createContainer.called.should.equal(false)
- })
-
- it('should call the callback with an error', function () {
- this.callback.calledWith(sinon.match(Error)).should.equal(true)
- })
- })
-
- describe('when a volume does not exist, but sibling-containers are used', function () {
- beforeEach(function () {
- this.fs.stat = sinon.stub().yields(new Error('no such path'))
- this.Settings.path.sandboxedCompilesHostDir = '/some/path'
- this.container.start = sinon.stub().yields()
- return this.DockerRunner.startContainer(
- this.options,
- this.volumes,
- () => {},
- this.callback
- )
- })
-
- afterEach(function () {
- return delete this.Settings.path.sandboxedCompilesHostDir
- })
-
- it('should start the container with the given name', function () {
- this.getContainer.calledWith(this.options.name).should.equal(true)
- return this.container.start.called.should.equal(true)
- })
-
- it('should not try to create the container', function () {
- return this.createContainer.called.should.equal(false)
- })
-
- return it('should call the callback', function () {
- this.callback.called.should.equal(true)
- return this.callback.calledWith(new Error()).should.equal(false)
- })
- })
-
return describe('when the container tries to be created, but already has been (race condition)', function () {})
})
diff --git a/services/clsi/test/unit/js/DraftModeManagerTests.js b/services/clsi/test/unit/js/DraftModeManagerTests.js
index acaae0f10a..eda83380e7 100644
--- a/services/clsi/test/unit/js/DraftModeManagerTests.js
+++ b/services/clsi/test/unit/js/DraftModeManagerTests.js
@@ -1,5 +1,5 @@
-const Path = require('path')
-const fsPromises = require('fs/promises')
+const Path = require('node:path')
+const fsPromises = require('node:fs/promises')
const { expect } = require('chai')
const mockFs = require('mock-fs')
const SandboxedModule = require('sandboxed-module')
diff --git a/services/clsi/test/unit/js/LatexRunnerTests.js b/services/clsi/test/unit/js/LatexRunnerTests.js
index ca36d7308f..0d250dd517 100644
--- a/services/clsi/test/unit/js/LatexRunnerTests.js
+++ b/services/clsi/test/unit/js/LatexRunnerTests.js
@@ -2,7 +2,7 @@ const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const MODULE_PATH = require('path').join(
+const MODULE_PATH = require('node:path').join(
__dirname,
'../../../app/js/LatexRunner'
)
diff --git a/services/clsi/test/unit/js/LockManagerTests.js b/services/clsi/test/unit/js/LockManagerTests.js
index cd0d34cd4a..7005b3e5a3 100644
--- a/services/clsi/test/unit/js/LockManagerTests.js
+++ b/services/clsi/test/unit/js/LockManagerTests.js
@@ -1,7 +1,7 @@
const { expect } = require('chai')
const sinon = require('sinon')
const SandboxedModule = require('sandboxed-module')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/LockManager'
)
@@ -21,6 +21,7 @@ describe('LockManager', function () {
compileConcurrencyLimit: 5,
}),
'./Errors': (this.Erros = Errors),
+ './RequestParser': { MAX_TIMEOUT: 600 },
},
})
})
diff --git a/services/clsi/test/unit/js/OutputControllerTests.js b/services/clsi/test/unit/js/OutputControllerTests.js
index 33367af26a..ee5c9c2a7a 100644
--- a/services/clsi/test/unit/js/OutputControllerTests.js
+++ b/services/clsi/test/unit/js/OutputControllerTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const MODULE_PATH = require('path').join(
+const MODULE_PATH = require('node:path').join(
__dirname,
'../../../app/js/OutputController'
)
@@ -19,7 +19,7 @@ describe('OutputController', function () {
'./OutputFileArchiveManager': {
archiveFilesForBuild: this.archiveFilesForBuild,
},
- 'node:stream/promises': {
+ 'stream/promises': {
pipeline: this.pipeline,
},
},
diff --git a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js b/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
index bd62b00825..d6817f3559 100644
--- a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
+++ b/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
@@ -2,7 +2,7 @@ const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const MODULE_PATH = require('path').join(
+const MODULE_PATH = require('node:path').join(
__dirname,
'../../../app/js/OutputFileArchiveManager'
)
@@ -50,7 +50,7 @@ describe('OutputFileArchiveManager', function () {
'./OutputFileFinder': this.OutputFileFinder,
'./OutputCacheManager': this.OutputCacheManger,
archiver: this.archiver,
- 'node:fs/promises': this.fs,
+ 'fs/promises': this.fs,
'@overleaf/settings': {
path: {
outputDir: this.outputDir,
diff --git a/services/clsi/test/unit/js/OutputFileFinderTests.js b/services/clsi/test/unit/js/OutputFileFinderTests.js
index cf596a6271..c9e1b443be 100644
--- a/services/clsi/test/unit/js/OutputFileFinderTests.js
+++ b/services/clsi/test/unit/js/OutputFileFinderTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/OutputFileFinder'
)
diff --git a/services/clsi/test/unit/js/OutputFileOptimiserTests.js b/services/clsi/test/unit/js/OutputFileOptimiserTests.js
index d74c6a3534..1dd1a751b9 100644
--- a/services/clsi/test/unit/js/OutputFileOptimiserTests.js
+++ b/services/clsi/test/unit/js/OutputFileOptimiserTests.js
@@ -12,13 +12,13 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/OutputFileOptimiser'
)
-const path = require('path')
+const path = require('node:path')
const { expect } = require('chai')
-const { EventEmitter } = require('events')
+const { EventEmitter } = require('node:events')
describe('OutputFileOptimiser', function () {
beforeEach(function () {
diff --git a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
index c5f30b1ea9..4f42411fba 100644
--- a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
+++ b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
@@ -13,7 +13,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const assert = require('chai').assert
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/ProjectPersistenceManager'
)
@@ -21,11 +21,16 @@ const tk = require('timekeeper')
describe('ProjectPersistenceManager', function () {
beforeEach(function () {
+ this.fsPromises = {
+ statfs: sinon.stub(),
+ }
+
this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
requires: {
+ '@overleaf/metrics': (this.Metrics = { gauge: sinon.stub() }),
'./UrlCache': (this.UrlCache = {}),
'./CompileManager': (this.CompileManager = {}),
- diskusage: (this.diskusage = { check: sinon.stub() }),
+ fs: { promises: this.fsPromises },
'@overleaf/settings': (this.settings = {
project_cache_length_ms: 1000,
path: {
@@ -43,12 +48,17 @@ describe('ProjectPersistenceManager', function () {
describe('refreshExpiryTimeout', function () {
it('should leave expiry alone if plenty of disk', function (done) {
- this.diskusage.check.resolves({
- available: 40,
- total: 100,
+ this.fsPromises.statfs.resolves({
+ blocks: 100,
+ bsize: 1,
+ bavail: 40,
})
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+ this.Metrics.gauge.should.have.been.calledWith(
+ 'disk_available_percent',
+ 40
+ )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
this.settings.project_cache_length_ms
)
@@ -57,31 +67,41 @@ describe('ProjectPersistenceManager', function () {
})
it('should drop EXPIRY_TIMEOUT 10% if low disk usage', function (done) {
- this.diskusage.check.resolves({
- available: 5,
- total: 100,
+ this.fsPromises.statfs.resolves({
+ blocks: 100,
+ bsize: 1,
+ bavail: 5,
})
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+ this.Metrics.gauge.should.have.been.calledWith(
+ 'disk_available_percent',
+ 5
+ )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
done()
})
})
it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', function (done) {
- this.diskusage.check.resolves({
- available: 5,
- total: 100,
+ this.fsPromises.statfs.resolves({
+ blocks: 100,
+ bsize: 1,
+ bavail: 5,
})
this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+ this.Metrics.gauge.should.have.been.calledWith(
+ 'disk_available_percent',
+ 5
+ )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
done()
})
})
it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', function (done) {
- this.diskusage.check.throws(new Error())
+ this.fsPromises.statfs.rejects(new Error())
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000)
done()
diff --git a/services/clsi/test/unit/js/RequestParserTests.js b/services/clsi/test/unit/js/RequestParserTests.js
index 4cf31a2b37..437c3c4fbe 100644
--- a/services/clsi/test/unit/js/RequestParserTests.js
+++ b/services/clsi/test/unit/js/RequestParserTests.js
@@ -1,7 +1,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/RequestParser'
)
@@ -30,6 +30,7 @@ describe('RequestParser', function () {
this.RequestParser = SandboxedModule.require(modulePath, {
requires: {
'@overleaf/settings': (this.settings = {}),
+ './OutputCacheManager': { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ },
},
})
})
@@ -274,6 +275,37 @@ describe('RequestParser', function () {
})
})
+ describe('with a valid buildId', function () {
+ beforeEach(function (done) {
+ this.validRequest.compile.options.buildId = '195a4869176-a4ad60bee7bf35e4'
+ this.RequestParser.parse(this.validRequest, (error, data) => {
+ if (error) return done(error)
+ this.data = data
+ done()
+ })
+ })
+
+ it('should return an error', function () {
+ this.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4')
+ })
+ })
+
+ describe('with a bad buildId', function () {
+ beforeEach(function () {
+ this.validRequest.compile.options.buildId = 'foo/bar'
+ this.RequestParser.parse(this.validRequest, this.callback)
+ })
+
+ it('should return an error', function () {
+ this.callback
+ .calledWithMatch({
+ message:
+ 'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/',
+ })
+ .should.equal(true)
+ })
+ })
+
describe('with a resource with a valid date', function () {
beforeEach(function () {
this.date = '12:00 01/02/03'
diff --git a/services/clsi/test/unit/js/ResourceStateManagerTests.js b/services/clsi/test/unit/js/ResourceStateManagerTests.js
index 0a97d7b705..823c81616f 100644
--- a/services/clsi/test/unit/js/ResourceStateManagerTests.js
+++ b/services/clsi/test/unit/js/ResourceStateManagerTests.js
@@ -12,11 +12,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/ResourceStateManager'
)
-const Path = require('path')
+const Path = require('node:path')
const Errors = require('../../../app/js/Errors')
describe('ResourceStateManager', function () {
diff --git a/services/clsi/test/unit/js/ResourceWriterTests.js b/services/clsi/test/unit/js/ResourceWriterTests.js
index 3b18a3b195..c2e09ce9cf 100644
--- a/services/clsi/test/unit/js/ResourceWriterTests.js
+++ b/services/clsi/test/unit/js/ResourceWriterTests.js
@@ -13,11 +13,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/ResourceWriter'
)
-const path = require('path')
+const path = require('node:path')
describe('ResourceWriter', function () {
beforeEach(function () {
@@ -378,12 +378,13 @@ describe('ResourceWriter', function () {
this.fs.mkdir = sinon.stub().callsArg(2)
this.resource = {
path: 'main.tex',
- url: 'http://www.example.com/main.tex',
+ url: 'http://www.example.com/primary/main.tex',
+ fallbackURL: 'http://fallback.example.com/fallback/main.tex',
modified: Date.now(),
}
this.UrlCache.downloadUrlToFile = sinon
.stub()
- .callsArgWith(4, 'fake error downloading file')
+ .callsArgWith(5, 'fake error downloading file')
return this.ResourceWriter._writeResourceToDisk(
this.project_id,
this.resource,
@@ -405,6 +406,7 @@ describe('ResourceWriter', function () {
.calledWith(
this.project_id,
this.resource.url,
+ this.resource.fallbackURL,
path.join(this.basePath, this.resource.path),
this.resource.modified
)
diff --git a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js b/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
index 0a3806a1e6..53507fe3f2 100644
--- a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
+++ b/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
@@ -10,8 +10,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const SandboxedModule = require('sandboxed-module')
-const assert = require('assert')
-const path = require('path')
+const assert = require('node:assert')
+const path = require('node:path')
const sinon = require('sinon')
const modulePath = path.join(
__dirname,
diff --git a/services/clsi/test/unit/js/SynctexOutputParserTests.js b/services/clsi/test/unit/js/SynctexOutputParserTests.js
index 19bf2f6d37..b999a6adb1 100644
--- a/services/clsi/test/unit/js/SynctexOutputParserTests.js
+++ b/services/clsi/test/unit/js/SynctexOutputParserTests.js
@@ -1,4 +1,4 @@
-const Path = require('path')
+const Path = require('node:path')
const SandboxedModule = require('sandboxed-module')
const { expect } = require('chai')
diff --git a/services/clsi/test/unit/js/TikzManager.js b/services/clsi/test/unit/js/TikzManager.js
index 8e01194955..ee651f6b1d 100644
--- a/services/clsi/test/unit/js/TikzManager.js
+++ b/services/clsi/test/unit/js/TikzManager.js
@@ -10,7 +10,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/TikzManager'
)
diff --git a/services/clsi/test/unit/js/UrlCacheTests.js b/services/clsi/test/unit/js/UrlCacheTests.js
index a5c1e60c0a..a3dc2fac3c 100644
--- a/services/clsi/test/unit/js/UrlCacheTests.js
+++ b/services/clsi/test/unit/js/UrlCacheTests.js
@@ -13,13 +13,17 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(__dirname, '../../../app/js/UrlCache')
+const modulePath = require('node:path').join(
+ __dirname,
+ '../../../app/js/UrlCache'
+)
describe('UrlCache', function () {
beforeEach(function () {
this.callback = sinon.stub()
this.url =
'http://filestore/project/60b0dd39c418bc00598a0d22/file/60ae721ffb1d920027d3201f'
+ this.fallbackURL = 'http://filestore/bucket/project-blobs/key/ab/cd/ef'
this.project_id = '60b0dd39c418bc00598a0d22'
return (this.UrlCache = SandboxedModule.require(modulePath, {
requires: {
@@ -51,6 +55,29 @@ describe('UrlCache', function () {
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
+ this.fallbackURL,
+ this.destPath,
+ this.lastModified,
+ error => {
+ expect(error).to.not.exist
+ expect(
+ this.UrlFetcher.promises.pipeUrlToFileWithRetry.called
+ ).to.equal(false)
+ done()
+ }
+ )
+ })
+
+ it('should not download on the semi-happy path', function (done) {
+ const codedError = new Error()
+ codedError.code = 'ENOENT'
+ this.fs.promises.copyFile.onCall(0).rejects(codedError)
+ this.fs.promises.copyFile.onCall(1).resolves()
+
+ this.UrlCache.downloadUrlToFile(
+ this.project_id,
+ this.url,
+ this.fallbackURL,
this.destPath,
this.lastModified,
error => {
@@ -67,11 +94,13 @@ describe('UrlCache', function () {
const codedError = new Error()
codedError.code = 'ENOENT'
this.fs.promises.copyFile.onCall(0).rejects(codedError)
- this.fs.promises.copyFile.onCall(1).resolves()
+ this.fs.promises.copyFile.onCall(1).rejects(codedError)
+ this.fs.promises.copyFile.onCall(2).resolves()
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
+ this.fallbackURL,
this.destPath,
this.lastModified,
error => {
@@ -91,6 +120,7 @@ describe('UrlCache', function () {
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
+ this.fallbackURL,
this.destPath,
this.lastModified,
error => {
diff --git a/services/clsi/test/unit/js/pdfjsTests.js b/services/clsi/test/unit/js/pdfjsTests.js
index ef85ccc704..bc8b775b43 100644
--- a/services/clsi/test/unit/js/pdfjsTests.js
+++ b/services/clsi/test/unit/js/pdfjsTests.js
@@ -1,5 +1,5 @@
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
const { expect } = require('chai')
const { parseXrefTable } = require('../../../app/js/XrefParser')
const { NoXrefTableError } = require('../../../app/js/Errors')
diff --git a/services/contacts/.nvmrc b/services/contacts/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/contacts/.nvmrc
+++ b/services/contacts/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/contacts/Dockerfile b/services/contacts/Dockerfile
index 2b2d8dae61..b59cada7b3 100644
--- a/services/contacts/Dockerfile
+++ b/services/contacts/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/contacts
diff --git a/services/contacts/Makefile b/services/contacts/Makefile
index 1f2b7a10fb..3309e298e8 100644
--- a/services/contacts/Makefile
+++ b/services/contacts/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -98,13 +116,6 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
- while ! mongosh --eval "db.version()" > /dev/null; do \
- echo "Waiting for Mongo..."; \
- sleep 1; \
- done; \
- mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -137,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/contacts/buildscript.txt b/services/contacts/buildscript.txt
index cfbc56040e..b20764246c 100644
--- a/services/contacts/buildscript.txt
+++ b/services/contacts/buildscript.txt
@@ -4,6 +4,6 @@ contacts
--env-add=
--env-pass-through=
--esmock-loader=True
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=False
---script-version=4.5.0
+--script-version=4.7.0
diff --git a/services/contacts/config/settings.defaults.cjs b/services/contacts/config/settings.defaults.cjs
index fb716d1812..7ffdb83ce5 100644
--- a/services/contacts/config/settings.defaults.cjs
+++ b/services/contacts/config/settings.defaults.cjs
@@ -1,5 +1,9 @@
-const http = require('http')
+const http = require('node:http')
+const https = require('node:https')
+
http.globalAgent.maxSockets = 300
+http.globalAgent.keepAlive = false
+https.globalAgent.keepAlive = false
module.exports = {
internal: {
diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml
index 6f1a608534..ca3303a079 100644
--- a/services/contacts/docker-compose.ci.yml
+++ b/services/contacts/docker-compose.ci.yml
@@ -24,10 +24,13 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
+ volumes:
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
user: node
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -39,9 +42,14 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml
index 71a8aaa5f9..474ea224f8 100644
--- a/services/contacts/docker-compose.yml
+++ b/services/contacts/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/contacts
- ../../node_modules:/overleaf/node_modules
@@ -14,37 +14,45 @@ services:
working_dir: /overleaf/services/contacts
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/contacts
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/contacts
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/contacts/package.json b/services/contacts/package.json
index 7906016a77..db707e55c9 100644
--- a/services/contacts/package.json
+++ b/services/contacts/package.json
@@ -6,14 +6,14 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
- "test:acceptance:_run": "LOG_LEVEL=fatal mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+ "test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
- "test:unit:_run": "LOG_LEVEL=fatal mocha --loader=esmock --recursive --reporter spec $@ test/unit/js",
+ "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -24,8 +24,8 @@
"async": "^3.2.5",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "express": "^4.21.0",
- "mongodb": "^6.1.0",
+ "express": "^4.21.2",
+ "mongodb": "6.12.0",
"request": "~2.88.2",
"underscore": "~1.13.1"
},
@@ -33,7 +33,7 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"esmock": "^2.6.3",
- "mocha": "^10.2.0",
+ "mocha": "^11.1.0",
"sinon": "~9.0.1",
"sinon-chai": "^3.7.0",
"typescript": "^5.0.4"
diff --git a/services/docstore/.gitignore b/services/docstore/.gitignore
deleted file mode 100644
index 84bf300f7f..0000000000
--- a/services/docstore/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-node_modules
-forever
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc
-
-# Jetbrains IDEs
-.idea
diff --git a/services/docstore/.nvmrc b/services/docstore/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/docstore/.nvmrc
+++ b/services/docstore/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/docstore/Dockerfile b/services/docstore/Dockerfile
index d1a661fc7b..f24f9ddaf7 100644
--- a/services/docstore/Dockerfile
+++ b/services/docstore/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/docstore
diff --git a/services/docstore/Makefile b/services/docstore/Makefile
index 25068f95e6..2b3596b0b4 100644
--- a/services/docstore/Makefile
+++ b/services/docstore/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -98,13 +116,6 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
- while ! mongosh --eval "db.version()" > /dev/null; do \
- echo "Waiting for Mongo..."; \
- sleep 1; \
- done; \
- mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -137,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/docstore/app.js b/services/docstore/app.js
index 51ad785065..ef755c4bb1 100644
--- a/services/docstore/app.js
+++ b/services/docstore/app.js
@@ -1,7 +1,7 @@
// Metrics must be initialized before importing anything else
require('@overleaf/metrics/initialize')
-const Events = require('events')
+const Events = require('node:events')
const Metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
@@ -50,6 +50,14 @@ app.param('doc_id', function (req, res, next, docId) {
app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
app.get('/project/:project_id/doc', HttpController.getAllDocs)
app.get('/project/:project_id/ranges', HttpController.getAllRanges)
+app.get(
+ '/project/:project_id/comment-thread-ids',
+ HttpController.getCommentThreadIds
+)
+app.get(
+ '/project/:project_id/tracked-changes-user-ids',
+ HttpController.getTrackedChangesUserIds
+)
app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges)
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted)
@@ -88,14 +96,17 @@ app.get('/status', (req, res) => res.send('docstore is alive'))
app.use(handleValidationErrors())
app.use(function (error, req, res, next) {
- logger.error({ err: error, req }, 'request errored')
if (error instanceof Errors.NotFoundError) {
+ logger.warn({ req }, 'not found')
res.sendStatus(404)
} else if (error instanceof Errors.DocModifiedError) {
+ logger.warn({ req }, 'conflict: doc modified')
res.sendStatus(409)
} else if (error instanceof Errors.DocVersionDecrementedError) {
+ logger.warn({ req }, 'conflict: doc version decremented')
res.sendStatus(409)
} else {
+ logger.error({ err: error, req }, 'request errored')
res.status(500).send('Oops, something went wrong')
}
})
diff --git a/services/docstore/app/js/DocArchiveManager.js b/services/docstore/app/js/DocArchiveManager.js
index 035448f4a1..d03ee161a8 100644
--- a/services/docstore/app/js/DocArchiveManager.js
+++ b/services/docstore/app/js/DocArchiveManager.js
@@ -1,35 +1,18 @@
-const { callbackify } = require('util')
-const MongoManager = require('./MongoManager').promises
+const MongoManager = require('./MongoManager')
const Errors = require('./Errors')
const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const { ReadableString } = require('@overleaf/stream-utils')
const RangeManager = require('./RangeManager')
const PersistorManager = require('./PersistorManager')
const pMap = require('p-map')
+const { streamToBuffer } = require('./StreamToBuffer')
const { BSON } = require('mongodb-legacy')
const PARALLEL_JOBS = Settings.parallelArchiveJobs
const UN_ARCHIVE_BATCH_SIZE = Settings.unArchiveBatchSize
-module.exports = {
- archiveAllDocs: callbackify(archiveAllDocs),
- archiveDoc: callbackify(archiveDoc),
- unArchiveAllDocs: callbackify(unArchiveAllDocs),
- unarchiveDoc: callbackify(unarchiveDoc),
- destroyProject: callbackify(destroyProject),
- getDoc: callbackify(getDoc),
- promises: {
- archiveAllDocs,
- archiveDoc,
- unArchiveAllDocs,
- unarchiveDoc,
- destroyProject,
- getDoc,
- },
-}
-
async function archiveAllDocs(projectId) {
if (!_isArchivingEnabled()) {
return
@@ -61,6 +44,8 @@ async function archiveDoc(projectId, docId) {
throw new Error('doc has no lines')
}
+ RangeManager.fixCommentIds(doc)
+
// warn about any oversized docs already in mongo
const linesSize = BSON.calculateObjectSize(doc.lines || {})
const rangesSize = BSON.calculateObjectSize(doc.ranges || {})
@@ -136,7 +121,7 @@ async function getDoc(projectId, docId) {
key
)
stream.resume()
- const buffer = await _streamToBuffer(projectId, docId, stream)
+ const buffer = await streamToBuffer(projectId, docId, stream)
const md5 = crypto.createHash('md5').update(buffer).digest('hex')
if (sourceMd5 !== md5) {
throw new Errors.Md5MismatchError('md5 mismatch when downloading doc', {
@@ -187,34 +172,6 @@ async function destroyProject(projectId) {
await Promise.all(tasks)
}
-async function _streamToBuffer(projectId, docId, stream) {
- const chunks = []
- let size = 0
- let logged = false
- const logIfTooLarge = finishedReading => {
- if (size <= Settings.max_doc_length) return
- // Log progress once and then again at the end.
- if (logged && !finishedReading) return
- logger.warn(
- { projectId, docId, size, finishedReading },
- 'potentially large doc pulled down from gcs'
- )
- logged = true
- }
- return await new Promise((resolve, reject) => {
- stream.on('data', chunk => {
- size += chunk.byteLength
- logIfTooLarge(false)
- chunks.push(chunk)
- })
- stream.on('error', reject)
- stream.on('end', () => {
- logIfTooLarge(true)
- resolve(Buffer.concat(chunks))
- })
- })
-}
-
function _deserializeArchivedDoc(buffer) {
const doc = JSON.parse(buffer)
@@ -252,3 +209,12 @@ function _isArchivingEnabled() {
return true
}
+
+module.exports = {
+ archiveAllDocs,
+ archiveDoc,
+ unArchiveAllDocs,
+ unarchiveDoc,
+ destroyProject,
+ getDoc,
+}
diff --git a/services/docstore/app/js/DocManager.js b/services/docstore/app/js/DocManager.js
index 80e8ae4527..c9e8dadc2c 100644
--- a/services/docstore/app/js/DocManager.js
+++ b/services/docstore/app/js/DocManager.js
@@ -5,8 +5,7 @@ const _ = require('lodash')
const DocArchive = require('./DocArchiveManager')
const RangeManager = require('./RangeManager')
const Settings = require('@overleaf/settings')
-const { callbackifyAll } = require('@overleaf/promise-utils')
-const { setTimeout } = require('timers/promises')
+const { setTimeout } = require('node:timers/promises')
/**
* @import { Document } from 'mongodb'
@@ -29,7 +28,7 @@ const DocManager = {
throw new Error('must include inS3 when getting doc')
}
- const doc = await MongoManager.promises.findDoc(projectId, docId, filter)
+ const doc = await MongoManager.findDoc(projectId, docId, filter)
if (doc == null) {
throw new Errors.NotFoundError(
@@ -38,15 +37,19 @@ const DocManager = {
}
if (doc.inS3) {
- await DocArchive.promises.unarchiveDoc(projectId, docId)
+ await DocArchive.unarchiveDoc(projectId, docId)
return await DocManager._getDoc(projectId, docId, filter)
}
+ if (filter.ranges) {
+ RangeManager.fixCommentIds(doc)
+ }
+
return doc
},
async isDocDeleted(projectId, docId) {
- const doc = await MongoManager.promises.findDoc(projectId, docId, {
+ const doc = await MongoManager.findDoc(projectId, docId, {
deleted: true,
})
@@ -74,7 +77,7 @@ const DocManager = {
// returns the doc without any version information
async _peekRawDoc(projectId, docId) {
- const doc = await MongoManager.promises.findDoc(projectId, docId, {
+ const doc = await MongoManager.findDoc(projectId, docId, {
lines: true,
rev: true,
deleted: true,
@@ -91,7 +94,7 @@ const DocManager = {
if (doc.inS3) {
// skip the unarchiving to mongo when getting a doc
- const archivedDoc = await DocArchive.promises.getDoc(projectId, docId)
+ const archivedDoc = await DocArchive.getDoc(projectId, docId)
Object.assign(doc, archivedDoc)
}
@@ -102,7 +105,7 @@ const DocManager = {
// without unarchiving it (avoids unnecessary writes to mongo)
async peekDoc(projectId, docId) {
const doc = await DocManager._peekRawDoc(projectId, docId)
- await MongoManager.promises.checkRevUnchanged(doc)
+ await MongoManager.checkRevUnchanged(doc)
return doc
},
@@ -111,16 +114,18 @@ const DocManager = {
lines: true,
inS3: true,
})
- return doc
+ if (!doc) throw new Errors.NotFoundError()
+ if (!Array.isArray(doc.lines)) throw new Errors.DocWithoutLinesError()
+ return doc.lines.join('\n')
},
async getAllDeletedDocs(projectId, filter) {
- return await MongoManager.promises.getProjectsDeletedDocs(projectId, filter)
+ return await MongoManager.getProjectsDeletedDocs(projectId, filter)
},
async getAllNonDeletedDocs(projectId, filter) {
- await DocArchive.promises.unArchiveAllDocs(projectId)
- const docs = await MongoManager.promises.getProjectsDocs(
+ await DocArchive.unArchiveAllDocs(projectId)
+ const docs = await MongoManager.getProjectsDocs(
projectId,
{ include_deleted: false },
filter
@@ -128,15 +133,46 @@ const DocManager = {
if (docs == null) {
throw new Errors.NotFoundError(`No docs for project ${projectId}`)
}
+ if (filter.ranges) {
+ for (const doc of docs) {
+ RangeManager.fixCommentIds(doc)
+ }
+ }
return docs
},
+ async getCommentThreadIds(projectId) {
+ const docs = await DocManager.getAllNonDeletedDocs(projectId, {
+ _id: true,
+ ranges: true,
+ })
+ const byDoc = new Map()
+ for (const doc of docs) {
+ const ids = new Set()
+ for (const comment of doc.ranges?.comments || []) {
+ ids.add(comment.op.t)
+ }
+ if (ids.size > 0) byDoc.set(doc._id.toString(), Array.from(ids))
+ }
+ return Object.fromEntries(byDoc.entries())
+ },
+
+ async getTrackedChangesUserIds(projectId) {
+ const docs = await DocManager.getAllNonDeletedDocs(projectId, {
+ ranges: true,
+ })
+ const userIds = new Set()
+ for (const doc of docs) {
+ for (const change of doc.ranges?.changes || []) {
+ if (change.metadata.user_id === 'anonymous-user') continue
+ userIds.add(change.metadata.user_id)
+ }
+ }
+ return Array.from(userIds)
+ },
+
async projectHasRanges(projectId) {
- const docs = await MongoManager.promises.getProjectsDocs(
- projectId,
- {},
- { _id: 1 }
- )
+ const docs = await MongoManager.getProjectsDocs(projectId, {}, { _id: 1 })
const docIds = docs.map(doc => doc._id)
for (const docId of docIds) {
const doc = await DocManager.peekDoc(projectId, docId)
@@ -247,7 +283,7 @@ const DocManager = {
}
modified = true
- await MongoManager.promises.upsertIntoDocCollection(
+ await MongoManager.upsertIntoDocCollection(
projectId,
docId,
doc?.rev,
@@ -262,11 +298,7 @@ const DocManager = {
async patchDoc(projectId, docId, meta) {
const projection = { _id: 1, deleted: true }
- const doc = await MongoManager.promises.findDoc(
- projectId,
- docId,
- projection
- )
+ const doc = await MongoManager.findDoc(projectId, docId, projection)
if (!doc) {
throw new Errors.NotFoundError(
`No such project/doc to delete: ${projectId}/${docId}`
@@ -275,7 +307,7 @@ const DocManager = {
if (meta.deleted && Settings.docstore.archiveOnSoftDelete) {
// The user will not read this doc anytime soon. Flush it out of mongo.
- DocArchive.promises.archiveDoc(projectId, docId).catch(err => {
+ DocArchive.archiveDoc(projectId, docId).catch(err => {
logger.warn(
{ projectId, docId, err },
'archiving a single doc in the background failed'
@@ -283,15 +315,8 @@ const DocManager = {
})
}
- await MongoManager.promises.patchDoc(projectId, docId, meta)
+ await MongoManager.patchDoc(projectId, docId, meta)
},
}
-module.exports = {
- ...callbackifyAll(DocManager, {
- multiResult: {
- updateDoc: ['modified', 'rev'],
- },
- }),
- promises: DocManager,
-}
+module.exports = DocManager
diff --git a/services/docstore/app/js/Errors.js b/services/docstore/app/js/Errors.js
index bbdbe75c08..7b150cc0db 100644
--- a/services/docstore/app/js/Errors.js
+++ b/services/docstore/app/js/Errors.js
@@ -10,10 +10,13 @@ class DocRevValueError extends OError {}
class DocVersionDecrementedError extends OError {}
+class DocWithoutLinesError extends OError {}
+
module.exports = {
Md5MismatchError,
DocModifiedError,
DocRevValueError,
DocVersionDecrementedError,
+ DocWithoutLinesError,
...Errors,
}
diff --git a/services/docstore/app/js/HealthChecker.js b/services/docstore/app/js/HealthChecker.js
index 65dc92cbba..a5b7ad7e9a 100644
--- a/services/docstore/app/js/HealthChecker.js
+++ b/services/docstore/app/js/HealthChecker.js
@@ -1,67 +1,35 @@
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
const { db, ObjectId } = require('./mongodb')
-const request = require('request')
-const async = require('async')
const _ = require('lodash')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const settings = require('@overleaf/settings')
const { port } = settings.internal.docstore
const logger = require('@overleaf/logger')
+const { fetchNothing, fetchJson } = require('@overleaf/fetch-utils')
-module.exports = {
- check(callback) {
- const docId = new ObjectId()
- const projectId = new ObjectId(settings.docstore.healthCheck.project_id)
- const url = `http://127.0.0.1:${port}/project/${projectId}/doc/${docId}`
- const lines = [
- 'smoke test - delete me',
- `${crypto.randomBytes(32).toString('hex')}`,
- ]
- const getOpts = () => ({
- url,
- timeout: 3000,
+async function check() {
+ const docId = new ObjectId()
+ const projectId = new ObjectId(settings.docstore.healthCheck.project_id)
+ const url = `http://127.0.0.1:${port}/project/${projectId}/doc/${docId}`
+ const lines = [
+ 'smoke test - delete me',
+ `${crypto.randomBytes(32).toString('hex')}`,
+ ]
+ logger.debug({ lines, url, docId, projectId }, 'running health check')
+ let body
+ try {
+ await fetchNothing(url, {
+ method: 'POST',
+ json: { lines, version: 42, ranges: {} },
+ signal: AbortSignal.timeout(3_000),
})
- logger.debug({ lines, url, docId, projectId }, 'running health check')
- const jobs = [
- function (cb) {
- const opts = getOpts()
- opts.json = { lines, version: 42, ranges: {} }
- return request.post(opts, cb)
- },
- function (cb) {
- const opts = getOpts()
- opts.json = true
- return request.get(opts, function (err, res, body) {
- if (err != null) {
- logger.err({ err }, 'docstore returned a error in health check get')
- return cb(err)
- } else if (res == null) {
- return cb(new Error('no response from docstore with get check'))
- } else if ((res != null ? res.statusCode : undefined) !== 200) {
- return cb(new Error(`status code not 200, its ${res.statusCode}`))
- } else if (
- _.isEqual(body != null ? body.lines : undefined, lines) &&
- (body != null ? body._id : undefined) === docId.toString()
- ) {
- return cb()
- } else {
- return cb(
- new Error(
- `health check lines not equal ${body.lines} != ${lines}`
- )
- )
- }
- })
- },
- cb => db.docs.deleteOne({ _id: docId, project_id: projectId }, cb),
- ]
- return async.series(jobs, callback)
- },
+ body = await fetchJson(url, { signal: AbortSignal.timeout(3_000) })
+ } finally {
+ await db.docs.deleteOne({ _id: docId, project_id: projectId })
+ }
+ if (!_.isEqual(body?.lines, lines)) {
+ throw new Error(`health check lines not equal ${body.lines} != ${lines}`)
+ }
+}
+module.exports = {
+ check,
}
diff --git a/services/docstore/app/js/HttpController.js b/services/docstore/app/js/HttpController.js
index 1c4e137033..50c4302aeb 100644
--- a/services/docstore/app/js/HttpController.js
+++ b/services/docstore/app/js/HttpController.js
@@ -4,143 +4,104 @@ const DocArchive = require('./DocArchiveManager')
const HealthChecker = require('./HealthChecker')
const Errors = require('./Errors')
const Settings = require('@overleaf/settings')
+const { expressify } = require('@overleaf/promise-utils')
-function getDoc(req, res, next) {
+async function getDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
const includeDeleted = req.query.include_deleted === 'true'
logger.debug({ projectId, docId }, 'getting doc')
- DocManager.getFullDoc(projectId, docId, function (error, doc) {
- if (error) {
- return next(error)
- }
- logger.debug({ docId, projectId }, 'got doc')
- if (doc == null) {
- res.sendStatus(404)
- } else if (doc.deleted && !includeDeleted) {
- res.sendStatus(404)
- } else {
- res.json(_buildDocView(doc))
- }
- })
+ const doc = await DocManager.getFullDoc(projectId, docId)
+ logger.debug({ docId, projectId }, 'got doc')
+ if (doc.deleted && !includeDeleted) {
+ res.sendStatus(404)
+ } else {
+ res.json(_buildDocView(doc))
+ }
}
-function peekDoc(req, res, next) {
+async function peekDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'peeking doc')
- DocManager.peekDoc(projectId, docId, function (error, doc) {
- if (error) {
- return next(error)
- }
- if (doc == null) {
- res.sendStatus(404)
- } else {
- res.setHeader('x-doc-status', doc.inS3 ? 'archived' : 'active')
- res.json(_buildDocView(doc))
- }
- })
+ const doc = await DocManager.peekDoc(projectId, docId)
+ res.setHeader('x-doc-status', doc.inS3 ? 'archived' : 'active')
+ res.json(_buildDocView(doc))
}
-function isDocDeleted(req, res, next) {
+async function isDocDeleted(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
- DocManager.isDocDeleted(projectId, docId, function (error, deleted) {
- if (error) {
- return next(error)
- }
- res.json({ deleted })
- })
+ const deleted = await DocManager.isDocDeleted(projectId, docId)
+ res.json({ deleted })
}
-function getRawDoc(req, res, next) {
+async function getRawDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'getting raw doc')
- DocManager.getDocLines(projectId, docId, function (error, doc) {
- if (error) {
- return next(error)
- }
- if (doc == null) {
- res.sendStatus(404)
- } else {
- res.setHeader('content-type', 'text/plain')
- res.send(_buildRawDocView(doc))
- }
- })
+ const content = await DocManager.getDocLines(projectId, docId)
+ res.setHeader('content-type', 'text/plain')
+ res.send(content)
}
-function getAllDocs(req, res, next) {
+async function getAllDocs(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'getting all docs')
- DocManager.getAllNonDeletedDocs(
- projectId,
- { lines: true, rev: true },
- function (error, docs) {
- if (docs == null) {
- docs = []
- }
- if (error) {
- return next(error)
- }
- const docViews = _buildDocsArrayView(projectId, docs)
- for (const docView of docViews) {
- if (!docView.lines) {
- logger.warn({ projectId, docId: docView._id }, 'missing doc lines')
- docView.lines = []
- }
- }
- res.json(docViews)
+ const docs = await DocManager.getAllNonDeletedDocs(projectId, {
+ lines: true,
+ rev: true,
+ })
+ const docViews = _buildDocsArrayView(projectId, docs)
+ for (const docView of docViews) {
+ if (!docView.lines) {
+ logger.warn({ projectId, docId: docView._id }, 'missing doc lines')
+ docView.lines = []
}
- )
+ }
+ res.json(docViews)
}
-function getAllDeletedDocs(req, res, next) {
+async function getAllDeletedDocs(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'getting all deleted docs')
- DocManager.getAllDeletedDocs(
- projectId,
- { name: true, deletedAt: true },
- function (error, docs) {
- if (error) {
- return next(error)
- }
- res.json(
- docs.map(doc => ({
- _id: doc._id.toString(),
- name: doc.name,
- deletedAt: doc.deletedAt,
- }))
- )
- }
+ const docs = await DocManager.getAllDeletedDocs(projectId, {
+ name: true,
+ deletedAt: true,
+ })
+ res.json(
+ docs.map(doc => ({
+ _id: doc._id.toString(),
+ name: doc.name,
+ deletedAt: doc.deletedAt,
+ }))
)
}
-function getAllRanges(req, res, next) {
+async function getAllRanges(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'getting all ranges')
- DocManager.getAllNonDeletedDocs(
- projectId,
- { ranges: true },
- function (error, docs) {
- if (docs == null) {
- docs = []
- }
- if (error) {
- return next(error)
- }
- res.json(_buildDocsArrayView(projectId, docs))
- }
- )
-}
-
-function projectHasRanges(req, res, next) {
- const { project_id: projectId } = req.params
- DocManager.projectHasRanges(projectId, (err, projectHasRanges) => {
- if (err) {
- return next(err)
- }
- res.json({ projectHasRanges })
+ const docs = await DocManager.getAllNonDeletedDocs(projectId, {
+ ranges: true,
})
+ res.json(_buildDocsArrayView(projectId, docs))
}
-function updateDoc(req, res, next) {
+async function getCommentThreadIds(req, res) {
+ const { project_id: projectId } = req.params
+ const threadIds = await DocManager.getCommentThreadIds(projectId)
+ res.json(threadIds)
+}
+
+async function getTrackedChangesUserIds(req, res) {
+ const { project_id: projectId } = req.params
+ const userIds = await DocManager.getTrackedChangesUserIds(projectId)
+ res.json(userIds)
+}
+
+async function projectHasRanges(req, res) {
+ const { project_id: projectId } = req.params
+ const projectHasRanges = await DocManager.projectHasRanges(projectId)
+ res.json({ projectHasRanges })
+}
+
+async function updateDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
const lines = req.body?.lines
const version = req.body?.version
@@ -172,25 +133,20 @@ function updateDoc(req, res, next) {
}
logger.debug({ projectId, docId }, 'got http request to update doc')
- DocManager.updateDoc(
+ const { modified, rev } = await DocManager.updateDoc(
projectId,
docId,
lines,
version,
- ranges,
- function (error, modified, rev) {
- if (error) {
- return next(error)
- }
- res.json({
- modified,
- rev,
- })
- }
+ ranges
)
+ res.json({
+ modified,
+ rev,
+ })
}
-function patchDoc(req, res, next) {
+async function patchDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'patching doc')
@@ -203,12 +159,8 @@ function patchDoc(req, res, next) {
logger.fatal({ field }, 'joi validation for pathDoc is broken')
}
})
- DocManager.patchDoc(projectId, docId, meta, function (error) {
- if (error) {
- return next(error)
- }
- res.sendStatus(204)
- })
+ await DocManager.patchDoc(projectId, docId, meta)
+ res.sendStatus(204)
}
function _buildDocView(doc) {
@@ -221,10 +173,6 @@ function _buildDocView(doc) {
return docView
}
-function _buildRawDocView(doc) {
- return (doc?.lines ?? []).join('\n')
-}
-
function _buildDocsArrayView(projectId, docs) {
const docViews = []
for (const doc of docs) {
@@ -241,79 +189,69 @@ function _buildDocsArrayView(projectId, docs) {
return docViews
}
-function archiveAllDocs(req, res, next) {
+async function archiveAllDocs(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'archiving all docs')
- DocArchive.archiveAllDocs(projectId, function (error) {
- if (error) {
- return next(error)
- }
- res.sendStatus(204)
- })
+ await DocArchive.archiveAllDocs(projectId)
+ res.sendStatus(204)
}
-function archiveDoc(req, res, next) {
+async function archiveDoc(req, res) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'archiving a doc')
- DocArchive.archiveDoc(projectId, docId, function (error) {
- if (error) {
- return next(error)
- }
- res.sendStatus(204)
- })
+ await DocArchive.archiveDoc(projectId, docId)
+ res.sendStatus(204)
}
-function unArchiveAllDocs(req, res, next) {
+async function unArchiveAllDocs(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'unarchiving all docs')
- DocArchive.unArchiveAllDocs(projectId, function (err) {
- if (err) {
- if (err instanceof Errors.DocRevValueError) {
- logger.warn({ err }, 'Failed to unarchive doc')
- return res.sendStatus(409)
- }
- return next(err)
+ try {
+ await DocArchive.unArchiveAllDocs(projectId)
+ } catch (err) {
+ if (err instanceof Errors.DocRevValueError) {
+ logger.warn({ err }, 'Failed to unarchive doc')
+ return res.sendStatus(409)
}
- res.sendStatus(200)
- })
+ throw err
+ }
+ res.sendStatus(200)
}
-function destroyProject(req, res, next) {
+async function destroyProject(req, res) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'destroying all docs')
- DocArchive.destroyProject(projectId, function (error) {
- if (error) {
- return next(error)
- }
- res.sendStatus(204)
- })
+ await DocArchive.destroyProject(projectId)
+ res.sendStatus(204)
}
-function healthCheck(req, res) {
- HealthChecker.check(function (err) {
- if (err) {
- logger.err({ err }, 'error performing health check')
- res.sendStatus(500)
- } else {
- res.sendStatus(200)
- }
- })
+async function healthCheck(req, res) {
+ try {
+ await HealthChecker.check()
+ } catch (err) {
+ logger.err({ err }, 'error performing health check')
+ res.sendStatus(500)
+ return
+ }
+ res.sendStatus(200)
}
module.exports = {
- getDoc,
- peekDoc,
- isDocDeleted,
- getRawDoc,
- getAllDocs,
- getAllDeletedDocs,
- getAllRanges,
- projectHasRanges,
- updateDoc,
- patchDoc,
- archiveAllDocs,
- archiveDoc,
- unArchiveAllDocs,
- destroyProject,
- healthCheck,
+ getDoc: expressify(getDoc),
+ peekDoc: expressify(peekDoc),
+ isDocDeleted: expressify(isDocDeleted),
+ getRawDoc: expressify(getRawDoc),
+ getAllDocs: expressify(getAllDocs),
+ getAllDeletedDocs: expressify(getAllDeletedDocs),
+ getAllRanges: expressify(getAllRanges),
+ getTrackedChangesUserIds: expressify(getTrackedChangesUserIds),
+ getCommentThreadIds: expressify(getCommentThreadIds),
+ projectHasRanges: expressify(projectHasRanges),
+ updateDoc: expressify(updateDoc),
+ patchDoc: expressify(patchDoc),
+ archiveAllDocs: expressify(archiveAllDocs),
+ archiveDoc: expressify(archiveDoc),
+ unArchiveAllDocs: expressify(unArchiveAllDocs),
+ destroyProject: expressify(destroyProject),
+ healthCheck: expressify(healthCheck),
}
diff --git a/services/docstore/app/js/MongoManager.js b/services/docstore/app/js/MongoManager.js
index 87d8af8a1b..ef101f91c0 100644
--- a/services/docstore/app/js/MongoManager.js
+++ b/services/docstore/app/js/MongoManager.js
@@ -1,7 +1,6 @@
const { db, ObjectId } = require('./mongodb')
const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
-const { callbackify } = require('util')
const ARCHIVING_LOCK_DURATION_MS = Settings.archivingLockDurationMs
@@ -241,34 +240,17 @@ async function destroyProject(projectId) {
}
module.exports = {
- findDoc: callbackify(findDoc),
- getProjectsDeletedDocs: callbackify(getProjectsDeletedDocs),
- getProjectsDocs: callbackify(getProjectsDocs),
- getArchivedProjectDocs: callbackify(getArchivedProjectDocs),
- getNonArchivedProjectDocIds: callbackify(getNonArchivedProjectDocIds),
- getNonDeletedArchivedProjectDocs: callbackify(
- getNonDeletedArchivedProjectDocs
- ),
- upsertIntoDocCollection: callbackify(upsertIntoDocCollection),
- restoreArchivedDoc: callbackify(restoreArchivedDoc),
- patchDoc: callbackify(patchDoc),
- getDocForArchiving: callbackify(getDocForArchiving),
- markDocAsArchived: callbackify(markDocAsArchived),
- checkRevUnchanged: callbackify(checkRevUnchanged),
- destroyProject: callbackify(destroyProject),
- promises: {
- findDoc,
- getProjectsDeletedDocs,
- getProjectsDocs,
- getArchivedProjectDocs,
- getNonArchivedProjectDocIds,
- getNonDeletedArchivedProjectDocs,
- upsertIntoDocCollection,
- restoreArchivedDoc,
- patchDoc,
- getDocForArchiving,
- markDocAsArchived,
- checkRevUnchanged,
- destroyProject,
- },
+ findDoc,
+ getProjectsDeletedDocs,
+ getProjectsDocs,
+ getArchivedProjectDocs,
+ getNonArchivedProjectDocIds,
+ getNonDeletedArchivedProjectDocs,
+ upsertIntoDocCollection,
+ restoreArchivedDoc,
+ patchDoc,
+ getDocForArchiving,
+ markDocAsArchived,
+ checkRevUnchanged,
+ destroyProject,
}
diff --git a/services/docstore/app/js/RangeManager.js b/services/docstore/app/js/RangeManager.js
index f36f68fe35..2fbadf9468 100644
--- a/services/docstore/app/js/RangeManager.js
+++ b/services/docstore/app/js/RangeManager.js
@@ -49,15 +49,25 @@ module.exports = RangeManager = {
updateMetadata(change.metadata)
}
for (const comment of Array.from(ranges.comments || [])) {
- comment.id = RangeManager._safeObjectId(comment.id)
- if ((comment.op != null ? comment.op.t : undefined) != null) {
- comment.op.t = RangeManager._safeObjectId(comment.op.t)
- }
+ // Two bugs resulted in mismatched ids, prefer the thread id from the op: https://github.com/overleaf/internal/issues/23272
+ comment.id = RangeManager._safeObjectId(comment.op?.t || comment.id)
+ if (comment.op) comment.op.t = comment.id
+
+ // resolved property is added to comments when they are obtained from history, but this state doesn't belong in mongo docs collection
+ // more info: https://github.com/overleaf/internal/issues/24371#issuecomment-2913095174
+ delete comment.op?.resolved
updateMetadata(comment.metadata)
}
return ranges
},
+ fixCommentIds(doc) {
+ for (const comment of doc?.ranges?.comments || []) {
+ // Two bugs resulted in mismatched ids, prefer the thread id from the op: https://github.com/overleaf/internal/issues/23272
+ if (comment.op?.t) comment.id = comment.op.t
+ }
+ },
+
_safeObjectId(data) {
try {
return new ObjectId(data)
diff --git a/services/docstore/app/js/StreamToBuffer.js b/services/docstore/app/js/StreamToBuffer.js
new file mode 100644
index 0000000000..09215a7367
--- /dev/null
+++ b/services/docstore/app/js/StreamToBuffer.js
@@ -0,0 +1,24 @@
+const { LoggerStream, WritableBuffer } = require('@overleaf/stream-utils')
+const Settings = require('@overleaf/settings')
+const logger = require('@overleaf/logger/logging-manager')
+const { pipeline } = require('node:stream/promises')
+
+module.exports = {
+ streamToBuffer,
+}
+
+async function streamToBuffer(projectId, docId, stream) {
+ const loggerTransform = new LoggerStream(
+ Settings.max_doc_length,
+ (size, isFlush) => {
+ logger.warn(
+ { projectId, docId, size, finishedReading: isFlush },
+ 'potentially large doc pulled down from gcs'
+ )
+ }
+ )
+
+ const buffer = new WritableBuffer()
+ await pipeline(stream, loggerTransform, buffer)
+ return buffer.contents()
+}
diff --git a/services/docstore/buildscript.txt b/services/docstore/buildscript.txt
index be2a937cd7..4526aa959c 100644
--- a/services/docstore/buildscript.txt
+++ b/services/docstore/buildscript.txt
@@ -4,6 +4,6 @@ docstore
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=True
---script-version=4.5.0
+--script-version=4.7.0
diff --git a/services/docstore/config/settings.defaults.js b/services/docstore/config/settings.defaults.js
index b91cdc87ec..9ad506a9bd 100644
--- a/services/docstore/config/settings.defaults.js
+++ b/services/docstore/config/settings.defaults.js
@@ -1,5 +1,9 @@
-const http = require('http')
+const http = require('node:http')
+const https = require('node:https')
+
http.globalAgent.maxSockets = 300
+http.globalAgent.keepAlive = false
+https.globalAgent.keepAlive = false
const Settings = {
internal: {
diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml
index a8847e8996..cdb4783c5a 100644
--- a/services/docstore/docker-compose.ci.yml
+++ b/services/docstore/docker-compose.ci.yml
@@ -27,12 +27,15 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
+ volumes:
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
gcs:
condition: service_healthy
user: node
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -44,12 +47,17 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml
index 4555d6ed54..a9099c7e7b 100644
--- a/services/docstore/docker-compose.yml
+++ b/services/docstore/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/docstore
- ../../node_modules:/overleaf/node_modules
@@ -14,17 +14,19 @@ services:
working_dir: /overleaf/services/docstore
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/docstore
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/docstore
environment:
ELASTIC_SEARCH_DSN: es:9200
@@ -34,24 +36,30 @@ services:
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
gcs:
condition: service_healthy
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
gcs:
image: fsouza/fake-gcs-server:1.45.2
diff --git a/services/docstore/package.json b/services/docstore/package.json
index 5ffd892f09..bf5857fd49 100644
--- a/services/docstore/package.json
+++ b/services/docstore/package.json
@@ -11,12 +11,13 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
"dependencies": {
+ "@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/o-error": "*",
@@ -28,9 +29,9 @@
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
"celebrate": "^15.0.3",
- "express": "^4.21.0",
+ "express": "^4.21.2",
"lodash": "^4.17.21",
- "mongodb-legacy": "^6.0.1",
+ "mongodb-legacy": "6.1.3",
"p-map": "^4.0.0",
"request": "^2.88.2"
},
@@ -38,7 +39,7 @@
"@google-cloud/storage": "^6.10.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^10.2.0",
+ "mocha": "^11.1.0",
"sandboxed-module": "~2.0.4",
"sinon": "~9.0.2",
"sinon-chai": "^3.7.0",
diff --git a/services/docstore/test/acceptance/deps/healthcheck.sh b/services/docstore/test/acceptance/deps/healthcheck.sh
index cd19cea637..675c205be6 100644
--- a/services/docstore/test/acceptance/deps/healthcheck.sh
+++ b/services/docstore/test/acceptance/deps/healthcheck.sh
@@ -1,9 +1,9 @@
#!/bin/sh
# health check to allow 404 status code as valid
-STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1)
+STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" "$1")
# will be 000 on non-http error (e.g. connection failure)
-if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then
+if test "$STATUSCODE" -ge 500 || test "$STATUSCODE" -lt 200; then
exit 1
fi
exit 0
diff --git a/services/docstore/test/acceptance/js/ArchiveDocsTests.js b/services/docstore/test/acceptance/js/ArchiveDocsTests.js
index f85b845a18..7e254c7e84 100644
--- a/services/docstore/test/acceptance/js/ArchiveDocsTests.js
+++ b/services/docstore/test/acceptance/js/ArchiveDocsTests.js
@@ -399,7 +399,7 @@ describe('Archiving', function () {
this.project_id = new ObjectId()
this.timeout(1000 * 30)
const quarterMegInBytes = 250000
- const bigLine = require('crypto')
+ const bigLine = require('node:crypto')
.randomBytes(quarterMegInBytes)
.toString('hex')
this.doc = {
@@ -1001,6 +1001,15 @@ describe('Archiving', function () {
},
version: 2,
}
+ this.fixedRanges = {
+ ...this.doc.ranges,
+ comments: [
+ {
+ ...this.doc.ranges.comments[0],
+ id: this.doc.ranges.comments[0].op.t,
+ },
+ ],
+ }
return DocstoreClient.createDoc(
this.project_id,
this.doc._id,
@@ -1048,7 +1057,7 @@ describe('Archiving', function () {
throw error
}
s3Doc.lines.should.deep.equal(this.doc.lines)
- const ranges = JSON.parse(JSON.stringify(this.doc.ranges)) // ObjectId -> String
+ const ranges = JSON.parse(JSON.stringify(this.fixedRanges)) // ObjectId -> String
s3Doc.ranges.should.deep.equal(ranges)
return done()
}
@@ -1075,7 +1084,7 @@ describe('Archiving', function () {
throw error
}
doc.lines.should.deep.equal(this.doc.lines)
- doc.ranges.should.deep.equal(this.doc.ranges)
+ doc.ranges.should.deep.equal(this.fixedRanges)
expect(doc.inS3).not.to.exist
return done()
})
diff --git a/services/docstore/test/acceptance/js/GettingAllDocsTests.js b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
index 8fe5e7d91b..57851b2c3b 100644
--- a/services/docstore/test/acceptance/js/GettingAllDocsTests.js
+++ b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
@@ -20,30 +20,73 @@ const DocstoreClient = require('./helpers/DocstoreClient')
describe('Getting all docs', function () {
beforeEach(function (done) {
this.project_id = new ObjectId()
+ this.threadId1 = new ObjectId().toString()
+ this.threadId2 = new ObjectId().toString()
this.docs = [
{
_id: new ObjectId(),
lines: ['one', 'two', 'three'],
- ranges: { mock: 'one' },
+ ranges: {
+ comments: [
+ { id: new ObjectId().toString(), op: { t: this.threadId1 } },
+ ],
+ changes: [
+ {
+ id: new ObjectId().toString(),
+ metadata: { user_id: 'user-id-1' },
+ },
+ ],
+ },
rev: 2,
},
{
_id: new ObjectId(),
lines: ['aaa', 'bbb', 'ccc'],
- ranges: { mock: 'two' },
+ ranges: {
+ changes: [
+ {
+ id: new ObjectId().toString(),
+ metadata: { user_id: 'user-id-2' },
+ },
+ ],
+ },
rev: 4,
},
{
_id: new ObjectId(),
lines: ['111', '222', '333'],
- ranges: { mock: 'three' },
+ ranges: {
+ comments: [
+ { id: new ObjectId().toString(), op: { t: this.threadId2 } },
+ ],
+ changes: [
+ {
+ id: new ObjectId().toString(),
+ metadata: { user_id: 'anonymous-user' },
+ },
+ ],
+ },
rev: 6,
},
]
+ this.fixedRanges = this.docs.map(doc => {
+ if (!doc.ranges?.comments?.length) return doc.ranges
+ return {
+ ...doc.ranges,
+ comments: [
+ { ...doc.ranges.comments[0], id: doc.ranges.comments[0].op.t },
+ ],
+ }
+ })
this.deleted_doc = {
_id: new ObjectId(),
lines: ['deleted'],
- ranges: { mock: 'four' },
+ ranges: {
+ comments: [{ id: new ObjectId().toString(), op: { t: 'thread-id-3' } }],
+ changes: [
+ { id: new ObjectId().toString(), metadata: { user_id: 'user-id-3' } },
+ ],
+ },
rev: 8,
}
const version = 42
@@ -96,7 +139,7 @@ describe('Getting all docs', function () {
})
})
- return it('getAllRanges should return all the (non-deleted) doc ranges', function (done) {
+ it('getAllRanges should return all the (non-deleted) doc ranges', function (done) {
return DocstoreClient.getAllRanges(this.project_id, (error, res, docs) => {
if (error != null) {
throw error
@@ -104,9 +147,38 @@ describe('Getting all docs', function () {
docs.length.should.equal(this.docs.length)
for (let i = 0; i < docs.length; i++) {
const doc = docs[i]
- doc.ranges.should.deep.equal(this.docs[i].ranges)
+ doc.ranges.should.deep.equal(this.fixedRanges[i])
}
return done()
})
})
+
+ it('getTrackedChangesUserIds should return all the user ids from (non-deleted) ranges', function (done) {
+ DocstoreClient.getTrackedChangesUserIds(
+ this.project_id,
+ (error, res, userIds) => {
+ if (error != null) {
+ throw error
+ }
+ userIds.should.deep.equal(['user-id-1', 'user-id-2'])
+ done()
+ }
+ )
+ })
+
+ it('getCommentThreadIds should return all the thread ids from (non-deleted) ranges', function (done) {
+ DocstoreClient.getCommentThreadIds(
+ this.project_id,
+ (error, res, threadIds) => {
+ if (error != null) {
+ throw error
+ }
+ threadIds.should.deep.equal({
+ [this.docs[0]._id.toString()]: [this.threadId1],
+ [this.docs[2]._id.toString()]: [this.threadId2],
+ })
+ done()
+ }
+ )
+ })
})
diff --git a/services/docstore/test/acceptance/js/GettingDocsTests.js b/services/docstore/test/acceptance/js/GettingDocsTests.js
index 121b3c1e24..1cfc53c5c6 100644
--- a/services/docstore/test/acceptance/js/GettingDocsTests.js
+++ b/services/docstore/test/acceptance/js/GettingDocsTests.js
@@ -28,10 +28,26 @@ describe('Getting a doc', function () {
op: { i: 'foo', p: 3 },
meta: {
user_id: new ObjectId().toString(),
- ts: new Date().toString(),
+ ts: new Date().toJSON(),
},
},
],
+ comments: [
+ {
+ id: new ObjectId().toString(),
+ op: { c: 'comment', p: 1, t: new ObjectId().toString() },
+ metadata: {
+ user_id: new ObjectId().toString(),
+ ts: new Date().toJSON(),
+ },
+ },
+ ],
+ }
+ this.fixedRanges = {
+ ...this.ranges,
+ comments: [
+ { ...this.ranges.comments[0], id: this.ranges.comments[0].op.t },
+ ],
}
return DocstoreApp.ensureRunning(() => {
return DocstoreClient.createDoc(
@@ -60,7 +76,7 @@ describe('Getting a doc', function () {
if (error) return done(error)
doc.lines.should.deep.equal(this.lines)
doc.version.should.equal(this.version)
- doc.ranges.should.deep.equal(this.ranges)
+ doc.ranges.should.deep.equal(this.fixedRanges)
return done()
}
)
@@ -114,7 +130,7 @@ describe('Getting a doc', function () {
if (error) return done(error)
doc.lines.should.deep.equal(this.lines)
doc.version.should.equal(this.version)
- doc.ranges.should.deep.equal(this.ranges)
+ doc.ranges.should.deep.equal(this.fixedRanges)
doc.deleted.should.equal(true)
return done()
}
diff --git a/services/docstore/test/acceptance/js/HealthCheckerTest.js b/services/docstore/test/acceptance/js/HealthCheckerTest.js
new file mode 100644
index 0000000000..b25a45312b
--- /dev/null
+++ b/services/docstore/test/acceptance/js/HealthCheckerTest.js
@@ -0,0 +1,28 @@
+const { db } = require('../../../app/js/mongodb')
+const DocstoreApp = require('./helpers/DocstoreApp')
+const DocstoreClient = require('./helpers/DocstoreClient')
+const { expect } = require('chai')
+
+describe('HealthChecker', function () {
+ beforeEach('start', function (done) {
+ DocstoreApp.ensureRunning(done)
+ })
+ beforeEach('clear docs collection', async function () {
+ await db.docs.deleteMany({})
+ })
+ let res
+ beforeEach('run health check', function (done) {
+ DocstoreClient.healthCheck((err, _res) => {
+ res = _res
+ done(err)
+ })
+ })
+
+ it('should return 200', function () {
+ res.statusCode.should.equal(200)
+ })
+
+ it('should not leave any cruft behind', async function () {
+ expect(await db.docs.find({}).toArray()).to.deep.equal([])
+ })
+})
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
index 03db0ea322..5e837b1277 100644
--- a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
@@ -1,5 +1,4 @@
const app = require('../../../../app')
-require('@overleaf/logger').logger.level('error')
const settings = require('@overleaf/settings')
module.exports = {
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
index 790ec8f237..cb8bce2579 100644
--- a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
@@ -100,6 +100,26 @@ module.exports = DocstoreClient = {
)
},
+ getCommentThreadIds(projectId, callback) {
+ request.get(
+ {
+ url: `http://127.0.0.1:${settings.internal.docstore.port}/project/${projectId}/comment-thread-ids`,
+ json: true,
+ },
+ callback
+ )
+ },
+
+ getTrackedChangesUserIds(projectId, callback) {
+ request.get(
+ {
+ url: `http://127.0.0.1:${settings.internal.docstore.port}/project/${projectId}/tracked-changes-user-ids`,
+ json: true,
+ },
+ callback
+ )
+ },
+
updateDoc(projectId, docId, lines, version, ranges, callback) {
return request.post(
{
@@ -181,6 +201,13 @@ module.exports = DocstoreClient = {
)
},
+ healthCheck(callback) {
+ request.get(
+ `http://127.0.0.1:${settings.internal.docstore.port}/health_check`,
+ callback
+ )
+ },
+
getS3Doc(projectId, docId, callback) {
getStringFromPersistor(
Persistor,
diff --git a/services/docstore/test/setup.js b/services/docstore/test/setup.js
index 809624a2ee..92b86c9384 100644
--- a/services/docstore/test/setup.js
+++ b/services/docstore/test/setup.js
@@ -3,7 +3,7 @@ const sinon = require('sinon')
const sinonChai = require('sinon-chai')
const chaiAsPromised = require('chai-as-promised')
const SandboxedModule = require('sandboxed-module')
-const timersPromises = require('timers/promises')
+const timersPromises = require('node:timers/promises')
// ensure every ObjectId has the id string as a property for correct comparisons
require('mongodb-legacy').ObjectId.cacheHexString = true
@@ -37,6 +37,11 @@ SandboxedModule.configure({
'mongodb-legacy': require('mongodb-legacy'),
},
globals: { Buffer, JSON, Math, console, process },
+ sourceTransformers: {
+ removeNodePrefix: function (source) {
+ return source.replace(/require\(['"]node:/g, "require('")
+ },
+ },
})
exports.mochaHooks = {
diff --git a/services/docstore/test/unit/js/DocArchiveManagerTests.js b/services/docstore/test/unit/js/DocArchiveManagerTests.js
index 13046d86fc..2ec1cb2016 100644
--- a/services/docstore/test/unit/js/DocArchiveManagerTests.js
+++ b/services/docstore/test/unit/js/DocArchiveManagerTests.js
@@ -4,6 +4,7 @@ const modulePath = '../../../app/js/DocArchiveManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb-legacy')
const Errors = require('../../../app/js/Errors')
+const StreamToBuffer = require('../../../app/js/StreamToBuffer')
describe('DocArchiveManager', function () {
let DocArchiveManager,
@@ -22,13 +23,15 @@ describe('DocArchiveManager', function () {
md5Sum,
projectId,
readStream,
- stream
+ stream,
+ streamToBuffer
beforeEach(function () {
md5Sum = 'decafbad'
RangeManager = {
jsonRangesToMongo: sinon.stub().returns({ mongo: 'ranges' }),
+ fixCommentIds: sinon.stub(),
}
Settings = {
docstore: {
@@ -140,17 +143,33 @@ describe('DocArchiveManager', function () {
}
MongoManager = {
- promises: {
- markDocAsArchived: sinon.stub().resolves(),
- restoreArchivedDoc: sinon.stub().resolves(),
- upsertIntoDocCollection: sinon.stub().resolves(),
- getProjectsDocs: sinon.stub().resolves(mongoDocs),
- getNonDeletedArchivedProjectDocs: getArchivedProjectDocs,
- getNonArchivedProjectDocIds,
- getArchivedProjectDocs,
- findDoc: sinon.stub().callsFake(fakeGetDoc),
- getDocForArchiving: sinon.stub().callsFake(fakeGetDoc),
- destroyProject: sinon.stub().resolves(),
+ markDocAsArchived: sinon.stub().resolves(),
+ restoreArchivedDoc: sinon.stub().resolves(),
+ upsertIntoDocCollection: sinon.stub().resolves(),
+ getProjectsDocs: sinon.stub().resolves(mongoDocs),
+ getNonDeletedArchivedProjectDocs: getArchivedProjectDocs,
+ getNonArchivedProjectDocIds,
+ getArchivedProjectDocs,
+ findDoc: sinon.stub().callsFake(fakeGetDoc),
+ getDocForArchiving: sinon.stub().callsFake(fakeGetDoc),
+ destroyProject: sinon.stub().resolves(),
+ }
+
+ // Wrap streamToBuffer so that we can pass in something that it expects (in
+ // this case, a Promise) rather than a stubbed stream object
+ streamToBuffer = {
+ streamToBuffer: async () => {
+ const inputStream = new Promise(resolve => {
+ stream.on('data', data => resolve(data))
+ })
+
+ const value = await StreamToBuffer.streamToBuffer(
+ 'testProjectId',
+ 'testDocId',
+ inputStream
+ )
+
+ return value
},
}
@@ -163,15 +182,20 @@ describe('DocArchiveManager', function () {
'./RangeManager': RangeManager,
'./PersistorManager': PersistorManager,
'./Errors': Errors,
+ './StreamToBuffer': streamToBuffer,
},
})
})
describe('archiveDoc', function () {
it('should resolve when passed a valid document', async function () {
- await expect(
- DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
- ).to.eventually.be.fulfilled
+ await expect(DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)).to
+ .eventually.be.fulfilled
+ })
+
+ it('should fix comment ids', async function () {
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[1]._id)
+ expect(RangeManager.fixCommentIds).to.have.been.called
})
it('should throw an error if the doc has no lines', async function () {
@@ -179,26 +203,26 @@ describe('DocArchiveManager', function () {
doc.lines = null
await expect(
- DocArchiveManager.promises.archiveDoc(projectId, doc._id)
+ DocArchiveManager.archiveDoc(projectId, doc._id)
).to.eventually.be.rejectedWith('doc has no lines')
})
it('should add the schema version', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[1]._id)
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[1]._id)
expect(StreamUtils.ReadableString).to.have.been.calledWith(
sinon.match(/"schema_v":1/)
)
})
it('should calculate the hex md5 sum of the content', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
expect(Crypto.createHash).to.have.been.calledWith('md5')
expect(HashUpdate).to.have.been.calledWith(archivedDocJson)
expect(HashDigest).to.have.been.calledWith('hex')
})
it('should pass the md5 hash to the object persistor for verification', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
expect(PersistorManager.sendStream).to.have.been.calledWith(
sinon.match.any,
@@ -209,7 +233,7 @@ describe('DocArchiveManager', function () {
})
it('should pass the correct bucket and key to the persistor', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
expect(PersistorManager.sendStream).to.have.been.calledWith(
Settings.docstore.bucket,
@@ -218,7 +242,7 @@ describe('DocArchiveManager', function () {
})
it('should create a stream from the encoded json and send it', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
expect(StreamUtils.ReadableString).to.have.been.calledWith(
archivedDocJson
)
@@ -230,8 +254,8 @@ describe('DocArchiveManager', function () {
})
it('should mark the doc as archived', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[0]._id,
mongoDocs[0].rev
@@ -244,8 +268,8 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.promises.getDocForArchiving).to.not.have.been.called
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.getDocForArchiving).to.not.have.been.called
})
})
@@ -262,7 +286,7 @@ describe('DocArchiveManager', function () {
it('should return an error', async function () {
await expect(
- DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
).to.eventually.be.rejectedWith('null bytes detected')
})
})
@@ -273,21 +297,19 @@ describe('DocArchiveManager', function () {
describe('when the doc is in S3', function () {
beforeEach(function () {
- MongoManager.promises.findDoc = sinon
- .stub()
- .resolves({ inS3: true, rev })
+ MongoManager.findDoc = sinon.stub().resolves({ inS3: true, rev })
docId = mongoDocs[0]._id
lines = ['doc', 'lines']
rev = 123
})
it('should resolve when passed a valid document', async function () {
- await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId))
- .to.eventually.be.fulfilled
+ await expect(DocArchiveManager.unarchiveDoc(projectId, docId)).to
+ .eventually.be.fulfilled
})
it('should test md5 validity with the raw buffer', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
expect(HashUpdate).to.have.been.calledWith(
sinon.match.instanceOf(Buffer)
)
@@ -296,15 +318,17 @@ describe('DocArchiveManager', function () {
it('should throw an error if the md5 does not match', async function () {
PersistorManager.getObjectMd5Hash.resolves('badf00d')
await expect(
- DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ DocArchiveManager.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.be.instanceof(Errors.Md5MismatchError)
})
it('should restore the doc in Mongo', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
- expect(
- MongoManager.promises.restoreArchivedDoc
- ).to.have.been.calledWith(projectId, docId, archivedDoc)
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
+ expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
+ projectId,
+ docId,
+ archivedDoc
+ )
})
describe('when archiving is not configured', function () {
@@ -314,15 +338,15 @@ describe('DocArchiveManager', function () {
it('should error out on archived doc', async function () {
await expect(
- DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ DocArchiveManager.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.match(
/found archived doc, but archiving backend is not configured/
)
})
it('should return early on non-archived doc', async function () {
- MongoManager.promises.findDoc = sinon.stub().resolves({ rev })
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ MongoManager.findDoc = sinon.stub().resolves({ rev })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
expect(PersistorManager.getObjectMd5Hash).to.not.have.been.called
})
})
@@ -340,10 +364,12 @@ describe('DocArchiveManager', function () {
})
it('should return the docs lines', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
- expect(
- MongoManager.promises.restoreArchivedDoc
- ).to.have.been.calledWith(projectId, docId, { lines, rev })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
+ expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
+ projectId,
+ docId,
+ { lines, rev }
+ )
})
})
@@ -362,14 +388,16 @@ describe('DocArchiveManager', function () {
})
it('should return the doc lines and ranges', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
- expect(
- MongoManager.promises.restoreArchivedDoc
- ).to.have.been.calledWith(projectId, docId, {
- lines,
- ranges: { mongo: 'ranges' },
- rev: 456,
- })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
+ expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
+ projectId,
+ docId,
+ {
+ lines,
+ ranges: { mongo: 'ranges' },
+ rev: 456,
+ }
+ )
})
})
@@ -383,10 +411,12 @@ describe('DocArchiveManager', function () {
})
it('should return only the doc lines', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
- expect(
- MongoManager.promises.restoreArchivedDoc
- ).to.have.been.calledWith(projectId, docId, { lines, rev: 456 })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
+ expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
+ projectId,
+ docId,
+ { lines, rev: 456 }
+ )
})
})
@@ -400,10 +430,12 @@ describe('DocArchiveManager', function () {
})
it('should use the rev obtained from Mongo', async function () {
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
- expect(
- MongoManager.promises.restoreArchivedDoc
- ).to.have.been.calledWith(projectId, docId, { lines, rev })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
+ expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
+ projectId,
+ docId,
+ { lines, rev }
+ )
})
})
@@ -418,7 +450,7 @@ describe('DocArchiveManager', function () {
it('should throw an error', async function () {
await expect(
- DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ DocArchiveManager.unarchiveDoc(projectId, docId)
).to.eventually.be.rejectedWith(
"I don't understand the doc format in s3"
)
@@ -428,8 +460,8 @@ describe('DocArchiveManager', function () {
})
it('should not do anything if the file is already unarchived', async function () {
- MongoManager.promises.findDoc.resolves({ inS3: false })
- await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ MongoManager.findDoc.resolves({ inS3: false })
+ await DocArchiveManager.unarchiveDoc(projectId, docId)
expect(PersistorManager.getObjectStream).not.to.have.been.called
})
@@ -438,7 +470,7 @@ describe('DocArchiveManager', function () {
.stub()
.rejects(new Errors.NotFoundError())
await expect(
- DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ DocArchiveManager.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
})
})
@@ -446,13 +478,11 @@ describe('DocArchiveManager', function () {
describe('destroyProject', function () {
describe('when archiving is enabled', function () {
beforeEach(async function () {
- await DocArchiveManager.promises.destroyProject(projectId)
+ await DocArchiveManager.destroyProject(projectId)
})
it('should delete the project in Mongo', function () {
- expect(MongoManager.promises.destroyProject).to.have.been.calledWith(
- projectId
- )
+ expect(MongoManager.destroyProject).to.have.been.calledWith(projectId)
})
it('should delete the project in the persistor', function () {
@@ -466,13 +496,11 @@ describe('DocArchiveManager', function () {
describe('when archiving is disabled', function () {
beforeEach(async function () {
Settings.docstore.backend = ''
- await DocArchiveManager.promises.destroyProject(projectId)
+ await DocArchiveManager.destroyProject(projectId)
})
it('should delete the project in Mongo', function () {
- expect(MongoManager.promises.destroyProject).to.have.been.calledWith(
- projectId
- )
+ expect(MongoManager.destroyProject).to.have.been.calledWith(projectId)
})
it('should not delete the project in the persistor', function () {
@@ -483,33 +511,35 @@ describe('DocArchiveManager', function () {
describe('archiveAllDocs', function () {
it('should resolve with valid arguments', async function () {
- await expect(DocArchiveManager.promises.archiveAllDocs(projectId)).to
- .eventually.be.fulfilled
+ await expect(DocArchiveManager.archiveAllDocs(projectId)).to.eventually.be
+ .fulfilled
})
it('should archive all project docs which are not in s3', async function () {
- await DocArchiveManager.promises.archiveAllDocs(projectId)
+ await DocArchiveManager.archiveAllDocs(projectId)
// not inS3
- expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[0]._id
)
- expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[1]._id
)
- expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[4]._id
)
// inS3
- expect(
- MongoManager.promises.markDocAsArchived
- ).not.to.have.been.calledWith(projectId, mongoDocs[2]._id)
- expect(
- MongoManager.promises.markDocAsArchived
- ).not.to.have.been.calledWith(projectId, mongoDocs[3]._id)
+ expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
+ projectId,
+ mongoDocs[2]._id
+ )
+ expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
+ projectId,
+ mongoDocs[3]._id
+ )
})
describe('when archiving is not configured', function () {
@@ -518,21 +548,20 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.promises.getNonArchivedProjectDocIds).to.not.have
- .been.called
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.getNonArchivedProjectDocIds).to.not.have.been.called
})
})
})
describe('unArchiveAllDocs', function () {
it('should resolve with valid arguments', async function () {
- await expect(DocArchiveManager.promises.unArchiveAllDocs(projectId)).to
- .eventually.be.fulfilled
+ await expect(DocArchiveManager.unArchiveAllDocs(projectId)).to.eventually
+ .be.fulfilled
})
it('should unarchive all inS3 docs', async function () {
- await DocArchiveManager.promises.unArchiveAllDocs(projectId)
+ await DocArchiveManager.unArchiveAllDocs(projectId)
for (const doc of archivedDocs) {
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
@@ -548,9 +577,9 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.promises.getNonDeletedArchivedProjectDocs).to.not
- .have.been.called
+ await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.getNonDeletedArchivedProjectDocs).to.not.have.been
+ .called
})
})
})
diff --git a/services/docstore/test/unit/js/DocManagerTests.js b/services/docstore/test/unit/js/DocManagerTests.js
index be833f4b3f..67a2f26547 100644
--- a/services/docstore/test/unit/js/DocManagerTests.js
+++ b/services/docstore/test/unit/js/DocManagerTests.js
@@ -1,7 +1,10 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('path').join(__dirname, '../../../app/js/DocManager')
+const modulePath = require('node:path').join(
+ __dirname,
+ '../../../app/js/DocManager'
+)
const { ObjectId } = require('mongodb-legacy')
const Errors = require('../../../app/js/Errors')
@@ -14,25 +17,22 @@ describe('DocManager', function () {
this.version = 42
this.MongoManager = {
- promises: {
- findDoc: sinon.stub(),
- getProjectsDocs: sinon.stub(),
- patchDoc: sinon.stub().resolves(),
- upsertIntoDocCollection: sinon.stub().resolves(),
- },
+ findDoc: sinon.stub(),
+ getProjectsDocs: sinon.stub(),
+ patchDoc: sinon.stub().resolves(),
+ upsertIntoDocCollection: sinon.stub().resolves(),
}
this.DocArchiveManager = {
- promises: {
- unarchiveDoc: sinon.stub(),
- unArchiveAllDocs: sinon.stub(),
- archiveDoc: sinon.stub().resolves(),
- },
+ unarchiveDoc: sinon.stub(),
+ unArchiveAllDocs: sinon.stub(),
+ archiveDoc: sinon.stub().resolves(),
}
this.RangeManager = {
jsonRangesToMongo(r) {
return r
},
shouldUpdateRanges: sinon.stub().returns(false),
+ fixCommentIds: sinon.stub(),
}
this.settings = { docstore: {} }
@@ -49,7 +49,7 @@ describe('DocManager', function () {
describe('getFullDoc', function () {
beforeEach(function () {
- this.DocManager.promises._getDoc = sinon.stub()
+ this.DocManager._getDoc = sinon.stub()
this.doc = {
_id: this.doc_id,
lines: ['2134'],
@@ -57,13 +57,10 @@ describe('DocManager', function () {
})
it('should call get doc with a quick filter', async function () {
- this.DocManager.promises._getDoc.resolves(this.doc)
- const doc = await this.DocManager.promises.getFullDoc(
- this.project_id,
- this.doc_id
- )
+ this.DocManager._getDoc.resolves(this.doc)
+ const doc = await this.DocManager.getFullDoc(this.project_id, this.doc_id)
doc.should.equal(this.doc)
- this.DocManager.promises._getDoc
+ this.DocManager._getDoc
.calledWith(this.project_id, this.doc_id, {
lines: true,
rev: true,
@@ -76,27 +73,27 @@ describe('DocManager', function () {
})
it('should return error when get doc errors', async function () {
- this.DocManager.promises._getDoc.rejects(this.stubbedError)
+ this.DocManager._getDoc.rejects(this.stubbedError)
await expect(
- this.DocManager.promises.getFullDoc(this.project_id, this.doc_id)
+ this.DocManager.getFullDoc(this.project_id, this.doc_id)
).to.be.rejectedWith(this.stubbedError)
})
})
describe('getRawDoc', function () {
beforeEach(function () {
- this.DocManager.promises._getDoc = sinon.stub()
+ this.DocManager._getDoc = sinon.stub()
this.doc = { lines: ['2134'] }
})
it('should call get doc with a quick filter', async function () {
- this.DocManager.promises._getDoc.resolves(this.doc)
- const doc = await this.DocManager.promises.getDocLines(
+ this.DocManager._getDoc.resolves(this.doc)
+ const content = await this.DocManager.getDocLines(
this.project_id,
this.doc_id
)
- doc.should.equal(this.doc)
- this.DocManager.promises._getDoc
+ content.should.equal(this.doc.lines.join('\n'))
+ this.DocManager._getDoc
.calledWith(this.project_id, this.doc_id, {
lines: true,
inS3: true,
@@ -105,11 +102,46 @@ describe('DocManager', function () {
})
it('should return error when get doc errors', async function () {
- this.DocManager.promises._getDoc.rejects(this.stubbedError)
+ this.DocManager._getDoc.rejects(this.stubbedError)
await expect(
- this.DocManager.promises.getDocLines(this.project_id, this.doc_id)
+ this.DocManager.getDocLines(this.project_id, this.doc_id)
).to.be.rejectedWith(this.stubbedError)
})
+
+ it('should return error when get doc does not exist', async function () {
+ this.DocManager._getDoc.resolves(null)
+ await expect(
+ this.DocManager.getDocLines(this.project_id, this.doc_id)
+ ).to.be.rejectedWith(Errors.NotFoundError)
+ })
+
+ it('should return error when get doc has no lines', async function () {
+ this.DocManager._getDoc.resolves({})
+ await expect(
+ this.DocManager.getDocLines(this.project_id, this.doc_id)
+ ).to.be.rejectedWith(Errors.DocWithoutLinesError)
+ })
+ })
+
+ describe('_getDoc', function () {
+ it('should return error when get doc does not exist', async function () {
+ this.MongoManager.findDoc.resolves(null)
+ await expect(
+ this.DocManager._getDoc(this.project_id, this.doc_id, { inS3: true })
+ ).to.be.rejectedWith(Errors.NotFoundError)
+ })
+
+ it('should fix comment ids', async function () {
+ this.MongoManager.findDoc.resolves({
+ _id: this.doc_id,
+ ranges: {},
+ })
+ await this.DocManager._getDoc(this.project_id, this.doc_id, {
+ inS3: true,
+ ranges: true,
+ })
+ expect(this.RangeManager.fixCommentIds).to.have.been.called
+ })
})
describe('getDoc', function () {
@@ -125,26 +157,25 @@ describe('DocManager', function () {
describe('when using a filter', function () {
beforeEach(function () {
- this.MongoManager.promises.findDoc.resolves(this.doc)
+ this.MongoManager.findDoc.resolves(this.doc)
})
it('should error if inS3 is not set to true', async function () {
await expect(
- this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
+ this.DocManager._getDoc(this.project_id, this.doc_id, {
inS3: false,
})
).to.be.rejected
})
it('should always get inS3 even when no filter is passed', async function () {
- await expect(
- this.DocManager.promises._getDoc(this.project_id, this.doc_id)
- ).to.be.rejected
- this.MongoManager.promises.findDoc.called.should.equal(false)
+ await expect(this.DocManager._getDoc(this.project_id, this.doc_id)).to
+ .be.rejected
+ this.MongoManager.findDoc.called.should.equal(false)
})
it('should not error if inS3 is set to true', async function () {
- await this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
+ await this.DocManager._getDoc(this.project_id, this.doc_id, {
inS3: true,
})
})
@@ -152,8 +183,8 @@ describe('DocManager', function () {
describe('when the doc is in the doc collection', function () {
beforeEach(async function () {
- this.MongoManager.promises.findDoc.resolves(this.doc)
- this.result = await this.DocManager.promises._getDoc(
+ this.MongoManager.findDoc.resolves(this.doc)
+ this.result = await this.DocManager._getDoc(
this.project_id,
this.doc_id,
{ version: true, inS3: true }
@@ -161,7 +192,7 @@ describe('DocManager', function () {
})
it('should get the doc from the doc collection', function () {
- this.MongoManager.promises.findDoc
+ this.MongoManager.findDoc
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
@@ -174,9 +205,9 @@ describe('DocManager', function () {
describe('when MongoManager.findDoc errors', function () {
it('should return the error', async function () {
- this.MongoManager.promises.findDoc.rejects(this.stubbedError)
+ this.MongoManager.findDoc.rejects(this.stubbedError)
await expect(
- this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
+ this.DocManager._getDoc(this.project_id, this.doc_id, {
version: true,
inS3: true,
})
@@ -199,15 +230,15 @@ describe('DocManager', function () {
version: 2,
inS3: false,
}
- this.MongoManager.promises.findDoc.resolves(this.doc)
- this.DocArchiveManager.promises.unarchiveDoc.callsFake(
+ this.MongoManager.findDoc.resolves(this.doc)
+ this.DocArchiveManager.unarchiveDoc.callsFake(
async (projectId, docId) => {
- this.MongoManager.promises.findDoc.resolves({
+ this.MongoManager.findDoc.resolves({
...this.unarchivedDoc,
})
}
)
- this.result = await this.DocManager.promises._getDoc(
+ this.result = await this.DocManager._getDoc(
this.project_id,
this.doc_id,
{
@@ -218,13 +249,13 @@ describe('DocManager', function () {
})
it('should call the DocArchive to unarchive the doc', function () {
- this.DocArchiveManager.promises.unarchiveDoc
+ this.DocArchiveManager.unarchiveDoc
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should look up the doc twice', function () {
- this.MongoManager.promises.findDoc.calledTwice.should.equal(true)
+ this.MongoManager.findDoc.calledTwice.should.equal(true)
})
it('should return the doc', function () {
@@ -236,9 +267,9 @@ describe('DocManager', function () {
describe('when the doc does not exist in the docs collection', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.promises.findDoc.resolves(null)
+ this.MongoManager.findDoc.resolves(null)
await expect(
- this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
+ this.DocManager._getDoc(this.project_id, this.doc_id, {
version: true,
inS3: true,
})
@@ -259,23 +290,27 @@ describe('DocManager', function () {
lines: ['mock-lines'],
},
]
- this.MongoManager.promises.getProjectsDocs.resolves(this.docs)
- this.DocArchiveManager.promises.unArchiveAllDocs.resolves(this.docs)
- this.filter = { lines: true }
- this.result = await this.DocManager.promises.getAllNonDeletedDocs(
+ this.MongoManager.getProjectsDocs.resolves(this.docs)
+ this.DocArchiveManager.unArchiveAllDocs.resolves(this.docs)
+ this.filter = { lines: true, ranges: true }
+ this.result = await this.DocManager.getAllNonDeletedDocs(
this.project_id,
this.filter
)
})
it('should get the project from the database', function () {
- this.MongoManager.promises.getProjectsDocs.should.have.been.calledWith(
+ this.MongoManager.getProjectsDocs.should.have.been.calledWith(
this.project_id,
{ include_deleted: false },
this.filter
)
})
+ it('should fix comment ids', async function () {
+ expect(this.RangeManager.fixCommentIds).to.have.been.called
+ })
+
it('should return the docs', function () {
expect(this.result).to.deep.equal(this.docs)
})
@@ -283,13 +318,10 @@ describe('DocManager', function () {
describe('when there are no docs for the project', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.promises.getProjectsDocs.resolves(null)
- this.DocArchiveManager.promises.unArchiveAllDocs.resolves(null)
+ this.MongoManager.getProjectsDocs.resolves(null)
+ this.DocArchiveManager.unArchiveAllDocs.resolves(null)
await expect(
- this.DocManager.promises.getAllNonDeletedDocs(
- this.project_id,
- this.filter
- )
+ this.DocManager.getAllNonDeletedDocs(this.project_id, this.filter)
).to.be.rejectedWith(`No docs for project ${this.project_id}`)
})
})
@@ -300,7 +332,7 @@ describe('DocManager', function () {
beforeEach(function () {
this.lines = ['mock', 'doc', 'lines']
this.rev = 77
- this.MongoManager.promises.findDoc.resolves({
+ this.MongoManager.findDoc.resolves({
_id: new ObjectId(this.doc_id),
})
this.meta = {}
@@ -308,7 +340,7 @@ describe('DocManager', function () {
describe('standard path', function () {
beforeEach(async function () {
- await this.DocManager.promises.patchDoc(
+ await this.DocManager.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -316,14 +348,14 @@ describe('DocManager', function () {
})
it('should get the doc', function () {
- expect(this.MongoManager.promises.findDoc).to.have.been.calledWith(
+ expect(this.MongoManager.findDoc).to.have.been.calledWith(
this.project_id,
this.doc_id
)
})
it('should persist the meta', function () {
- expect(this.MongoManager.promises.patchDoc).to.have.been.calledWith(
+ expect(this.MongoManager.patchDoc).to.have.been.calledWith(
this.project_id,
this.doc_id,
this.meta
@@ -336,7 +368,7 @@ describe('DocManager', function () {
this.settings.docstore.archiveOnSoftDelete = false
this.meta.deleted = true
- await this.DocManager.promises.patchDoc(
+ await this.DocManager.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -344,8 +376,7 @@ describe('DocManager', function () {
})
it('should not flush the doc out of mongo', function () {
- expect(this.DocArchiveManager.promises.archiveDoc).to.not.have.been
- .called
+ expect(this.DocArchiveManager.archiveDoc).to.not.have.been.called
})
})
@@ -353,7 +384,7 @@ describe('DocManager', function () {
beforeEach(async function () {
this.settings.docstore.archiveOnSoftDelete = false
this.meta.deleted = false
- await this.DocManager.promises.patchDoc(
+ await this.DocManager.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -361,8 +392,7 @@ describe('DocManager', function () {
})
it('should not flush the doc out of mongo', function () {
- expect(this.DocArchiveManager.promises.archiveDoc).to.not.have.been
- .called
+ expect(this.DocArchiveManager.archiveDoc).to.not.have.been.called
})
})
@@ -374,7 +404,7 @@ describe('DocManager', function () {
describe('when the background flush succeeds', function () {
beforeEach(async function () {
- await this.DocManager.promises.patchDoc(
+ await this.DocManager.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -386,17 +416,18 @@ describe('DocManager', function () {
})
it('should flush the doc out of mongo', function () {
- expect(
- this.DocArchiveManager.promises.archiveDoc
- ).to.have.been.calledWith(this.project_id, this.doc_id)
+ expect(this.DocArchiveManager.archiveDoc).to.have.been.calledWith(
+ this.project_id,
+ this.doc_id
+ )
})
})
describe('when the background flush fails', function () {
beforeEach(async function () {
this.err = new Error('foo')
- this.DocArchiveManager.promises.archiveDoc.rejects(this.err)
- await this.DocManager.promises.patchDoc(
+ this.DocArchiveManager.archiveDoc.rejects(this.err)
+ await this.DocManager.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -419,9 +450,9 @@ describe('DocManager', function () {
describe('when the doc does not exist', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.promises.findDoc.resolves(null)
+ this.MongoManager.findDoc.resolves(null)
await expect(
- this.DocManager.promises.patchDoc(this.project_id, this.doc_id, {})
+ this.DocManager.patchDoc(this.project_id, this.doc_id, {})
).to.be.rejectedWith(
`No such project/doc to delete: ${this.project_id}/${this.doc_id}`
)
@@ -467,13 +498,13 @@ describe('DocManager', function () {
ranges: this.originalRanges,
}
- this.DocManager.promises._getDoc = sinon.stub()
+ this.DocManager._getDoc = sinon.stub()
})
describe('when only the doc lines have changed', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.promises.updateDoc(
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -483,7 +514,7 @@ describe('DocManager', function () {
})
it('should get the existing doc', function () {
- this.DocManager.promises._getDoc
+ this.DocManager._getDoc
.calledWith(this.project_id, this.doc_id, {
version: true,
rev: true,
@@ -495,7 +526,7 @@ describe('DocManager', function () {
})
it('should upsert the document to the doc collection', function () {
- this.MongoManager.promises.upsertIntoDocCollection
+ this.MongoManager.upsertIntoDocCollection
.calledWith(this.project_id, this.doc_id, this.rev, {
lines: this.newDocLines,
})
@@ -509,9 +540,9 @@ describe('DocManager', function () {
describe('when the doc ranges have changed', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
this.RangeManager.shouldUpdateRanges.returns(true)
- this.result = await this.DocManager.promises.updateDoc(
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -521,7 +552,7 @@ describe('DocManager', function () {
})
it('should upsert the ranges', function () {
- this.MongoManager.promises.upsertIntoDocCollection
+ this.MongoManager.upsertIntoDocCollection
.calledWith(this.project_id, this.doc_id, this.rev, {
ranges: this.newRanges,
})
@@ -535,8 +566,8 @@ describe('DocManager', function () {
describe('when only the version has changed', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.promises.updateDoc(
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -546,7 +577,7 @@ describe('DocManager', function () {
})
it('should update the version', function () {
- this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
this.rev,
@@ -561,8 +592,8 @@ describe('DocManager', function () {
describe('when the doc has not changed at all', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.promises.updateDoc(
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -572,9 +603,7 @@ describe('DocManager', function () {
})
it('should not update the ranges or lines or version', function () {
- this.MongoManager.promises.upsertIntoDocCollection.called.should.equal(
- false
- )
+ this.MongoManager.upsertIntoDocCollection.called.should.equal(false)
})
it('should return the old rev and modified == false', function () {
@@ -585,7 +614,7 @@ describe('DocManager', function () {
describe('when the version is null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.promises.updateDoc(
+ this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -599,7 +628,7 @@ describe('DocManager', function () {
describe('when the lines are null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.promises.updateDoc(
+ this.DocManager.updateDoc(
this.project_id,
this.doc_id,
null,
@@ -613,7 +642,7 @@ describe('DocManager', function () {
describe('when the ranges are null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.promises.updateDoc(
+ this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -627,9 +656,9 @@ describe('DocManager', function () {
describe('when there is a generic error getting the doc', function () {
beforeEach(async function () {
this.error = new Error('doc could not be found')
- this.DocManager.promises._getDoc = sinon.stub().rejects(this.error)
+ this.DocManager._getDoc = sinon.stub().rejects(this.error)
await expect(
- this.DocManager.promises.updateDoc(
+ this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -640,16 +669,15 @@ describe('DocManager', function () {
})
it('should not upsert the document to the doc collection', function () {
- this.MongoManager.promises.upsertIntoDocCollection.should.not.have.been
- .called
+ this.MongoManager.upsertIntoDocCollection.should.not.have.been.called
})
})
describe('when the version was decremented', function () {
it('should return an error', async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
await expect(
- this.DocManager.promises.updateDoc(
+ this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -662,8 +690,8 @@ describe('DocManager', function () {
describe('when the doc lines have not changed', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.promises.updateDoc(
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines.slice(),
@@ -673,9 +701,7 @@ describe('DocManager', function () {
})
it('should not update the doc', function () {
- this.MongoManager.promises.upsertIntoDocCollection.called.should.equal(
- false
- )
+ this.MongoManager.upsertIntoDocCollection.called.should.equal(false)
})
it('should return the existing rev', function () {
@@ -685,8 +711,8 @@ describe('DocManager', function () {
describe('when the doc does not exist', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(null)
- this.result = await this.DocManager.promises.updateDoc(
+ this.DocManager._getDoc = sinon.stub().resolves(null)
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -696,7 +722,7 @@ describe('DocManager', function () {
})
it('should upsert the document to the doc collection', function () {
- this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
undefined,
@@ -715,12 +741,12 @@ describe('DocManager', function () {
describe('when another update is racing', function () {
beforeEach(async function () {
- this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
- this.MongoManager.promises.upsertIntoDocCollection
+ this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.MongoManager.upsertIntoDocCollection
.onFirstCall()
.rejects(new Errors.DocRevValueError())
this.RangeManager.shouldUpdateRanges.returns(true)
- this.result = await this.DocManager.promises.updateDoc(
+ this.result = await this.DocManager.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -730,7 +756,7 @@ describe('DocManager', function () {
})
it('should upsert the doc twice', function () {
- this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
this.rev,
@@ -740,8 +766,7 @@ describe('DocManager', function () {
version: this.version + 1,
}
)
- this.MongoManager.promises.upsertIntoDocCollection.should.have.been
- .calledTwice
+ this.MongoManager.upsertIntoDocCollection.should.have.been.calledTwice
})
it('should return the new rev', function () {
diff --git a/services/docstore/test/unit/js/HttpControllerTests.js b/services/docstore/test/unit/js/HttpControllerTests.js
index 2ed46a9d70..ab491ec150 100644
--- a/services/docstore/test/unit/js/HttpControllerTests.js
+++ b/services/docstore/test/unit/js/HttpControllerTests.js
@@ -1,7 +1,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/HttpController'
)
@@ -14,7 +14,7 @@ describe('HttpController', function () {
max_doc_length: 2 * 1024 * 1024,
}
this.DocArchiveManager = {
- unArchiveAllDocs: sinon.stub().yields(),
+ unArchiveAllDocs: sinon.stub().returns(),
}
this.DocManager = {}
this.HttpController = SandboxedModule.require(modulePath, {
@@ -54,15 +54,13 @@ describe('HttpController', function () {
describe('getDoc', function () {
describe('without deleted docs', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getFullDoc = sinon
- .stub()
- .callsArgWith(2, null, this.doc)
- this.HttpController.getDoc(this.req, this.res, this.next)
+ this.DocManager.getFullDoc = sinon.stub().resolves(this.doc)
+ await this.HttpController.getDoc(this.req, this.res, this.next)
})
it('should get the document with the version (including deleted)', function () {
@@ -89,26 +87,24 @@ describe('HttpController', function () {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getFullDoc = sinon
- .stub()
- .callsArgWith(2, null, this.deletedDoc)
+ this.DocManager.getFullDoc = sinon.stub().resolves(this.deletedDoc)
})
- it('should get the doc from the doc manager', function () {
- this.HttpController.getDoc(this.req, this.res, this.next)
+ it('should get the doc from the doc manager', async function () {
+ await this.HttpController.getDoc(this.req, this.res, this.next)
this.DocManager.getFullDoc
.calledWith(this.projectId, this.docId)
.should.equal(true)
})
- it('should return 404 if the query string delete is not set ', function () {
- this.HttpController.getDoc(this.req, this.res, this.next)
+ it('should return 404 if the query string delete is not set ', async function () {
+ await this.HttpController.getDoc(this.req, this.res, this.next)
this.res.sendStatus.calledWith(404).should.equal(true)
})
- it('should return the doc as JSON if include_deleted is set to true', function () {
+ it('should return the doc as JSON if include_deleted is set to true', async function () {
this.req.query.include_deleted = 'true'
- this.HttpController.getDoc(this.req, this.res, this.next)
+ await this.HttpController.getDoc(this.req, this.res, this.next)
this.res.json
.calledWith({
_id: this.docId,
@@ -123,13 +119,15 @@ describe('HttpController', function () {
})
describe('getRawDoc', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getDocLines = sinon.stub().callsArgWith(2, null, this.doc)
- this.HttpController.getRawDoc(this.req, this.res, this.next)
+ this.DocManager.getDocLines = sinon
+ .stub()
+ .resolves(this.doc.lines.join('\n'))
+ await this.HttpController.getRawDoc(this.req, this.res, this.next)
})
it('should get the document without the version', function () {
@@ -154,7 +152,7 @@ describe('HttpController', function () {
describe('getAllDocs', function () {
describe('normally', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -168,10 +166,8 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon
- .stub()
- .callsArgWith(2, null, this.docs)
- this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
+ await this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should get all the (non-deleted) docs', function () {
@@ -199,7 +195,7 @@ describe('HttpController', function () {
})
describe('with null lines', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -213,10 +209,8 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon
- .stub()
- .callsArgWith(2, null, this.docs)
- this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
+ await this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should return the doc with fallback lines', function () {
@@ -238,7 +232,7 @@ describe('HttpController', function () {
})
describe('with a null doc', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -253,10 +247,8 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon
- .stub()
- .callsArgWith(2, null, this.docs)
- this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
+ await this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should return the non null docs as JSON', function () {
@@ -292,7 +284,7 @@ describe('HttpController', function () {
describe('getAllRanges', function () {
describe('normally', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -304,10 +296,8 @@ describe('HttpController', function () {
ranges: { mock_ranges: 'two' },
},
]
- this.DocManager.getAllNonDeletedDocs = sinon
- .stub()
- .callsArgWith(2, null, this.docs)
- this.HttpController.getAllRanges(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
+ await this.HttpController.getAllRanges(this.req, this.res, this.next)
})
it('should get all the (non-deleted) doc ranges', function () {
@@ -342,16 +332,17 @@ describe('HttpController', function () {
})
describe('when the doc lines exist and were updated', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = {
lines: (this.lines = ['hello', 'world']),
version: (this.version = 42),
ranges: (this.ranges = { changes: 'mock' }),
}
+ this.rev = 5
this.DocManager.updateDoc = sinon
.stub()
- .yields(null, true, (this.rev = 5))
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ .resolves({ modified: true, rev: this.rev })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should update the document', function () {
@@ -374,16 +365,17 @@ describe('HttpController', function () {
})
describe('when the doc lines exist and were not updated', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = {
lines: (this.lines = ['hello', 'world']),
version: (this.version = 42),
ranges: {},
}
+ this.rev = 5
this.DocManager.updateDoc = sinon
.stub()
- .yields(null, false, (this.rev = 5))
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ .resolves({ modified: false, rev: this.rev })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should return a modified status', function () {
@@ -394,10 +386,12 @@ describe('HttpController', function () {
})
describe('when the doc lines are not provided', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = { version: 42, ranges: {} }
- this.DocManager.updateDoc = sinon.stub().yields(null, false)
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .resolves({ modified: false, rev: 0 })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -410,10 +404,12 @@ describe('HttpController', function () {
})
describe('when the doc version are not provided', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = { version: 42, lines: ['hello world'] }
- this.DocManager.updateDoc = sinon.stub().yields(null, false)
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .resolves({ modified: false, rev: 0 })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -426,10 +422,12 @@ describe('HttpController', function () {
})
describe('when the doc ranges is not provided', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = { lines: ['foo'], version: 42 }
- this.DocManager.updateDoc = sinon.stub().yields(null, false)
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .resolves({ modified: false, rev: 0 })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -442,13 +440,20 @@ describe('HttpController', function () {
})
describe('when the doc body is too large', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = {
lines: (this.lines = Array(2049).fill('a'.repeat(1024))),
version: (this.version = 42),
ranges: (this.ranges = { changes: 'mock' }),
}
- this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon
+ .stub()
+ .resolves({ modified: false, rev: 0 })
+ await this.HttpController.updateDoc(this.req, this.res, this.next)
+ })
+
+ it('should not update the document', function () {
+ this.DocManager.updateDoc.called.should.equal(false)
})
it('should return a 413 (too large) response', function () {
@@ -462,14 +467,14 @@ describe('HttpController', function () {
})
describe('patchDoc', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
this.req.body = { name: 'foo.tex' }
- this.DocManager.patchDoc = sinon.stub().yields(null)
- this.HttpController.patchDoc(this.req, this.res, this.next)
+ this.DocManager.patchDoc = sinon.stub().resolves()
+ await this.HttpController.patchDoc(this.req, this.res, this.next)
})
it('should delete the document', function () {
@@ -484,11 +489,11 @@ describe('HttpController', function () {
})
describe('with an invalid payload', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.body = { cannot: 'happen' }
- this.DocManager.patchDoc = sinon.stub().yields(null)
- this.HttpController.patchDoc(this.req, this.res, this.next)
+ this.DocManager.patchDoc = sinon.stub().resolves()
+ await this.HttpController.patchDoc(this.req, this.res, this.next)
})
it('should log a message', function () {
@@ -509,10 +514,10 @@ describe('HttpController', function () {
})
describe('archiveAllDocs', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
- this.DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1)
- this.HttpController.archiveAllDocs(this.req, this.res, this.next)
+ this.DocArchiveManager.archiveAllDocs = sinon.stub().resolves()
+ await this.HttpController.archiveAllDocs(this.req, this.res, this.next)
})
it('should archive the project', function () {
@@ -532,9 +537,12 @@ describe('HttpController', function () {
})
describe('on success', function () {
- beforeEach(function (done) {
- this.res.sendStatus.callsFake(() => done())
- this.HttpController.unArchiveAllDocs(this.req, this.res, this.next)
+ beforeEach(async function () {
+ await this.HttpController.unArchiveAllDocs(
+ this.req,
+ this.res,
+ this.next
+ )
})
it('returns a 200', function () {
@@ -543,12 +551,15 @@ describe('HttpController', function () {
})
describe("when the archived rev doesn't match", function () {
- beforeEach(function (done) {
- this.res.sendStatus.callsFake(() => done())
- this.DocArchiveManager.unArchiveAllDocs.yields(
+ beforeEach(async function () {
+ this.DocArchiveManager.unArchiveAllDocs.rejects(
new Errors.DocRevValueError('bad rev')
)
- this.HttpController.unArchiveAllDocs(this.req, this.res, this.next)
+ await this.HttpController.unArchiveAllDocs(
+ this.req,
+ this.res,
+ this.next
+ )
})
it('returns a 409', function () {
@@ -558,10 +569,10 @@ describe('HttpController', function () {
})
describe('destroyProject', function () {
- beforeEach(function () {
+ beforeEach(async function () {
this.req.params = { project_id: this.projectId }
- this.DocArchiveManager.destroyProject = sinon.stub().callsArg(1)
- this.HttpController.destroyProject(this.req, this.res, this.next)
+ this.DocArchiveManager.destroyProject = sinon.stub().resolves()
+ await this.HttpController.destroyProject(this.req, this.res, this.next)
})
it('should destroy the docs', function () {
diff --git a/services/docstore/test/unit/js/MongoManagerTests.js b/services/docstore/test/unit/js/MongoManagerTests.js
index da8f59ae07..b96b661df4 100644
--- a/services/docstore/test/unit/js/MongoManagerTests.js
+++ b/services/docstore/test/unit/js/MongoManagerTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/MongoManager'
)
@@ -41,7 +41,7 @@ describe('MongoManager', function () {
this.doc = { name: 'mock-doc' }
this.db.docs.findOne = sinon.stub().resolves(this.doc)
this.filter = { lines: true }
- this.result = await this.MongoManager.promises.findDoc(
+ this.result = await this.MongoManager.findDoc(
this.projectId,
this.docId,
this.filter
@@ -70,11 +70,7 @@ describe('MongoManager', function () {
describe('patchDoc', function () {
beforeEach(async function () {
this.meta = { name: 'foo.tex' }
- await this.MongoManager.promises.patchDoc(
- this.projectId,
- this.docId,
- this.meta
- )
+ await this.MongoManager.patchDoc(this.projectId, this.docId, this.meta)
})
it('should pass the parameter along', function () {
@@ -104,7 +100,7 @@ describe('MongoManager', function () {
describe('with included_deleted = false', function () {
beforeEach(async function () {
- this.result = await this.MongoManager.promises.getProjectsDocs(
+ this.result = await this.MongoManager.getProjectsDocs(
this.projectId,
{ include_deleted: false },
this.filter
@@ -132,7 +128,7 @@ describe('MongoManager', function () {
describe('with included_deleted = true', function () {
beforeEach(async function () {
- this.result = await this.MongoManager.promises.getProjectsDocs(
+ this.result = await this.MongoManager.getProjectsDocs(
this.projectId,
{ include_deleted: true },
this.filter
@@ -167,7 +163,7 @@ describe('MongoManager', function () {
this.db.docs.find = sinon.stub().returns({
toArray: sinon.stub().resolves([this.doc1, this.doc2, this.doc3]),
})
- this.result = await this.MongoManager.promises.getProjectsDeletedDocs(
+ this.result = await this.MongoManager.getProjectsDeletedDocs(
this.projectId,
this.filter
)
@@ -203,7 +199,7 @@ describe('MongoManager', function () {
})
it('should upsert the document', async function () {
- await this.MongoManager.promises.upsertIntoDocCollection(
+ await this.MongoManager.upsertIntoDocCollection(
this.projectId,
this.docId,
this.oldRev,
@@ -223,7 +219,7 @@ describe('MongoManager', function () {
it('should handle update error', async function () {
this.db.docs.updateOne.rejects(this.stubbedErr)
await expect(
- this.MongoManager.promises.upsertIntoDocCollection(
+ this.MongoManager.upsertIntoDocCollection(
this.projectId,
this.docId,
this.rev,
@@ -235,7 +231,7 @@ describe('MongoManager', function () {
})
it('should insert without a previous rev', async function () {
- await this.MongoManager.promises.upsertIntoDocCollection(
+ await this.MongoManager.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -254,7 +250,7 @@ describe('MongoManager', function () {
it('should handle generic insert error', async function () {
this.db.docs.insertOne.rejects(this.stubbedErr)
await expect(
- this.MongoManager.promises.upsertIntoDocCollection(
+ this.MongoManager.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -266,7 +262,7 @@ describe('MongoManager', function () {
it('should handle duplicate insert error', async function () {
this.db.docs.insertOne.rejects({ code: 11000 })
await expect(
- this.MongoManager.promises.upsertIntoDocCollection(
+ this.MongoManager.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -280,7 +276,7 @@ describe('MongoManager', function () {
beforeEach(async function () {
this.projectId = new ObjectId()
this.db.docs.deleteMany = sinon.stub().resolves()
- await this.MongoManager.promises.destroyProject(this.projectId)
+ await this.MongoManager.destroyProject(this.projectId)
})
it('should destroy all docs', function () {
@@ -297,13 +293,13 @@ describe('MongoManager', function () {
it('should not error when the rev has not changed', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 1 })
- await this.MongoManager.promises.checkRevUnchanged(this.doc)
+ await this.MongoManager.checkRevUnchanged(this.doc)
})
it('should return an error when the rev has changed', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 2 })
await expect(
- this.MongoManager.promises.checkRevUnchanged(this.doc)
+ this.MongoManager.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocModifiedError)
})
@@ -311,14 +307,14 @@ describe('MongoManager', function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 2 })
this.doc = { _id: new ObjectId(), name: 'mock-doc', rev: NaN }
await expect(
- this.MongoManager.promises.checkRevUnchanged(this.doc)
+ this.MongoManager.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocRevValueError)
})
it('should return a value error if checked doc rev is NaN', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: NaN })
await expect(
- this.MongoManager.promises.checkRevUnchanged(this.doc)
+ this.MongoManager.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocRevValueError)
})
})
@@ -334,7 +330,7 @@ describe('MongoManager', function () {
describe('complete doc', function () {
beforeEach(async function () {
- await this.MongoManager.promises.restoreArchivedDoc(
+ await this.MongoManager.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
@@ -364,7 +360,7 @@ describe('MongoManager', function () {
describe('without ranges', function () {
beforeEach(async function () {
delete this.archivedDoc.ranges
- await this.MongoManager.promises.restoreArchivedDoc(
+ await this.MongoManager.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
@@ -395,7 +391,7 @@ describe('MongoManager', function () {
it('throws a DocRevValueError', async function () {
this.db.docs.updateOne.resolves({ matchedCount: 0 })
await expect(
- this.MongoManager.promises.restoreArchivedDoc(
+ this.MongoManager.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
diff --git a/services/docstore/test/unit/js/RangeManagerTests.js b/services/docstore/test/unit/js/RangeManagerTests.js
index 0d2a353558..ba99280a7a 100644
--- a/services/docstore/test/unit/js/RangeManagerTests.js
+++ b/services/docstore/test/unit/js/RangeManagerTests.js
@@ -12,7 +12,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../app/js/RangeManager'
)
@@ -30,7 +30,7 @@ describe('RangeManager', function () {
})
describe('jsonRangesToMongo', function () {
- it('should convert ObjectIds and dates to proper objects', function () {
+ it('should convert ObjectIds and dates to proper objects and fix comment id', function () {
const changeId = new ObjectId().toString()
const commentId = new ObjectId().toString()
const userId = new ObjectId().toString()
@@ -66,7 +66,7 @@ describe('RangeManager', function () {
],
comments: [
{
- id: new ObjectId(commentId),
+ id: new ObjectId(threadId),
op: { c: 'foo', p: 3, t: new ObjectId(threadId) },
},
],
@@ -110,7 +110,6 @@ describe('RangeManager', function () {
return it('should be consistent when transformed through json -> mongo -> json', function () {
const changeId = new ObjectId().toString()
- const commentId = new ObjectId().toString()
const userId = new ObjectId().toString()
const threadId = new ObjectId().toString()
const ts = new Date().toJSON()
@@ -127,7 +126,7 @@ describe('RangeManager', function () {
],
comments: [
{
- id: commentId,
+ id: threadId,
op: { c: 'foo', p: 3, t: threadId },
},
],
@@ -142,6 +141,7 @@ describe('RangeManager', function () {
return describe('shouldUpdateRanges', function () {
beforeEach(function () {
+ const threadId = new ObjectId()
this.ranges = {
changes: [
{
@@ -155,8 +155,8 @@ describe('RangeManager', function () {
],
comments: [
{
- id: new ObjectId(),
- op: { c: 'foo', p: 3, t: new ObjectId() },
+ id: threadId,
+ op: { c: 'foo', p: 3, t: threadId },
},
],
}
diff --git a/services/document-updater/.gitignore b/services/document-updater/.gitignore
deleted file mode 100644
index 624e78f096..0000000000
--- a/services/document-updater/.gitignore
+++ /dev/null
@@ -1,52 +0,0 @@
-compileFolder
-
-Compiled source #
-###################
-*.com
-*.class
-*.dll
-*.exe
-*.o
-*.so
-
-# Packages #
-############
-# it's better to unpack these files and commit the raw source
-# git has its own built in compression methods
-*.7z
-*.dmg
-*.gz
-*.iso
-*.jar
-*.rar
-*.tar
-*.zip
-
-# Logs and databases #
-######################
-*.log
-*.sql
-*.sqlite
-
-# OS generated files #
-######################
-.DS_Store?
-ehthumbs.db
-Icon?
-Thumbs.db
-
-/node_modules/*
-
-
-
-forever/
-
-**.swp
-
-# Redis cluster
-**/appendonly.aof
-**/dump.rdb
-**/nodes.conf
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc
diff --git a/services/document-updater/.nvmrc b/services/document-updater/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/document-updater/.nvmrc
+++ b/services/document-updater/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile
index 06c1610c66..720b619c41 100644
--- a/services/document-updater/Dockerfile
+++ b/services/document-updater/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/document-updater
diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile
index de0cf0eabe..46dfced5c9 100644
--- a/services/document-updater/Makefile
+++ b/services/document-updater/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -98,13 +116,6 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
- while ! mongosh --eval "db.version()" > /dev/null; do \
- echo "Waiting for Mongo..."; \
- sleep 1; \
- done; \
- mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -137,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/document-updater/app.js b/services/document-updater/app.js
index fab33150c3..65c9895377 100644
--- a/services/document-updater/app.js
+++ b/services/document-updater/app.js
@@ -9,10 +9,6 @@ logger.initialize('document-updater')
logger.logger.addSerializers(require('./app/js/LoggerSerializers'))
-if (Settings.sentry != null && Settings.sentry.dsn != null) {
- logger.initializeErrorReporting(Settings.sentry.dsn)
-}
-
const RedisManager = require('./app/js/RedisManager')
const DispatchManager = require('./app/js/DispatchManager')
const DeleteQueueManager = require('./app/js/DeleteQueueManager')
@@ -139,6 +135,10 @@ app.use((req, res, next) => {
})
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
+app.get(
+ '/project/:project_id/doc/:doc_id/comment/:comment_id',
+ HttpController.getComment
+)
app.get('/project/:project_id/doc/:doc_id/peek', HttpController.peekDoc)
// temporarily keep the GET method for backwards compatibility
app.get('/project/:project_id/doc', HttpController.getProjectDocsAndFlushIfOld)
@@ -147,8 +147,13 @@ app.post(
'/project/:project_id/get_and_flush_if_old',
HttpController.getProjectDocsAndFlushIfOld
)
+app.get(
+ '/project/:project_id/last_updated_at',
+ HttpController.getProjectLastUpdatedAt
+)
app.post('/project/:project_id/clearState', HttpController.clearProjectState)
app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
+app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)
app.post(
'/project/:project_id/doc/:doc_id/flush',
HttpController.flushDocIfLoaded
diff --git a/services/document-updater/app/js/DiffCodec.js b/services/document-updater/app/js/DiffCodec.js
index 245903ca13..17da409386 100644
--- a/services/document-updater/app/js/DiffCodec.js
+++ b/services/document-updater/app/js/DiffCodec.js
@@ -1,4 +1,6 @@
+const OError = require('@overleaf/o-error')
const DMP = require('diff-match-patch')
+const { TextOperation } = require('overleaf-editor-core')
const dmp = new DMP()
// Do not attempt to produce a diff for more than 100ms
@@ -16,8 +18,7 @@ module.exports = {
const ops = []
let position = 0
for (const diff of diffs) {
- const type = diff[0]
- const content = diff[1]
+ const [type, content] = diff
if (type === this.ADDED) {
ops.push({
i: content,
@@ -37,4 +38,63 @@ module.exports = {
}
return ops
},
+
+ /**
+ * @param {import("overleaf-editor-core").StringFileData} file
+ * @param {string} after
+ * @return {TextOperation}
+ */
+ diffAsHistoryOTEditOperation(file, after) {
+ const beforeWithoutTrackedDeletes = file.getContent({
+ filterTrackedDeletes: true,
+ })
+ const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after)
+ dmp.diff_cleanupSemantic(diffs)
+
+ const trackedChanges = file.trackedChanges.asSorted()
+ let nextTc = trackedChanges.shift()
+
+ const op = new TextOperation()
+ for (const diff of diffs) {
+ let [type, content] = diff
+ if (type === this.ADDED) {
+ op.insert(content)
+ } else if (type === this.REMOVED || type === this.UNCHANGED) {
+ while (op.baseLength + content.length > nextTc?.range.start) {
+ if (nextTc.tracking.type === 'delete') {
+ const untilRange = nextTc.range.start - op.baseLength
+ if (type === this.REMOVED) {
+ op.remove(untilRange)
+ } else if (type === this.UNCHANGED) {
+ op.retain(untilRange)
+ }
+ op.retain(nextTc.range.end - nextTc.range.start)
+ content = content.slice(untilRange)
+ }
+ nextTc = trackedChanges.shift()
+ }
+ if (type === this.REMOVED) {
+ op.remove(content.length)
+ } else if (type === this.UNCHANGED) {
+ op.retain(content.length)
+ }
+ } else {
+ throw new Error('Unknown type')
+ }
+ }
+ while (nextTc) {
+ if (
+ nextTc.tracking.type !== 'delete' ||
+ nextTc.range.start !== op.baseLength
+ ) {
+ throw new OError(
+ 'StringFileData.trackedChanges out of sync: unexpected range after end of diff',
+ { nextTc, baseLength: op.baseLength }
+ )
+ }
+ op.retain(nextTc.range.end - nextTc.range.start)
+ nextTc = trackedChanges.shift()
+ }
+ return op
+ },
}
diff --git a/services/document-updater/app/js/DocumentManager.js b/services/document-updater/app/js/DocumentManager.js
index 157a99ccf6..3fb3d10a6e 100644
--- a/services/document-updater/app/js/DocumentManager.js
+++ b/services/document-updater/app/js/DocumentManager.js
@@ -9,10 +9,18 @@ const HistoryManager = require('./HistoryManager')
const Errors = require('./Errors')
const RangesManager = require('./RangesManager')
const { extractOriginOrSource } = require('./Utils')
+const { getTotalSizeOfLines } = require('./Limits')
+const Settings = require('@overleaf/settings')
+const { StringFileData } = require('overleaf-editor-core')
const MAX_UNFLUSHED_AGE = 300 * 1000 // 5 mins, document should be flushed to mongo this time after a change
const DocumentManager = {
+ /**
+ * @param {string} projectId
+ * @param {string} docId
+ * @return {Promise<{lines: (string[] | StringFileRawData), version: number, ranges: Ranges, resolvedCommentIds: any[], pathname: string, projectHistoryId: string, unflushedTime: any, alreadyLoaded: boolean, historyRangesSupport: boolean, type: OTType}>}
+ */
async getDoc(projectId, docId) {
const {
lines,
@@ -73,6 +81,7 @@ const DocumentManager = {
unflushedTime: null,
alreadyLoaded: false,
historyRangesSupport,
+ type: Array.isArray(lines) ? 'sharejs-text-ot' : 'history-ot',
}
} else {
return {
@@ -85,16 +94,25 @@ const DocumentManager = {
unflushedTime,
alreadyLoaded: true,
historyRangesSupport,
+ type: Array.isArray(lines) ? 'sharejs-text-ot' : 'history-ot',
}
}
},
async getDocAndRecentOps(projectId, docId, fromVersion) {
- const { lines, version, ranges, pathname, projectHistoryId } =
+ const { lines, version, ranges, pathname, projectHistoryId, type } =
await DocumentManager.getDoc(projectId, docId)
if (fromVersion === -1) {
- return { lines, version, ops: [], ranges, pathname, projectHistoryId }
+ return {
+ lines,
+ version,
+ ops: [],
+ ranges,
+ pathname,
+ projectHistoryId,
+ type,
+ }
} else {
const ops = await RedisManager.promises.getPreviousDocOps(
docId,
@@ -108,39 +126,90 @@ const DocumentManager = {
ranges,
pathname,
projectHistoryId,
+ type,
}
}
},
- async setDoc(projectId, docId, newLines, originOrSource, userId, undoing) {
+ async appendToDoc(projectId, docId, linesToAppend, originOrSource, userId) {
+ let { lines: currentLines, type } = await DocumentManager.getDoc(
+ projectId,
+ docId
+ )
+ if (type === 'history-ot') {
+ const file = StringFileData.fromRaw(currentLines)
+ // TODO(24596): tc support for history-ot
+ currentLines = file.getLines()
+ }
+ const currentLineSize = getTotalSizeOfLines(currentLines)
+ const addedSize = getTotalSizeOfLines(linesToAppend)
+ const newlineSize = '\n'.length
+
+ if (currentLineSize + newlineSize + addedSize > Settings.max_doc_length) {
+ throw new Errors.FileTooLargeError(
+ 'doc would become too large if appending this text'
+ )
+ }
+
+ return await DocumentManager.setDoc(
+ projectId,
+ docId,
+ currentLines.concat(linesToAppend),
+ originOrSource,
+ userId,
+ false,
+ false
+ )
+ },
+
+ async setDoc(
+ projectId,
+ docId,
+ newLines,
+ originOrSource,
+ userId,
+ undoing,
+ external
+ ) {
if (newLines == null) {
throw new Error('No lines were provided to setDoc')
}
+ // Circular dependencies. Import at runtime.
+ const HistoryOTUpdateManager = require('./HistoryOTUpdateManager')
const UpdateManager = require('./UpdateManager')
+
const {
lines: oldLines,
version,
alreadyLoaded,
+ type,
} = await DocumentManager.getDoc(projectId, docId)
- if (oldLines != null && oldLines.length > 0 && oldLines[0].text != null) {
- logger.debug(
- { docId, projectId, oldLines, newLines },
- 'document is JSON so not updating'
- )
- return
- }
-
logger.debug(
{ docId, projectId, oldLines, newLines },
'setting a document via http'
)
- const op = DiffCodec.diffAsShareJsOp(oldLines, newLines)
- if (undoing) {
- for (const o of op || []) {
- o.u = true
- } // Turn on undo flag for each op for track changes
+
+ let op
+ if (type === 'history-ot') {
+ const file = StringFileData.fromRaw(oldLines)
+ const operation = DiffCodec.diffAsHistoryOTEditOperation(
+ file,
+ newLines.join('\n')
+ )
+ if (operation.isNoop()) {
+ op = []
+ } else {
+ op = [operation.toJSON()]
+ }
+ } else {
+ op = DiffCodec.diffAsShareJsOp(oldLines, newLines)
+ if (undoing) {
+ for (const o of op || []) {
+ o.u = true
+ } // Turn on undo flag for each op for track changes
+ }
}
const { origin, source } = extractOriginOrSource(originOrSource)
@@ -150,10 +219,12 @@ const DocumentManager = {
op,
v: version,
meta: {
- type: 'external',
user_id: userId,
},
}
+ if (external) {
+ update.meta.type = 'external'
+ }
if (origin) {
update.meta.origin = origin
} else if (source) {
@@ -173,7 +244,11 @@ const DocumentManager = {
// this update, otherwise the doc would never be
// removed from redis.
if (op.length > 0) {
- await UpdateManager.promises.applyUpdate(projectId, docId, update)
+ if (type === 'history-ot') {
+ await HistoryOTUpdateManager.applyUpdate(projectId, docId, update)
+ } else {
+ await UpdateManager.promises.applyUpdate(projectId, docId, update)
+ }
}
// If the document was loaded already, then someone has it open
@@ -194,7 +269,7 @@ const DocumentManager = {
},
async flushDocIfLoaded(projectId, docId) {
- const {
+ let {
lines,
version,
ranges,
@@ -215,6 +290,11 @@ const DocumentManager = {
logger.debug({ projectId, docId, version }, 'flushing doc')
Metrics.inc('flush-doc-if-loaded', 1, { status: 'modified' })
+ if (!Array.isArray(lines)) {
+ const file = StringFileData.fromRaw(lines)
+ // TODO(24596): tc support for history-ot
+ lines = file.getLines()
+ }
const result = await PersistenceManager.promises.setDoc(
projectId,
docId,
@@ -264,7 +344,14 @@ const DocumentManager = {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
- const newRanges = RangesManager.acceptChanges(changeIds, ranges)
+ // TODO(24596): tc support for history-ot
+ const newRanges = RangesManager.acceptChanges(
+ projectId,
+ docId,
+ changeIds,
+ ranges,
+ lines
+ )
await RedisManager.promises.updateDocument(
projectId,
@@ -323,6 +410,22 @@ const DocumentManager = {
}
},
+ async getComment(projectId, docId, commentId) {
+ // TODO(24596): tc support for history-ot
+ const { ranges } = await DocumentManager.getDoc(projectId, docId)
+
+ const comment = ranges?.comments?.find(comment => comment.id === commentId)
+
+ if (!comment) {
+ throw new Errors.NotFoundError({
+ message: 'comment not found',
+ info: { commentId },
+ })
+ }
+
+ return { comment }
+ },
+
async deleteComment(projectId, docId, commentId, userId) {
const { lines, version, ranges, pathname, historyRangesSupport } =
await DocumentManager.getDoc(projectId, docId)
@@ -330,6 +433,7 @@ const DocumentManager = {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
+ // TODO(24596): tc support for history-ot
const newRanges = RangesManager.deleteComment(commentId, ranges)
await RedisManager.promises.updateDocument(
@@ -369,7 +473,7 @@ const DocumentManager = {
},
async getDocAndFlushIfOld(projectId, docId) {
- const { lines, version, unflushedTime, alreadyLoaded } =
+ let { lines, version, unflushedTime, alreadyLoaded } =
await DocumentManager.getDoc(projectId, docId)
// if doc was already loaded see if it needs to be flushed
@@ -381,6 +485,12 @@ const DocumentManager = {
await DocumentManager.flushDocIfLoaded(projectId, docId)
}
+ if (!Array.isArray(lines)) {
+ const file = StringFileData.fromRaw(lines)
+ // TODO(24596): tc support for history-ot
+ lines = file.getLines()
+ }
+
return { lines, version }
},
@@ -456,6 +566,16 @@ const DocumentManager = {
)
},
+ async getCommentWithLock(projectId, docId, commentId) {
+ const UpdateManager = require('./UpdateManager')
+ return await UpdateManager.promises.lockUpdatesAndDo(
+ DocumentManager.getComment,
+ projectId,
+ docId,
+ commentId
+ )
+ },
+
async getDocAndRecentOpsWithLock(projectId, docId, fromVersion) {
const UpdateManager = require('./UpdateManager')
return await UpdateManager.promises.lockUpdatesAndDo(
@@ -475,7 +595,15 @@ const DocumentManager = {
)
},
- async setDocWithLock(projectId, docId, lines, source, userId, undoing) {
+ async setDocWithLock(
+ projectId,
+ docId,
+ lines,
+ source,
+ userId,
+ undoing,
+ external
+ ) {
const UpdateManager = require('./UpdateManager')
return await UpdateManager.promises.lockUpdatesAndDo(
DocumentManager.setDoc,
@@ -484,7 +612,20 @@ const DocumentManager = {
lines,
source,
userId,
- undoing
+ undoing,
+ external
+ )
+ },
+
+ async appendToDocWithLock(projectId, docId, lines, source, userId) {
+ const UpdateManager = require('./UpdateManager')
+ return await UpdateManager.promises.lockUpdatesAndDo(
+ DocumentManager.appendToDoc,
+ projectId,
+ docId,
+ lines,
+ source,
+ userId
)
},
@@ -602,6 +743,7 @@ module.exports = {
'ranges',
'pathname',
'projectHistoryId',
+ 'type',
],
getDocAndRecentOpsWithLock: [
'lines',
@@ -610,7 +752,9 @@ module.exports = {
'ranges',
'pathname',
'projectHistoryId',
+ 'type',
],
+ getCommentWithLock: ['comment'],
},
}),
promises: DocumentManager,
diff --git a/services/document-updater/app/js/Errors.js b/services/document-updater/app/js/Errors.js
index a43f69ad35..ac1f5875fa 100644
--- a/services/document-updater/app/js/Errors.js
+++ b/services/document-updater/app/js/Errors.js
@@ -5,6 +5,15 @@ class OpRangeNotAvailableError extends OError {}
class ProjectStateChangedError extends OError {}
class DeleteMismatchError extends OError {}
class FileTooLargeError extends OError {}
+class OTTypeMismatchError extends OError {
+ /**
+ * @param {OTType} got
+ * @param {OTType} want
+ */
+ constructor(got, want) {
+ super('ot type mismatch', { got, want })
+ }
+}
module.exports = {
NotFoundError,
@@ -12,4 +21,5 @@ module.exports = {
ProjectStateChangedError,
DeleteMismatchError,
FileTooLargeError,
+ OTTypeMismatchError,
}
diff --git a/services/document-updater/app/js/HistoryManager.js b/services/document-updater/app/js/HistoryManager.js
index 3a91b29cb8..d9a8459525 100644
--- a/services/document-updater/app/js/HistoryManager.js
+++ b/services/document-updater/app/js/HistoryManager.js
@@ -62,6 +62,7 @@ const HistoryManager = {
// record updates for project history
if (
HistoryManager.shouldFlushHistoryOps(
+ projectId,
projectOpsLength,
ops.length,
HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
@@ -77,7 +78,8 @@ const HistoryManager = {
}
},
- shouldFlushHistoryOps(length, opsLength, threshold) {
+ shouldFlushHistoryOps(projectId, length, opsLength, threshold) {
+ if (Settings.shortHistoryQueues.includes(projectId)) return true
if (!length) {
return false
} // don't flush unless we know the length
@@ -106,10 +108,12 @@ const HistoryManager = {
projectHistoryId,
docs,
files,
+ opts,
function (error) {
if (error) {
return callback(error)
}
+ if (opts.resyncProjectStructureOnly) return callback()
const DocumentManager = require('./DocumentManager')
const resyncDoc = (doc, cb) => {
DocumentManager.resyncDocContentsWithLock(
diff --git a/services/document-updater/app/js/HistoryOTUpdateManager.js b/services/document-updater/app/js/HistoryOTUpdateManager.js
new file mode 100644
index 0000000000..5a8b92099e
--- /dev/null
+++ b/services/document-updater/app/js/HistoryOTUpdateManager.js
@@ -0,0 +1,158 @@
+// @ts-check
+
+const Profiler = require('./Profiler')
+const DocumentManager = require('./DocumentManager')
+const Errors = require('./Errors')
+const RedisManager = require('./RedisManager')
+const {
+ EditOperationBuilder,
+ StringFileData,
+ EditOperationTransformer,
+} = require('overleaf-editor-core')
+const Metrics = require('./Metrics')
+const ProjectHistoryRedisManager = require('./ProjectHistoryRedisManager')
+const HistoryManager = require('./HistoryManager')
+const RealTimeRedisManager = require('./RealTimeRedisManager')
+
+/**
+ * @typedef {import("./types").Update} Update
+ * @typedef {import("./types").HistoryOTEditOperationUpdate} HistoryOTEditOperationUpdate
+ */
+
+/**
+ * @param {Update} update
+ * @return {update is HistoryOTEditOperationUpdate}
+ */
+function isHistoryOTEditOperationUpdate(update) {
+ return (
+ update &&
+ 'doc' in update &&
+ 'op' in update &&
+ 'v' in update &&
+ Array.isArray(update.op) &&
+ EditOperationBuilder.isValid(update.op[0])
+ )
+}
+
+/**
+ * Try to apply an update to the given document
+ *
+ * @param {string} projectId
+ * @param {string} docId
+ * @param {HistoryOTEditOperationUpdate} update
+ * @param {Profiler} profiler
+ */
+async function tryApplyUpdate(projectId, docId, update, profiler) {
+ let { lines, version, pathname, type } =
+ await DocumentManager.promises.getDoc(projectId, docId)
+ profiler.log('getDoc')
+
+ if (lines == null || version == null) {
+ throw new Errors.NotFoundError(`document not found: ${docId}`)
+ }
+ if (type !== 'history-ot') {
+ throw new Errors.OTTypeMismatchError(type, 'history-ot')
+ }
+
+ let op = EditOperationBuilder.fromJSON(update.op[0])
+ if (version !== update.v) {
+ const transformUpdates = await RedisManager.promises.getPreviousDocOps(
+ docId,
+ update.v,
+ version
+ )
+ for (const transformUpdate of transformUpdates) {
+ if (!isHistoryOTEditOperationUpdate(transformUpdate)) {
+ throw new Errors.OTTypeMismatchError('sharejs-text-ot', 'history-ot')
+ }
+
+ if (
+ transformUpdate.meta.source &&
+ update.dupIfSource?.includes(transformUpdate.meta.source)
+ ) {
+ update.dup = true
+ break
+ }
+ const other = EditOperationBuilder.fromJSON(transformUpdate.op[0])
+ op = EditOperationTransformer.transform(op, other)[0]
+ }
+ update.op = [op.toJSON()]
+ }
+
+ if (!update.dup) {
+ const file = StringFileData.fromRaw(lines)
+ file.edit(op)
+ version += 1
+ update.meta.ts = Date.now()
+ await RedisManager.promises.updateDocument(
+ projectId,
+ docId,
+ file.toRaw(),
+ version,
+ [update],
+ {},
+ update.meta
+ )
+
+ Metrics.inc('history-queue', 1, { status: 'project-history' })
+ try {
+ const projectOpsLength =
+ await ProjectHistoryRedisManager.promises.queueOps(projectId, [
+ JSON.stringify({
+ ...update,
+ meta: {
+ ...update.meta,
+ pathname,
+ },
+ }),
+ ])
+ HistoryManager.recordAndFlushHistoryOps(
+ projectId,
+ [update],
+ projectOpsLength
+ )
+ profiler.log('recordAndFlushHistoryOps')
+ } catch (err) {
+ // The full project history can re-sync a project in case
+ // updates went missing.
+ // Just record the error here and acknowledge the write-op.
+ Metrics.inc('history-queue-error')
+ }
+ }
+ RealTimeRedisManager.sendData({
+ project_id: projectId,
+ doc_id: docId,
+ op: update,
+ })
+}
+
+/**
+ * Apply an update to the given document
+ *
+ * @param {string} projectId
+ * @param {string} docId
+ * @param {HistoryOTEditOperationUpdate} update
+ */
+async function applyUpdate(projectId, docId, update) {
+ const profiler = new Profiler('applyUpdate', {
+ project_id: projectId,
+ doc_id: docId,
+ type: 'history-ot',
+ })
+
+ try {
+ await tryApplyUpdate(projectId, docId, update, profiler)
+ } catch (error) {
+ RealTimeRedisManager.sendData({
+ project_id: projectId,
+ doc_id: docId,
+ error: error instanceof Error ? error.message : error,
+ })
+ profiler.log('sendData')
+ throw error
+ } finally {
+ profiler.end()
+ }
+}
+
+module.exports = { isHistoryOTEditOperationUpdate, applyUpdate }
diff --git a/services/document-updater/app/js/HttpController.js b/services/document-updater/app/js/HttpController.js
index a3c7b6cd44..0a6ae3b2b4 100644
--- a/services/document-updater/app/js/HttpController.js
+++ b/services/document-updater/app/js/HttpController.js
@@ -9,6 +9,7 @@ const Metrics = require('./Metrics')
const DeleteQueueManager = require('./DeleteQueueManager')
const { getTotalSizeOfLines } = require('./Limits')
const async = require('async')
+const { StringFileData } = require('overleaf-editor-core')
function getDoc(req, res, next) {
let fromVersion
@@ -27,7 +28,7 @@ function getDoc(req, res, next) {
projectId,
docId,
fromVersion,
- (error, lines, version, ops, ranges, pathname) => {
+ (error, lines, version, ops, ranges, pathname, _projectHistoryId, type) => {
timer.done()
if (error) {
return next(error)
@@ -36,6 +37,11 @@ function getDoc(req, res, next) {
if (lines == null || version == null) {
return next(new Errors.NotFoundError('document not found'))
}
+ if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
+ const file = StringFileData.fromRaw(lines)
+ // TODO(24596): tc support for history-ot
+ lines = file.getLines()
+ }
res.json({
id: docId,
lines,
@@ -44,11 +50,35 @@ function getDoc(req, res, next) {
ranges,
pathname,
ttlInS: RedisManager.DOC_OPS_TTL,
+ type,
})
}
)
}
+function getComment(req, res, next) {
+ const docId = req.params.doc_id
+ const projectId = req.params.project_id
+ const commentId = req.params.comment_id
+
+ logger.debug({ projectId, docId, commentId }, 'getting comment via http')
+
+ DocumentManager.getCommentWithLock(
+ projectId,
+ docId,
+ commentId,
+ (error, comment) => {
+ if (error) {
+ return next(error)
+ }
+ if (comment == null) {
+ return next(new Errors.NotFoundError('comment not found'))
+ }
+ res.json(comment)
+ }
+ )
+}
+
// return the doc from redis if present, but don't load it from mongo
function peekDoc(req, res, next) {
const docId = req.params.doc_id
@@ -61,6 +91,11 @@ function peekDoc(req, res, next) {
if (lines == null || version == null) {
return next(new Errors.NotFoundError('document not found'))
}
+ if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
+ const file = StringFileData.fromRaw(lines)
+ // TODO(24596): tc support for history-ot
+ lines = file.getLines()
+ }
res.json({ id: docId, lines, version })
})
}
@@ -106,6 +141,22 @@ function getProjectDocsAndFlushIfOld(req, res, next) {
)
}
+function getProjectLastUpdatedAt(req, res, next) {
+ const projectId = req.params.project_id
+ ProjectManager.getProjectDocsTimestamps(projectId, (err, timestamps) => {
+ if (err) return next(err)
+
+ // Filter out nulls. This can happen when
+ // - docs get flushed between the listing and getting the individual docs ts
+ // - a doc flush failed half way (doc keys removed, project tracking not updated)
+ timestamps = timestamps.filter(ts => !!ts)
+
+ timestamps = timestamps.map(ts => parseInt(ts, 10))
+ timestamps.sort((a, b) => (a > b ? 1 : -1))
+ res.json({ lastUpdatedAt: timestamps.pop() })
+ })
+}
+
function clearProjectState(req, res, next) {
const projectId = req.params.project_id
const timer = new Metrics.Timer('http.clearProjectState')
@@ -144,6 +195,7 @@ function setDoc(req, res, next) {
source,
userId,
undoing,
+ true,
(error, result) => {
timer.done()
if (error) {
@@ -155,6 +207,35 @@ function setDoc(req, res, next) {
)
}
+function appendToDoc(req, res, next) {
+ const docId = req.params.doc_id
+ const projectId = req.params.project_id
+ const { lines, source, user_id: userId } = req.body
+ const timer = new Metrics.Timer('http.appendToDoc')
+ DocumentManager.appendToDocWithLock(
+ projectId,
+ docId,
+ lines,
+ source,
+ userId,
+ (error, result) => {
+ timer.done()
+ if (error instanceof Errors.FileTooLargeError) {
+ logger.warn('refusing to append to file, it would become too large')
+ return res.sendStatus(422)
+ }
+ if (error) {
+ return next(error)
+ }
+ logger.debug(
+ { projectId, docId, lines, source, userId },
+ 'appending to doc via http'
+ )
+ res.json(result)
+ }
+ )
+}
+
function flushDocIfLoaded(req, res, next) {
const docId = req.params.doc_id
const projectId = req.params.project_id
@@ -380,7 +461,13 @@ function updateProject(req, res, next) {
function resyncProjectHistory(req, res, next) {
const projectId = req.params.project_id
- const { projectHistoryId, docs, files, historyRangesMigration } = req.body
+ const {
+ projectHistoryId,
+ docs,
+ files,
+ historyRangesMigration,
+ resyncProjectStructureOnly,
+ } = req.body
logger.debug(
{ projectId, docs, files },
@@ -391,6 +478,9 @@ function resyncProjectHistory(req, res, next) {
if (historyRangesMigration) {
opts.historyRangesMigration = historyRangesMigration
}
+ if (resyncProjectStructureOnly) {
+ opts.resyncProjectStructureOnly = resyncProjectStructureOnly
+ }
HistoryManager.resyncProjectHistory(
projectId,
@@ -459,7 +549,9 @@ module.exports = {
getDoc,
peekDoc,
getProjectDocsAndFlushIfOld,
+ getProjectLastUpdatedAt,
clearProjectState,
+ appendToDoc,
setDoc,
flushDocIfLoaded,
deleteDoc,
@@ -475,4 +567,5 @@ module.exports = {
flushQueuedProjects,
blockProject,
unblockProject,
+ getComment,
}
diff --git a/services/document-updater/app/js/Limits.js b/services/document-updater/app/js/Limits.js
index 268ccd3f9b..cbd9293042 100644
--- a/services/document-updater/app/js/Limits.js
+++ b/services/document-updater/app/js/Limits.js
@@ -28,4 +28,19 @@ module.exports = {
// since we didn't hit the limit in the loop, the document is within the allowed length
return false
},
+
+ /**
+ * @param {StringFileRawData} raw
+ * @param {number} maxDocLength
+ */
+ stringFileDataContentIsTooLarge(raw, maxDocLength) {
+ let n = raw.content.length
+ if (n <= maxDocLength) return false // definitely under the limit, no need to calculate the total size
+ for (const tc of raw.trackedChanges ?? []) {
+ if (tc.tracking.type !== 'delete') continue
+ n -= tc.range.length
+ if (n <= maxDocLength) return false // under the limit now, no need to calculate the exact size
+ }
+ return true
+ },
}
diff --git a/services/document-updater/app/js/PersistenceManager.js b/services/document-updater/app/js/PersistenceManager.js
index 484b9bab2a..6e832f9aa7 100644
--- a/services/document-updater/app/js/PersistenceManager.js
+++ b/services/document-updater/app/js/PersistenceManager.js
@@ -1,4 +1,4 @@
-const { promisify } = require('util')
+const { promisify } = require('node:util')
const { promisifyMultiResult } = require('@overleaf/promise-utils')
const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
@@ -95,6 +95,13 @@ function getDoc(projectId, docId, options = {}, _callback) {
status: body.pathname === '' ? 'zero-length' : 'undefined',
})
}
+
+ if (body.otMigrationStage > 0) {
+ // Use history-ot
+ body.lines = { content: body.lines.join('\n') }
+ body.ranges = {}
+ }
+
callback(
null,
body.lines,
diff --git a/services/document-updater/app/js/Profiler.js b/services/document-updater/app/js/Profiler.js
index 8daac4ca41..aac8a9706e 100644
--- a/services/document-updater/app/js/Profiler.js
+++ b/services/document-updater/app/js/Profiler.js
@@ -1,68 +1,52 @@
-/* eslint-disable
- no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS206: Consider reworking classes to avoid initClass
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-let Profiler
-const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
-const deltaMs = function (ta, tb) {
+function deltaMs(ta, tb) {
const nanoSeconds = (ta[0] - tb[0]) * 1e9 + (ta[1] - tb[1])
const milliSeconds = Math.floor(nanoSeconds * 1e-6)
return milliSeconds
}
-module.exports = Profiler = (function () {
- Profiler = class Profiler {
- static initClass() {
- this.prototype.LOG_CUTOFF_TIME = 15 * 1000
- this.prototype.LOG_SYNC_CUTOFF_TIME = 1000
- }
+class Profiler {
+ LOG_CUTOFF_TIME = 15 * 1000
+ LOG_SYNC_CUTOFF_TIME = 1000
- constructor(name, args) {
- this.name = name
- this.args = args
- this.t0 = this.t = process.hrtime()
- this.start = new Date()
- this.updateTimes = []
- this.totalSyncTime = 0
- }
-
- log(label, options = {}) {
- const t1 = process.hrtime()
- const dtMilliSec = deltaMs(t1, this.t)
- this.t = t1
- this.totalSyncTime += options.sync ? dtMilliSec : 0
- this.updateTimes.push([label, dtMilliSec]) // timings in ms
- return this // make it chainable
- }
-
- end(message) {
- const totalTime = deltaMs(this.t, this.t0)
- const exceedsCutoff = totalTime > this.LOG_CUTOFF_TIME
- const exceedsSyncCutoff = this.totalSyncTime > this.LOG_SYNC_CUTOFF_TIME
- if (exceedsCutoff || exceedsSyncCutoff) {
- // log anything greater than cutoffs
- const args = {}
- for (const k in this.args) {
- const v = this.args[k]
- args[k] = v
- }
- args.updateTimes = this.updateTimes
- args.start = this.start
- args.end = new Date()
- args.status = { exceedsCutoff, exceedsSyncCutoff }
- logger.warn(args, this.name)
- }
- return totalTime
- }
+ constructor(name, args) {
+ this.name = name
+ this.args = args
+ this.t0 = this.t = process.hrtime()
+ this.start = new Date()
+ this.updateTimes = []
+ this.totalSyncTime = 0
}
- Profiler.initClass()
- return Profiler
-})()
+
+ log(label, options = {}) {
+ const t1 = process.hrtime()
+ const dtMilliSec = deltaMs(t1, this.t)
+ this.t = t1
+ this.totalSyncTime += options.sync ? dtMilliSec : 0
+ this.updateTimes.push([label, dtMilliSec]) // timings in ms
+ return this // make it chainable
+ }
+
+ end() {
+ const totalTime = deltaMs(this.t, this.t0)
+ const exceedsCutoff = totalTime > this.LOG_CUTOFF_TIME
+ const exceedsSyncCutoff = this.totalSyncTime > this.LOG_SYNC_CUTOFF_TIME
+ if (exceedsCutoff || exceedsSyncCutoff) {
+ // log anything greater than cutoffs
+ const args = {}
+ for (const k in this.args) {
+ const v = this.args[k]
+ args[k] = v
+ }
+ args.updateTimes = this.updateTimes
+ args.start = this.start
+ args.end = new Date()
+ args.status = { exceedsCutoff, exceedsSyncCutoff }
+ logger.warn(args, this.name)
+ }
+ return totalTime
+ }
+}
+
+module.exports = Profiler
diff --git a/services/document-updater/app/js/ProjectHistoryRedisManager.js b/services/document-updater/app/js/ProjectHistoryRedisManager.js
index b84edfe74a..78e9c2ea4c 100644
--- a/services/document-updater/app/js/ProjectHistoryRedisManager.js
+++ b/services/document-updater/app/js/ProjectHistoryRedisManager.js
@@ -8,13 +8,14 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
)
const logger = require('@overleaf/logger')
const metrics = require('./Metrics')
-const { docIsTooLarge } = require('./Limits')
+const { docIsTooLarge, stringFileDataContentIsTooLarge } = require('./Limits')
const { addTrackedDeletesToContent, extractOriginOrSource } = require('./Utils')
const HistoryConversions = require('./HistoryConversions')
const OError = require('@overleaf/o-error')
/**
* @import { Ranges } from './types'
+ * @import { StringFileRawData } from 'overleaf-editor-core/lib/types'
*/
const ProjectHistoryRedisManager = {
@@ -122,6 +123,7 @@ const ProjectHistoryRedisManager = {
hash: projectUpdate.hash,
metadata: projectUpdate.metadata,
projectHistoryId,
+ createdBlob: projectUpdate.createdBlob ?? false,
}
if (ranges) {
projectUpdate.ranges = ranges
@@ -151,7 +153,13 @@ const ProjectHistoryRedisManager = {
return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
},
- async queueResyncProjectStructure(projectId, projectHistoryId, docs, files) {
+ async queueResyncProjectStructure(
+ projectId,
+ projectHistoryId,
+ docs,
+ files,
+ opts
+ ) {
logger.debug({ projectId, docs, files }, 'queue project structure resync')
const projectUpdate = {
resyncProjectStructure: { docs, files },
@@ -160,6 +168,9 @@ const ProjectHistoryRedisManager = {
ts: new Date(),
},
}
+ if (opts.resyncProjectStructureOnly) {
+ projectUpdate.resyncProjectStructureOnly = opts.resyncProjectStructureOnly
+ }
const jsonUpdate = JSON.stringify(projectUpdate)
return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
},
@@ -170,7 +181,7 @@ const ProjectHistoryRedisManager = {
* @param {string} projectId
* @param {string} projectHistoryId
* @param {string} docId
- * @param {string[]} lines
+ * @param {string[] | StringFileRawData} lines
* @param {Ranges} ranges
* @param {string[]} resolvedCommentIds
* @param {number} version
@@ -194,13 +205,8 @@ const ProjectHistoryRedisManager = {
'queue doc content resync'
)
- let content = lines.join('\n')
- if (historyRangesSupport) {
- content = addTrackedDeletesToContent(content, ranges.changes ?? [])
- }
-
const projectUpdate = {
- resyncDocContent: { content, version },
+ resyncDocContent: { version },
projectHistoryId,
path: pathname,
doc: docId,
@@ -209,17 +215,38 @@ const ProjectHistoryRedisManager = {
},
}
- if (historyRangesSupport) {
- projectUpdate.resyncDocContent.ranges =
- HistoryConversions.toHistoryRanges(ranges)
- projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
+ let content = ''
+ if (Array.isArray(lines)) {
+ content = lines.join('\n')
+ if (historyRangesSupport) {
+ content = addTrackedDeletesToContent(content, ranges.changes ?? [])
+ projectUpdate.resyncDocContent.ranges =
+ HistoryConversions.toHistoryRanges(ranges)
+ projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
+ }
+ } else {
+ content = lines.content
+ projectUpdate.resyncDocContent.historyOTRanges = {
+ comments: lines.comments,
+ trackedChanges: lines.trackedChanges,
+ }
}
+ projectUpdate.resyncDocContent.content = content
const jsonUpdate = JSON.stringify(projectUpdate)
// Do an optimised size check on the docLines using the serialised
// project update length as an upper bound
const sizeBound = jsonUpdate.length
- if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
+ if (Array.isArray(lines)) {
+ if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
+ throw new OError(
+ 'blocking resync doc content insert into project history queue: doc is too large',
+ { projectId, docId, docSize: sizeBound }
+ )
+ }
+ } else if (
+ stringFileDataContentIsTooLarge(lines, Settings.max_doc_length)
+ ) {
throw new OError(
'blocking resync doc content insert into project history queue: doc is too large',
{ projectId, docId, docSize: sizeBound }
diff --git a/services/document-updater/app/js/ProjectManager.js b/services/document-updater/app/js/ProjectManager.js
index 781ed0e168..cdd4c11482 100644
--- a/services/document-updater/app/js/ProjectManager.js
+++ b/services/document-updater/app/js/ProjectManager.js
@@ -317,6 +317,7 @@ function updateProjectWithLocks(
}
if (
HistoryManager.shouldFlushHistoryOps(
+ projectId,
projectOpsLength,
updates.length,
HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
diff --git a/services/document-updater/app/js/RangesManager.js b/services/document-updater/app/js/RangesManager.js
index 7dfa462ce4..c146afda60 100644
--- a/services/document-updater/app/js/RangesManager.js
+++ b/services/document-updater/app/js/RangesManager.js
@@ -127,7 +127,7 @@ const RangesManager = {
return { newRanges, rangesWereCollapsed, historyUpdates }
},
- acceptChanges(changeIds, ranges) {
+ acceptChanges(projectId, docId, changeIds, ranges, lines) {
const { changes, comments } = ranges
logger.debug(`accepting ${changeIds.length} changes in ranges`)
const rangesTracker = new RangesTracker(changes, comments)
@@ -339,6 +339,12 @@ function getHistoryOpForInsert(op, comments, changes) {
}
}
+ // If it's determined that the op is a tracked delete rejection, we have to
+ // calculate its proper history position. If multiple tracked deletes are
+ // found at the same position as the insert, the tracked deletes that come
+ // before the tracked delete that was actually rejected offset the history
+ // position.
+ let trackedDeleteRejectionOffset = 0
for (const change of changes) {
if (!isDelete(change.op)) {
// We're only interested in tracked deletes
@@ -349,14 +355,25 @@ function getHistoryOpForInsert(op, comments, changes) {
// Tracked delete is before the op. Move the op forward.
hpos += change.op.d.length
} else if (change.op.p === op.p) {
- // Tracked delete is at the same position as the op. The insert comes before
- // the tracked delete so it doesn't move.
+ // Tracked delete is at the same position as the op.
if (op.u && change.op.d.startsWith(op.i)) {
// We're undoing and the insert matches the start of the tracked
// delete. RangesManager treats this as a tracked delete rejection. We
// will note this in the op so that project-history can take the
// appropriate action.
trackedDeleteRejection = true
+
+ // The history must be updated to take into account all preceding
+ // tracked deletes at the same position
+ hpos += trackedDeleteRejectionOffset
+
+ // No need to continue. All subsequent tracked deletes are after the
+ // insert.
+ break
+ } else {
+ // This tracked delete does not match the insert. Note its length in
+ // case we find a tracked delete that matches later.
+ trackedDeleteRejectionOffset += change.op.d.length
}
} else {
// Tracked delete is after the insert. Tracked deletes are ordered, so
diff --git a/services/document-updater/app/js/RealTimeRedisManager.js b/services/document-updater/app/js/RealTimeRedisManager.js
index 9f7465acb8..2b67971c5c 100644
--- a/services/document-updater/app/js/RealTimeRedisManager.js
+++ b/services/document-updater/app/js/RealTimeRedisManager.js
@@ -20,8 +20,8 @@ const pubsubClient = require('@overleaf/redis-wrapper').createClient(
)
const Keys = Settings.redis.documentupdater.key_schema
const logger = require('@overleaf/logger')
-const os = require('os')
-const crypto = require('crypto')
+const os = require('node:os')
+const crypto = require('node:crypto')
const metrics = require('./Metrics')
const HOST = os.hostname()
@@ -49,7 +49,7 @@ const RealTimeRedisManager = {
MAX_OPS_PER_ITERATION,
-1
)
- return multi.exec(function (error, replys) {
+ multi.exec(function (error, replys) {
if (error != null) {
return callback(error)
}
@@ -80,7 +80,7 @@ const RealTimeRedisManager = {
},
getUpdatesLength(docId, callback) {
- return rclient.llen(Keys.pendingUpdates({ doc_id: docId }), callback)
+ rclient.llen(Keys.pendingUpdates({ doc_id: docId }), callback)
},
sendCanaryAppliedOp({ projectId, docId, op }) {
@@ -132,5 +132,5 @@ const RealTimeRedisManager = {
module.exports = RealTimeRedisManager
module.exports.promises = promisifyAll(RealTimeRedisManager, {
- without: ['sendData'],
+ without: ['sendCanaryAppliedOp', 'sendData'],
})
diff --git a/services/document-updater/app/js/RedisManager.js b/services/document-updater/app/js/RedisManager.js
index 0d2dad1047..7f86036427 100644
--- a/services/document-updater/app/js/RedisManager.js
+++ b/services/document-updater/app/js/RedisManager.js
@@ -7,7 +7,7 @@ const OError = require('@overleaf/o-error')
const { promisifyAll } = require('@overleaf/promise-utils')
const metrics = require('./Metrics')
const Errors = require('./Errors')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const async = require('async')
const { docIsTooLarge } = require('./Limits')
@@ -48,6 +48,7 @@ const RedisManager = {
timer.done()
_callback(error)
}
+ const shareJSTextOT = Array.isArray(docLines)
const docLinesArray = docLines
docLines = JSON.stringify(docLines)
if (docLines.indexOf('\u0000') !== -1) {
@@ -60,7 +61,10 @@ const RedisManager = {
// Do an optimised size check on the docLines using the serialised
// length as an upper bound
const sizeBound = docLines.length
- if (docIsTooLarge(sizeBound, docLinesArray, Settings.max_doc_length)) {
+ if (
+ shareJSTextOT && // editor-core has a size check in TextOperation.apply and TextOperation.applyToLength.
+ docIsTooLarge(sizeBound, docLinesArray, Settings.max_doc_length)
+ ) {
const docSize = docLines.length
const err = new Error('blocking doc insert into redis: doc is too large')
logger.error({ projectId, docId, err, docSize }, err.message)
@@ -461,6 +465,7 @@ const RedisManager = {
if (appliedOps == null) {
appliedOps = []
}
+ const shareJSTextOT = Array.isArray(docLines)
RedisManager.getDocVersion(docId, (error, currentVersion) => {
if (error) {
return callback(error)
@@ -500,7 +505,10 @@ const RedisManager = {
// Do an optimised size check on the docLines using the serialised
// length as an upper bound
const sizeBound = newDocLines.length
- if (docIsTooLarge(sizeBound, docLines, Settings.max_doc_length)) {
+ if (
+ shareJSTextOT && // editor-core has a size check in TextOperation.apply and TextOperation.applyToLength.
+ docIsTooLarge(sizeBound, docLines, Settings.max_doc_length)
+ ) {
const err = new Error('blocking doc update: doc is too large')
const docSize = newDocLines.length
logger.error({ projectId, docId, err, docSize }, err.message)
diff --git a/services/document-updater/app/js/ShareJsUpdateManager.js b/services/document-updater/app/js/ShareJsUpdateManager.js
index edac3700b7..933ded1ce1 100644
--- a/services/document-updater/app/js/ShareJsUpdateManager.js
+++ b/services/document-updater/app/js/ShareJsUpdateManager.js
@@ -16,10 +16,10 @@ const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
const { promisifyAll } = require('@overleaf/promise-utils')
const Keys = require('./UpdateKeys')
-const { EventEmitter } = require('events')
-const util = require('util')
+const { EventEmitter } = require('node:events')
+const util = require('node:util')
const RealTimeRedisManager = require('./RealTimeRedisManager')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const metrics = require('./Metrics')
const Errors = require('./Errors')
diff --git a/services/document-updater/app/js/UpdateManager.js b/services/document-updater/app/js/UpdateManager.js
index b23522a2cb..e5df48575e 100644
--- a/services/document-updater/app/js/UpdateManager.js
+++ b/services/document-updater/app/js/UpdateManager.js
@@ -14,10 +14,11 @@ const DocumentManager = require('./DocumentManager')
const RangesManager = require('./RangesManager')
const SnapshotManager = require('./SnapshotManager')
const Profiler = require('./Profiler')
-const { isInsert, isDelete, getDocLength } = require('./Utils')
+const { isInsert, isDelete, getDocLength, computeDocHash } = require('./Utils')
+const HistoryOTUpdateManager = require('./HistoryOTUpdateManager')
/**
- * @import { DeleteOp, InsertOp, Op, Ranges, Update, HistoryUpdate } from "./types"
+ * @import { Ranges, Update, HistoryUpdate } from "./types"
*/
const UpdateManager = {
@@ -80,7 +81,11 @@ const UpdateManager = {
profile.log('getPendingUpdatesForDoc')
for (const update of updates) {
- await UpdateManager.applyUpdate(projectId, docId, update)
+ if (HistoryOTUpdateManager.isHistoryOTEditOperationUpdate(update)) {
+ await HistoryOTUpdateManager.applyUpdate(projectId, docId, update)
+ } else {
+ await UpdateManager.applyUpdate(projectId, docId, update)
+ }
profile.log('applyUpdate')
}
profile.log('async done').end()
@@ -110,12 +115,16 @@ const UpdateManager = {
pathname,
projectHistoryId,
historyRangesSupport,
+ type,
} = await DocumentManager.promises.getDoc(projectId, docId)
profile.log('getDoc')
if (lines == null || version == null) {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
+ if (type !== 'sharejs-text-ot') {
+ throw new Errors.OTTypeMismatchError(type, 'sharejs-text-ot')
+ }
const previousVersion = version
const incomingUpdateVersion = update.v
@@ -162,6 +171,7 @@ const UpdateManager = {
projectHistoryId,
lines,
ranges,
+ updatedDocLines,
historyRangesSupport
)
@@ -290,8 +300,9 @@ const UpdateManager = {
* @param {HistoryUpdate[]} updates
* @param {string} pathname
* @param {string} projectHistoryId
- * @param {string[]} lines
- * @param {Ranges} ranges
+ * @param {string[]} lines - document lines before updates were applied
+ * @param {Ranges} ranges - ranges before updates were applied
+ * @param {string[]} newLines - document lines after updates were applied
* @param {boolean} historyRangesSupport
*/
_adjustHistoryUpdatesMetadata(
@@ -300,6 +311,7 @@ const UpdateManager = {
projectHistoryId,
lines,
ranges,
+ newLines,
historyRangesSupport
) {
let docLength = getDocLength(lines)
@@ -363,6 +375,12 @@ const UpdateManager = {
delete update.meta.tc
}
}
+
+ if (historyRangesSupport && updates.length > 0) {
+ const lastUpdate = updates[updates.length - 1]
+ lastUpdate.meta ??= {}
+ lastUpdate.meta.doc_hash = computeDocHash(newLines)
+ }
},
}
diff --git a/services/document-updater/app/js/Utils.js b/services/document-updater/app/js/Utils.js
index 4e9e60ba06..a632cf32eb 100644
--- a/services/document-updater/app/js/Utils.js
+++ b/services/document-updater/app/js/Utils.js
@@ -1,4 +1,5 @@
// @ts-check
+const { createHash } = require('node:crypto')
const _ = require('lodash')
/**
@@ -79,6 +80,27 @@ function addTrackedDeletesToContent(content, trackedChanges) {
return result
}
+/**
+ * Compute the content hash for a doc
+ *
+ * This hash is sent to the history to validate updates.
+ *
+ * @param {string[]} lines
+ * @return {string} the doc hash
+ */
+function computeDocHash(lines) {
+ const hash = createHash('sha1')
+ if (lines.length > 0) {
+ for (const line of lines.slice(0, lines.length - 1)) {
+ hash.update(line)
+ hash.update('\n')
+ }
+ // The last line doesn't end with a newline
+ hash.update(lines[lines.length - 1])
+ }
+ return hash.digest('hex')
+}
+
/**
* checks if the given originOrSource should be treated as a source or origin
* TODO: remove this hack and remove all "source" references
@@ -102,5 +124,6 @@ module.exports = {
isComment,
addTrackedDeletesToContent,
getDocLength,
+ computeDocHash,
extractOriginOrSource,
}
diff --git a/services/document-updater/app/js/mongodb.js b/services/document-updater/app/js/mongodb.js
index 6c4415adcc..6e38993bc4 100644
--- a/services/document-updater/app/js/mongodb.js
+++ b/services/document-updater/app/js/mongodb.js
@@ -24,5 +24,5 @@ module.exports = {
db,
ObjectId,
mongoClient,
- healthCheck: require('util').callbackify(healthCheck),
+ healthCheck: require('node:util').callbackify(healthCheck),
}
diff --git a/services/document-updater/app/js/sharejs/server/model.js b/services/document-updater/app/js/sharejs/server/model.js
index 370208cc37..a646b22492 100644
--- a/services/document-updater/app/js/sharejs/server/model.js
+++ b/services/document-updater/app/js/sharejs/server/model.js
@@ -22,7 +22,7 @@
// Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
let Model
-const { EventEmitter } = require('events')
+const { EventEmitter } = require('node:events')
const queue = require('./syncqueue')
const types = require('../types')
diff --git a/services/document-updater/app/js/sharejs/types/model.js b/services/document-updater/app/js/sharejs/types/model.js
index d927811895..af9fd0ad18 100644
--- a/services/document-updater/app/js/sharejs/types/model.js
+++ b/services/document-updater/app/js/sharejs/types/model.js
@@ -22,7 +22,7 @@
// Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
let Model
-const { EventEmitter } = require('events')
+const { EventEmitter } = require('node:events')
const queue = require('./syncqueue')
const types = require('../types')
diff --git a/services/document-updater/app/js/types.ts b/services/document-updater/app/js/types.ts
index d635ab31ca..851e62d8c8 100644
--- a/services/document-updater/app/js/types.ts
+++ b/services/document-updater/app/js/types.ts
@@ -1,12 +1,17 @@
import {
TrackingPropsRawData,
ClearTrackingPropsRawData,
+ RawEditOperation,
} from 'overleaf-editor-core/lib/types'
+export type OTType = 'sharejs-text-ot' | 'history-ot'
+
/**
* An update coming from the editor
*/
export type Update = {
+ dup?: boolean
+ dupIfSource?: string[]
doc: string
op: Op[]
v: number
@@ -18,6 +23,11 @@ export type Update = {
projectHistoryId?: string
}
+export type HistoryOTEditOperationUpdate = Omit & {
+ op: RawEditOperation[]
+ meta: Update['meta'] & { source: string }
+}
+
export type Op = InsertOp | DeleteOp | CommentOp | RetainOp
export type InsertOp = {
@@ -84,6 +94,7 @@ export type HistoryUpdate = {
pathname?: string
doc_length?: number
history_doc_length?: number
+ doc_hash?: string
tc?: boolean
user_id?: string
}
diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt
index 3221114171..98d10a8e55 100644
--- a/services/document-updater/buildscript.txt
+++ b/services/document-updater/buildscript.txt
@@ -4,6 +4,6 @@ document-updater
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=True
---script-version=4.5.0
+--script-version=4.7.0
diff --git a/services/document-updater/config/settings.defaults.js b/services/document-updater/config/settings.defaults.js
index d8c2ada6d3..9ed59de6c4 100755
--- a/services/document-updater/config/settings.defaults.js
+++ b/services/document-updater/config/settings.defaults.js
@@ -1,3 +1,9 @@
+const http = require('node:http')
+const https = require('node:https')
+
+http.globalAgent.keepAlive = false
+https.globalAgent.keepAlive = false
+
module.exports = {
internal: {
documentupdater: {
@@ -170,10 +176,6 @@ module.exports = {
},
},
- sentry: {
- dsn: process.env.SENTRY_DSN,
- },
-
publishOnIndividualChannels:
process.env.PUBLISH_ON_INDIVIDUAL_CHANNELS === 'true',
@@ -182,4 +184,8 @@ module.exports = {
smoothingOffset: process.env.SMOOTHING_OFFSET || 1000, // milliseconds
gracefulShutdownDelayInMs:
parseInt(process.env.GRACEFUL_SHUTDOWN_DELAY_SECONDS ?? '10', 10) * 1000,
+
+ shortHistoryQueues: (process.env.SHORT_HISTORY_QUEUES || '')
+ .split(',')
+ .filter(s => !!s),
}
diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml
index 332a9710ca..c6ec24a84b 100644
--- a/services/document-updater/docker-compose.ci.yml
+++ b/services/document-updater/docker-compose.ci.yml
@@ -21,18 +21,22 @@ services:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
+ HISTORY_REDIS_HOST: redis
ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
+ volumes:
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
redis:
condition: service_healthy
user: node
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -44,16 +48,21 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
- image: redis
+ image: redis:7.4.3
healthcheck:
test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml
index 83f5d2efcc..c1b23c11c5 100644
--- a/services/document-updater/docker-compose.yml
+++ b/services/document-updater/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/document-updater
- ../../node_modules:/overleaf/node_modules
@@ -14,49 +14,58 @@ services:
working_dir: /overleaf/services/document-updater
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:18.20.2
+ image: node:22.17.0
volumes:
- .:/overleaf/services/document-updater
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/document-updater
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
+ HISTORY_REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_healthy
+ condition: service_started
redis:
condition: service_healthy
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
redis:
- image: redis
+ image: redis:7.4.3
healthcheck:
test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
mongo:
- image: mongo:6.0.13
+ image: mongo:8.0.11
command: --replSet overleaf
- healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
- interval: 1s
- retries: 20
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
diff --git a/services/document-updater/package.json b/services/document-updater/package.json
index bf49f53b21..7d892689e9 100644
--- a/services/document-updater/package.json
+++ b/services/document-updater/package.json
@@ -12,8 +12,8 @@
"nodemon": "node --watch app.js",
"benchmark:apply": "node benchmarks/apply",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -30,10 +30,11 @@
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
"diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
- "express": "^4.21.0",
+ "express": "^4.21.2",
"lodash": "^4.17.21",
"minimist": "^1.2.8",
- "mongodb-legacy": "^6.0.1",
+ "mongodb-legacy": "6.1.3",
+ "overleaf-editor-core": "*",
"request": "^2.88.2",
"requestretry": "^7.1.0"
},
@@ -41,7 +42,7 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"cluster-key-slot": "^1.0.5",
- "mocha": "^10.2.0",
+ "mocha": "^11.1.0",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
"sinon-chai": "^3.7.0",
diff --git a/services/document-updater/scripts/check_redis_mongo_sync_state.js b/services/document-updater/scripts/check_redis_mongo_sync_state.js
index 0ca2bb4e8b..51db47af4d 100644
--- a/services/document-updater/scripts/check_redis_mongo_sync_state.js
+++ b/services/document-updater/scripts/check_redis_mongo_sync_state.js
@@ -1,5 +1,5 @@
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
const _ = require('lodash')
const logger = require('@overleaf/logger')
const OError = require('@overleaf/o-error')
@@ -15,6 +15,7 @@ const request = require('requestretry').defaults({
retryDelay: 10,
})
+const ONLY_PROJECT_ID = process.env.ONLY_PROJECT_ID
const AUTO_FIX_VERSION_MISMATCH =
process.env.AUTO_FIX_VERSION_MISMATCH === 'true'
const AUTO_FIX_PARTIALLY_DELETED_DOC_METADATA =
@@ -319,10 +320,12 @@ async function processProject(projectId) {
* @return {Promise<{perIterationOutOfSync: number, done: boolean}>}
*/
async function scanOnce(processed, outOfSync) {
- const projectIds = await ProjectFlusher.promises.flushAllProjects({
- limit: LIMIT,
- dryRun: true,
- })
+ const projectIds = ONLY_PROJECT_ID
+ ? [ONLY_PROJECT_ID]
+ : await ProjectFlusher.promises.flushAllProjects({
+ limit: LIMIT,
+ dryRun: true,
+ })
let perIterationOutOfSync = 0
for (const projectId of projectIds) {
diff --git a/services/document-updater/scripts/fix_docs_with_empty_pathnames.js b/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
index abbfb01ba7..e3e034157b 100644
--- a/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
+++ b/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
@@ -6,7 +6,7 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('app/js/ProjectFlusher')
const DocumentManager = require('app/js/DocumentManager')
-const util = require('util')
+const util = require('node:util')
const flushAndDeleteDocWithLock = util.promisify(
DocumentManager.flushAndDeleteDocWithLock
)
diff --git a/services/document-updater/scripts/fix_docs_with_missing_project.js b/services/document-updater/scripts/fix_docs_with_missing_project.js
index 0a9a05eaaa..d3022127b3 100644
--- a/services/document-updater/scripts/fix_docs_with_missing_project.js
+++ b/services/document-updater/scripts/fix_docs_with_missing_project.js
@@ -7,7 +7,7 @@ const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('../app/js/ProjectFlusher')
const DocumentManager = require('../app/js/DocumentManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
-const util = require('util')
+const util = require('node:util')
const flushAndDeleteDocWithLock = util.promisify(
DocumentManager.flushAndDeleteDocWithLock
)
diff --git a/services/document-updater/scripts/flush_projects_with_no_history_id.js b/services/document-updater/scripts/flush_projects_with_no_history_id.js
new file mode 100644
index 0000000000..aa912b4b66
--- /dev/null
+++ b/services/document-updater/scripts/flush_projects_with_no_history_id.js
@@ -0,0 +1,211 @@
+// @ts-check
+
+const Settings = require('@overleaf/settings')
+const logger = require('@overleaf/logger')
+const RedisManager = require('../app/js/RedisManager')
+const minimist = require('minimist')
+const { db, ObjectId } = require('../app/js/mongodb')
+const ProjectManager = require('../app/js/ProjectManager')
+const OError = require('@overleaf/o-error')
+
+const docUpdaterKeys = Settings.redis.documentupdater.key_schema
+
+const rclient = RedisManager.rclient
+
+const { verbose, commit, ...args } = minimist(process.argv.slice(2), {
+ boolean: ['verbose', 'commit'],
+ string: ['batchSize'],
+ default: {
+ batchSize: '1000',
+ },
+})
+
+logger.logger.level(verbose ? 'debug' : 'warn')
+
+const batchSize = parseInt(args.batchSize, 10)
+
+/**
+ * @typedef {import('ioredis').Redis} Redis
+ */
+
+/**
+ *
+ * @param {string} key
+ * @return {string|void}
+ */
+function extractDocId(key) {
+ const matches = key.match(/ProjectHistoryId:\{(.*?)\}/)
+ if (matches) {
+ return matches[1]
+ }
+}
+
+/**
+ *
+ * @param {string} docId
+ * @return {Promise<{projectId: string, historyId: string}>}
+ */
+async function getHistoryId(docId) {
+ const doc = await db.docs.findOne(
+ { _id: new ObjectId(docId) },
+ { projection: { project_id: 1 }, readPreference: 'secondaryPreferred' }
+ )
+
+ if (!doc) {
+ throw new OError('Doc not present in mongo', { docId })
+ }
+
+ const project = await db.projects.findOne(
+ { _id: doc.project_id },
+ {
+ projection: { 'overleaf.history': 1 },
+ readPreference: 'secondaryPreferred',
+ }
+ )
+
+ if (!project?.overleaf?.history?.id) {
+ throw new OError('Project not present in mongo (or has no history id)', {
+ docId,
+ project,
+ doc,
+ })
+ }
+
+ return {
+ historyId: project?.overleaf?.history?.id,
+ projectId: doc.project_id.toString(),
+ }
+}
+
+/**
+ * @typedef {Object} UpdateableDoc
+ * @property {string} docId
+ * @property {string} projectId
+ * @property {string} historyId
+ */
+
+/**
+ *
+ * @param {Redis} node
+ * @param {Array} docIds
+ * @return {Promise>}
+ */
+async function findDocsWithMissingHistoryIds(node, docIds) {
+ const historyIds = await node.mget(
+ docIds.map(docId => docUpdaterKeys.projectHistoryId({ doc_id: docId }))
+ )
+
+ const results = []
+
+ for (const index in docIds) {
+ const historyId = historyIds[index]
+ const docId = docIds[index]
+ if (!historyId) {
+ try {
+ const { projectId, historyId } = await getHistoryId(docId)
+ results.push({ projectId, historyId, docId })
+ } catch (error) {
+ logger.warn(
+ { error },
+ 'Error gathering data for doc with missing history id'
+ )
+ }
+ }
+ }
+ return results
+}
+
+/**
+ *
+ * @param {Array} updates
+ * @return {Promise}
+ */
+async function fixAndFlushProjects(updates) {
+ for (const update of updates) {
+ if (commit) {
+ try {
+ await rclient.set(
+ docUpdaterKeys.projectHistoryId({ doc_id: update.docId }),
+ update.historyId
+ )
+ logger.debug({ ...update }, 'Set history id in redis')
+ await ProjectManager.promises.flushAndDeleteProjectWithLocks(
+ update.projectId,
+ {}
+ )
+ logger.debug({ ...update }, 'Flushed project')
+ } catch (err) {
+ logger.error({ err, ...update }, 'Error fixing and flushing project')
+ }
+ } else {
+ logger.debug(
+ { ...update },
+ 'Would have set history id in redis and flushed'
+ )
+ }
+ }
+}
+
+/**
+ *
+ * @param {Array} nodes
+ * @param {number} batchSize
+ * @return {Promise}
+ */
+async function scanNodes(nodes, batchSize = 1000) {
+ let scanned = 0
+
+ for (const node of nodes) {
+ const stream = node.scanStream({
+ match: docUpdaterKeys.projectHistoryId({ doc_id: '*' }),
+ count: batchSize,
+ })
+
+ for await (const docKeys of stream) {
+ if (docKeys.length === 0) {
+ continue
+ }
+ stream.pause()
+ scanned += docKeys.length
+
+ const docIds = docKeys
+ .map((/** @type {string} */ docKey) => extractDocId(docKey))
+ .filter(Boolean)
+
+ try {
+ const updates = await findDocsWithMissingHistoryIds(node, docIds)
+ if (updates.length > 0) {
+ logger.info({ updates }, 'Found doc(s) with missing history ids')
+ await fixAndFlushProjects(updates)
+ }
+ } catch (error) {
+ logger.error({ docKeys }, 'Error processing batch')
+ } finally {
+ stream.resume()
+ }
+ }
+
+ logger.info({ scanned, server: node.serverInfo.role }, 'Scanned node')
+ }
+}
+
+async function main({ batchSize }) {
+ const nodes = (typeof rclient.nodes === 'function'
+ ? rclient.nodes('master')
+ : undefined) || [rclient]
+ await scanNodes(nodes, batchSize)
+}
+
+let code = 0
+
+main({ batchSize })
+ .then(() => {
+ logger.info({}, 'done')
+ })
+ .catch(error => {
+ logger.error({ error }, 'error')
+ code = 1
+ })
+ .finally(() => {
+ rclient.quit().then(() => process.exit(code))
+ })
diff --git a/services/document-updater/scripts/remove_deleted_docs.js b/services/document-updater/scripts/remove_deleted_docs.js
index 1d0844872e..a69807119f 100644
--- a/services/document-updater/scripts/remove_deleted_docs.js
+++ b/services/document-updater/scripts/remove_deleted_docs.js
@@ -7,7 +7,7 @@ const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('../app/js/ProjectFlusher')
const RedisManager = require('../app/js/RedisManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
-const util = require('util')
+const util = require('node:util')
const getDoc = util.promisify((projectId, docId, cb) =>
RedisManager.getDoc(projectId, docId, (err, ...args) => cb(err, args))
)
diff --git a/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js b/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
index 8de7f091a8..39ec6c2ac7 100644
--- a/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
+++ b/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
@@ -16,27 +16,36 @@ const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Applying updates to a doc', function () {
- before(function (done) {
+ beforeEach(function (done) {
+ sinon.spy(MockWebApi, 'getDocument')
this.lines = ['one', 'two', 'three']
this.version = 42
this.op = {
i: 'one and a half\n',
p: 4,
}
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
this.update = {
doc: this.doc_id,
op: [this.op],
v: this.version,
}
+ this.historyOTUpdate = {
+ doc: this.doc_id,
+ op: [{ textOperation: [4, 'one and a half\n', 9] }],
+ v: this.version,
+ meta: { source: 'random-publicId' },
+ }
this.result = ['one', 'one and a half', 'two', 'three']
DocUpdaterApp.ensureRunning(done)
})
+ afterEach(function () {
+ sinon.restore()
+ })
describe('when the document is not loaded', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
- sinon.spy(MockWebApi, 'getDocument')
+ beforeEach(function (done) {
this.startTime = Date.now()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
@@ -50,15 +59,25 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- setTimeout(done, 200)
+ setTimeout(() => {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
+ if (error != null) {
+ throw error
+ }
+ result = parseInt(result, 10)
+ this.firstOpTimestamp = result
+ done()
+ }
+ )
+ }, 200)
}
)
})
- after(function () {
- MockWebApi.getDocument.restore()
- })
-
it('should load the document from the web API', function () {
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
@@ -92,28 +111,44 @@ describe('Applying updates to a doc', function () {
)
})
- it('should set the first op timestamp', function (done) {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
+ it('should set the first op timestamp', function () {
+ this.firstOpTimestamp.should.be.within(this.startTime, Date.now())
+ })
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
if (error != null) {
throw error
}
- result = parseInt(result, 10)
- result.should.be.within(this.startTime, Date.now())
- this.firstOpTimestamp = result
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
+ done()
+ }
+ )
+ })
+
+ it('should yield no last updated time for another project', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ DocUpdaterClient.randomId(),
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.should.deep.equal({})
done()
}
)
})
describe('when sending another update', function () {
- before(function (done) {
- this.timeout = 10000
- this.second_update = Object.create(this.update)
+ beforeEach(function (done) {
+ this.timeout(10000)
+ this.second_update = Object.assign({}, this.update)
this.second_update.v = this.version + 1
+ this.secondStartTime = Date.now()
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -127,6 +162,24 @@ describe('Applying updates to a doc', function () {
)
})
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) done(error)
+ doc.lines.should.deep.equal([
+ 'one',
+ 'one and a half',
+ 'one and a half',
+ 'two',
+ 'three',
+ ])
+ done()
+ }
+ )
+ })
+
it('should not change the first op timestamp', function (done) {
rclientProjectHistory.get(
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
@@ -142,14 +195,357 @@ describe('Applying updates to a doc', function () {
}
)
})
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(
+ this.secondStartTime,
+ Date.now()
+ )
+ done()
+ }
+ )
+ })
+ })
+
+ describe('when another client is sending a concurrent update', function () {
+ beforeEach(function (done) {
+ this.timeout(10000)
+ this.otherUpdate = {
+ doc: this.doc_id,
+ op: [{ p: 8, i: 'two and a half\n' }],
+ v: this.version,
+ meta: { source: 'other-random-publicId' },
+ }
+ this.secondStartTime = Date.now()
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.otherUpdate,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) done(error)
+ doc.lines.should.deep.equal([
+ 'one',
+ 'one and a half',
+ 'two',
+ 'two and a half',
+ 'three',
+ ])
+ done()
+ }
+ )
+ })
+
+ it('should not change the first op timestamp', function (done) {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
+ if (error != null) {
+ throw error
+ }
+ result = parseInt(result, 10)
+ result.should.equal(this.firstOpTimestamp)
+ done()
+ }
+ )
+ })
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(
+ this.secondStartTime,
+ Date.now()
+ )
+ done()
+ }
+ )
+ })
+ })
+ })
+
+ describe('when the document is not loaded (history-ot)', function () {
+ beforeEach(function (done) {
+ this.startTime = Date.now()
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.historyOTUpdate,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(() => {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
+ if (error != null) {
+ throw error
+ }
+ result = parseInt(result, 10)
+ this.firstOpTimestamp = result
+ done()
+ }
+ )
+ }, 200)
+ }
+ )
+ })
+
+ it('should load the document from the web API', function () {
+ MockWebApi.getDocument
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) done(error)
+ doc.lines.should.deep.equal(this.result)
+ done()
+ }
+ )
+ })
+
+ it('should push the applied updates to the project history changes api', function (done) {
+ rclientProjectHistory.lrange(
+ ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
+ 0,
+ -1,
+ (error, updates) => {
+ if (error != null) {
+ throw error
+ }
+ JSON.parse(updates[0]).op.should.deep.equal(this.historyOTUpdate.op)
+ JSON.parse(updates[0]).meta.pathname.should.equal('/a/b/c.tex')
+
+ done()
+ }
+ )
+ })
+
+ it('should set the first op timestamp', function () {
+ this.firstOpTimestamp.should.be.within(this.startTime, Date.now())
+ })
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
+ done()
+ }
+ )
+ })
+
+ it('should yield no last updated time for another project', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ DocUpdaterClient.randomId(),
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.should.deep.equal({})
+ done()
+ }
+ )
+ })
+
+ describe('when sending another update', function () {
+ beforeEach(function (done) {
+ this.timeout(10000)
+ this.second_update = Object.assign({}, this.historyOTUpdate)
+ this.second_update.op = [
+ {
+ textOperation: [4, 'one and a half\n', 24],
+ },
+ ]
+ this.second_update.v = this.version + 1
+ this.secondStartTime = Date.now()
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.second_update,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) done(error)
+ doc.lines.should.deep.equal([
+ 'one',
+ 'one and a half',
+ 'one and a half',
+ 'two',
+ 'three',
+ ])
+ done()
+ }
+ )
+ })
+
+ it('should not change the first op timestamp', function (done) {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
+ if (error != null) {
+ throw error
+ }
+ result = parseInt(result, 10)
+ result.should.equal(this.firstOpTimestamp)
+ done()
+ }
+ )
+ })
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(
+ this.secondStartTime,
+ Date.now()
+ )
+ done()
+ }
+ )
+ })
+ })
+
+ describe('when another client is sending a concurrent update', function () {
+ beforeEach(function (done) {
+ this.timeout(10000)
+ this.otherUpdate = {
+ doc: this.doc_id,
+ op: [{ textOperation: [8, 'two and a half\n', 5] }],
+ v: this.version,
+ meta: { source: 'other-random-publicId' },
+ }
+ this.secondStartTime = Date.now()
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.otherUpdate,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) done(error)
+ doc.lines.should.deep.equal([
+ 'one',
+ 'one and a half',
+ 'two',
+ 'two and a half',
+ 'three',
+ ])
+ done()
+ }
+ )
+ })
+
+ it('should not change the first op timestamp', function (done) {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
+ if (error != null) {
+ throw error
+ }
+ result = parseInt(result, 10)
+ result.should.equal(this.firstOpTimestamp)
+ done()
+ }
+ )
+ })
+
+ it('should yield last updated time', function (done) {
+ DocUpdaterClient.getProjectLastUpdatedAt(
+ this.project_id,
+ (error, res, body) => {
+ if (error != null) {
+ throw error
+ }
+ res.statusCode.should.equal(200)
+ body.lastUpdatedAt.should.be.within(
+ this.secondStartTime,
+ Date.now()
+ )
+ done()
+ }
+ )
+ })
})
})
describe('when the document is loaded', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
-
+ beforeEach(function (done) {
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -158,7 +554,7 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- sinon.spy(MockWebApi, 'getDocument')
+ sinon.resetHistory()
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -173,10 +569,6 @@ describe('Applying updates to a doc', function () {
})
})
- after(function () {
- MockWebApi.getDocument.restore()
- })
-
it('should not need to call the web api', function () {
MockWebApi.getDocument.called.should.equal(false)
})
@@ -208,10 +600,7 @@ describe('Applying updates to a doc', function () {
})
describe('when the document is loaded and is using project-history only', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
-
+ beforeEach(function (done) {
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -220,7 +609,7 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- sinon.spy(MockWebApi, 'getDocument')
+ sinon.resetHistory()
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -235,10 +624,6 @@ describe('Applying updates to a doc', function () {
})
})
- after(function () {
- MockWebApi.getDocument.restore()
- })
-
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
@@ -265,11 +650,61 @@ describe('Applying updates to a doc', function () {
})
})
+ describe('when the document is loaded (history-ot)', function () {
+ beforeEach(function (done) {
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+ DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
+ if (error != null) {
+ throw error
+ }
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.historyOTUpdate,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.result)
+ done()
+ }
+ )
+ })
+
+ it('should push the applied updates to the project history changes api', function (done) {
+ rclientProjectHistory.lrange(
+ ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
+ 0,
+ -1,
+ (error, updates) => {
+ if (error) return done(error)
+ JSON.parse(updates[0]).op.should.deep.equal(this.historyOTUpdate.op)
+ JSON.parse(updates[0]).meta.pathname.should.equal('/a/b/c.tex')
+ done()
+ }
+ )
+ })
+ })
+
describe('when the document has been deleted', function () {
describe('when the ops come in a single linear order', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
@@ -289,54 +724,49 @@ describe('Applying updates to a doc', function () {
{ doc_id: this.doc_id, v: 10, op: [{ i: 'd', p: 10 }] },
]
this.my_result = ['hello world', '', '']
- done()
- })
-
- it('should be able to continue applying updates when the project has been deleted', function (done) {
- let update
const actions = []
- for (update of this.updates.slice(0, 6)) {
- ;(update => {
- actions.push(callback =>
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- update,
- callback
- )
+ for (const update of this.updates.slice(0, 6)) {
+ actions.push(callback =>
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ update,
+ callback
)
- })(update)
+ )
}
actions.push(callback =>
DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)
)
- for (update of this.updates.slice(6)) {
- ;(update => {
- actions.push(callback =>
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- update,
- callback
- )
+ for (const update of this.updates.slice(6)) {
+ actions.push(callback =>
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ update,
+ callback
)
- })(update)
+ )
}
- async.series(actions, error => {
- if (error != null) {
- throw error
+ // process updates
+ actions.push(cb =>
+ DocUpdaterClient.getDoc(this.project_id, this.doc_id, cb)
+ )
+
+ async.series(actions, done)
+ })
+
+ it('should be able to continue applying updates when the project has been deleted', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.my_result)
+ done()
}
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.my_result)
- done()
- }
- )
- })
+ )
})
it('should store the doc ops in the correct order', function (done) {
@@ -358,9 +788,7 @@ describe('Applying updates to a doc', function () {
})
describe('when older ops come in after the delete', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
@@ -428,11 +856,9 @@ describe('Applying updates to a doc', function () {
})
describe('with a broken update', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
this.broken_update = {
- doc_id: this.doc_id,
+ doc: this.doc_id,
v: this.version,
op: [{ d: 'not the correct content', p: 0 }],
}
@@ -482,10 +908,162 @@ describe('Applying updates to a doc', function () {
})
})
+ describe('with a broken update (history-ot)', function () {
+ beforeEach(function (done) {
+ this.broken_update = {
+ doc: this.doc_id,
+ v: this.version,
+ op: [{ textOperation: [99, -1] }],
+ meta: { source: '42' },
+ }
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+
+ DocUpdaterClient.subscribeToAppliedOps(
+ (this.messageCallback = sinon.stub())
+ )
+
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.broken_update,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should not update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.lines)
+ done()
+ }
+ )
+ })
+
+ it('should send a message with an error', function () {
+ this.messageCallback.called.should.equal(true)
+ const [channel, message] = this.messageCallback.args[0]
+ channel.should.equal('applied-ops')
+ JSON.parse(message).should.deep.include({
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ error:
+ "The operation's base length must be equal to the string's length.",
+ })
+ })
+ })
+
+ describe('when mixing ot types (sharejs-text-ot -> history-ot)', function () {
+ beforeEach(function (done) {
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 0,
+ })
+
+ DocUpdaterClient.subscribeToAppliedOps(
+ (this.messageCallback = sinon.stub())
+ )
+
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.historyOTUpdate,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should not update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.lines)
+ done()
+ }
+ )
+ })
+
+ it('should send a message with an error', function () {
+ this.messageCallback.called.should.equal(true)
+ const [channel, message] = this.messageCallback.args[0]
+ channel.should.equal('applied-ops')
+ JSON.parse(message).should.deep.include({
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ error: 'ot type mismatch',
+ })
+ })
+ })
+
+ describe('when mixing ot types (history-ot -> sharejs-text-ot)', function () {
+ beforeEach(function (done) {
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+
+ DocUpdaterClient.subscribeToAppliedOps(
+ (this.messageCallback = sinon.stub())
+ )
+
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.update,
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ })
+
+ it('should not update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.lines)
+ done()
+ }
+ )
+ })
+
+ it('should send a message with an error', function () {
+ this.messageCallback.called.should.equal(true)
+ const [channel, message] = this.messageCallback.args[0]
+ channel.should.equal('applied-ops')
+ JSON.parse(message).should.deep.include({
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ error: 'ot type mismatch',
+ })
+ })
+ })
+
describe('when there is no version in Mongo', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
})
@@ -522,9 +1100,7 @@ describe('Applying updates to a doc', function () {
})
describe('when the sending duplicate ops', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -606,12 +1182,88 @@ describe('Applying updates to a doc', function () {
})
})
+ describe('when sending duplicate ops (history-ot)', function () {
+ beforeEach(function (done) {
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+
+ DocUpdaterClient.subscribeToAppliedOps(
+ (this.messageCallback = sinon.stub())
+ )
+
+ // One user delete 'one', the next turns it into 'once'. The second becomes a NOP.
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ {
+ doc: this.doc_id,
+ op: [{ textOperation: [4, 'one and a half\n', 9] }],
+ v: this.version,
+ meta: {
+ source: 'ikHceq3yfAdQYzBo4-xZ',
+ },
+ },
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(() => {
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ {
+ doc: this.doc_id,
+ op: [
+ {
+ textOperation: [4, 'one and a half\n', 9],
+ },
+ ],
+ v: this.version,
+ dupIfSource: ['ikHceq3yfAdQYzBo4-xZ'],
+ meta: {
+ source: 'ikHceq3yfAdQYzBo4-xZ',
+ },
+ },
+ error => {
+ if (error != null) {
+ throw error
+ }
+ setTimeout(done, 200)
+ }
+ )
+ }, 200)
+ }
+ )
+ })
+
+ it('should update the doc', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.result)
+ done()
+ }
+ )
+ })
+
+ it('should return a message about duplicate ops', function () {
+ this.messageCallback.calledTwice.should.equal(true)
+ this.messageCallback.args[0][0].should.equal('applied-ops')
+ expect(JSON.parse(this.messageCallback.args[0][1]).op.dup).to.be.undefined
+ this.messageCallback.args[1][0].should.equal('applied-ops')
+ expect(JSON.parse(this.messageCallback.args[1][1]).op.dup).to.equal(true)
+ })
+ })
+
describe('when sending updates for a non-existing doc id', function () {
- before(function (done) {
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
+ beforeEach(function (done) {
this.non_existing = {
- doc_id: this.doc_id,
+ doc: this.doc_id,
v: this.version,
op: [{ d: 'content', p: 0 }],
}
diff --git a/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js b/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
index cd4f07bde7..ebbc015a58 100644
--- a/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
+++ b/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
@@ -1,12 +1,12 @@
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
-const { promisify } = require('util')
-const { exec } = require('child_process')
+const { promisify } = require('node:util')
+const { exec } = require('node:child_process')
const { expect } = require('chai')
const Settings = require('@overleaf/settings')
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
const MockDocstoreApi = require('./helpers/MockDocstoreApi')
const sinon = require('sinon')
diff --git a/services/document-updater/test/acceptance/js/SettingADocumentTests.js b/services/document-updater/test/acceptance/js/SettingADocumentTests.js
index 5b0c4ab281..e1bc54dc90 100644
--- a/services/document-updater/test/acceptance/js/SettingADocumentTests.js
+++ b/services/document-updater/test/acceptance/js/SettingADocumentTests.js
@@ -196,6 +196,167 @@ describe('Setting a document', function () {
})
})
+ describe('when the updated doc exists in the doc updater (history-ot)', function () {
+ before(function (done) {
+ numberOfReceivedUpdates = 0
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
+ this.historyOTUpdate = {
+ doc: this.doc_id,
+ op: [{ textOperation: [4, 'one and a half\n', 9] }],
+ v: this.version,
+ meta: { source: 'random-publicId' },
+ }
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines: this.lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+ DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
+ if (error) {
+ throw error
+ }
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.historyOTUpdate,
+ error => {
+ if (error) {
+ throw error
+ }
+ setTimeout(() => {
+ DocUpdaterClient.setDocLines(
+ this.project_id,
+ this.doc_id,
+ this.newLines,
+ this.source,
+ this.user_id,
+ false,
+ (error, res, body) => {
+ if (error) {
+ return done(error)
+ }
+ this.statusCode = res.statusCode
+ this.body = body
+ done()
+ }
+ )
+ }, 200)
+ }
+ )
+ })
+ })
+
+ after(function () {
+ MockProjectHistoryApi.flushProject.resetHistory()
+ MockWebApi.setDocument.resetHistory()
+ })
+
+ it('should return a 200 status code', function () {
+ this.statusCode.should.equal(200)
+ })
+
+ it('should emit two updates (from sendUpdate and setDocLines)', function () {
+ expect(numberOfReceivedUpdates).to.equal(2)
+ })
+
+ it('should send the updated doc lines and version to the web api', function () {
+ MockWebApi.setDocument
+ .calledWith(this.project_id, this.doc_id, this.newLines)
+ .should.equal(true)
+ })
+
+ it('should update the lines in the doc updater', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) {
+ return done(error)
+ }
+ doc.lines.should.deep.equal(this.newLines)
+ done()
+ }
+ )
+ })
+
+ it('should bump the version in the doc updater', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) {
+ return done(error)
+ }
+ doc.version.should.equal(this.version + 2)
+ done()
+ }
+ )
+ })
+
+ it('should leave the document in redis', function (done) {
+ docUpdaterRedis.get(
+ Keys.docLines({ doc_id: this.doc_id }),
+ (error, lines) => {
+ if (error) {
+ throw error
+ }
+ expect(JSON.parse(lines)).to.deep.equal({
+ content: this.newLines.join('\n'),
+ })
+ done()
+ }
+ )
+ })
+
+ it('should return the mongo rev in the json response', function () {
+ this.body.should.deep.equal({ rev: '123' })
+ })
+
+ describe('when doc has the same contents', function () {
+ beforeEach(function (done) {
+ numberOfReceivedUpdates = 0
+ DocUpdaterClient.setDocLines(
+ this.project_id,
+ this.doc_id,
+ this.newLines,
+ this.source,
+ this.user_id,
+ false,
+ (error, res, body) => {
+ if (error) {
+ return done(error)
+ }
+ this.statusCode = res.statusCode
+ this.body = body
+ done()
+ }
+ )
+ })
+
+ it('should not bump the version in doc updater', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) {
+ return done(error)
+ }
+ doc.version.should.equal(this.version + 2)
+ done()
+ }
+ )
+ })
+
+ it('should not emit any updates', function (done) {
+ setTimeout(() => {
+ expect(numberOfReceivedUpdates).to.equal(0)
+ done()
+ }, 100) // delay by 100ms: make sure we do not check too early!
+ })
+ })
+ })
+
describe('when the updated doc does not exist in the doc updater', function () {
before(function (done) {
this.project_id = DocUpdaterClient.randomId()
@@ -525,4 +686,285 @@ describe('Setting a document', function () {
})
})
})
+
+ describe('with track changes (history-ot)', function () {
+ const lines = ['one', 'one and a half', 'two', 'three']
+ const userId = DocUpdaterClient.randomId()
+ const ts = new Date().toISOString()
+ beforeEach(function (done) {
+ numberOfReceivedUpdates = 0
+ this.newLines = ['one', 'two', 'three']
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
+ this.historyOTUpdate = {
+ doc: this.doc_id,
+ op: [
+ {
+ textOperation: [
+ 4,
+ {
+ r: 'one and a half\n'.length,
+ tracking: {
+ type: 'delete',
+ userId,
+ ts,
+ },
+ },
+ 9,
+ ],
+ },
+ ],
+ v: this.version,
+ meta: { source: 'random-publicId' },
+ }
+ MockWebApi.insertDoc(this.project_id, this.doc_id, {
+ lines,
+ version: this.version,
+ otMigrationStage: 1,
+ })
+ DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
+ if (error) {
+ throw error
+ }
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ this.historyOTUpdate,
+ error => {
+ if (error) {
+ throw error
+ }
+ DocUpdaterClient.waitForPendingUpdates(
+ this.project_id,
+ this.doc_id,
+ done
+ )
+ }
+ )
+ })
+ })
+
+ afterEach(function () {
+ MockProjectHistoryApi.flushProject.resetHistory()
+ MockWebApi.setDocument.resetHistory()
+ })
+ it('should record tracked changes', function (done) {
+ docUpdaterRedis.get(
+ Keys.docLines({ doc_id: this.doc_id }),
+ (error, data) => {
+ if (error) {
+ throw error
+ }
+ expect(JSON.parse(data)).to.deep.equal({
+ content: lines.join('\n'),
+ trackedChanges: [
+ {
+ range: {
+ pos: 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ })
+ done()
+ }
+ )
+ })
+
+ it('should apply the change', function (done) {
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, data) => {
+ if (error) {
+ throw error
+ }
+ expect(data.lines).to.deep.equal(this.newLines)
+ done()
+ }
+ )
+ })
+ const cases = [
+ {
+ name: 'when resetting the content',
+ lines,
+ want: {
+ content: 'one\none and a half\none and a half\ntwo\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 'one and a half\n'.length + 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when adding content before a tracked delete',
+ lines: ['one', 'INSERT', 'two', 'three'],
+ want: {
+ content: 'one\nINSERT\none and a half\ntwo\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 'INSERT\n'.length + 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when adding content after a tracked delete',
+ lines: ['one', 'two', 'INSERT', 'three'],
+ want: {
+ content: 'one\none and a half\ntwo\nINSERT\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when deleting content before a tracked delete',
+ lines: ['two', 'three'],
+ want: {
+ content: 'one and a half\ntwo\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 0,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when deleting content after a tracked delete',
+ lines: ['one', 'two'],
+ want: {
+ content: 'one\none and a half\ntwo',
+ trackedChanges: [
+ {
+ range: {
+ pos: 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when deleting content immediately after a tracked delete',
+ lines: ['one', 'three'],
+ want: {
+ content: 'one\none and a half\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 4,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ {
+ name: 'when deleting content across a tracked delete',
+ lines: ['onethree'],
+ want: {
+ content: 'oneone and a half\nthree',
+ trackedChanges: [
+ {
+ range: {
+ pos: 3,
+ length: 15,
+ },
+ tracking: {
+ ts,
+ type: 'delete',
+ userId,
+ },
+ },
+ ],
+ },
+ },
+ ]
+
+ for (const { name, lines, want } of cases) {
+ describe(name, function () {
+ beforeEach(function (done) {
+ DocUpdaterClient.setDocLines(
+ this.project_id,
+ this.doc_id,
+ lines,
+ this.source,
+ userId,
+ false,
+ (error, res, body) => {
+ if (error) {
+ return done(error)
+ }
+ this.statusCode = res.statusCode
+ this.body = body
+ done()
+ }
+ )
+ })
+ it('should update accordingly', function (done) {
+ docUpdaterRedis.get(
+ Keys.docLines({ doc_id: this.doc_id }),
+ (error, data) => {
+ if (error) {
+ throw error
+ }
+ expect(JSON.parse(data)).to.deep.equal(want)
+ done()
+ }
+ )
+ })
+ })
+ }
+ })
})
diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
index 33c6882138..d34996ca7c 100644
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
@@ -9,7 +9,6 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
-require('@overleaf/logger').logger.level('fatal')
module.exports = {
running: false,
diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
index 4ed4f929de..0a4ec8922e 100644
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
@@ -119,6 +119,18 @@ module.exports = DocUpdaterClient = {
)
},
+ getProjectLastUpdatedAt(projectId, callback) {
+ request.get(
+ `http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
+ (error, res, body) => {
+ if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+ body = JSON.parse(body)
+ }
+ callback(error, res, body)
+ }
+ )
+ },
+
preloadDoc(projectId, docId, callback) {
DocUpdaterClient.getDoc(projectId, docId, callback)
},
diff --git a/services/document-updater/test/setup.js b/services/document-updater/test/setup.js
index f7d47e7b75..8ba17d922f 100644
--- a/services/document-updater/test/setup.js
+++ b/services/document-updater/test/setup.js
@@ -31,8 +31,14 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
'mongodb-legacy': require('mongodb-legacy'), // for ObjectId comparisons
+ 'overleaf-editor-core': require('overleaf-editor-core'), // does not play nice with sandbox
},
globals: { Buffer, JSON, Math, console, process },
+ sourceTransformers: {
+ removeNodePrefix: function (source) {
+ return source.replace(/require\(['"]node:/g, "require('")
+ },
+ },
})
// Mocha hooks
diff --git a/services/document-updater/test/stress/js/run.js b/services/document-updater/test/stress/js/run.js
index 4d2614eac4..1bda73c1aa 100644
--- a/services/document-updater/test/stress/js/run.js
+++ b/services/document-updater/test/stress/js/run.js
@@ -15,7 +15,7 @@
*/
const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient')
// MockWebApi = require "../../acceptance/js/helpers/MockWebApi"
-const assert = require('assert')
+const assert = require('node:assert')
const async = require('async')
const insert = function (string, pos, content) {
diff --git a/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js b/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
index a0c379a5be..1816579103 100644
--- a/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
+++ b/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
@@ -49,10 +49,16 @@ describe('DocumentManager', function () {
applyUpdate: sinon.stub().resolves(),
},
}
+ this.HistoryOTUpdateManager = {
+ applyUpdate: sinon.stub().resolves(),
+ }
this.RangesManager = {
acceptChanges: sinon.stub(),
deleteComment: sinon.stub(),
}
+ this.Settings = {
+ max_doc_length: 2 * 1024 * 1024, // 2mb
+ }
this.DocumentManager = SandboxedModule.require(modulePath, {
requires: {
@@ -63,8 +69,10 @@ describe('DocumentManager', function () {
'./Metrics': this.Metrics,
'./DiffCodec': this.DiffCodec,
'./UpdateManager': this.UpdateManager,
+ './HistoryOTUpdateManager': this.HistoryOTUpdateManager,
'./RangesManager': this.RangesManager,
'./Errors': Errors,
+ '@overleaf/settings': this.Settings,
},
})
this.project_id = 'project-id-123'
@@ -218,6 +226,7 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
+ type: 'sharejs-text-ot',
})
this.RedisManager.promises.getPreviousDocOps.resolves(this.ops)
this.result = await this.DocumentManager.promises.getDocAndRecentOps(
@@ -247,6 +256,7 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
+ type: 'sharejs-text-ot',
})
})
})
@@ -259,6 +269,7 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
+ type: 'sharejs-text-ot',
})
this.RedisManager.promises.getPreviousDocOps.resolves(this.ops)
this.result = await this.DocumentManager.promises.getDocAndRecentOps(
@@ -286,6 +297,7 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
+ type: 'sharejs-text-ot',
})
})
})
@@ -329,6 +341,7 @@ describe('DocumentManager', function () {
unflushedTime: this.unflushedTime,
alreadyLoaded: true,
historyRangesSupport: this.historyRangesSupport,
+ type: 'sharejs-text-ot',
})
})
})
@@ -396,6 +409,7 @@ describe('DocumentManager', function () {
unflushedTime: null,
alreadyLoaded: false,
historyRangesSupport: this.historyRangesSupport,
+ type: 'sharejs-text-ot',
})
})
})
@@ -446,7 +460,8 @@ describe('DocumentManager', function () {
this.beforeLines,
this.source,
this.user_id,
- false
+ false,
+ true
)
})
@@ -480,7 +495,8 @@ describe('DocumentManager', function () {
this.beforeLines,
this.source,
this.user_id,
- false
+ false,
+ true
)
})
@@ -513,7 +529,8 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
- false
+ false,
+ true
)
})
@@ -582,7 +599,8 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
- false
+ false,
+ true
)
})
@@ -618,7 +636,8 @@ describe('DocumentManager', function () {
null,
this.source,
this.user_id,
- false
+ false,
+ true
)
).to.be.rejectedWith('No lines were provided to setDoc')
})
@@ -642,6 +661,7 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
+ true,
true
)
})
@@ -650,6 +670,77 @@ describe('DocumentManager', function () {
this.ops.map(op => op.u.should.equal(true))
})
})
+
+ describe('with the external flag', function () {
+ beforeEach(async function () {
+ this.undoing = false
+ // Copy ops so we don't interfere with other tests
+ this.ops = [
+ { i: 'foo', p: 4 },
+ { d: 'bar', p: 42 },
+ ]
+ this.DiffCodec.diffAsShareJsOp.returns(this.ops)
+ await this.DocumentManager.promises.setDoc(
+ this.project_id,
+ this.doc_id,
+ this.afterLines,
+ this.source,
+ this.user_id,
+ this.undoing,
+ true
+ )
+ })
+
+ it('should add the external type to update metadata', function () {
+ this.UpdateManager.promises.applyUpdate
+ .calledWith(this.project_id, this.doc_id, {
+ doc: this.doc_id,
+ v: this.version,
+ op: this.ops,
+ meta: {
+ type: 'external',
+ source: this.source,
+ user_id: this.user_id,
+ },
+ })
+ .should.equal(true)
+ })
+ })
+
+ describe('without the external flag', function () {
+ beforeEach(async function () {
+ this.undoing = false
+ // Copy ops so we don't interfere with other tests
+ this.ops = [
+ { i: 'foo', p: 4 },
+ { d: 'bar', p: 42 },
+ ]
+ this.DiffCodec.diffAsShareJsOp.returns(this.ops)
+ await this.DocumentManager.promises.setDoc(
+ this.project_id,
+ this.doc_id,
+ this.afterLines,
+ this.source,
+ this.user_id,
+ this.undoing,
+ false
+ )
+ })
+
+ it('should not add the external type to update metadata', function () {
+ this.UpdateManager.promises.applyUpdate
+ .calledWith(this.project_id, this.doc_id, {
+ doc: this.doc_id,
+ v: this.version,
+ op: this.ops,
+ meta: {
+ source: this.source,
+ user_id: this.user_id,
+ },
+ })
+ .should.equal(true)
+ })
+ })
})
})
@@ -691,6 +782,8 @@ describe('DocumentManager', function () {
it('should apply the accept change to the ranges', function () {
this.RangesManager.acceptChanges.should.have.been.calledWith(
+ this.project_id,
+ this.doc_id,
[this.change_id],
this.ranges
)
@@ -722,7 +815,12 @@ describe('DocumentManager', function () {
it('should apply the accept change to the ranges', function () {
this.RangesManager.acceptChanges
- .calledWith(this.change_ids, this.ranges)
+ .calledWith(
+ this.project_id,
+ this.doc_id,
+ this.change_ids,
+ this.ranges
+ )
.should.equal(true)
})
})
@@ -747,6 +845,77 @@ describe('DocumentManager', function () {
})
})
+ describe('getComment', function () {
+ beforeEach(function () {
+ this.ranges.comments = [
+ {
+ id: 'mock-comment-id-1',
+ },
+ {
+ id: 'mock-comment-id-2',
+ },
+ ]
+ this.DocumentManager.promises.getDoc = sinon.stub().resolves({
+ lines: this.lines,
+ version: this.version,
+ ranges: this.ranges,
+ })
+ })
+
+ describe('when comment exists', function () {
+ beforeEach(async function () {
+ await expect(
+ this.DocumentManager.promises.getComment(
+ this.project_id,
+ this.doc_id,
+ 'mock-comment-id-1'
+ )
+ ).to.eventually.deep.equal({
+ comment: { id: 'mock-comment-id-1' },
+ })
+ })
+
+ it("should get the document's current ranges", function () {
+ this.DocumentManager.promises.getDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+ })
+
+ describe('when comment doesnt exists', function () {
+ beforeEach(async function () {
+ await expect(
+ this.DocumentManager.promises.getComment(
+ this.project_id,
+ this.doc_id,
+ 'mock-comment-id-x'
+ )
+ ).to.be.rejectedWith(Errors.NotFoundError)
+ })
+
+ it("should get the document's current ranges", function () {
+ this.DocumentManager.promises.getDoc
+ .calledWith(this.project_id, this.doc_id)
+ .should.equal(true)
+ })
+ })
+
+ describe('when the doc is not found', function () {
+ beforeEach(async function () {
+ this.DocumentManager.promises.getDoc = sinon
+ .stub()
+ .resolves({ lines: null, version: null, ranges: null })
+ await expect(
+ this.DocumentManager.promises.acceptChanges(
+ this.project_id,
+ this.doc_id,
+ [this.change_id]
+ )
+ ).to.be.rejectedWith(Errors.NotFoundError)
+ })
+ })
+ })
+
describe('deleteComment', function () {
beforeEach(function () {
this.comment_id = 'mock-comment-id'
@@ -1129,4 +1298,68 @@ describe('DocumentManager', function () {
})
})
})
+
+ describe('appendToDoc', function () {
+ describe('sucessfully', function () {
+ beforeEach(async function () {
+ this.lines = ['one', 'two', 'three']
+ this.DocumentManager.promises.setDoc = sinon
+ .stub()
+ .resolves({ rev: '123' })
+ this.DocumentManager.promises.getDoc = sinon.stub().resolves({
+ lines: this.lines,
+ })
+ this.result = await this.DocumentManager.promises.appendToDoc(
+ this.project_id,
+ this.doc_id,
+ ['four', 'five', 'six'],
+ this.source,
+ this.user_id
+ )
+ })
+
+ it('should call setDoc with concatenated lines', function () {
+ this.DocumentManager.promises.setDoc
+ .calledWith(
+ this.project_id,
+ this.doc_id,
+ ['one', 'two', 'three', 'four', 'five', 'six'],
+ this.source,
+ this.user_id,
+ false,
+ false
+ )
+ .should.equal(true)
+ })
+
+ it('should return output from setDoc', function () {
+ this.result.should.deep.equal({ rev: '123' })
+ })
+ })
+
+ describe('when doc would become too big', function () {
+ beforeEach(async function () {
+ this.Settings.max_doc_length = 100
+ this.lines = ['one', 'two', 'three']
+ this.DocumentManager.promises.setDoc = sinon
+ .stub()
+ .resolves({ rev: '123' })
+ this.DocumentManager.promises.getDoc = sinon.stub().resolves({
+ lines: this.lines,
+ })
+ })
+
+ it('should fail with FileTooLarge error', async function () {
+ expect(
+ this.DocumentManager.promises.appendToDoc(
+ this.project_id,
+ this.doc_id,
+ ['x'.repeat(1000)],
+ this.source,
+ this.user_id
+ )
+ ).to.eventually.be.rejectedWith(Errors.FileTooLargeError)
+ })
+ })
+ })
})
diff --git a/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js b/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
index ce98c4eaa8..2a5fb29b6d 100644
--- a/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
+++ b/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
@@ -3,7 +3,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('path').join(
+const modulePath = require('node:path').join(
__dirname,
'../../../../app/js/HistoryManager'
)
@@ -14,6 +14,7 @@ describe('HistoryManager', function () {
requires: {
request: (this.request = {}),
'@overleaf/settings': (this.Settings = {
+ shortHistoryQueues: [],
apis: {
project_history: {
url: 'http://project_history.example.com',
@@ -118,7 +119,7 @@ describe('HistoryManager', function () {
beforeEach(function () {
this.HistoryManager.shouldFlushHistoryOps = sinon.stub()
this.HistoryManager.shouldFlushHistoryOps
- .withArgs(this.project_ops_length)
+ .withArgs(this.project_id, this.project_ops_length)
.returns(true)
this.HistoryManager.recordAndFlushHistoryOps(
@@ -139,7 +140,7 @@ describe('HistoryManager', function () {
beforeEach(function () {
this.HistoryManager.shouldFlushHistoryOps = sinon.stub()
this.HistoryManager.shouldFlushHistoryOps
- .withArgs(this.project_ops_length)
+ .withArgs(this.project_id, this.project_ops_length)
.returns(false)
this.HistoryManager.recordAndFlushHistoryOps(
@@ -157,6 +158,7 @@ describe('HistoryManager', function () {
describe('shouldFlushHistoryOps', function () {
it('should return false if the number of ops is not known', function () {
this.HistoryManager.shouldFlushHistoryOps(
+ this.project_id,
null,
['a', 'b', 'c'].length,
1
@@ -168,6 +170,7 @@ describe('HistoryManager', function () {
// Previously we were on 11 ops
// We didn't pass over a multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
+ this.project_id,
14,
['a', 'b', 'c'].length,
5
@@ -178,6 +181,7 @@ describe('HistoryManager', function () {
// Previously we were on 12 ops
// We've reached a new multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
+ this.project_id,
15,
['a', 'b', 'c'].length,
5
@@ -189,11 +193,22 @@ describe('HistoryManager', function () {
// Previously we were on 16 ops
// We didn't pass over a multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
+ this.project_id,
17,
['a', 'b', 'c'].length,
5
).should.equal(true)
})
+
+ it('should return true if the project has a short queue', function () {
+ this.Settings.shortHistoryQueues = [this.project_id]
+ this.HistoryManager.shouldFlushHistoryOps(
+ this.project_id,
+ 14,
+ ['a', 'b', 'c'].length,
+ 5
+ ).should.equal(true)
+ })
})
})
@@ -217,34 +232,75 @@ describe('HistoryManager', function () {
.stub()
.yields()
this.DocumentManager.resyncDocContentsWithLock = sinon.stub().yields()
- this.HistoryManager.resyncProjectHistory(
- this.project_id,
- this.projectHistoryId,
- this.docs,
- this.files,
- this.callback
- )
})
- it('should queue a project structure reync', function () {
- this.ProjectHistoryRedisManager.queueResyncProjectStructure
- .calledWith(
+ describe('full sync', function () {
+ beforeEach(function () {
+ this.HistoryManager.resyncProjectHistory(
this.project_id,
this.projectHistoryId,
this.docs,
- this.files
+ this.files,
+ {},
+ this.callback
)
- .should.equal(true)
+ })
+
+ it('should queue a project structure reync', function () {
+ this.ProjectHistoryRedisManager.queueResyncProjectStructure
+ .calledWith(
+ this.project_id,
+ this.projectHistoryId,
+ this.docs,
+ this.files
+ )
+ .should.equal(true)
+ })
+
+ it('should queue doc content reyncs', function () {
+ this.DocumentManager.resyncDocContentsWithLock
+ .calledWith(this.project_id, this.docs[0].doc, this.docs[0].path)
+ .should.equal(true)
+ })
+
+ it('should call the callback', function () {
+ this.callback.called.should.equal(true)
+ })
})
- it('should queue doc content reyncs', function () {
- this.DocumentManager.resyncDocContentsWithLock
- .calledWith(this.project_id, this.docs[0].doc, this.docs[0].path)
- .should.equal(true)
- })
+ describe('resyncProjectStructureOnly=true', function () {
+ beforeEach(function () {
+ this.HistoryManager.resyncProjectHistory(
+ this.project_id,
+ this.projectHistoryId,
+ this.docs,
+ this.files,
+ { resyncProjectStructureOnly: true },
+ this.callback
+ )
+ })
- it('should call the callback', function () {
- this.callback.called.should.equal(true)
+ it('should queue a project structure reync', function () {
+ this.ProjectHistoryRedisManager.queueResyncProjectStructure
+ .calledWith(
+ this.project_id,
+ this.projectHistoryId,
+ this.docs,
+ this.files,
+ { resyncProjectStructureOnly: true }
+ )
+ .should.equal(true)
+ })
+
+ it('should not queue doc content reyncs', function () {
+ this.DocumentManager.resyncDocContentsWithLock.called.should.equal(
+ false
+ )
+ })
+
+ it('should call the callback', function () {
+ this.callback.called.should.equal(true)
+ })
})
})
})
diff --git a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
index c422b8c2c0..333da10d15 100644
--- a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
+++ b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
@@ -26,6 +26,7 @@ describe('HttpController', function () {
this.Metrics.Timer.prototype.done = sinon.stub()
this.project_id = 'project-id-123'
+ this.projectHistoryId = '123'
this.doc_id = 'doc-id-123'
this.source = 'editor'
this.next = sinon.stub()
@@ -65,7 +66,9 @@ describe('HttpController', function () {
this.version,
[],
this.ranges,
- this.pathname
+ this.pathname,
+ this.projectHistoryId,
+ 'sharejs-text-ot'
)
this.HttpController.getDoc(this.req, this.res, this.next)
})
@@ -77,17 +80,16 @@ describe('HttpController', function () {
})
it('should return the doc as JSON', function () {
- this.res.json
- .calledWith({
- id: this.doc_id,
- lines: this.lines,
- version: this.version,
- ops: [],
- ranges: this.ranges,
- pathname: this.pathname,
- ttlInS: 42,
- })
- .should.equal(true)
+ this.res.json.should.have.been.calledWith({
+ id: this.doc_id,
+ lines: this.lines,
+ version: this.version,
+ ops: [],
+ ranges: this.ranges,
+ pathname: this.pathname,
+ ttlInS: 42,
+ type: 'sharejs-text-ot',
+ })
})
it('should log the request', function () {
@@ -115,7 +117,9 @@ describe('HttpController', function () {
this.version,
this.ops,
this.ranges,
- this.pathname
+ this.pathname,
+ this.projectHistoryId,
+ 'sharejs-text-ot'
)
this.req.query = { fromVersion: `${this.fromVersion}` }
this.HttpController.getDoc(this.req, this.res, this.next)
@@ -128,17 +132,16 @@ describe('HttpController', function () {
})
it('should return the doc as JSON', function () {
- this.res.json
- .calledWith({
- id: this.doc_id,
- lines: this.lines,
- version: this.version,
- ops: this.ops,
- ranges: this.ranges,
- pathname: this.pathname,
- ttlInS: 42,
- })
- .should.equal(true)
+ this.res.json.should.have.been.calledWith({
+ id: this.doc_id,
+ lines: this.lines,
+ version: this.version,
+ ops: this.ops,
+ ranges: this.ranges,
+ pathname: this.pathname,
+ ttlInS: 42,
+ type: 'sharejs-text-ot',
+ })
})
it('should log the request', function () {
@@ -184,6 +187,65 @@ describe('HttpController', function () {
})
})
+ describe('getComment', function () {
+ beforeEach(function () {
+ this.ranges = {
+ changes: 'mock',
+ comments: [
+ {
+ id: 'comment-id-1',
+ },
+ {
+ id: 'comment-id-2',
+ },
+ ],
+ }
+ this.req = {
+ params: {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ comment_id: this.comment_id,
+ },
+ query: {},
+ body: {},
+ }
+ })
+
+ beforeEach(function () {
+ this.DocumentManager.getCommentWithLock = sinon
+ .stub()
+ .callsArgWith(3, null, this.ranges.comments[0])
+ this.HttpController.getComment(this.req, this.res, this.next)
+ })
+
+ it('should get the comment', function () {
+ this.DocumentManager.getCommentWithLock
+ .calledWith(this.project_id, this.doc_id, this.comment_id)
+ .should.equal(true)
+ })
+
+ it('should return the comment as JSON', function () {
+ this.res.json
+ .calledWith({
+ id: 'comment-id-1',
+ })
+ .should.equal(true)
+ })
+
+ it('should log the request', function () {
+ this.logger.debug
+ .calledWith(
+ {
+ projectId: this.project_id,
+ docId: this.doc_id,
+ commentId: this.comment_id,
+ },
+ 'getting comment via http'
+ )
+ .should.equal(true)
+ })
+ })
+
describe('setDoc', function () {
beforeEach(function () {
this.lines = ['one', 'two', 'three']
@@ -209,7 +271,7 @@ describe('HttpController', function () {
beforeEach(function () {
this.DocumentManager.setDocWithLock = sinon
.stub()
- .callsArgWith(6, null, { rev: '123' })
+ .callsArgWith(7, null, { rev: '123' })
this.HttpController.setDoc(this.req, this.res, this.next)
})
@@ -221,7 +283,8 @@ describe('HttpController', function () {
this.lines,
this.source,
this.user_id,
- this.undoing
+ this.undoing,
+ true
)
.should.equal(true)
})
@@ -255,7 +318,7 @@ describe('HttpController', function () {
beforeEach(function () {
this.DocumentManager.setDocWithLock = sinon
.stub()
- .callsArgWith(6, new Error('oops'))
+ .callsArgWith(7, new Error('oops'))
this.HttpController.setDoc(this.req, this.res, this.next)
})
@@ -1103,4 +1166,96 @@ describe('HttpController', function () {
})
})
})
+
+ describe('appendToDoc', function () {
+ beforeEach(function () {
+ this.lines = ['one', 'two', 'three']
+ this.source = 'dropbox'
+ this.user_id = 'user-id-123'
+ this.req = {
+ headers: {},
+ params: {
+ project_id: this.project_id,
+ doc_id: this.doc_id,
+ },
+ query: {},
+ body: {
+ lines: this.lines,
+ source: this.source,
+ user_id: this.user_id,
+ undoing: (this.undoing = true),
+ },
+ }
+ })
+
+ describe('successfully', function () {
+ beforeEach(function () {
+ this.DocumentManager.appendToDocWithLock = sinon
+ .stub()
+ .callsArgWith(5, null, { rev: '123' })
+ this.HttpController.appendToDoc(this.req, this.res, this.next)
+ })
+
+ it('should append to the doc', function () {
+ this.DocumentManager.appendToDocWithLock
+ .calledWith(
+ this.project_id,
+ this.doc_id,
+ this.lines,
+ this.source,
+ this.user_id
+ )
+ .should.equal(true)
+ })
+
+ it('should return a json response with the document rev from web', function () {
+ this.res.json.calledWithMatch({ rev: '123' }).should.equal(true)
+ })
+
+ it('should log the request', function () {
+ this.logger.debug
+ .calledWith(
+ {
+ docId: this.doc_id,
+ projectId: this.project_id,
+ lines: this.lines,
+ source: this.source,
+ userId: this.user_id,
+ },
+ 'appending to doc via http'
+ )
+ .should.equal(true)
+ })
+
+ it('should time the request', function () {
+ this.Metrics.Timer.prototype.done.called.should.equal(true)
+ })
+ })
+
+ describe('when an errors occurs', function () {
+ beforeEach(function () {
+ this.DocumentManager.appendToDocWithLock = sinon
+ .stub()
+ .callsArgWith(5, new Error('oops'))
+ this.HttpController.appendToDoc(this.req, this.res, this.next)
+ })
+
+ it('should call next with the error', function () {
+ this.next.calledWith(sinon.match.instanceOf(Error)).should.equal(true)
+ })
+ })
+
+ describe('when the payload is too large', function () {
+ beforeEach(function () {
+ this.DocumentManager.appendToDocWithLock = sinon
+ .stub()
+ .callsArgWith(5, new Errors.FileTooLargeError())
+ this.HttpController.appendToDoc(this.req, this.res, this.next)
+ })
+
+ it('should send back a 422 response', function () {
+ this.res.sendStatus.calledWith(422).should.equal(true)
+ })
+ })
+ })
})
diff --git a/services/document-updater/test/unit/js/Limits/LimitsTests.js b/services/document-updater/test/unit/js/Limits/LimitsTests.js
index 34a5c13c26..11ca38746a 100644
--- a/services/document-updater/test/unit/js/Limits/LimitsTests.js
+++ b/services/document-updater/test/unit/js/Limits/LimitsTests.js
@@ -81,4 +81,88 @@ describe('Limits', function () {
})
})
})
+
+ describe('stringFileDataContentIsTooLarge', function () {
+ it('should handle small docs', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge({ content: '' }, 123)
+ ).to.equal(false)
+ })
+ it('should handle docs at the limit', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge(
+ { content: 'x'.repeat(123) },
+ 123
+ )
+ ).to.equal(false)
+ })
+ it('should handle docs above the limit', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge(
+ { content: 'x'.repeat(123 + 1) },
+ 123
+ )
+ ).to.equal(true)
+ })
+ it('should handle docs above the limit and below with tracked-deletes removed', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge(
+ {
+ content: 'x'.repeat(123 + 1),
+ trackedChanges: [
+ {
+ range: { pos: 1, length: 1 },
+ tracking: {
+ type: 'delete',
+ ts: '2025-06-16T14:31:44.910Z',
+ userId: 'user-id',
+ },
+ },
+ ],
+ },
+ 123
+ )
+ ).to.equal(false)
+ })
+ it('should handle docs above the limit and above with tracked-deletes removed', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge(
+ {
+ content: 'x'.repeat(123 + 2),
+ trackedChanges: [
+ {
+ range: { pos: 1, length: 1 },
+ tracking: {
+ type: 'delete',
+ ts: '2025-06-16T14:31:44.910Z',
+ userId: 'user-id',
+ },
+ },
+ ],
+ },
+ 123
+ )
+ ).to.equal(true)
+ })
+ it('should handle docs above the limit and with tracked-inserts', function () {
+ expect(
+ this.Limits.stringFileDataContentIsTooLarge(
+ {
+ content: 'x'.repeat(123 + 1),
+ trackedChanges: [
+ {
+ range: { pos: 1, length: 1 },
+ tracking: {
+ type: 'insert',
+ ts: '2025-06-16T14:31:44.910Z',
+ userId: 'user-id',
+ },
+ },
+ ],
+ },
+ 123
+ )
+ ).to.equal(true)
+ })
+ })
})
diff --git a/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js b/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
index eb5b44532a..575ed90702 100644
--- a/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
+++ b/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
@@ -10,8 +10,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
-const assert = require('assert')
-const path = require('path')
+const assert = require('node:assert')
+const path = require('node:path')
const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js')
const projectId = 1234
const docId = 5678
diff --git a/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js b/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
index 0413e268d6..a39a3b4fff 100644
--- a/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
+++ b/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
@@ -11,8 +11,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
-const assert = require('assert')
-const path = require('path')
+const assert = require('node:assert')
+const path = require('node:path')
const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js')
const projectId = 1234
const docId = 5678
diff --git a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
index 0f5df2e29f..ad6c121dfb 100644
--- a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
+++ b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
@@ -15,6 +15,7 @@ describe('ProjectHistoryRedisManager', function () {
this.Limits = {
docIsTooLarge: sinon.stub().returns(false),
+ stringFileDataContentIsTooLarge: sinon.stub().returns(false),
}
this.ProjectHistoryRedisManager = SandboxedModule.require(modulePath, {
@@ -61,22 +62,18 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
- this.multi.rpush
- .calledWithExactly(
- `ProjectHistory:Ops:${this.project_id}`,
- this.ops[0],
- this.ops[1]
- )
- .should.equal(true)
+ this.multi.rpush.should.have.been.calledWithExactly(
+ `ProjectHistory:Ops:${this.project_id}`,
+ this.ops[0],
+ this.ops[1]
+ )
})
it('should set the queue timestamp if not present', function () {
- this.multi.setnx
- .calledWithExactly(
- `ProjectHistory:FirstOpTimestamp:${this.project_id}`,
- Date.now()
- )
- .should.equal(true)
+ this.multi.setnx.should.have.been.calledWithExactly(
+ `ProjectHistory:FirstOpTimestamp:${this.project_id}`,
+ Date.now()
+ )
})
})
@@ -118,9 +115,10 @@ describe('ProjectHistoryRedisManager', function () {
file: this.file_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
})
})
@@ -162,12 +160,14 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
})
it('should queue an update with file metadata', async function () {
@@ -207,6 +207,7 @@ describe('ProjectHistoryRedisManager', function () {
hash: '1337',
metadata,
projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
file: fileId,
}
@@ -291,6 +292,7 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
ranges: historyCompatibleRanges,
doc: this.doc_id,
}
@@ -343,12 +345,14 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
})
it('should not forward ranges if history ranges support is undefined', async function () {
@@ -394,12 +398,77 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
+ })
+
+ it('should pass "false" as the createdBlob field if not provided', async function () {
+ await this.ProjectHistoryRedisManager.promises.queueAddEntity(
+ this.project_id,
+ this.projectHistoryId,
+ 'doc',
+ this.doc_id,
+ this.user_id,
+ this.rawUpdate,
+ this.source
+ )
+
+ const update = {
+ pathname: this.pathname,
+ docLines: this.docLines,
+ meta: {
+ user_id: this.user_id,
+ ts: new Date(),
+ source: this.source,
+ },
+ version: this.version,
+ projectHistoryId: this.projectHistoryId,
+ createdBlob: false,
+ doc: this.doc_id,
+ }
+
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
+ })
+
+ it('should pass through the value of the createdBlob field', async function () {
+ this.rawUpdate.createdBlob = true
+ await this.ProjectHistoryRedisManager.promises.queueAddEntity(
+ this.project_id,
+ this.projectHistoryId,
+ 'doc',
+ this.doc_id,
+ this.user_id,
+ this.rawUpdate,
+ this.source
+ )
+
+ const update = {
+ pathname: this.pathname,
+ docLines: this.docLines,
+ meta: {
+ user_id: this.user_id,
+ ts: new Date(),
+ source: this.source,
+ },
+ version: this.version,
+ projectHistoryId: this.projectHistoryId,
+ createdBlob: true,
+ doc: this.doc_id,
+ }
+
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(update)
+ )
})
})
@@ -427,8 +496,8 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
- content: 'one\ntwo',
version: this.version,
+ content: 'one\ntwo',
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -450,19 +519,18 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should check if the doc is too large', function () {
- this.Limits.docIsTooLarge
- .calledWith(
- JSON.stringify(this.update).length,
- this.lines,
- this.settings.max_doc_length
- )
- .should.equal(true)
+ this.Limits.docIsTooLarge.should.have.been.calledWith(
+ JSON.stringify(this.update).length,
+ this.lines,
+ this.settings.max_doc_length
+ )
})
it('should queue an update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(this.update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(this.update)
+ )
})
})
@@ -485,9 +553,8 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should not queue an update if the doc is too large', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.called.should.equal(
- false
- )
+ this.ProjectHistoryRedisManager.promises.queueOps.should.not.have.been
+ .called
})
})
@@ -495,10 +562,10 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
- content: 'onedeleted\ntwo',
version: this.version,
ranges: this.ranges,
resolvedCommentIds: this.resolvedCommentIds,
+ content: 'onedeleted\ntwo',
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -535,9 +602,76 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps
- .calledWithExactly(this.project_id, JSON.stringify(this.update))
- .should.equal(true)
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(this.update)
+ )
+ })
+ })
+
+ describe('history-ot', function () {
+ beforeEach(async function () {
+ this.lines = {
+ content: 'onedeleted\ntwo',
+ comments: [{ id: 'id1', ranges: [{ pos: 0, length: 3 }] }],
+ trackedChanges: [
+ {
+ range: { pos: 3, length: 7 },
+ tracking: {
+ type: 'delete',
+ userId: 'user-id',
+ ts: '2025-06-16T14:31:44.910Z',
+ },
+ },
+ ],
+ }
+ this.update = {
+ resyncDocContent: {
+ version: this.version,
+ historyOTRanges: {
+ comments: this.lines.comments,
+ trackedChanges: this.lines.trackedChanges,
+ },
+ content: this.lines.content,
+ },
+ projectHistoryId: this.projectHistoryId,
+ path: this.pathname,
+ doc: this.doc_id,
+ meta: { ts: new Date() },
+ }
+
+ await this.ProjectHistoryRedisManager.promises.queueResyncDocContent(
+ this.project_id,
+ this.projectHistoryId,
+ this.doc_id,
+ this.lines,
+ this.ranges,
+ this.resolvedCommentIds,
+ this.version,
+ this.pathname,
+ true
+ )
+ })
+
+ it('should include tracked deletes in the update', function () {
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(this.update)
+ )
+ })
+
+ it('should check the doc length without tracked deletes', function () {
+ this.Limits.stringFileDataContentIsTooLarge.should.have.been.calledWith(
+ this.lines,
+ this.settings.max_doc_length
+ )
+ })
+
+ it('should queue an update', function () {
+ this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
+ this.project_id,
+ JSON.stringify(this.update)
+ )
})
})
})
diff --git a/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js b/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
index 67ac6cc175..4053aafb01 100644
--- a/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
+++ b/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
@@ -323,6 +323,44 @@ describe('RangesManager', function () {
})
})
+ describe('tracked delete rejections with multiple tracked deletes at the same position', function () {
+ beforeEach(function () {
+ // original text is "one [two ][three ][four ]five"
+ // [] denotes tracked deletes
+ this.ranges = {
+ changes: makeRanges([
+ { d: 'two ', p: 4 },
+ { d: 'three ', p: 4 },
+ { d: 'four ', p: 4 },
+ ]),
+ }
+ this.updates = makeUpdates([{ i: 'three ', p: 4, u: true }])
+ this.newDocLines = ['one three five']
+ this.result = this.RangesManager.applyUpdate(
+ this.project_id,
+ this.doc_id,
+ this.ranges,
+ this.updates,
+ this.newDocLines,
+ { historyRangesSupport: true }
+ )
+ })
+
+ it('should insert the text at the right history position', function () {
+ expect(this.result.historyUpdates.map(x => x.op)).to.deep.equal([
+ [
+ {
+ i: 'three ',
+ p: 4,
+ hpos: 8,
+ u: true,
+ trackedDeleteRejection: true,
+ },
+ ],
+ ])
+ })
+ })
+
describe('deletes over tracked changes', function () {
beforeEach(function () {
// original text is "on[1]e [22](three) f[333]ou[4444]r [55555]five"
@@ -658,6 +696,7 @@ describe('RangesManager', function () {
{ i: 'amet', p: 40 },
]),
}
+ this.lines = ['lorem xxx', 'ipsum yyy', 'dolor zzz', 'sit wwwww', 'amet']
this.removeChangeIdsSpy = sinon.spy(
this.RangesTracker.prototype,
'removeChangeIds'
@@ -668,8 +707,11 @@ describe('RangesManager', function () {
beforeEach(function () {
this.change_ids = [this.ranges.changes[1].id]
this.result = this.RangesManager.acceptChanges(
+ this.project_id,
+ this.doc_id,
this.change_ids,
- this.ranges
+ this.ranges,
+ this.lines
)
})
@@ -714,8 +756,11 @@ describe('RangesManager', function () {
this.ranges.changes[4].id,
]
this.result = this.RangesManager.acceptChanges(
+ this.project_id,
+ this.doc_id,
this.change_ids,
- this.ranges
+ this.ranges,
+ this.lines
)
})
diff --git a/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js b/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
index e23964eaba..125dd3d08c 100644
--- a/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
+++ b/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
@@ -2,7 +2,7 @@ const sinon = require('sinon')
const { expect } = require('chai')
const SandboxedModule = require('sandboxed-module')
const Errors = require('../../../../app/js/Errors')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const tk = require('timekeeper')
const MODULE_PATH = '../../../../app/js/RedisManager.js'
diff --git a/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js b/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
index 347ac7dea6..e699439a74 100644
--- a/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
+++ b/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
@@ -12,7 +12,7 @@
const sinon = require('sinon')
const modulePath = '../../../../app/js/ShareJsUpdateManager.js'
const SandboxedModule = require('sandboxed-module')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
describe('ShareJsUpdateManager', function () {
beforeEach(function () {
diff --git a/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js b/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
index 16ee0b12e1..912707e01d 100644
--- a/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
+++ b/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
@@ -1,5 +1,4 @@
-// @ts-check
-
+const { createHash } = require('node:crypto')
const sinon = require('sinon')
const { expect } = require('chai')
const SandboxedModule = require('sandboxed-module')
@@ -332,6 +331,7 @@ describe('UpdateManager', function () {
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: false,
+ type: 'sharejs-text-ot',
})
this.RangesManager.applyUpdate.returns({
newRanges: this.updated_ranges,
@@ -399,7 +399,9 @@ describe('UpdateManager', function () {
this.historyUpdates,
this.pathname,
this.projectHistoryId,
- this.lines
+ this.lines,
+ this.ranges,
+ this.updatedDocLines
)
})
@@ -501,6 +503,7 @@ describe('UpdateManager', function () {
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: true,
+ type: 'sharejs-text-ot',
})
await this.UpdateManager.promises.applyUpdate(
this.project_id,
@@ -526,6 +529,7 @@ describe('UpdateManager', function () {
describe('_adjustHistoryUpdatesMetadata', function () {
beforeEach(function () {
this.lines = ['some', 'test', 'data']
+ this.updatedDocLines = ['after', 'updates']
this.historyUpdates = [
{
v: 42,
@@ -570,6 +574,7 @@ describe('UpdateManager', function () {
this.pathname,
this.projectHistoryId,
this.lines,
+ this.updatedDocLines,
this.ranges,
false
)
@@ -632,6 +637,7 @@ describe('UpdateManager', function () {
this.projectHistoryId,
this.lines,
this.ranges,
+ this.updatedDocLines,
true
)
this.historyUpdates.should.deep.equal([
@@ -685,6 +691,7 @@ describe('UpdateManager', function () {
meta: {
pathname: this.pathname,
doc_length: 21, // 23 - 'so'
+ doc_hash: stringHash(this.updatedDocLines.join('\n')),
history_doc_length: 28, // 30 - 'so'
},
},
@@ -699,6 +706,7 @@ describe('UpdateManager', function () {
this.projectHistoryId,
[],
{},
+ ['foobar'],
false
)
this.historyUpdates.should.deep.equal([
@@ -822,3 +830,9 @@ describe('UpdateManager', function () {
})
})
})
+
+function stringHash(s) {
+ const hash = createHash('sha1')
+ hash.update(s)
+ return hash.digest('hex')
+}
diff --git a/services/document-updater/test/unit/js/UtilsTests.js b/services/document-updater/test/unit/js/UtilsTests.js
index 553b90159a..5d0f03ca64 100644
--- a/services/document-updater/test/unit/js/UtilsTests.js
+++ b/services/document-updater/test/unit/js/UtilsTests.js
@@ -1,5 +1,6 @@
// @ts-check
+const { createHash } = require('node:crypto')
const { expect } = require('chai')
const Utils = require('../../../app/js/Utils')
@@ -24,4 +25,30 @@ describe('Utils', function () {
expect(result).to.equal('the quick brown fox jumps over the lazy dog')
})
})
+
+ describe('computeDocHash', function () {
+ it('computes the hash for an empty doc', function () {
+ const actual = Utils.computeDocHash([])
+ const expected = stringHash('')
+ expect(actual).to.equal(expected)
+ })
+
+ it('computes the hash for a single-line doc', function () {
+ const actual = Utils.computeDocHash(['hello'])
+ const expected = stringHash('hello')
+ expect(actual).to.equal(expected)
+ })
+
+ it('computes the hash for a multiline doc', function () {
+ const actual = Utils.computeDocHash(['hello', 'there', 'world'])
+ const expected = stringHash('hello\nthere\nworld')
+ expect(actual).to.equal(expected)
+ })
+ })
})
+
+function stringHash(s) {
+ const hash = createHash('sha1')
+ hash.update(s)
+ return hash.digest('hex')
+}
diff --git a/services/filestore/.gitignore b/services/filestore/.gitignore
index a2f4b5afb2..1772191882 100644
--- a/services/filestore/.gitignore
+++ b/services/filestore/.gitignore
@@ -1,54 +1,3 @@
-compileFolder
-
-Compiled source #
-###################
-*.com
-*.class
-*.dll
-*.exe
-*.o
-*.so
-
-# Packages #
-############
-# it's better to unpack these files and commit the raw source
-# git has its own built in compression methods
-*.7z
-*.dmg
-*.gz
-*.iso
-*.jar
-*.rar
-*.tar
-*.zip
-
-# Logs and databases #
-######################
-*.log
-*.sql
-*.sqlite
-
-# OS generated files #
-######################
-.DS_Store?
-ehthumbs.db
-Icon?
-Thumbs.db
-
-/node_modules/*
-data/*/*
-
-**/*.map
-cookies.txt
uploads/*
-
user_files/*
template_files/*
-
-**.swp
-
-/log.json
-hash_folder
-
-# managed by dev-environment$ bin/update_build_scripts
-.npmrc
diff --git a/services/filestore/.nvmrc b/services/filestore/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/filestore/.nvmrc
+++ b/services/filestore/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile
index 6593f60161..33de01c80f 100644
--- a/services/filestore/Dockerfile
+++ b/services/filestore/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/filestore
COPY services/filestore/install_deps.sh /overleaf/services/filestore/
diff --git a/services/filestore/Makefile b/services/filestore/Makefile
index 941f989d64..69d7f85bf4 100644
--- a/services/filestore/Makefile
+++ b/services/filestore/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -130,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/filestore/app.js b/services/filestore/app.js
index 74b5fff246..178e8c7ff0 100644
--- a/services/filestore/app.js
+++ b/services/filestore/app.js
@@ -1,7 +1,7 @@
// Metrics must be initialized before importing anything else
require('@overleaf/metrics/initialize')
-const Events = require('events')
+const Events = require('node:events')
const Metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
@@ -23,10 +23,6 @@ const app = express()
app.use(RequestLogger.middleware)
-if (settings.sentry && settings.sentry.dsn) {
- logger.initializeErrorReporting(settings.sentry.dsn)
-}
-
Metrics.open_sockets.monitor(true)
Metrics.memory.monitor(logger)
if (Metrics.event_loop) {
@@ -54,64 +50,73 @@ app.use((req, res, next) => {
Metrics.injectMetricsRoute(app)
-app.head(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.getFileHead
-)
-app.get(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.getFile
-)
-app.post(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.insertFile
-)
-app.put(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- bodyParser.json(),
- fileController.copyFile
-)
-app.delete(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.deleteFile
-)
-app.delete(
- '/project/:project_id',
- keyBuilder.userProjectKeyMiddleware,
- fileController.deleteProject
-)
+if (settings.filestore.stores.user_files) {
+ app.head(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.getFileHead
+ )
+ app.get(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.getFile
+ )
+ app.post(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.insertFile
+ )
+ app.put(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ bodyParser.json(),
+ fileController.copyFile
+ )
+ app.delete(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.deleteFile
+ )
+ app.delete(
+ '/project/:project_id',
+ keyBuilder.userProjectKeyMiddleware,
+ fileController.deleteProject
+ )
-app.get(
- '/project/:project_id/size',
- keyBuilder.userProjectKeyMiddleware,
- fileController.directorySize
-)
+ app.get(
+ '/project/:project_id/size',
+ keyBuilder.userProjectKeyMiddleware,
+ fileController.directorySize
+ )
+}
-app.head(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFileHead
-)
-app.get(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFile
-)
-app.get(
- '/template/:template_id/v/:version/:format/:sub_type',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFile
-)
-app.post(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.insertFile
-)
+if (settings.filestore.stores.template_files) {
+ app.head(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFileHead
+ )
+ app.get(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFile
+ )
+ app.get(
+ '/template/:template_id/v/:version/:format/:sub_type',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFile
+ )
+ app.post(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.insertFile
+ )
+ app.delete(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.deleteFile
+ )
+}
app.get(
'/bucket/:bucket/key/*',
@@ -166,7 +171,10 @@ function handleShutdownSignal(signal) {
// stop accepting new connections, the callback is called when existing connections have finished
server.close(() => {
logger.info({ signal }, 'server closed')
- process.exit()
+ // exit after a short delay so logs can be flushed
+ setTimeout(() => {
+ process.exit()
+ }, 100)
})
// close idle http keep-alive connections
server.closeIdleConnections()
@@ -178,7 +186,7 @@ function handleShutdownSignal(signal) {
setTimeout(() => {
process.exit()
}, 100)
- }, settings.delayShutdownMs)
+ }, settings.gracefulShutdownDelayInMs)
}
process.on('SIGTERM', handleShutdownSignal)
diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js
index a97869258a..127bbcc20f 100644
--- a/services/filestore/app/js/FileController.js
+++ b/services/filestore/app/js/FileController.js
@@ -1,9 +1,8 @@
-const PersistorManager = require('./PersistorManager')
const FileHandler = require('./FileHandler')
const metrics = require('@overleaf/metrics')
const parseRange = require('range-parser')
const Errors = require('./Errors')
-const { pipeline } = require('stream')
+const { pipeline } = require('node:stream')
const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
@@ -139,17 +138,17 @@ function copyFile(req, res, next) {
})
req.requestLogger.setMessage('copying file')
- PersistorManager.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key)
- .then(() => res.sendStatus(200))
- .catch(err => {
- if (err) {
- if (err instanceof Errors.NotFoundError) {
- res.sendStatus(404)
- } else {
- next(err)
- }
+ FileHandler.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key, err => {
+ if (err) {
+ if (err instanceof Errors.NotFoundError) {
+ res.sendStatus(404)
+ } else {
+ next(err)
}
- })
+ } else {
+ res.sendStatus(200)
+ }
+ })
}
function deleteFile(req, res, next) {
diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js
index cb21a482e3..bfc34314e9 100644
--- a/services/filestore/app/js/FileConverter.js
+++ b/services/filestore/app/js/FileConverter.js
@@ -1,11 +1,11 @@
const metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
-const { callbackify } = require('util')
+const { callbackify } = require('node:util')
const safeExec = require('./SafeExec').promises
const { ConversionError } = require('./Errors')
-const APPROVED_FORMATS = ['png']
+const APPROVED_FORMATS = ['png', 'jpg']
const FOURTY_SECONDS = 40 * 1000
const KILL_SIGNAL = 'SIGTERM'
@@ -34,16 +34,14 @@ async function convert(sourcePath, requestedFormat) {
}
async function thumbnail(sourcePath) {
- const width = '260x'
- return await convert(sourcePath, 'png', [
+ const width = '548x'
+ return await _convert(sourcePath, 'jpg', [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
- '-define',
- `pdf:fit-page=${width}`,
`${sourcePath}[0]`,
'-resize',
width,
@@ -51,16 +49,14 @@ async function thumbnail(sourcePath) {
}
async function preview(sourcePath) {
- const width = '548x'
- return await convert(sourcePath, 'png', [
+ const width = '794x'
+ return await _convert(sourcePath, 'jpg', [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
- '-define',
- `pdf:fit-page=${width}`,
`${sourcePath}[0]`,
'-resize',
width,
diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js
index b52418f07d..0c092c85cd 100644
--- a/services/filestore/app/js/FileHandler.js
+++ b/services/filestore/app/js/FileHandler.js
@@ -1,7 +1,7 @@
const Settings = require('@overleaf/settings')
-const { callbackify } = require('util')
-const fs = require('fs')
-const PersistorManager = require('./PersistorManager')
+const { callbackify } = require('node:util')
+const fs = require('node:fs')
+let PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
@@ -10,6 +10,7 @@ const { ConversionError, InvalidParametersError } = require('./Errors')
const metrics = require('@overleaf/metrics')
module.exports = {
+ copyObject: callbackify(copyObject),
insertFile: callbackify(insertFile),
deleteFile: callbackify(deleteFile),
deleteProject: callbackify(deleteProject),
@@ -18,6 +19,7 @@ module.exports = {
getFileSize: callbackify(getFileSize),
getDirectorySize: callbackify(getDirectorySize),
promises: {
+ copyObject,
getFile,
getRedirectUrl,
insertFile,
@@ -28,6 +30,16 @@ module.exports = {
},
}
+if (process.env.NODE_ENV === 'test') {
+ module.exports._TESTONLYSwapPersistorManager = _PersistorManager => {
+ PersistorManager = _PersistorManager
+ }
+}
+
+async function copyObject(bucket, sourceKey, destinationKey) {
+ await PersistorManager.copyObject(bucket, sourceKey, destinationKey)
+}
+
async function insertFile(bucket, key, stream) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
@@ -37,9 +49,6 @@ async function insertFile(bucket, key, stream) {
convertedKey,
})
}
- if (Settings.enableConversions) {
- await PersistorManager.deleteDirectory(bucket, convertedKey)
- }
await PersistorManager.sendStream(bucket, key, stream)
}
@@ -53,7 +62,10 @@ async function deleteFile(bucket, key) {
})
}
const jobs = [PersistorManager.deleteObject(bucket, key)]
- if (Settings.enableConversions) {
+ if (
+ Settings.enableConversions &&
+ bucket === Settings.filestore.stores.template_files
+ ) {
jobs.push(PersistorManager.deleteDirectory(bucket, convertedKey))
}
await Promise.all(jobs)
@@ -118,7 +130,7 @@ async function getFileSize(bucket, key) {
}
async function getDirectorySize(bucket, projectId) {
- return PersistorManager.directorySize(bucket, projectId)
+ return await PersistorManager.directorySize(bucket, projectId)
}
async function _getConvertedFile(bucket, key, opts) {
@@ -138,7 +150,9 @@ async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) {
let convertedFsPath
try {
convertedFsPath = await _convertFile(bucket, key, opts)
- await ImageOptimiser.promises.compressPng(convertedFsPath)
+ if (convertedFsPath.toLowerCase().endsWith(".png")) {
+ await ImageOptimiser.promises.compressPng(convertedFsPath)
+ }
await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath)
} catch (err) {
LocalFileWriter.deleteFile(convertedFsPath, () => {})
diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js
index fb296513b5..e9b739a971 100644
--- a/services/filestore/app/js/HealthCheckController.js
+++ b/services/filestore/app/js/HealthCheckController.js
@@ -1,9 +1,9 @@
-const fs = require('fs')
-const path = require('path')
+const fs = require('node:fs')
+const path = require('node:path')
const Settings = require('@overleaf/settings')
const { WritableBuffer } = require('@overleaf/stream-utils')
-const { promisify } = require('util')
-const Stream = require('stream')
+const { promisify } = require('node:util')
+const Stream = require('node:stream')
const pipeline = promisify(Stream.pipeline)
const fsCopy = promisify(fs.copyFile)
diff --git a/services/filestore/app/js/ImageOptimiser.js b/services/filestore/app/js/ImageOptimiser.js
index a46a2857e1..6ed29e1c6d 100644
--- a/services/filestore/app/js/ImageOptimiser.js
+++ b/services/filestore/app/js/ImageOptimiser.js
@@ -1,6 +1,6 @@
const logger = require('@overleaf/logger')
const metrics = require('@overleaf/metrics')
-const { callbackify } = require('util')
+const { callbackify } = require('node:util')
const safeExec = require('./SafeExec').promises
module.exports = {
diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js
index c5b8ea41f1..fe55bdc138 100644
--- a/services/filestore/app/js/LocalFileWriter.js
+++ b/services/filestore/app/js/LocalFileWriter.js
@@ -1,8 +1,8 @@
-const fs = require('fs')
-const crypto = require('crypto')
-const path = require('path')
-const Stream = require('stream')
-const { callbackify, promisify } = require('util')
+const fs = require('node:fs')
+const crypto = require('node:crypto')
+const path = require('node:path')
+const Stream = require('node:stream')
+const { callbackify, promisify } = require('node:util')
const metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
const { WriteError } = require('./Errors')
diff --git a/services/filestore/app/js/SafeExec.js b/services/filestore/app/js/SafeExec.js
index 63177b8057..16ebcf126b 100644
--- a/services/filestore/app/js/SafeExec.js
+++ b/services/filestore/app/js/SafeExec.js
@@ -1,5 +1,5 @@
const lodashOnce = require('lodash.once')
-const childProcess = require('child_process')
+const childProcess = require('node:child_process')
const Settings = require('@overleaf/settings')
const { ConversionsDisabledError, FailedCommandError } = require('./Errors')
diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt
index 147e8b4a46..bd4d2116f6 100644
--- a/services/filestore/buildscript.txt
+++ b/services/filestore/buildscript.txt
@@ -2,11 +2,11 @@ filestore
--data-dirs=uploads,user_files,template_files
--dependencies=s3,gcs
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,GCS_USER_FILES_BUCKET_NAME=fake_userfiles,GCS_TEMPLATE_FILES_BUCKET_NAME=fake_templatefiles
+--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=True
---script-version=4.5.0
+--script-version=4.7.0
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
--use-large-ci-runner=True
diff --git a/services/filestore/config/settings.defaults.js b/services/filestore/config/settings.defaults.js
index 08fa66408f..9a08bb197e 100644
--- a/services/filestore/config/settings.defaults.js
+++ b/services/filestore/config/settings.defaults.js
@@ -1,4 +1,4 @@
-const Path = require('path')
+const Path = require('node:path')
// environment variables renamed for consistency
// use AWS_ACCESS_KEY_ID-style going forward
@@ -73,6 +73,10 @@ const settings = {
stores: {
user_files: process.env.USER_FILES_BUCKET_NAME,
template_files: process.env.TEMPLATE_FILES_BUCKET_NAME,
+
+ // allow signed links to be generated for these buckets
+ project_blobs: process.env.OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET,
+ global_blobs: process.env.OVERLEAF_EDITOR_BLOBS_BUCKET,
},
fallback: process.env.FALLBACK_BACKEND
@@ -99,10 +103,6 @@ const settings = {
enableConversions: process.env.ENABLE_CONVERSIONS === 'true',
- sentry: {
- dsn: process.env.SENTRY_DSN,
- },
-
gracefulShutdownDelayInMs:
parseInt(process.env.GRACEFUL_SHUTDOWN_DELAY_SECONDS ?? '30', 10) * 1000,
}
diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml
index aaa11bea6b..fdf860b511 100644
--- a/services/filestore/docker-compose.ci.yml
+++ b/services/filestore/docker-compose.ci.yml
@@ -21,10 +21,12 @@ services:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: http://s3:9090
+ AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: fake
- AWS_SECRET_ACCESS_KEY: fake
+ AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@@ -33,13 +35,21 @@ services:
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
- AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
- AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
- GCS_USER_FILES_BUCKET_NAME: fake_userfiles
- GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+ AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
+ AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
+ AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
+ AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
+ GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
+ GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
+ volumes:
+ - ./test/acceptance/certs:/certs
depends_on:
- s3:
- condition: service_healthy
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ minio_setup:
+ condition: service_completed_successfully
gcs:
condition: service_healthy
user: node
@@ -53,14 +63,137 @@ services:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
- s3:
- image: adobe/s3mock:2.4.14
+ certs:
+ image: node:22.17.0
+ volumes:
+ - ./test/acceptance/certs:/certs
+ working_dir: /certs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ if [ ! -f ./certgen ]; then
+ wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+ chmod +x ./certgen
+ fi
+ if [ ! -f private.key ] || [ ! -f public.crt ]; then
+ ./certgen -host minio
+ fi
+
+ minio:
+ image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+ command: server /data
+ volumes:
+ - ./test/acceptance/certs:/root/.minio/certs
environment:
- - initialBuckets=fake_user_files,fake_template_files,bucket
- healthcheck:
- test: wget --quiet --output-document=/dev/null http://localhost:9090
- interval: 1s
- retries: 20
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+
+ minio_setup:
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+ volumes:
+ - ./test/acceptance/certs:/root/.mc/certs/CAs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ sleep 1
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+ mc mb --ignore-existing s3/fake-user-files
+ mc mb --ignore-existing s3/fake-user-files-dek
+ mc mb --ignore-existing s3/fake-template-files
+ mc admin user add s3 \
+ OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+ OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+ echo '
+ {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files-dek"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files-dek/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-template-files"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-template-files/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::random-bucket-*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::random-bucket-*"
+ }
+ ]
+ }' > policy-filestore.json
+
+ mc admin policy create s3 overleaf-filestore policy-filestore.json
+ mc admin policy attach s3 overleaf-filestore \
+ --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml
index 427384e452..971d35b708 100644
--- a/services/filestore/docker-compose.yml
+++ b/services/filestore/docker-compose.yml
@@ -17,6 +17,7 @@ services:
working_dir: /overleaf/services/filestore
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
@@ -31,44 +32,176 @@ services:
- .:/overleaf/services/filestore
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ./test/acceptance/certs:/certs
working_dir: /overleaf/services/filestore
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: http://s3:9090
+ AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: fake
- AWS_SECRET_ACCESS_KEY: fake
+ AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
- AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
- AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
- GCS_USER_FILES_BUCKET_NAME: fake_userfiles
- GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
+ AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
+ AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
+ AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
+ AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
+ GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
+ GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
user: node
depends_on:
- s3:
- condition: service_healthy
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ minio_setup:
+ condition: service_completed_successfully
gcs:
condition: service_healthy
command: npm run --silent test:acceptance
- s3:
- image: adobe/s3mock:2.4.14
+ certs:
+ image: node:22.17.0
+ volumes:
+ - ./test/acceptance/certs:/certs
+ working_dir: /certs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ if [ ! -f ./certgen ]; then
+ wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+ chmod +x ./certgen
+ fi
+ if [ ! -f private.key ] || [ ! -f public.crt ]; then
+ ./certgen -host minio
+ fi
+
+ minio:
+ image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+ command: server /data
+ volumes:
+ - ./test/acceptance/certs:/root/.minio/certs
environment:
- - initialBuckets=fake_user_files,fake_template_files,bucket
- healthcheck:
- test: wget --quiet --output-document=/dev/null http://localhost:9090
- interval: 1s
- retries: 20
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+
+ minio_setup:
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+ volumes:
+ - ./test/acceptance/certs:/root/.mc/certs/CAs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ sleep 1
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+ mc mb --ignore-existing s3/fake-user-files
+ mc mb --ignore-existing s3/fake-user-files-dek
+ mc mb --ignore-existing s3/fake-template-files
+ mc admin user add s3 \
+ OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
+ OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
+
+ echo '
+ {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files-dek"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-user-files-dek/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::fake-template-files"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::fake-template-files/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::random-bucket-*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::random-bucket-*"
+ }
+ ]
+ }' > policy-filestore.json
+
+ mc admin policy create s3 overleaf-filestore policy-filestore.json
+ mc admin policy attach s3 overleaf-filestore \
+ --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/filestore/package.json b/services/filestore/package.json
index e680f54b2b..4b9043aed7 100644
--- a/services/filestore/package.json
+++ b/services/filestore/package.json
@@ -11,8 +11,8 @@
"start": "node app.js",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"lint:fix": "eslint --fix .",
@@ -27,7 +27,7 @@
"@overleaf/stream-utils": "^0.1.0",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "express": "^4.21.0",
+ "express": "^4.21.2",
"glob": "^7.1.6",
"lodash.once": "^4.1.1",
"node-fetch": "^2.7.0",
@@ -36,17 +36,14 @@
},
"devDependencies": {
"@google-cloud/storage": "^6.10.1",
- "aws-sdk": "^2.718.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "disrequire": "^1.1.0",
- "mocha": "^10.2.0",
- "mongodb": "^6.1.0",
+ "mocha": "^11.1.0",
+ "mongodb": "6.12.0",
"sandboxed-module": "2.0.4",
"sinon": "9.0.2",
"sinon-chai": "^3.7.0",
"streamifier": "^0.1.1",
- "timekeeper": "^2.2.0",
"typescript": "^5.0.4"
}
}
diff --git a/services/filestore/test/acceptance/certs/.gitignore b/services/filestore/test/acceptance/certs/.gitignore
new file mode 100644
index 0000000000..d6b7ef32c8
--- /dev/null
+++ b/services/filestore/test/acceptance/certs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/services/filestore/test/acceptance/deps/healthcheck.sh b/services/filestore/test/acceptance/deps/healthcheck.sh
index cd19cea637..675c205be6 100755
--- a/services/filestore/test/acceptance/deps/healthcheck.sh
+++ b/services/filestore/test/acceptance/deps/healthcheck.sh
@@ -1,9 +1,9 @@
#!/bin/sh
# health check to allow 404 status code as valid
-STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1)
+STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" "$1")
# will be 000 on non-http error (e.g. connection failure)
-if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then
+if test "$STATUSCODE" -ge 500 || test "$STATUSCODE" -lt 200; then
exit 1
fi
exit 0
diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js
index 924f852eea..61e9a29b7d 100644
--- a/services/filestore/test/acceptance/js/FilestoreApp.js
+++ b/services/filestore/test/acceptance/js/FilestoreApp.js
@@ -1,64 +1,31 @@
-const logger = require('@overleaf/logger')
+const ObjectPersistor = require('@overleaf/object-persistor')
const Settings = require('@overleaf/settings')
-const fs = require('fs')
-const Path = require('path')
-const { promisify } = require('util')
-const disrequire = require('disrequire')
-const AWS = require('aws-sdk')
-
-logger.logger.level('info')
-
-const fsReaddir = promisify(fs.readdir)
-const sleep = promisify(setTimeout)
+const { promisify } = require('node:util')
+const App = require('../../../app')
+const FileHandler = require('../../../app/js/FileHandler')
class FilestoreApp {
- constructor() {
- this.running = false
- this.initing = false
- }
-
async runServer() {
- if (this.running) {
- return
- }
-
- if (this.initing) {
- return await this.waitForInit()
- }
- this.initing = true
-
- this.app = await FilestoreApp.requireApp()
-
- await new Promise((resolve, reject) => {
- this.server = this.app.listen(
- Settings.internal.filestore.port,
- '127.0.0.1',
- err => {
- if (err) {
- return reject(err)
+ if (!this.server) {
+ await new Promise((resolve, reject) => {
+ this.server = App.listen(
+ Settings.internal.filestore.port,
+ '127.0.0.1',
+ err => {
+ if (err) {
+ return reject(err)
+ }
+ resolve()
}
- resolve()
- }
- )
+ )
+ })
+ }
+
+ this.persistor = ObjectPersistor({
+ ...Settings.filestore,
+ paths: Settings.path,
})
-
- if (Settings.filestore.backend === 's3') {
- try {
- await FilestoreApp.waitForS3()
- } catch (err) {
- await this.stop()
- throw err
- }
- }
-
- this.initing = false
- this.persistor = require('../../../app/js/PersistorManager')
- }
-
- async waitForInit() {
- while (this.initing) {
- await sleep(1000)
- }
+ FileHandler._TESTONLYSwapPersistorManager(this.persistor)
}
async stop() {
@@ -70,54 +37,6 @@ class FilestoreApp {
delete this.server
}
}
-
- static async waitForS3() {
- let tries = 0
- if (!Settings.filestore.s3.endpoint) {
- return
- }
-
- const s3 = new AWS.S3({
- accessKeyId: Settings.filestore.s3.key,
- secretAccessKey: Settings.filestore.s3.secret,
- endpoint: Settings.filestore.s3.endpoint,
- s3ForcePathStyle: true,
- signatureVersion: 'v4',
- })
-
- while (true) {
- try {
- return await s3
- .putObject({
- Key: 'startup',
- Body: '42',
- Bucket: Settings.filestore.stores.user_files,
- })
- .promise()
- } catch (err) {
- // swallow errors, as we may experience them until fake-s3 is running
- if (tries === 9) {
- // throw just before hitting the 10s test timeout
- throw err
- }
- tries++
- await sleep(1000)
- }
- }
- }
-
- static async requireApp() {
- // unload the app, as we may be doing this on multiple runs with
- // different settings, which affect startup in some cases
- const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js'))
- files.forEach(file => {
- disrequire(Path.resolve(__dirname, '../../../app/js', file))
- })
- disrequire(Path.resolve(__dirname, '../../../app'))
- disrequire('@overleaf/object-persistor')
-
- return require('../../../app')
- }
}
module.exports = FilestoreApp
diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js
index 441a179c59..28f90d49b6 100644
--- a/services/filestore/test/acceptance/js/FilestoreTests.js
+++ b/services/filestore/test/acceptance/js/FilestoreTests.js
@@ -1,19 +1,18 @@
const chai = require('chai')
const { expect } = chai
-const fs = require('fs')
+const fs = require('node:fs')
+const Stream = require('node:stream')
const Settings = require('@overleaf/settings')
-const Path = require('path')
+const Path = require('node:path')
const FilestoreApp = require('./FilestoreApp')
const TestHelper = require('./TestHelper')
const fetch = require('node-fetch')
-const S3 = require('aws-sdk/clients/s3')
-const { promisify } = require('util')
+const { promisify } = require('node:util')
const { Storage } = require('@google-cloud/storage')
const streamifier = require('streamifier')
chai.use(require('chai-as-promised'))
const { ObjectId } = require('mongodb')
-const tk = require('timekeeper')
-const ChildProcess = require('child_process')
+const ChildProcess = require('node:child_process')
const fsWriteFile = promisify(fs.writeFile)
const fsStat = promisify(fs.stat)
@@ -32,7 +31,26 @@ process.on('unhandledRejection', e => {
// store settings for multiple backends, so that we can test each one.
// fs will always be available - add others if they are configured
-const BackendSettings = require('./TestConfig')
+const {
+ BackendSettings,
+ s3Config,
+ s3SSECConfig,
+ AWS_S3_USER_FILES_STORAGE_CLASS,
+} = require('./TestConfig')
+const {
+ AlreadyWrittenError,
+ NotFoundError,
+ NotImplementedError,
+ NoKEKMatchedError,
+} = require('@overleaf/object-persistor/src/Errors')
+const {
+ PerProjectEncryptedS3Persistor,
+ RootKeyEncryptionKey,
+} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
+const { S3Persistor } = require('@overleaf/object-persistor/src/S3Persistor')
+const crypto = require('node:crypto')
+const { WritableBuffer } = require('@overleaf/stream-utils')
+const { gzipSync } = require('node:zlib')
describe('Filestore', function () {
this.timeout(1000 * 10)
@@ -77,8 +95,10 @@ describe('Filestore', function () {
}
// redefine the test suite for every available backend
- Object.keys(BackendSettings).forEach(backend => {
- describe(backend, function () {
+ for (const [backendVariantWithShardNumber, backendSettings] of Object.entries(
+ BackendSettings
+ )) {
+ describe(backendVariantWithShardNumber, function () {
let app,
previousEgress,
previousIngress,
@@ -86,6 +106,9 @@ describe('Filestore', function () {
projectId,
otherProjectId
+ const dataEncryptionKeySize =
+ backendSettings.backend === 's3SSEC' ? 32 : 0
+
const BUCKET_NAMES = [
process.env.GCS_USER_FILES_BUCKET_NAME,
process.env.GCS_TEMPLATE_FILES_BUCKET_NAME,
@@ -93,15 +116,15 @@ describe('Filestore', function () {
`${process.env.GCS_TEMPLATE_FILES_BUCKET_NAME}-deleted`,
]
- before(async function () {
+ before('start filestore with new settings', async function () {
// create the app with the relevant filestore settings
- Settings.filestore = BackendSettings[backend]
+ Settings.filestore = backendSettings
app = new FilestoreApp()
await app.runServer()
})
- if (BackendSettings[backend].gcs) {
- before(async function () {
+ if (backendSettings.gcs) {
+ before('create gcs buckets', async function () {
// create test buckets for gcs
const storage = new Storage(Settings.filestore.gcs.endpoint)
for (const bucketName of BUCKET_NAMES) {
@@ -109,7 +132,7 @@ describe('Filestore', function () {
}
})
- after(async function () {
+ after('delete gcs buckets', async function () {
// tear down all the gcs buckets
const storage = new Storage(Settings.filestore.gcs.endpoint)
for (const bucketName of BUCKET_NAMES) {
@@ -120,15 +143,14 @@ describe('Filestore', function () {
})
}
- after(async function () {
- await msleep(3000)
+ after('stop filestore app', async function () {
await app.stop()
})
- beforeEach(async function () {
+ beforeEach('fetch previous egress metric', async function () {
// retrieve previous metrics from the app
- if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
- metricPrefix = Settings.filestore.backend
+ if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
+ metricPrefix = Settings.filestore.backend.replace('SSEC', '')
previousEgress = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
@@ -152,7 +174,7 @@ describe('Filestore', function () {
const localFileReadPath =
'/tmp/filestore_acceptance_tests_file_read.txt'
- beforeEach(async function () {
+ beforeEach('upload file', async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
constantFileContent = [
@@ -164,14 +186,15 @@ describe('Filestore', function () {
await fsWriteFile(localFileReadPath, constantFileContent)
const readStream = fs.createReadStream(localFileReadPath)
- await fetch(fileUrl, { method: 'POST', body: readStream })
+ const res = await fetch(fileUrl, { method: 'POST', body: readStream })
+ if (!res.ok) throw new Error(res.statusText)
})
- beforeEach(async function retrievePreviousIngressMetrics() {
+ beforeEach('retrieve previous ingress metric', async function () {
// The upload request can bump the ingress metric.
// The content hash validation might require a full download
// in case the ETag field of the upload response is not a md5 sum.
- if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
+ if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
previousIngress = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
@@ -212,7 +235,9 @@ describe('Filestore', function () {
})
it('should not leak a socket', async function () {
- await fetch(fileUrl)
+ const res = await fetch(fileUrl)
+ if (!res.ok) throw new Error(res.statusText)
+ await res.text()
await expectNoSockets()
})
@@ -269,24 +294,61 @@ describe('Filestore', function () {
expect(body).to.equal(newContent)
})
- if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
+ describe('IfNoneMatch', function () {
+ if (backendSettings.backend === 'fs') {
+ it('should refuse to handle IfNoneMatch', async function () {
+ await expect(
+ app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${fileId}`,
+ fs.createReadStream(localFileReadPath),
+ { ifNoneMatch: '*' }
+ )
+ ).to.be.rejectedWith(NotImplementedError)
+ })
+ } else {
+ it('should reject sendStream on the same key with IfNoneMatch', async function () {
+ await expect(
+ app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${fileId}`,
+ fs.createReadStream(localFileReadPath),
+ { ifNoneMatch: '*' }
+ )
+ ).to.be.rejectedWith(AlreadyWrittenError)
+ })
+ it('should allow sendStream on a different key with IfNoneMatch', async function () {
+ await app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${fileId}-other`,
+ fs.createReadStream(localFileReadPath),
+ { ifNoneMatch: '*' }
+ )
+ })
+ }
+ })
+
+ if (backendSettings.backend !== 'fs') {
it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
)
- expect(metric - previousEgress).to.equal(constantFileContent.length)
+ expect(metric - previousEgress).to.equal(
+ constantFileContent.length + dataEncryptionKeySize
+ )
})
it('should record an ingress metric when downloading the file', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
+ await response.text()
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
)
expect(metric - previousIngress).to.equal(
- constantFileContent.length
+ constantFileContent.length + dataEncryptionKeySize
)
})
@@ -295,11 +357,12 @@ describe('Filestore', function () {
headers: { Range: 'bytes=0-8' },
})
expect(response.ok).to.be.true
+ await response.text()
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
)
- expect(metric - previousIngress).to.equal(9)
+ expect(metric - previousIngress).to.equal(9 + dataEncryptionKeySize)
})
}
})
@@ -329,7 +392,7 @@ describe('Filestore', function () {
].join('\n'),
]
- before(async function () {
+ before('create local files', async function () {
return await Promise.all([
fsWriteFile(localFileReadPaths[0], constantFileContents[0]),
fsWriteFile(localFileReadPaths[1], constantFileContents[1]),
@@ -337,7 +400,7 @@ describe('Filestore', function () {
])
})
- beforeEach(async function () {
+ beforeEach('upload two files', async function () {
projectUrl = `${filestoreUrl}/project/${projectId}`
otherProjectUrl = `${filestoreUrl}/project/${otherProjectId}`
fileIds = [
@@ -411,9 +474,10 @@ describe('Filestore', function () {
})
describe('with a large file', function () {
+ this.timeout(1000 * 20)
let fileId, fileUrl, largeFileContent, error
- beforeEach(async function () {
+ beforeEach('upload large file', async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
@@ -421,7 +485,8 @@ describe('Filestore', function () {
largeFileContent += Math.random()
const readStream = streamifier.createReadStream(largeFileContent)
- await fetch(fileUrl, { method: 'POST', body: readStream })
+ const res = await fetch(fileUrl, { method: 'POST', body: readStream })
+ if (!res.ok) throw new Error(res.statusText)
})
it('should be able to get the file back', async function () {
@@ -449,27 +514,25 @@ describe('Filestore', function () {
})
})
- if (backend === 'S3Persistor' || backend === 'FallbackGcsToS3Persistor') {
+ if (
+ (backendSettings.backend === 's3' && !backendSettings.fallback) ||
+ (backendSettings.backend === 'gcs' &&
+ backendSettings.fallback?.backend === 's3')
+ ) {
describe('with a file in a specific bucket', function () {
let constantFileContent, fileId, fileUrl, bucketName
- beforeEach(async function () {
+ beforeEach('upload file into random bucket', async function () {
constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
fileId = new ObjectId().toString()
- bucketName = new ObjectId().toString()
+ bucketName = `random-bucket-${new ObjectId().toString()}`
fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
- const s3ClientSettings = {
- credentials: {
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
- secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
- },
- endpoint: process.env.AWS_S3_ENDPOINT,
- sslEnabled: false,
- s3ForcePathStyle: true,
- }
-
- const s3 = new S3(s3ClientSettings)
+ const s3 = new S3Persistor({
+ ...s3Config(),
+ key: process.env.MINIO_ROOT_USER,
+ secret: process.env.MINIO_ROOT_PASSWORD,
+ })._getClientForBucket(bucketName)
await s3
.createBucket({
Bucket: bucketName,
@@ -492,25 +555,23 @@ describe('Filestore', function () {
})
}
- if (backend === 'GcsPersistor') {
+ if (backendSettings.backend === 'gcs') {
describe('when deleting a file in GCS', function () {
- let fileId, fileUrl, content, error, date
+ let fileId, fileUrl, content, error, dateBefore, dateAfter
- beforeEach(async function () {
- date = new Date()
- tk.freeze(date)
+ beforeEach('upload and delete file', async function () {
fileId = new ObjectId()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
content = '_wombat_' + Math.random()
const readStream = streamifier.createReadStream(content)
- await fetch(fileUrl, { method: 'POST', body: readStream })
- await fetch(fileUrl, { method: 'DELETE' })
- })
-
- afterEach(function () {
- tk.reset()
+ let res = await fetch(fileUrl, { method: 'POST', body: readStream })
+ if (!res.ok) throw new Error(res.statusText)
+ dateBefore = new Date()
+ res = await fetch(fileUrl, { method: 'DELETE' })
+ dateAfter = new Date()
+ if (!res.ok) throw new Error(res.statusText)
})
it('should not throw an error', function () {
@@ -518,10 +579,16 @@ describe('Filestore', function () {
})
it('should copy the file to the deleted-files bucket', async function () {
- await TestHelper.expectPersistorToHaveFile(
+ let date = dateBefore
+ const keys = []
+ while (date <= dateAfter) {
+ keys.push(`${projectId}/${fileId}-${date.toISOString()}`)
+ date = new Date(date.getTime() + 1)
+ }
+ await TestHelper.expectPersistorToHaveSomeFile(
app.persistor,
`${Settings.filestore.stores.user_files}-deleted`,
- `${projectId}/${fileId}-${date.toISOString()}`,
+ keys,
content
)
})
@@ -536,7 +603,7 @@ describe('Filestore', function () {
})
}
- if (BackendSettings[backend].fallback) {
+ if (backendSettings.fallback) {
describe('with a fallback', function () {
let constantFileContent,
fileId,
@@ -545,7 +612,7 @@ describe('Filestore', function () {
bucket,
fallbackBucket
- beforeEach(function () {
+ beforeEach('prepare fallback', function () {
constantFileContent = `This is yet more file content ${Math.random()}`
fileId = new ObjectId().toString()
fileKey = `${projectId}/${fileId}`
@@ -556,7 +623,7 @@ describe('Filestore', function () {
})
describe('with a file in the fallback bucket', function () {
- beforeEach(async function () {
+ beforeEach('upload into fallback', async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor,
fallbackBucket,
@@ -583,7 +650,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is disabled', function () {
- beforeEach(function () {
+ beforeEach('swap copyOnMiss=false', function () {
app.persistor.settings.copyOnMiss = false
})
@@ -596,6 +663,7 @@ describe('Filestore', function () {
it('should not copy the file to the primary', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
+ await response.text()
await TestHelper.expectPersistorNotToHaveFile(
app.persistor.primaryPersistor,
@@ -606,7 +674,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is enabled', function () {
- beforeEach(function () {
+ beforeEach('swap copyOnMiss=true', function () {
app.persistor.settings.copyOnMiss = true
})
@@ -619,6 +687,7 @@ describe('Filestore', function () {
it('copies the file to the primary', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
+ await response.text()
// wait for the file to copy in the background
await msleep(1000)
@@ -634,7 +703,7 @@ describe('Filestore', function () {
describe('when copying a file', function () {
let newFileId, newFileUrl, newFileKey, opts
- beforeEach(function () {
+ beforeEach('prepare to copy file', function () {
const newProjectID = new ObjectId().toString()
newFileId = new ObjectId().toString()
newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
@@ -655,7 +724,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is false', function () {
- beforeEach(async function () {
+ beforeEach('copy with copyOnMiss=false', async function () {
app.persistor.settings.copyOnMiss = false
const response = await fetch(newFileUrl, opts)
@@ -701,7 +770,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is true', function () {
- beforeEach(async function () {
+ beforeEach('copy with copyOnMiss=false', async function () {
app.persistor.settings.copyOnMiss = true
const response = await fetch(newFileUrl, opts)
@@ -750,10 +819,14 @@ describe('Filestore', function () {
})
describe('when sending a file', function () {
- beforeEach(async function () {
+ beforeEach('upload file', async function () {
const readStream =
streamifier.createReadStream(constantFileContent)
- await fetch(fileUrl, { method: 'POST', body: readStream })
+ const res = await fetch(fileUrl, {
+ method: 'POST',
+ body: readStream,
+ })
+ if (!res.ok) throw new Error(res.statusText)
})
it('should store the file on the primary', async function () {
@@ -776,7 +849,7 @@ describe('Filestore', function () {
describe('when deleting a file', function () {
describe('when the file exists on the primary', function () {
- beforeEach(async function () {
+ beforeEach('upload into primary', async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.primaryPersistor,
bucket,
@@ -794,7 +867,7 @@ describe('Filestore', function () {
})
describe('when the file exists on the fallback', function () {
- beforeEach(async function () {
+ beforeEach('upload into fallback', async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor,
fallbackBucket,
@@ -812,20 +885,23 @@ describe('Filestore', function () {
})
describe('when the file exists on both the primary and the fallback', function () {
- beforeEach(async function () {
- await TestHelper.uploadStringToPersistor(
- app.persistor.primaryPersistor,
- bucket,
- fileKey,
- constantFileContent
- )
- await TestHelper.uploadStringToPersistor(
- app.persistor.fallbackPersistor,
- fallbackBucket,
- fileKey,
- constantFileContent
- )
- })
+ beforeEach(
+ 'upload into both primary and fallback',
+ async function () {
+ await TestHelper.uploadStringToPersistor(
+ app.persistor.primaryPersistor,
+ bucket,
+ fileKey,
+ constantFileContent
+ )
+ await TestHelper.uploadStringToPersistor(
+ app.persistor.fallbackPersistor,
+ fallbackBucket,
+ fileKey,
+ constantFileContent
+ )
+ }
+ )
it('should delete the files', async function () {
const response1 = await fetch(fileUrl, { method: 'DELETE' })
@@ -854,13 +930,14 @@ describe('Filestore', function () {
'../../fixtures/test.pdf'
)
- beforeEach(async function () {
+ beforeEach('upload test.pdf', async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
const stat = await fsStat(localFileReadPath)
localFileSize = stat.size
const readStream = fs.createReadStream(localFileReadPath)
- await fetch(fileUrl, { method: 'POST', body: readStream })
+ const res = await fetch(fileUrl, { method: 'POST', body: readStream })
+ if (!res.ok) throw new Error(res.statusText)
})
it('should be able get the file back', async function () {
@@ -869,13 +946,15 @@ describe('Filestore', function () {
expect(body.substring(0, 8)).to.equal('%PDF-1.5')
})
- if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
+ if (backendSettings.backend !== 'fs') {
it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
)
- expect(metric - previousEgress).to.equal(localFileSize)
+ expect(metric - previousEgress).to.equal(
+ localFileSize + dataEncryptionKeySize
+ )
})
}
@@ -883,18 +962,20 @@ describe('Filestore', function () {
this.timeout(1000 * 20)
let previewFileUrl
- beforeEach(function () {
+ beforeEach('prepare previewFileUrl for preview', function () {
previewFileUrl = `${fileUrl}?style=preview`
})
it('should not time out', async function () {
const response = await fetch(previewFileUrl)
expect(response.status).to.equal(200)
+ await response.arrayBuffer()
})
it('should respond with image data', async function () {
// note: this test relies of the imagemagick conversion working
const response = await fetch(previewFileUrl)
+ expect(response.status).to.equal(200)
const body = await response.text()
expect(body.length).to.be.greaterThan(400)
expect(body.substr(1, 3)).to.equal('PNG')
@@ -905,20 +986,23 @@ describe('Filestore', function () {
this.timeout(1000 * 20)
let previewFileUrl
- beforeEach(function () {
+ beforeEach('prepare previewFileUrl for cacheWarn', function () {
previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
})
it('should not time out', async function () {
const response = await fetch(previewFileUrl)
expect(response.status).to.equal(200)
+ await response.arrayBuffer()
})
it('should not leak sockets', async function () {
const response1 = await fetch(previewFileUrl)
expect(response1.status).to.equal(200)
+ // do not read the response body, should be destroyed immediately
const response2 = await fetch(previewFileUrl)
expect(response2.status).to.equal(200)
+ // do not read the response body, should be destroyed immediately
await expectNoSockets()
})
@@ -930,6 +1014,551 @@ describe('Filestore', function () {
})
})
})
+
+ describe('with server side encryption', function () {
+ if (backendSettings.backend !== 's3SSEC') return
+
+ before('sanity check top-level variable', function () {
+ expect(dataEncryptionKeySize).to.equal(32)
+ })
+
+ let fileId1,
+ fileId2,
+ fileKey1,
+ fileKey2,
+ fileKeyOtherProject,
+ fileUrl1,
+ fileUrl2
+ beforeEach('prepare ids', function () {
+ fileId1 = new ObjectId().toString()
+ fileId2 = new ObjectId().toString()
+ fileKey1 = `${projectId}/${fileId1}`
+ fileKey2 = `${projectId}/${fileId2}`
+ fileKeyOtherProject = `${new ObjectId().toString()}/${new ObjectId().toString()}`
+ fileUrl1 = `${filestoreUrl}/project/${projectId}/file/${fileId1}`
+ fileUrl2 = `${filestoreUrl}/project/${projectId}/file/${fileId2}`
+ })
+
+ beforeEach('ensure DEK is missing', async function () {
+ // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
+ await expect(
+ app.persistor.getDataEncryptionKeySize(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ ).to.rejectedWith(NotFoundError)
+ })
+
+ async function createRandomContent(url, suffix = '') {
+ const content = Math.random().toString() + suffix
+ const res = await fetch(url, {
+ method: 'POST',
+ body: Stream.Readable.from([content]),
+ })
+ if (!res.ok) throw new Error(res.statusText)
+ return async () => {
+ const res = await fetch(url, { method: 'GET' })
+ if (!res.ok) throw new Error(res.statusText)
+ expect(await res.text()).to.equal(content)
+ }
+ }
+
+ it('should create a DEK when asked explicitly', async function () {
+ await app.persistor.generateDataEncryptionKey(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ expect(
+ await app.persistor.getDataEncryptionKeySize(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ ).to.equal(32)
+ })
+
+ it('should create a DEK from writes', async function () {
+ await createRandomContent(fileUrl1)
+ expect(
+ await app.persistor.getDataEncryptionKeySize(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ ).to.equal(32)
+ })
+
+ it('should not create a DEK from reads', async function () {
+ const res = await fetch(fileUrl1, {
+ method: 'GET',
+ })
+ if (res.status !== 404) throw new Error(`${res.status} should be 404`)
+
+ // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
+ await expect(
+ app.persistor.getDataEncryptionKeySize(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ ).to.rejectedWith(NotFoundError)
+ })
+
+ it('should never overwrite a data encryption key', async function () {
+ const checkGET = await createRandomContent(fileUrl1)
+
+ await expect(
+ app.persistor.generateDataEncryptionKey(
+ backendSettings.stores.user_files,
+ fileKey1
+ )
+ ).to.rejectedWith(AlreadyWrittenError)
+
+ await checkGET()
+ })
+
+ it('should re-use the data encryption key after a write', async function () {
+ const checkGET1 = await createRandomContent(fileUrl1, '1')
+ const checkGET2 = await createRandomContent(fileUrl2, '2')
+ await checkGET1()
+ await checkGET2()
+ })
+
+ describe('kek rotation', function () {
+ const newKEK = new RootKeyEncryptionKey(
+ crypto.generateKeySync('aes', { length: 256 }).export(),
+ Buffer.alloc(32)
+ )
+ const oldKEK = new RootKeyEncryptionKey(
+ crypto.generateKeySync('aes', { length: 256 }).export(),
+ Buffer.alloc(32)
+ )
+ const migrationStep0 = new PerProjectEncryptedS3Persistor({
+ ...s3SSECConfig(),
+ automaticallyRotateDEKEncryption: false,
+ async getRootKeyEncryptionKeys() {
+ return [oldKEK] // only old key
+ },
+ })
+ const migrationStep1 = new PerProjectEncryptedS3Persistor({
+ ...s3SSECConfig(),
+ automaticallyRotateDEKEncryption: false,
+ async getRootKeyEncryptionKeys() {
+ return [oldKEK, newKEK] // new key as fallback
+ },
+ })
+ const migrationStep2 = new PerProjectEncryptedS3Persistor({
+ ...s3SSECConfig(),
+ automaticallyRotateDEKEncryption: true, // <- different compared to partiallyRotated
+ async getRootKeyEncryptionKeys() {
+ return [newKEK, oldKEK] // old keys as fallback
+ },
+ })
+ const migrationStep3 = new PerProjectEncryptedS3Persistor({
+ ...s3SSECConfig(),
+ automaticallyRotateDEKEncryption: true,
+ async getRootKeyEncryptionKeys() {
+ return [newKEK] // only new key
+ },
+ })
+
+ async function checkWrites(
+ fileKey,
+ writer,
+ readersSuccess,
+ readersFailed
+ ) {
+ const content = Math.random().toString()
+ await writer.sendStream(
+ Settings.filestore.stores.user_files,
+ fileKey,
+ Stream.Readable.from([content])
+ )
+
+ for (const persistor of readersSuccess) {
+ await TestHelper.expectPersistorToHaveFile(
+ persistor,
+ backendSettings.stores.user_files,
+ fileKey,
+ content
+ )
+ }
+
+ for (const persistor of readersFailed) {
+ await expect(
+ TestHelper.expectPersistorToHaveFile(
+ persistor,
+ backendSettings.stores.user_files,
+ fileKey,
+ content
+ )
+ ).to.be.rejectedWith(NoKEKMatchedError)
+ }
+ }
+
+ const stages = [
+ {
+ name: 'stage 0 - [old]',
+ prev: migrationStep0,
+ cur: migrationStep0,
+ fail: [migrationStep3],
+ },
+ {
+ name: 'stage 1 - [old,new]',
+ prev: migrationStep0,
+ cur: migrationStep1,
+ fail: [],
+ },
+ {
+ name: 'stage 2 - [new,old]',
+ prev: migrationStep1,
+ cur: migrationStep2,
+ fail: [],
+ },
+ {
+ name: 'stage 3 - [new]',
+ prev: migrationStep2,
+ cur: migrationStep3,
+ fail: [migrationStep0],
+ },
+ ]
+
+ for (const { name, prev, cur, fail } of stages) {
+ describe(name, function () {
+ this.timeout(1000 * 30)
+
+ it('can read old writes', async function () {
+ await checkWrites(fileKey1, prev, [prev, cur], fail)
+ await checkWrites(fileKey2, prev, [prev, cur], fail) // check again after access
+ await checkWrites(fileKeyOtherProject, prev, [prev, cur], fail)
+ })
+ it('can read new writes', async function () {
+ await checkWrites(fileKey1, prev, [prev, cur], fail)
+ await checkWrites(fileKey2, cur, [prev, cur], fail) // check again after access
+ await checkWrites(fileKeyOtherProject, cur, [prev, cur], fail)
+ })
+ })
+ }
+
+ describe('full migration', function () {
+ it('can read old writes if rotated in sequence', async function () {
+ await checkWrites(
+ fileKey1,
+ migrationStep0,
+ [
+ migrationStep0,
+ migrationStep1,
+ migrationStep2, // migrates
+ migrationStep3,
+ ],
+ []
+ )
+ })
+ it('cannot read/write if not rotated', async function () {
+ await checkWrites(
+ fileKey1,
+ migrationStep0,
+ [migrationStep0],
+ [migrationStep3]
+ )
+ })
+ })
+ })
+
+ /** @type {import('aws-sdk/clients/s3')} */
+ let s3Client
+ before('create s3 client', function () {
+ s3Client = new S3Persistor(s3Config())._getClientForBucket('')
+ })
+
+ async function checkDEKStorage({
+ dekBucketKeys = [],
+ userFilesBucketKeys = [],
+ }) {
+ await createRandomContent(fileUrl1)
+
+ const { Contents: dekEntries } = await s3Client
+ .listObjectsV2({
+ Bucket: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME,
+ Prefix: `${projectId}/`,
+ })
+ .promise()
+ expect(dekEntries).to.have.length(dekBucketKeys.length)
+ // Order is not predictable, use members
+ expect(dekEntries.map(o => o.Key)).to.have.members(dekBucketKeys)
+
+ const { Contents: userFilesEntries } = await s3Client
+ .listObjectsV2({
+ Bucket: backendSettings.stores.user_files,
+ Prefix: `${projectId}/`,
+ })
+ .promise()
+ expect(userFilesEntries).to.have.length(userFilesBucketKeys.length)
+ // Order is not predictable, use members
+ expect(userFilesEntries.map(o => o.Key)).to.have.members(
+ userFilesBucketKeys
+ )
+ }
+
+ it('should use a custom bucket for DEKs', async function () {
+ await checkDEKStorage({
+ dekBucketKeys: [`${projectId}/dek`],
+ userFilesBucketKeys: [fileKey1],
+ })
+ })
+
+ describe('deleteDirectory', function () {
+ let checkGET1, checkGET2
+ beforeEach('create files', async function () {
+ checkGET1 = await createRandomContent(fileUrl1, '1')
+ checkGET2 = await createRandomContent(fileUrl2, '2')
+ })
+ it('should refuse to delete top-level prefix', async function () {
+ await expect(
+ app.persistor.deleteDirectory(
+ Settings.filestore.stores.user_files,
+ projectId.slice(0, 3)
+ )
+ ).to.be.rejectedWith('not a project-folder')
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey1
+ )
+ ).to.equal(true)
+ await checkGET1()
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.equal(true)
+ expect(
+ await app.persistor.getDataEncryptionKeySize(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.equal(32)
+ await checkGET2()
+ })
+ it('should delete sub-folder and keep DEK', async function () {
+ await app.persistor.deleteDirectory(
+ Settings.filestore.stores.user_files,
+ fileKey1 // not really a sub-folder, but it will do for this test.
+ )
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey1
+ )
+ ).to.equal(false)
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.equal(true)
+ expect(
+ await app.persistor.getDataEncryptionKeySize(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.equal(32)
+ await checkGET2()
+ })
+ it('should delete project folder and DEK', async function () {
+ await app.persistor.deleteDirectory(
+ Settings.filestore.stores.user_files,
+ `${projectId}/`
+ )
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey1
+ )
+ ).to.equal(false)
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.equal(false)
+ await expect(
+ app.persistor.getDataEncryptionKeySize(
+ Settings.filestore.stores.user_files,
+ fileKey2
+ )
+ ).to.rejectedWith(NotFoundError)
+ })
+ })
+ })
+
+ describe('getObjectSize', function () {
+ it('should return a number', async function () {
+ const buf = Buffer.from('hello')
+ const fileId = new ObjectId().toString()
+ const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
+ const res = await fetch(fileUrl, {
+ method: 'POST',
+ body: Stream.Readable.from([buf]),
+ })
+ if (!res.ok) throw new Error(res.statusText)
+ expect(
+ await app.persistor.getObjectSize(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${fileId}`
+ )
+ ).to.equal(buf.byteLength)
+ })
+ })
+
+ describe('checkIfObjectExists', function () {
+ it('should return false when the object does not exist', async function () {
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${new ObjectId().toString()}`
+ )
+ ).to.equal(false)
+ })
+ it('should return true when the object exists', async function () {
+ const fileId = new ObjectId().toString()
+ const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
+ const res = await fetch(fileUrl, {
+ method: 'POST',
+ body: Stream.Readable.from(['hello']),
+ })
+ if (!res.ok) throw new Error(res.statusText)
+ expect(
+ await app.persistor.checkIfObjectExists(
+ Settings.filestore.stores.user_files,
+ `${projectId}/${fileId}`
+ )
+ ).to.equal(true)
+ })
+ })
+
+ if (backendSettings.backend === 's3SSEC') {
+ describe('storageClass', function () {
+ it('should use the default storage class for dek', async function () {
+ const key = `${projectId}/${new ObjectId()}`
+ const dekBucket = process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME
+ await app.persistor.sendStream(
+ dekBucket,
+ key,
+ Stream.Readable.from(['hello'])
+ )
+ expect(
+ await app.persistor.getObjectStorageClass(dekBucket, key)
+ ).to.equal(undefined)
+ })
+
+ it('should use the custom storage class for user files', async function () {
+ const key = `${projectId}/${new ObjectId()}`
+ await app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ key,
+ Stream.Readable.from(['hello'])
+ )
+ const sc = AWS_S3_USER_FILES_STORAGE_CLASS
+ expect(sc).to.exist
+ expect(
+ await app.persistor.getObjectStorageClass(
+ Settings.filestore.stores.user_files,
+ key
+ )
+ ).to.equal(sc)
+ })
+ })
+ }
+
+ describe('autoGunzip', function () {
+ let key
+ beforeEach('new key', function () {
+ key = `${projectId}/${new ObjectId().toString()}`
+ })
+ this.timeout(60 * 1000)
+ const body = Buffer.alloc(10 * 1024 * 1024, 'hello')
+ const gzippedBody = gzipSync(body)
+
+ /**
+ * @param {string} key
+ * @param {Buffer} wantBody
+ * @param {boolean} autoGunzip
+ * @return {Promise}
+ */
+ async function checkBodyIsTheSame(key, wantBody, autoGunzip) {
+ const s = await app.persistor.getObjectStream(
+ Settings.filestore.stores.user_files,
+ key,
+ { autoGunzip }
+ )
+ const buf = new WritableBuffer()
+ await Stream.promises.pipeline(s, buf)
+ expect(buf.getContents()).to.deep.equal(wantBody)
+ }
+
+ if (backendSettings.backend === 'fs') {
+ it('should refuse to handle autoGunzip', async function () {
+ await expect(
+ app.persistor.getObjectStream(
+ Settings.filestore.stores.user_files,
+ key,
+ { autoGunzip: true }
+ )
+ ).to.be.rejectedWith(NotImplementedError)
+ })
+ } else {
+ it('should return the raw body with gzip', async function () {
+ await app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ key,
+ Stream.Readable.from([gzippedBody]),
+ { contentEncoding: 'gzip' }
+ )
+ expect(
+ await app.persistor.getObjectSize(
+ Settings.filestore.stores.user_files,
+ key
+ )
+ ).to.equal(gzippedBody.byteLength)
+ // raw body with autoGunzip=true
+ await checkBodyIsTheSame(key, body, true)
+ // gzip body without autoGunzip=false
+ await checkBodyIsTheSame(key, gzippedBody, false)
+ })
+ it('should return the raw body without gzip compression', async function () {
+ await app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ key,
+ Stream.Readable.from([body])
+ )
+ expect(
+ await app.persistor.getObjectSize(
+ Settings.filestore.stores.user_files,
+ key
+ )
+ ).to.equal(body.byteLength)
+ // raw body with both autoGunzip options
+ await checkBodyIsTheSame(key, body, true)
+ await checkBodyIsTheSame(key, body, false)
+ })
+
+ it('should return the gzip body without gzip header', async function () {
+ await app.persistor.sendStream(
+ Settings.filestore.stores.user_files,
+ key,
+ Stream.Readable.from([gzippedBody])
+ )
+ expect(
+ await app.persistor.getObjectSize(
+ Settings.filestore.stores.user_files,
+ key
+ )
+ ).to.equal(gzippedBody.byteLength)
+ // gzip body with both autoGunzip options
+ await checkBodyIsTheSame(key, gzippedBody, true)
+ await checkBodyIsTheSame(key, gzippedBody, false)
+ })
+ }
+ })
})
- })
+ }
})
diff --git a/services/filestore/test/acceptance/js/TestConfig.js b/services/filestore/test/acceptance/js/TestConfig.js
index 4b72bc971d..3ad4ba423d 100644
--- a/services/filestore/test/acceptance/js/TestConfig.js
+++ b/services/filestore/test/acceptance/js/TestConfig.js
@@ -1,22 +1,63 @@
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
+const crypto = require('node:crypto')
+const {
+ RootKeyEncryptionKey,
+} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
+
+const AWS_S3_USER_FILES_STORAGE_CLASS =
+ process.env.AWS_S3_USER_FILES_STORAGE_CLASS
// use functions to get a fresh copy, not a reference, each time
+function s3BaseConfig() {
+ return {
+ endpoint: process.env.AWS_S3_ENDPOINT,
+ pathStyle: true,
+ partSize: 100 * 1024 * 1024,
+ ca: [fs.readFileSync('/certs/public.crt')],
+ }
+}
+
function s3Config() {
return {
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
- endpoint: process.env.AWS_S3_ENDPOINT,
- pathStyle: true,
- partSize: 100 * 1024 * 1024,
+ ...s3BaseConfig(),
+ }
+}
+
+const S3SSECKeys = [
+ new RootKeyEncryptionKey(
+ crypto.generateKeySync('aes', { length: 256 }).export(),
+ Buffer.alloc(32)
+ ),
+]
+
+function s3SSECConfig() {
+ return {
+ ...s3Config(),
+ ignoreErrorsFromDEKReEncryption: false,
+ automaticallyRotateDEKEncryption: true,
+ dataEncryptionKeyBucketName: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME,
+ pathToProjectFolder(_bucketName, path) {
+ const match = path.match(/^[a-f0-9]{24}\//)
+ if (!match) throw new Error('not a project-folder')
+ const [projectFolder] = match
+ return projectFolder
+ },
+ async getRootKeyEncryptionKeys() {
+ return S3SSECKeys
+ },
+ storageClass: {
+ [process.env.AWS_S3_USER_FILES_BUCKET_NAME]:
+ AWS_S3_USER_FILES_STORAGE_CLASS,
+ },
}
}
function s3ConfigDefaultProviderCredentials() {
return {
- endpoint: process.env.AWS_S3_ENDPOINT,
- pathStyle: true,
- partSize: 100 * 1024 * 1024,
+ ...s3BaseConfig(),
}
}
@@ -60,7 +101,7 @@ function fallbackStores(primaryConfig, fallbackConfig) {
}
}
-module.exports = {
+const BackendSettings = {
SHARD_01_FSPersistor: {
backend: 'fs',
stores: fsStores(),
@@ -80,6 +121,11 @@ module.exports = {
gcs: gcsConfig(),
stores: gcsStores(),
},
+ SHARD_01_PerProjectEncryptedS3Persistor: {
+ backend: 's3SSEC',
+ s3SSEC: s3SSECConfig(),
+ stores: s3Stores(),
+ },
SHARD_02_FallbackS3ToFSPersistor: {
backend: 's3',
s3: s3Config(),
@@ -137,3 +183,10 @@ function checkForUnexpectedTestFile() {
}
}
checkForUnexpectedTestFile()
+
+module.exports = {
+ AWS_S3_USER_FILES_STORAGE_CLASS,
+ BackendSettings,
+ s3Config,
+ s3SSECConfig,
+}
diff --git a/services/filestore/test/acceptance/js/TestHelper.js b/services/filestore/test/acceptance/js/TestHelper.js
index 1b7eae4d0a..384f8aab6f 100644
--- a/services/filestore/test/acceptance/js/TestHelper.js
+++ b/services/filestore/test/acceptance/js/TestHelper.js
@@ -1,5 +1,6 @@
const streamifier = require('streamifier')
const fetch = require('node-fetch')
+const ObjectPersistor = require('@overleaf/object-persistor')
const { expect } = require('chai')
@@ -7,6 +8,7 @@ module.exports = {
uploadStringToPersistor,
getStringFromPersistor,
expectPersistorToHaveFile,
+ expectPersistorToHaveSomeFile,
expectPersistorNotToHaveFile,
streamToString,
getMetric,
@@ -15,10 +17,14 @@ module.exports = {
async function getMetric(filestoreUrl, metric) {
const res = await fetch(`${filestoreUrl}/metrics`)
expect(res.status).to.equal(200)
- const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
+ const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'gm')
const body = await res.text()
- const found = metricRegex.exec(body)
- return parseInt(found ? found[1] : 0) || 0
+ let v = 0
+ // Sum up size="lt-128KiB" and size="gte-128KiB"
+ for (const [, found] of body.matchAll(metricRegex)) {
+ v += parseInt(found, 10) || 0
+ }
+ return v
}
function streamToString(stream) {
@@ -46,6 +52,25 @@ async function expectPersistorToHaveFile(persistor, bucket, key, content) {
expect(foundContent).to.equal(content)
}
+async function expectPersistorToHaveSomeFile(persistor, bucket, keys, content) {
+ let foundContent
+ for (const key of keys) {
+ try {
+ foundContent = await getStringFromPersistor(persistor, bucket, key)
+ break
+ } catch (err) {
+ if (err instanceof ObjectPersistor.Errors.NotFoundError) {
+ continue
+ }
+ throw err
+ }
+ }
+ if (foundContent === undefined) {
+ expect.fail(`Could not find any of the specified keys: ${keys}`)
+ }
+ expect(foundContent).to.equal(content)
+}
+
async function expectPersistorNotToHaveFile(persistor, bucket, key) {
await expect(
getStringFromPersistor(persistor, bucket, key)
diff --git a/services/filestore/test/setup.js b/services/filestore/test/setup.js
index 57a462390f..744ab9130f 100644
--- a/services/filestore/test/setup.js
+++ b/services/filestore/test/setup.js
@@ -21,6 +21,11 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
},
+ sourceTransformers: {
+ removeNodePrefix: function (source) {
+ return source.replace(/require\(['"]node:/g, "require('")
+ },
+ },
})
exports.mochaHooks = {
diff --git a/services/filestore/test/unit/js/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js
index 508db8b153..ec562116a0 100644
--- a/services/filestore/test/unit/js/FileControllerTests.js
+++ b/services/filestore/test/unit/js/FileControllerTests.js
@@ -6,14 +6,7 @@ const Errors = require('../../../app/js/Errors')
const modulePath = '../../../app/js/FileController.js'
describe('FileController', function () {
- let PersistorManager,
- FileHandler,
- LocalFileWriter,
- FileController,
- req,
- res,
- next,
- stream
+ let FileHandler, LocalFileWriter, FileController, req, res, next, stream
const settings = {
s3: {
buckets: {
@@ -32,13 +25,8 @@ describe('FileController', function () {
const error = new Error('incorrect utensil')
beforeEach(function () {
- PersistorManager = {
- sendStream: sinon.stub().yields(),
- copyObject: sinon.stub().resolves(),
- deleteObject: sinon.stub().yields(),
- }
-
FileHandler = {
+ copyObject: sinon.stub().yields(),
getFile: sinon.stub().yields(null, fileStream),
getFileSize: sinon.stub().yields(null, fileSize),
deleteFile: sinon.stub().yields(),
@@ -57,7 +45,6 @@ describe('FileController', function () {
requires: {
'./LocalFileWriter': LocalFileWriter,
'./FileHandler': FileHandler,
- './PersistorManager': PersistorManager,
'./Errors': Errors,
stream,
'@overleaf/settings': settings,
@@ -239,7 +226,7 @@ describe('FileController', function () {
})
describe('insertFile', function () {
- it('should send bucket name key and res to PersistorManager', function (done) {
+ it('should send bucket name key and res to FileHandler', function (done) {
res.sendStatus = code => {
expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req)
expect(code).to.equal(200)
@@ -263,10 +250,10 @@ describe('FileController', function () {
}
})
- it('should send bucket name and both keys to PersistorManager', function (done) {
+ it('should send bucket name and both keys to FileHandler', function (done) {
res.sendStatus = code => {
code.should.equal(200)
- expect(PersistorManager.copyObject).to.have.been.calledWith(
+ expect(FileHandler.copyObject).to.have.been.calledWith(
bucket,
oldKey,
key
@@ -277,7 +264,7 @@ describe('FileController', function () {
})
it('should send a 404 if the original file was not found', function (done) {
- PersistorManager.copyObject.rejects(
+ FileHandler.copyObject.yields(
new Errors.NotFoundError({ message: 'not found', info: {} })
)
res.sendStatus = code => {
@@ -288,7 +275,7 @@ describe('FileController', function () {
})
it('should send an error if there was an error', function (done) {
- PersistorManager.copyObject.rejects(error)
+ FileHandler.copyObject.yields(error)
FileController.copyFile(req, res, err => {
expect(err).to.equal(error)
done()
diff --git a/services/filestore/test/unit/js/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js
index d4899dd20a..12a23667b4 100644
--- a/services/filestore/test/unit/js/FileHandlerTests.js
+++ b/services/filestore/test/unit/js/FileHandlerTests.js
@@ -67,7 +67,11 @@ describe('FileHandler', function () {
compressPng: sinon.stub().resolves(),
},
}
- Settings = {}
+ Settings = {
+ filestore: {
+ stores: { template_files: 'template_files', user_files: 'user_files' },
+ },
+ }
fs = {
createReadStream: sinon.stub().returns(readStream),
}
@@ -89,7 +93,7 @@ describe('FileHandler', function () {
},
fs,
},
- globals: { console },
+ globals: { console, process },
})
})
@@ -133,23 +137,6 @@ describe('FileHandler', function () {
done()
})
})
-
- describe('when conversions are enabled', function () {
- beforeEach(function () {
- Settings.enableConversions = true
- })
-
- it('should delete the convertedKey folder', function (done) {
- FileHandler.insertFile(bucket, key, stream, err => {
- expect(err).not.to.exist
- expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
- bucket,
- convertedFolderKey
- )
- done()
- })
- })
- })
})
describe('deleteFile', function () {
@@ -195,15 +182,31 @@ describe('FileHandler', function () {
Settings.enableConversions = true
})
- it('should delete the convertedKey folder', function (done) {
- FileHandler.deleteFile(bucket, key, err => {
- expect(err).not.to.exist
- expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
- bucket,
- convertedFolderKey
- )
- done()
- })
+ it('should delete the convertedKey folder for template files', function (done) {
+ FileHandler.deleteFile(
+ Settings.filestore.stores.template_files,
+ key,
+ err => {
+ expect(err).not.to.exist
+ expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
+ Settings.filestore.stores.template_files,
+ convertedFolderKey
+ )
+ done()
+ }
+ )
+ })
+
+ it('should not delete the convertedKey folder for user files', function (done) {
+ FileHandler.deleteFile(
+ Settings.filestore.stores.user_files,
+ key,
+ err => {
+ expect(err).not.to.exist
+ expect(PersistorManager.deleteDirectory).to.not.have.been.called
+ done()
+ }
+ )
})
})
})
diff --git a/services/git-bridge/.gitignore b/services/git-bridge/.gitignore
index 74a7f43d6e..f35e2ee038 100644
--- a/services/git-bridge/.gitignore
+++ b/services/git-bridge/.gitignore
@@ -1,53 +1,6 @@
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# Let's not share anything because we're using Maven.
-
-.idea
-*.iml
-
-# User-specific stuff:
-.idea/workspace.xml
-.idea/tasks.xml
-.idea/dictionaries
-.idea/vcs.xml
-.idea/jsLibraryMappings.xml
-
-# Sensitive or high-churn files:
-.idea/dataSources.ids
-.idea/dataSources.xml
-.idea/dataSources.local.xml
-.idea/sqlDataSources.xml
-.idea/dynamic.xml
-.idea/uiDesigner.xml
-
-# Gradle:
-.idea/gradle.xml
-.idea/libraries
-
-# Mongo Explorer plugin:
-.idea/mongoSettings.xml
-
-## File-based project format:
-*.iws
-
-## Plugin-specific files:
-
-# IntelliJ
+# Build output
/out/
target/
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
# Local configuration files
conf/runtime.json
diff --git a/services/git-bridge/Dockerfile b/services/git-bridge/Dockerfile
index 0d8b1e43e5..48579b9494 100644
--- a/services/git-bridge/Dockerfile
+++ b/services/git-bridge/Dockerfile
@@ -1,11 +1,17 @@
-# Dockerfile for git-bridge
+# Build the a8m/envsubst binary, as it supports default values,
+# which the gnu envsubst (from gettext-base) does not.
+FROM golang:1.24.3-alpine AS envsubst_builder
+
+WORKDIR /build
+
+RUN go install github.com/a8m/envsubst/cmd/envsubst@latest
FROM maven:3-amazoncorretto-21-debian AS base
RUN apt-get update && apt-get install -y make git sqlite3 \
&& rm -rf /var/lib/apt/lists
-COPY vendor/envsubst /opt/envsubst
+COPY --from=envsubst_builder /go/bin/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
RUN useradd --create-home node
@@ -29,16 +35,11 @@ RUN apk add --update --no-cache bash git sqlite procps htop net-tools jemalloc u
ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2
-# Install Google Cloud Profiler agent
-RUN mkdir -p /opt/cprof && \
- wget -q -O- https://storage.googleapis.com/cloud-profiler/java/latest/profiler_java_agent.tar.gz \
- | tar xzv -C /opt/cprof
-
RUN adduser -D node
COPY --from=builder /git-bridge.jar /
-COPY vendor/envsubst /opt/envsubst
+COPY --from=envsubst_builder /go/bin/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
COPY conf/envsubst_template.json envsubst_template.json
diff --git a/services/git-bridge/README.md b/services/git-bridge/README.md
index 13b24cc6d0..eadc2abc4f 100644
--- a/services/git-bridge/README.md
+++ b/services/git-bridge/README.md
@@ -76,12 +76,10 @@ The configuration file is in `.json` format.
"postbackBaseUrl" (string): the postback url,
"serviceName" (string): current name of writeLaTeX
in case it ever changes,
- "oauth2" (object): { null or missing if oauth2 shouldn't be used
- "oauth2ClientID" (string): oauth2 client ID,
- "oauth2ClientSecret" (string): oauth2 client secret,
- "oauth2Server" (string): oauth2 server,
- with protocol and
- without trailing slash
+ "oauth2Server" (string): oauth2 server,
+ with protocol and
+ without trailing slash,
+ null or missing if oauth2 shouldn't be used
},
"repoStore" (object, optional): { configure the repo store
"maxFileSize" (long, optional): maximum size of a file, inclusive
diff --git a/services/git-bridge/conf/envsubst_template.json b/services/git-bridge/conf/envsubst_template.json
index 1f52ffbaef..4ede5bab7f 100644
--- a/services/git-bridge/conf/envsubst_template.json
+++ b/services/git-bridge/conf/envsubst_template.json
@@ -3,14 +3,11 @@
"bindIp": "${GIT_BRIDGE_BIND_IP:-0.0.0.0}",
"idleTimeout": ${GIT_BRIDGE_IDLE_TIMEOUT:-30000},
"rootGitDirectory": "${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}",
+ "allowedCorsOrigins": "${GIT_BRIDGE_ALLOWED_CORS_ORIGINS:-https://localhost}",
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
- "oauth2": {
- "oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
- "oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
- "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
- },
+ "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}",
"userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
"repoStore": {
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json
index bfad73f461..76b82eb6a0 100644
--- a/services/git-bridge/conf/example_config.json
+++ b/services/git-bridge/conf/example_config.json
@@ -3,14 +3,11 @@
"bindIp": "127.0.0.1",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
+ "allowedCorsOrigins": "https://localhost",
"apiBaseUrl": "https://localhost/api/v0",
"postbackBaseUrl": "https://localhost",
"serviceName": "Overleaf",
- "oauth2": {
- "oauth2ClientID": "asdf",
- "oauth2ClientSecret": "asdf",
- "oauth2Server": "https://localhost"
- },
+ "oauth2Server": "https://localhost",
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
diff --git a/services/git-bridge/conf/local.json b/services/git-bridge/conf/local.json
index 03ce4febe4..c4de48d819 100644
--- a/services/git-bridge/conf/local.json
+++ b/services/git-bridge/conf/local.json
@@ -3,14 +3,11 @@
"bindIp": "0.0.0.0",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
+ "allowedCorsOrigins": "http://v2.overleaf.test",
"apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
"postbackBaseUrl": "http://git-bridge:8000",
"serviceName": "Overleaf",
- "oauth2": {
- "oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
- "oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
- "oauth2Server": "http://v2.overleaf.test:3000"
- },
+ "oauth2Server": "http://v2.overleaf.test:3000",
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
diff --git a/services/git-bridge/pom.xml b/services/git-bridge/pom.xml
index 809676d769..3feb4dd860 100644
--- a/services/git-bridge/pom.xml
+++ b/services/git-bridge/pom.xml
@@ -16,24 +16,24 @@
2.23
4.13.2
2.8.4
- 9.4.51.v20230217
+ 9.4.57.v20241219
2.9.0
- 2.12.3
- 6.6.1.202309021850-r
+ 3.0.2
+ 6.10.1.202505221210-r
3.41.2.2
2.9.9
- 1.34.1
+ 1.37.0
1.23.0
- 3.12.0
- 1.2.3
+ 3.17.0
+ 1.2.13
5.12.0
5.12.0
- 1.11.274
+ 1.12.780
${jaxb.runtime.version}
2.3.2
4.5.14
- 2.10.0
- 1.24.0
+ 2.18.0
+ 1.27.1
0.10.0
1.70
@@ -206,7 +206,7 @@
com.amazonaws
- aws-java-sdk
+ aws-java-sdk-s3
${aws.java.sdk.version}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
index cf36916600..d5b530100e 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
@@ -26,10 +26,11 @@ public class Config implements JSONSource {
config.bindIp,
config.idleTimeout,
config.rootGitDirectory,
+ config.allowedCorsOrigins,
config.apiBaseURL,
config.postbackURL,
config.serviceName,
- Oauth2.asSanitised(config.oauth2),
+ config.oauth2Server,
config.userPasswordEnabled,
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
@@ -41,10 +42,11 @@ public class Config implements JSONSource {
private String bindIp;
private int idleTimeout;
private String rootGitDirectory;
+ private String[] allowedCorsOrigins;
private String apiBaseURL;
private String postbackURL;
private String serviceName;
- @Nullable private Oauth2 oauth2;
+ @Nullable private String oauth2Server;
private boolean userPasswordEnabled;
@Nullable private RepoStoreConfig repoStore;
@Nullable private SwapStoreConfig swapStore;
@@ -64,10 +66,11 @@ public class Config implements JSONSource {
String bindIp,
int idleTimeout,
String rootGitDirectory,
+ String[] allowedCorsOrigins,
String apiBaseURL,
String postbackURL,
String serviceName,
- Oauth2 oauth2,
+ String oauth2Server,
boolean userPasswordEnabled,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
@@ -77,10 +80,11 @@ public class Config implements JSONSource {
this.bindIp = bindIp;
this.idleTimeout = idleTimeout;
this.rootGitDirectory = rootGitDirectory;
+ this.allowedCorsOrigins = allowedCorsOrigins;
this.apiBaseURL = apiBaseURL;
this.postbackURL = postbackURL;
this.serviceName = serviceName;
- this.oauth2 = oauth2;
+ this.oauth2Server = oauth2Server;
this.userPasswordEnabled = userPasswordEnabled;
this.repoStore = repoStore;
this.swapStore = swapStore;
@@ -101,11 +105,18 @@ public class Config implements JSONSource {
}
this.apiBaseURL = apiBaseURL;
serviceName = getElement(configObject, "serviceName").getAsString();
+ final String rawAllowedCorsOrigins =
+ getOptionalString(configObject, "allowedCorsOrigins").trim();
+ if (rawAllowedCorsOrigins.isEmpty()) {
+ allowedCorsOrigins = new String[] {};
+ } else {
+ allowedCorsOrigins = rawAllowedCorsOrigins.split(",");
+ }
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
if (!postbackURL.endsWith("/")) {
postbackURL += "/";
}
- oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
+ oauth2Server = getOptionalString(configObject, "oauth2Server");
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
@@ -139,6 +150,10 @@ public class Config implements JSONSource {
return this.sqliteHeapLimitBytes;
}
+ public String[] getAllowedCorsOrigins() {
+ return allowedCorsOrigins;
+ }
+
public String getAPIBaseURL() {
return apiBaseURL;
}
@@ -151,19 +166,12 @@ public class Config implements JSONSource {
return postbackURL;
}
- public boolean isUsingOauth2() {
- return oauth2 != null;
- }
-
public boolean isUserPasswordEnabled() {
return userPasswordEnabled;
}
- public Oauth2 getOauth2() {
- if (!isUsingOauth2()) {
- throw new AssertionError("Getting oauth2 when not using it");
- }
- return oauth2;
+ public String getOauth2Server() {
+ return oauth2Server;
}
public Optional getRepoStore() {
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java
deleted file mode 100644
index 1db7d3b4d2..0000000000
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package uk.ac.ic.wlgitbridge.application.config;
-
-/*
- * Created by winston on 25/10/15.
- */
-public class Oauth2 {
-
- private final String oauth2ClientID;
- private final String oauth2ClientSecret;
- private final String oauth2Server;
-
- public Oauth2(String oauth2ClientID, String oauth2ClientSecret, String oauth2Server) {
- this.oauth2ClientID = oauth2ClientID;
- this.oauth2ClientSecret = oauth2ClientSecret;
- this.oauth2Server = oauth2Server;
- }
-
- public String getOauth2ClientID() {
- return oauth2ClientID;
- }
-
- public String getOauth2ClientSecret() {
- return oauth2ClientSecret;
- }
-
- public String getOauth2Server() {
- return oauth2Server;
- }
-
- public static Oauth2 asSanitised(Oauth2 oauth2) {
- return new Oauth2("", "", oauth2.oauth2Server);
- }
-}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java
new file mode 100644
index 0000000000..10d978c352
--- /dev/null
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java
@@ -0,0 +1,47 @@
+package uk.ac.ic.wlgitbridge.server;
+
+import java.io.IOException;
+import java.util.Set;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+public class CORSHandler extends AbstractHandler {
+ private final Set allowedCorsOrigins;
+
+ public CORSHandler(String[] allowedCorsOrigins) {
+ this.allowedCorsOrigins = Set.of(allowedCorsOrigins);
+ }
+
+ @Override
+ public void handle(
+ String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+
+ String origin = request.getHeader("Origin");
+ if (origin == null) {
+ return; // Not a CORS request
+ }
+
+ final boolean ok = allowedCorsOrigins.contains(origin);
+ if (ok) {
+ response.setHeader("Access-Control-Allow-Origin", origin);
+ response.setHeader("Access-Control-Allow-Credentials", "true");
+ response.setHeader("Access-Control-Allow-Methods", "GET, HEAD, PUT, POST, DELETE");
+ response.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type");
+ response.setHeader("Access-Control-Max-Age", "86400"); // cache for 24h
+ }
+ String method = baseRequest.getMethod();
+ if ("OPTIONS".equals(method)) {
+ Log.debug("OPTIONS <- {}", target);
+ baseRequest.setHandled(true);
+ if (ok) {
+ response.setStatus(200);
+ } else {
+ response.setStatus(403);
+ }
+ }
+ }
+}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
index 30c5039212..57d1b34a7b 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
@@ -110,6 +110,7 @@ public class GitBridgeServer {
this.jettyServer.addConnector(connector);
HandlerCollection handlers = new HandlerList();
+ handlers.addHandler(new CORSHandler(config.getAllowedCorsOrigins()));
handlers.addHandler(initApiHandler());
handlers.addHandler(initBaseHandler());
handlers.addHandler(initGitHandler(config, repoStore, snapshotApi));
@@ -150,9 +151,9 @@ public class GitBridgeServer {
throws ServletException {
final ServletContextHandler servletContextHandler =
new ServletContextHandler(ServletContextHandler.SESSIONS);
- if (config.isUsingOauth2()) {
+ if (config.getOauth2Server() != null) {
Filter filter =
- new Oauth2Filter(snapshotApi, config.getOauth2(), config.isUserPasswordEnabled());
+ new Oauth2Filter(snapshotApi, config.getOauth2Server(), config.isUserPasswordEnabled());
servletContextHandler.addFilter(
new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
index 5bd3904e47..586a21ab3f 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
@@ -13,7 +13,6 @@ import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.codec.binary.Base64;
-import uk.ac.ic.wlgitbridge.application.config.Oauth2;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.util.Instance;
import uk.ac.ic.wlgitbridge.util.Log;
@@ -28,13 +27,13 @@ public class Oauth2Filter implements Filter {
private final SnapshotApi snapshotApi;
- private final Oauth2 oauth2;
+ private final String oauth2Server;
private final boolean isUserPasswordEnabled;
- public Oauth2Filter(SnapshotApi snapshotApi, Oauth2 oauth2, boolean isUserPasswordEnabled) {
+ public Oauth2Filter(SnapshotApi snapshotApi, String oauth2Server, boolean isUserPasswordEnabled) {
this.snapshotApi = snapshotApi;
- this.oauth2 = oauth2;
+ this.oauth2Server = oauth2Server;
this.isUserPasswordEnabled = isUserPasswordEnabled;
}
@@ -108,7 +107,7 @@ public class Oauth2Filter implements Filter {
// fail later (for example, in the unlikely event that the token
// expired between the two requests). In that case, JGit will
// return a 401 without a custom error message.
- int statusCode = checkAccessToken(oauth2, password, getClientIp(request));
+ int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
if (statusCode == 429) {
handleRateLimit(projectId, username, request, response);
return;
@@ -238,10 +237,9 @@ public class Oauth2Filter implements Filter {
"your Overleaf Account Settings."));
}
- private int checkAccessToken(Oauth2 oauth2, String accessToken, String clientIp)
+ private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
throws IOException {
- GenericUrl url =
- new GenericUrl(oauth2.getOauth2Server() + "/oauth/token/info?client_ip=" + clientIp);
+ GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
HttpHeaders headers = new HttpHeaders();
headers.setAuthorization("Bearer " + accessToken);
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
index 878adde27d..512babf9c7 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
@@ -5,6 +5,7 @@ import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -147,7 +148,7 @@ public class Tar {
throws IOException {
Preconditions.checkArgument(dir.isDirectory());
String name = base.relativize(Paths.get(dir.getAbsolutePath())).toString();
- ArchiveEntry entry = tout.createArchiveEntry(dir, name);
+ TarArchiveEntry entry = tout.createArchiveEntry(dir, name);
tout.putArchiveEntry(entry);
tout.closeArchiveEntry();
for (File f : dir.listFiles()) {
@@ -160,7 +161,7 @@ public class Tar {
Preconditions.checkArgument(file.isFile(), "given file" + " is not file: %s", file);
checkFileSize(file.length());
String name = base.relativize(Paths.get(file.getAbsolutePath())).toString();
- ArchiveEntry entry = tout.createArchiveEntry(file, name);
+ TarArchiveEntry entry = tout.createArchiveEntry(file, name);
tout.putArchiveEntry(entry);
try (InputStream in = new FileInputStream(file)) {
IOUtils.copy(in, tout);
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
index e250798652..f706d98edf 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
@@ -465,8 +465,12 @@ public class WLGitBridgeIntegrationTest {
@After
public void tearDown() {
- server.stop();
- wlgb.stop();
+ if (server != null) {
+ server.stop();
+ }
+ if (wlgb != null) {
+ wlgb.stop();
+ }
}
private void gitConfig(File dir) throws IOException, InterruptedException {
@@ -1391,6 +1395,80 @@ public class WLGitBridgeIntegrationTest {
assertTrue(f.exists());
}
+ @Test
+ public void noCors() throws IOException, ExecutionException, InterruptedException {
+
+ int gitBridgePort = 33893;
+ int mockServerPort = 3893;
+
+ server = new MockSnapshotServer(mockServerPort, getResource("/canServePushedFiles").toFile());
+ server.start();
+ server.setState(states.get("canServePushedFiles").get("state"));
+
+ wlgb = new GitBridgeApp(new String[] {makeConfigFile(gitBridgePort, mockServerPort)});
+ wlgb.run();
+
+ String url = "http://127.0.0.1:" + gitBridgePort + "/status";
+ Response response = asyncHttpClient().prepareGet(url).execute().get();
+ assertEquals(200, response.getStatusCode());
+ assertEquals("ok\n", response.getResponseBody());
+ assertNull(response.getHeader("Access-Control-Allow-Origin"));
+ }
+
+ @Test
+ public void cors() throws IOException, ExecutionException, InterruptedException {
+
+ int gitBridgePort = 33894;
+ int mockServerPort = 3894;
+
+ server = new MockSnapshotServer(mockServerPort, getResource("/canServePushedFiles").toFile());
+ server.start();
+ server.setState(states.get("canServePushedFiles").get("state"));
+
+ wlgb = new GitBridgeApp(new String[] {makeConfigFile(gitBridgePort, mockServerPort)});
+ wlgb.run();
+
+ String url = "http://127.0.0.1:" + gitBridgePort + "/status";
+
+ // Success
+ Response response =
+ asyncHttpClient()
+ .prepareOptions(url)
+ .setHeader("Origin", "https://localhost")
+ .execute()
+ .get();
+ assertEquals(200, response.getStatusCode());
+ assertEquals("", response.getResponseBody());
+ assertEquals("https://localhost", response.getHeader("Access-Control-Allow-Origin"));
+
+ response =
+ asyncHttpClient().prepareGet(url).setHeader("Origin", "https://localhost").execute().get();
+ assertEquals(200, response.getStatusCode());
+ assertEquals("ok\n", response.getResponseBody());
+ assertEquals("https://localhost", response.getHeader("Access-Control-Allow-Origin"));
+
+ // Deny
+ response =
+ asyncHttpClient()
+ .prepareOptions(url)
+ .setHeader("Origin", "https://not-localhost")
+ .execute()
+ .get();
+ assertEquals(403, response.getStatusCode());
+ assertEquals("", response.getResponseBody());
+ assertNull(response.getHeader("Access-Control-Allow-Origin"));
+
+ response =
+ asyncHttpClient()
+ .prepareGet(url)
+ .setHeader("Origin", "https://not-localhost")
+ .execute()
+ .get();
+ assertEquals(200, response.getStatusCode());
+ assertEquals("ok\n", response.getResponseBody());
+ assertNull(response.getHeader("Access-Control-Allow-Origin"));
+ }
+
private String makeConfigFile(int port, int apiPort) throws IOException {
return makeConfigFile(port, apiPort, null);
}
@@ -1409,6 +1487,7 @@ public class WLGitBridgeIntegrationTest {
+ " \"rootGitDirectory\": \""
+ wlgb.getAbsolutePath()
+ "\",\n"
+ + " \"allowedCorsOrigins\": \"https://localhost\",\n"
+ " \"apiBaseUrl\": \"http://127.0.0.1:"
+ apiPort
+ "/api/v0\",\n"
@@ -1416,13 +1495,9 @@ public class WLGitBridgeIntegrationTest {
+ port
+ "\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2\": {\n"
- + " \"oauth2ClientID\": \"clientID\",\n"
- + " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
- + " \"oauth2Server\": \"http://127.0.0.1:"
+ + " \"oauth2Server\": \"http://127.0.0.1:"
+ apiPort
- + "\"\n"
- + " }";
+ + "\"";
if (swapCfg != null) {
cfgStr +=
",\n"
@@ -1445,7 +1520,6 @@ public class WLGitBridgeIntegrationTest {
+ ",\n"
+ " \"intervalMillis\": "
+ swapCfg.getIntervalMillis()
- + "\n"
+ " }\n";
}
cfgStr += "}\n";
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
index ddafc621d6..8c102dbda3 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
@@ -23,11 +23,7 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2\": {\n"
- + " \"oauth2ClientID\": \"clientID\",\n"
- + " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
- + " }\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ "}\n");
Config config = new Config(reader);
assertEquals(80, config.getPort());
@@ -35,10 +31,7 @@ public class ConfigTest {
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
assertEquals("http://127.0.0.1/", config.getPostbackURL());
assertEquals("Overleaf", config.getServiceName());
- assertTrue(config.isUsingOauth2());
- assertEquals("clientID", config.getOauth2().getOauth2ClientID());
- assertEquals("oauth2 client secret", config.getOauth2().getOauth2ClientSecret());
- assertEquals("https://www.overleaf.com", config.getOauth2().getOauth2Server());
+ assertEquals("https://www.overleaf.com", config.getOauth2Server());
}
@Test(expected = AssertionError.class)
@@ -53,7 +46,7 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2\": null\n"
+ + " \"oauth2Server\": null\n"
+ "}\n");
Config config = new Config(reader);
assertEquals(80, config.getPort());
@@ -61,8 +54,7 @@ public class ConfigTest {
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
assertEquals("http://127.0.0.1/", config.getPostbackURL());
assertEquals("Overleaf", config.getServiceName());
- assertFalse(config.isUsingOauth2());
- config.getOauth2();
+ assertNull(config.getOauth2Server());
}
@Test
@@ -77,11 +69,7 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2\": {\n"
- + " \"oauth2ClientID\": \"my oauth2 client id\",\n"
- + " \"oauth2ClientSecret\": \"my oauth2 client secret\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
- + " }\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ "}\n");
Config config = new Config(reader);
String expected =
@@ -90,14 +78,11 @@ public class ConfigTest {
+ " \"bindIp\": \"127.0.0.1\",\n"
+ " \"idleTimeout\": 30000,\n"
+ " \"rootGitDirectory\": \"/var/wlgb/git\",\n"
+ + " \"allowedCorsOrigins\": [],\n"
+ " \"apiBaseURL\": \"http://127.0.0.1:60000/api/v0/\",\n"
+ " \"postbackURL\": \"http://127.0.0.1/\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2\": {\n"
- + " \"oauth2ClientID\": \"\",\n"
- + " \"oauth2ClientSecret\": \"\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
- + " },\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\",\n"
+ " \"userPasswordEnabled\": false,\n"
+ " \"repoStore\": null,\n"
+ " \"swapStore\": null,\n"
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
index f749dea357..e27c3488c0 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
@@ -50,7 +50,7 @@ public class BridgeTest {
gcJob = mock(GcJob.class);
bridge =
new Bridge(
- new Config(0, "", 0, "", "", "", "", null, false, null, null, null, 0),
+ new Config(0, "", 0, "", null, "", "", "", null, false, null, null, null, 0),
lock,
repoStore,
dbStore,
diff --git a/services/git-bridge/vendor/envsubst b/services/git-bridge/vendor/envsubst
deleted file mode 100755
index f7ad8081d0..0000000000
Binary files a/services/git-bridge/vendor/envsubst and /dev/null differ
diff --git a/services/history-v1/.gitignore b/services/history-v1/.gitignore
deleted file mode 100644
index edb0f85350..0000000000
--- a/services/history-v1/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc
diff --git a/services/history-v1/.nvmrc b/services/history-v1/.nvmrc
index 123b052798..fc37597bcc 100644
--- a/services/history-v1/.nvmrc
+++ b/services/history-v1/.nvmrc
@@ -1 +1 @@
-18.20.2
+22.17.0
diff --git a/services/history-v1/Dockerfile b/services/history-v1/Dockerfile
index cf6d0b3aaf..322ab67ff8 100644
--- a/services/history-v1/Dockerfile
+++ b/services/history-v1/Dockerfile
@@ -2,9 +2,11 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:18.20.2 AS base
+FROM node:22.17.0 AS base
WORKDIR /overleaf/services/history-v1
+COPY services/history-v1/install_deps.sh /overleaf/services/history-v1/
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile
index 6ba72740b0..7e62ba1812 100644
--- a/services/history-v1/Makefile
+++ b/services/history-v1/Makefile
@@ -32,12 +32,30 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
+
+SHELLCHECK_OPTS = \
+ --shell=bash \
+ --external-sources
+SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
+SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
+
+shellcheck:
+ @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
+ koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
+
+shellcheck_fix:
+ @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
+ diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
+ if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
+ elif [ -n "$$diff" ]; then echo "$$file"; \
+ else echo "\033[2m$$file\033[0m"; fi \
+ done
format:
$(RUN_LINTING) format
@@ -63,7 +81,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck test_unit test_acceptance
+test: format lint typecheck shellcheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -98,13 +116,6 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
- $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
- while ! mongosh --eval "db.version()" > /dev/null; do \
- echo "Waiting for Mongo..."; \
- sleep 1; \
- done; \
- mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -137,6 +148,7 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
+ shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/history-v1/api/app/rollout.js b/services/history-v1/api/app/rollout.js
new file mode 100644
index 0000000000..24ca0409f8
--- /dev/null
+++ b/services/history-v1/api/app/rollout.js
@@ -0,0 +1,76 @@
+const crypto = require('node:crypto')
+
+class Rollout {
+ constructor(config) {
+ // The history buffer level is used to determine whether to queue changes
+ // in Redis or persist them directly to the chunk store.
+ // If defaults to 0 (no queuing) if not set.
+ this.historyBufferLevel = config.has('historyBufferLevel')
+ ? parseInt(config.get('historyBufferLevel'), 10)
+ : 0
+ // The forcePersistBuffer flag will ensure the buffer is fully persisted before
+ // any persist operation. Set this to true if you want to make the persisted-version
+ // in Redis match the endVersion of the latest chunk. This should be set to true
+ // when downgrading from a history buffer level that queues changes in Redis
+ // without persisting them immediately.
+ this.forcePersistBuffer = config.has('forcePersistBuffer')
+ ? config.get('forcePersistBuffer') === 'true'
+ : false
+
+ // Support gradual rollout of the next history buffer level
+ // with a percentage of projects using it.
+ this.nextHistoryBufferLevel = config.has('nextHistoryBufferLevel')
+ ? parseInt(config.get('nextHistoryBufferLevel'), 10)
+ : null
+ this.nextHistoryBufferLevelRolloutPercentage = config.has(
+ 'nextHistoryBufferLevelRolloutPercentage'
+ )
+ ? parseInt(config.get('nextHistoryBufferLevelRolloutPercentage'), 10)
+ : 0
+ }
+
+ report(logger) {
+ logger.info(
+ {
+ historyBufferLevel: this.historyBufferLevel,
+ forcePersistBuffer: this.forcePersistBuffer,
+ nextHistoryBufferLevel: this.nextHistoryBufferLevel,
+ nextHistoryBufferLevelRolloutPercentage:
+ this.nextHistoryBufferLevelRolloutPercentage,
+ },
+ this.historyBufferLevel > 0 || this.forcePersistBuffer
+ ? 'using history buffer'
+ : 'history buffer disabled'
+ )
+ }
+
+ /**
+ * Get the history buffer level for a project.
+ * @param {string} projectId
+ * @returns {Object} - An object containing the history buffer level and force persist buffer flag.
+ * @property {number} historyBufferLevel - The history buffer level to use for processing changes.
+ * @property {boolean} forcePersistBuffer - If true, forces the buffer to be persisted before any operation.
+ */
+ getHistoryBufferLevelOptions(projectId) {
+ if (
+ this.nextHistoryBufferLevel > this.historyBufferLevel &&
+ this.nextHistoryBufferLevelRolloutPercentage > 0
+ ) {
+ const hash = crypto.createHash('sha1').update(projectId).digest('hex')
+ const percentage = parseInt(hash.slice(0, 8), 16) % 100
+ // If the project is in the rollout percentage, we use the next history buffer level.
+ if (percentage < this.nextHistoryBufferLevelRolloutPercentage) {
+ return {
+ historyBufferLevel: this.nextHistoryBufferLevel,
+ forcePersistBuffer: this.forcePersistBuffer,
+ }
+ }
+ }
+ return {
+ historyBufferLevel: this.historyBufferLevel,
+ forcePersistBuffer: this.forcePersistBuffer,
+ }
+ }
+}
+
+module.exports = Rollout
diff --git a/services/history-v1/api/app/security.js b/services/history-v1/api/app/security.js
index c82c3d2683..08d6f030dc 100644
--- a/services/history-v1/api/app/security.js
+++ b/services/history-v1/api/app/security.js
@@ -105,6 +105,8 @@ function handleJWTAuth(req, authOrSecDef, scopesOrApiKey, next) {
next()
}
+exports.hasValidBasicAuthCredentials = hasValidBasicAuthCredentials
+
/**
* Verify and decode the given JSON Web Token
*/
diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js
index ec4aa317b0..02fb793c87 100644
--- a/services/history-v1/api/controllers/project_import.js
+++ b/services/history-v1/api/controllers/project_import.js
@@ -1,6 +1,10 @@
+// @ts-check
+
'use strict'
-const BPromise = require('bluebird')
+const config = require('config')
+const { expressify } = require('@overleaf/promise-utils')
+
const HTTPStatus = require('http-status')
const core = require('overleaf-editor-core')
@@ -18,11 +22,18 @@ const BatchBlobStore = storage.BatchBlobStore
const BlobStore = storage.BlobStore
const chunkStore = storage.chunkStore
const HashCheckBlobStore = storage.HashCheckBlobStore
-const persistChanges = storage.persistChanges
+const commitChanges = storage.commitChanges
+const persistBuffer = storage.persistBuffer
+const InvalidChangeError = storage.InvalidChangeError
const render = require('./render')
+const Rollout = require('../app/rollout')
+const redisBackend = require('../../storage/lib/chunk_store/redis')
-exports.importSnapshot = function importSnapshot(req, res, next) {
+const rollout = new Rollout(config)
+rollout.report(logger) // display the rollout configuration in the logs
+
+async function importSnapshot(req, res) {
const projectId = req.swagger.params.project_id.value
const rawSnapshot = req.swagger.params.snapshot.value
@@ -31,24 +42,26 @@ exports.importSnapshot = function importSnapshot(req, res, next) {
try {
snapshot = Snapshot.fromRaw(rawSnapshot)
} catch (err) {
+ logger.warn({ err, projectId }, 'failed to import snapshot')
return render.unprocessableEntity(res)
}
- return chunkStore
- .initializeProject(projectId, snapshot)
- .then(function (projectId) {
- res.status(HTTPStatus.OK).json({ projectId })
- })
- .catch(err => {
- if (err instanceof chunkStore.AlreadyInitialized) {
- render.conflict(res)
- } else {
- next(err)
- }
- })
+ let historyId
+ try {
+ historyId = await chunkStore.initializeProject(projectId, snapshot)
+ } catch (err) {
+ if (err instanceof chunkStore.AlreadyInitialized) {
+ logger.warn({ err, projectId }, 'already initialized')
+ return render.conflict(res)
+ } else {
+ throw err
+ }
+ }
+
+ res.status(HTTPStatus.OK).json({ projectId: historyId })
}
-exports.importChanges = function importChanges(req, res, next) {
+async function importChanges(req, res, next) {
const projectId = req.swagger.params.project_id.value
const rawChanges = req.swagger.params.changes.value
const endVersion = req.swagger.params.end_version.value
@@ -59,7 +72,7 @@ exports.importChanges = function importChanges(req, res, next) {
try {
changes = rawChanges.map(Change.fromRaw)
} catch (err) {
- logger.error(err)
+ logger.warn({ err, projectId }, 'failed to parse changes')
return render.unprocessableEntity(res)
}
@@ -76,65 +89,102 @@ exports.importChanges = function importChanges(req, res, next) {
const batchBlobStore = new BatchBlobStore(blobStore)
const hashCheckBlobStore = new HashCheckBlobStore(blobStore)
- function loadFiles() {
+ async function loadFiles() {
const blobHashes = new Set()
- changes.forEach(function findBlobHashesToPreload(change) {
+ for (const change of changes) {
+ // This populates the set blobHashes with blobs referred to in the change
change.findBlobHashes(blobHashes)
- })
-
- function lazyLoadChangeFiles(change) {
- return change.loadFiles('lazy', batchBlobStore)
}
- return batchBlobStore
- .preload(Array.from(blobHashes))
- .then(function lazyLoadChangeFilesWithBatching() {
- return BPromise.each(changes, lazyLoadChangeFiles)
- })
+ await batchBlobStore.preload(Array.from(blobHashes))
+
+ for (const change of changes) {
+ await change.loadFiles('lazy', batchBlobStore)
+ }
}
- function buildResultSnapshot(resultChunk) {
- return BPromise.resolve(
- resultChunk || chunkStore.loadLatest(projectId)
- ).then(function (chunk) {
- const snapshot = chunk.getSnapshot()
- snapshot.applyAll(chunk.getChanges())
- return snapshot.store(hashCheckBlobStore)
- })
+ async function buildResultSnapshot(resultChunk) {
+ const chunk =
+ resultChunk ||
+ (await chunkStore.loadLatest(projectId, { persistedOnly: true }))
+ const snapshot = chunk.getSnapshot()
+ snapshot.applyAll(chunk.getChanges())
+ const rawSnapshot = await snapshot.store(hashCheckBlobStore)
+ return rawSnapshot
}
- return loadFiles()
- .then(function () {
- return persistChanges(projectId, changes, limits, endVersion)
+ await loadFiles()
+
+ let result
+ try {
+ const { historyBufferLevel, forcePersistBuffer } =
+ rollout.getHistoryBufferLevelOptions(projectId)
+ result = await commitChanges(projectId, changes, limits, endVersion, {
+ historyBufferLevel,
+ forcePersistBuffer,
})
- .then(function (result) {
- if (returnSnapshot === 'none') {
- res.status(HTTPStatus.CREATED).json({})
- } else {
- return buildResultSnapshot(result && result.currentChunk).then(
- function (rawSnapshot) {
- res.status(HTTPStatus.CREATED).json(rawSnapshot)
- }
- )
- }
- })
- .catch(err => {
- if (
- err instanceof Chunk.ConflictingEndVersion ||
- err instanceof TextOperation.UnprocessableError ||
- err instanceof File.NotEditableError ||
- err instanceof FileMap.PathnameError ||
- err instanceof Snapshot.EditMissingFileError ||
- err instanceof chunkStore.ChunkVersionConflictError
- ) {
- // If we failed to apply operations, that's probably because they were
- // invalid.
- logger.error(err)
- render.unprocessableEntity(res)
- } else if (err instanceof Chunk.NotFoundError) {
- render.notFound(res)
- } else {
- next(err)
- }
+ } catch (err) {
+ if (
+ err instanceof Chunk.ConflictingEndVersion ||
+ err instanceof TextOperation.UnprocessableError ||
+ err instanceof File.NotEditableError ||
+ err instanceof FileMap.PathnameError ||
+ err instanceof Snapshot.EditMissingFileError ||
+ err instanceof chunkStore.ChunkVersionConflictError ||
+ err instanceof InvalidChangeError
+ ) {
+ // If we failed to apply operations, that's probably because they were
+ // invalid.
+ logger.warn({ err, projectId, endVersion }, 'changes rejected by history')
+ return render.unprocessableEntity(res)
+ } else if (err instanceof Chunk.NotFoundError) {
+ logger.warn({ err, projectId }, 'chunk not found')
+ return render.notFound(res)
+ } else {
+ throw err
+ }
+ }
+
+ if (returnSnapshot === 'none') {
+ res.status(HTTPStatus.CREATED).json({
+ resyncNeeded: result.resyncNeeded,
})
+ } else {
+ const rawSnapshot = await buildResultSnapshot(result && result.currentChunk)
+ res.status(HTTPStatus.CREATED).json(rawSnapshot)
+ }
}
+
+async function flushChanges(req, res, next) {
+ const projectId = req.swagger.params.project_id.value
+ // Use the same limits importChanges, since these are passed to persistChanges
+ const farFuture = new Date()
+ farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+ const limits = {
+ maxChanges: 0,
+ minChangeTimestamp: farFuture,
+ maxChangeTimestamp: farFuture,
+ autoResync: true,
+ }
+ try {
+ await persistBuffer(projectId, limits)
+ res.status(HTTPStatus.OK).end()
+ } catch (err) {
+ if (err instanceof Chunk.NotFoundError) {
+ render.notFound(res)
+ } else {
+ throw err
+ }
+ }
+}
+
+async function expireProject(req, res, next) {
+ const projectId = req.swagger.params.project_id.value
+ await redisBackend.expireProject(projectId)
+ res.status(HTTPStatus.OK).end()
+}
+
+exports.importSnapshot = expressify(importSnapshot)
+exports.importChanges = expressify(importChanges)
+exports.flushChanges = expressify(flushChanges)
+exports.expireProject = expressify(expireProject)
diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js
index 3e00035d39..b7f07c4834 100644
--- a/services/history-v1/api/controllers/projects.js
+++ b/services/history-v1/api/controllers/projects.js
@@ -1,12 +1,13 @@
'use strict'
const _ = require('lodash')
-const Path = require('path')
-const Stream = require('stream')
+const Path = require('node:path')
+const Stream = require('node:stream')
const HTTPStatus = require('http-status')
-const fs = require('fs')
-const { promisify } = require('util')
+const fs = require('node:fs')
+const { promisify } = require('node:util')
const config = require('config')
+const OError = require('@overleaf/o-error')
const logger = require('@overleaf/logger')
const { Chunk, ChunkResponse, Blob } = require('overleaf-editor-core')
@@ -14,6 +15,7 @@ const {
BlobStore,
blobHash,
chunkStore,
+ redisBuffer,
HashCheckBlobStore,
ProjectArchive,
zipStore,
@@ -33,6 +35,7 @@ async function initializeProject(req, res, next) {
res.status(HTTPStatus.OK).json({ projectId })
} catch (err) {
if (err instanceof chunkStore.AlreadyInitialized) {
+ logger.warn({ err, projectId }, 'failed to initialize')
render.conflict(res)
} else {
throw err
@@ -85,6 +88,26 @@ async function getLatestHistory(req, res, next) {
}
}
+async function getLatestHistoryRaw(req, res, next) {
+ const projectId = req.swagger.params.project_id.value
+ const readOnly = req.swagger.params.readOnly.value
+ try {
+ const { startVersion, endVersion, endTimestamp } =
+ await chunkStore.getLatestChunkMetadata(projectId, { readOnly })
+ res.json({
+ startVersion,
+ endVersion,
+ endTimestamp,
+ })
+ } catch (err) {
+ if (err instanceof Chunk.NotFoundError) {
+ render.notFound(res)
+ } else {
+ throw err
+ }
+ }
+}
+
async function getHistory(req, res, next) {
const projectId = req.swagger.params.project_id.value
const version = req.swagger.params.version.value
@@ -117,6 +140,43 @@ async function getHistoryBefore(req, res, next) {
}
}
+/**
+ * Get all changes since the beginning of history or since a given version
+ */
+async function getChanges(req, res, next) {
+ const projectId = req.swagger.params.project_id.value
+ const since = req.swagger.params.since.value ?? 0
+
+ if (since < 0) {
+ // Negative values would cause an infinite loop
+ return res.status(400).json({
+ error: `Version out of bounds: ${since}`,
+ })
+ }
+
+ let chunk
+ try {
+ chunk = await chunkStore.loadAtVersion(projectId, since, {
+ preferNewer: true,
+ })
+ } catch (err) {
+ if (err instanceof Chunk.VersionNotFoundError) {
+ return res.status(400).json({
+ error: `Version out of bounds: ${since}`,
+ })
+ }
+ throw err
+ }
+
+ const latestChunkMetadata = await chunkStore.getLatestChunkMetadata(projectId)
+
+ // Extract the relevant changes from the chunk that contains the start version
+ const changes = chunk.getChanges().slice(since - chunk.getStartVersion())
+ const hasMore = latestChunkMetadata.endVersion > chunk.getEndVersion()
+
+ res.json({ changes: changes.map(change => change.toRaw()), hasMore })
+}
+
async function getZip(req, res, next) {
const projectId = req.swagger.params.project_id.value
const version = req.swagger.params.version.value
@@ -167,7 +227,9 @@ async function createZip(req, res, next) {
async function deleteProject(req, res, next) {
const projectId = req.swagger.params.project_id.value
const blobStore = new BlobStore(projectId)
+
await Promise.all([
+ redisBuffer.hardDeleteProject(projectId),
chunkStore.deleteProjectChunks(projectId),
blobStore.deleteBlobs(),
])
@@ -184,44 +246,127 @@ async function createProjectBlob(req, res, next) {
const sizeLimit = new StreamSizeLimit(maxUploadSize)
await pipeline(req, sizeLimit, fs.createWriteStream(tmpPath))
if (sizeLimit.sizeLimitExceeded) {
+ logger.warn(
+ { projectId, expectedHash, maxUploadSize },
+ 'blob exceeds size threshold'
+ )
return render.requestEntityTooLarge(res)
}
const hash = await blobHash.fromFile(tmpPath)
if (hash !== expectedHash) {
- logger.debug({ hash, expectedHash }, 'Hash mismatch')
+ logger.warn({ projectId, hash, expectedHash }, 'Hash mismatch')
return render.conflict(res, 'File hash mismatch')
}
const blobStore = new BlobStore(projectId)
- await blobStore.putFile(tmpPath)
+ const newBlob = await blobStore.putFile(tmpPath)
+
+ if (config.has('backupStore')) {
+ try {
+ const { backupBlob } = await import('../../storage/lib/backupBlob.mjs')
+ await backupBlob(projectId, newBlob, tmpPath)
+ } catch (error) {
+ logger.warn({ error, projectId, hash }, 'Failed to backup blob')
+ }
+ }
res.status(HTTPStatus.CREATED).end()
})
}
+async function headProjectBlob(req, res) {
+ const projectId = req.swagger.params.project_id.value
+ const hash = req.swagger.params.hash.value
+
+ const blobStore = new BlobStore(projectId)
+ const blob = await blobStore.getBlob(hash)
+ if (blob) {
+ res.set('Content-Length', blob.getByteLength())
+ res.status(200).end()
+ } else {
+ res.status(404).end()
+ }
+}
+
+// Support simple, singular ranges starting from zero only, up-to 2MB = 2_000_000, 7 digits
+const RANGE_HEADER = /^bytes=0-(\d{1,7})$/
+
+/**
+ * @param {string} header
+ * @return {{}|{start: number, end: number}}
+ * @private
+ */
+function _getRangeOpts(header) {
+ if (!header) return {}
+ const match = header.match(RANGE_HEADER)
+ if (match) {
+ const end = parseInt(match[1], 10)
+ return { start: 0, end }
+ }
+ return {}
+}
+
async function getProjectBlob(req, res, next) {
const projectId = req.swagger.params.project_id.value
const hash = req.swagger.params.hash.value
+ const opts = _getRangeOpts(req.swagger.params.range.value || '')
const blobStore = new BlobStore(projectId)
logger.debug({ projectId, hash }, 'getProjectBlob started')
try {
let stream
try {
- stream = await blobStore.getStream(hash)
+ stream = await blobStore.getStream(hash, opts)
} catch (err) {
if (err instanceof Blob.NotFoundError) {
- return render.notFound(res)
+ logger.warn({ projectId, hash }, 'Blob not found')
+ return res.status(404).end()
} else {
throw err
}
}
res.set('Content-Type', 'application/octet-stream')
- await pipeline(stream, res)
+ try {
+ await pipeline(stream, res)
+ } catch (err) {
+ if (err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
+ res.end()
+ } else {
+ throw OError.tag(err, 'error transferring stream', { projectId, hash })
+ }
+ }
} finally {
logger.debug({ projectId, hash }, 'getProjectBlob finished')
}
}
+async function copyProjectBlob(req, res, next) {
+ const sourceProjectId = req.swagger.params.copyFrom.value
+ const targetProjectId = req.swagger.params.project_id.value
+ const blobHash = req.swagger.params.hash.value
+ // Check that blob exists in source project
+ const sourceBlobStore = new BlobStore(sourceProjectId)
+ const targetBlobStore = new BlobStore(targetProjectId)
+ const [sourceBlob, targetBlob] = await Promise.all([
+ sourceBlobStore.getBlob(blobHash),
+ targetBlobStore.getBlob(blobHash),
+ ])
+ if (!sourceBlob) {
+ logger.warn(
+ { sourceProjectId, targetProjectId, blobHash },
+ 'missing source blob when copying across projects'
+ )
+ return render.notFound(res)
+ }
+ // Exit early if the blob exists in the target project.
+ // This will also catch global blobs, which always exist.
+ if (targetBlob) {
+ return res.status(HTTPStatus.NO_CONTENT).end()
+ }
+ // Otherwise, copy blob from source project to target project
+ await sourceBlobStore.copyBlob(sourceBlob, targetProjectId)
+ res.status(HTTPStatus.CREATED).end()
+}
+
async function getSnapshotAtVersion(projectId, version) {
const chunk = await chunkStore.loadAtVersion(projectId, version)
const snapshot = chunk.getSnapshot()
@@ -240,11 +385,15 @@ module.exports = {
getLatestHashedContent: expressify(getLatestHashedContent),
getLatestPersistedHistory: expressify(getLatestHistory),
getLatestHistory: expressify(getLatestHistory),
+ getLatestHistoryRaw: expressify(getLatestHistoryRaw),
getHistory: expressify(getHistory),
getHistoryBefore: expressify(getHistoryBefore),
+ getChanges: expressify(getChanges),
getZip: expressify(getZip),
createZip: expressify(createZip),
deleteProject: expressify(deleteProject),
createProjectBlob: expressify(createProjectBlob),
getProjectBlob: expressify(getProjectBlob),
+ headProjectBlob: expressify(headProjectBlob),
+ copyProjectBlob: expressify(copyProjectBlob),
}
diff --git a/services/history-v1/api/controllers/stream_size_limit.js b/services/history-v1/api/controllers/stream_size_limit.js
index fbb2ab8030..f3a14959f6 100644
--- a/services/history-v1/api/controllers/stream_size_limit.js
+++ b/services/history-v1/api/controllers/stream_size_limit.js
@@ -1,4 +1,4 @@
-const stream = require('stream')
+const stream = require('node:stream')
/**
* Transform stream that stops passing bytes through after some threshold has
diff --git a/services/history-v1/api/controllers/with_tmp_dir.js b/services/history-v1/api/controllers/with_tmp_dir.js
index ab2279ce17..2e0737ba69 100644
--- a/services/history-v1/api/controllers/with_tmp_dir.js
+++ b/services/history-v1/api/controllers/with_tmp_dir.js
@@ -1,15 +1,15 @@
-const fs = require('fs')
+const fs = require('node:fs')
const fsExtra = require('fs-extra')
const logger = require('@overleaf/logger')
-const os = require('os')
-const path = require('path')
+const os = require('node:os')
+const path = require('node:path')
/**
* Create a temporary directory before executing a function and cleaning up
* after.
*
* @param {string} prefix - prefix for the temporary directory name
- * @param {Function} fn - async function to call
+ * @param {(tmpDir: string) => Promise} fn - async function to call
*/
async function withTmpDir(prefix, fn) {
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix))
diff --git a/services/history-v1/api/swagger/index.js b/services/history-v1/api/swagger/index.js
index edfb68f4e4..3702c6ec07 100644
--- a/services/history-v1/api/swagger/index.js
+++ b/services/history-v1/api/swagger/index.js
@@ -84,6 +84,19 @@ module.exports = {
},
},
},
+ ChunkResponseRaw: {
+ properties: {
+ startVersion: {
+ type: 'number',
+ },
+ endVersion: {
+ type: 'number',
+ },
+ endTimestamp: {
+ type: 'string',
+ },
+ },
+ },
History: {
properties: {
snapshot: {
diff --git a/services/history-v1/api/swagger/project_import.js b/services/history-v1/api/swagger/project_import.js
index 60eb47fce4..043dc70667 100644
--- a/services/history-v1/api/swagger/project_import.js
+++ b/services/history-v1/api/swagger/project_import.js
@@ -100,9 +100,120 @@ const importChanges = {
],
}
+const getChanges = {
+ 'x-swagger-router-controller': 'projects',
+ operationId: 'getChanges',
+ tags: ['Project'],
+ description: 'Get changes applied to a project',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'project id',
+ required: true,
+ type: 'string',
+ },
+ {
+ name: 'since',
+ in: 'query',
+ description: 'start version',
+ required: false,
+ type: 'number',
+ },
+ ],
+ responses: {
+ 200: {
+ description: 'Success',
+ schema: {
+ type: 'array',
+ items: {
+ $ref: '#/definitions/Change',
+ },
+ },
+ },
+ },
+ security: [
+ {
+ basic: [],
+ },
+ ],
+}
+
+const flushChanges = {
+ 'x-swagger-router-controller': 'project_import',
+ operationId: 'flushChanges',
+ tags: ['ProjectImport'],
+ description: 'Flush project changes from buffer to the chunk store.',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'project id',
+ required: true,
+ type: 'string',
+ },
+ ],
+ responses: {
+ 200: {
+ description: 'Success',
+ schema: {
+ $ref: '#/definitions/Project',
+ },
+ },
+ 404: {
+ description: 'Not Found',
+ schema: {
+ $ref: '#/definitions/Error',
+ },
+ },
+ },
+ security: [
+ {
+ basic: [],
+ },
+ ],
+}
+
+const expireProject = {
+ 'x-swagger-router-controller': 'project_import',
+ operationId: 'expireProject',
+ tags: ['ProjectImport'],
+ description: 'Expire project changes from buffer.',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'project id',
+ required: true,
+ type: 'string',
+ },
+ ],
+ responses: {
+ 200: {
+ description: 'Success',
+ schema: {
+ $ref: '#/definitions/Project',
+ },
+ },
+ 404: {
+ description: 'Not Found',
+ schema: {
+ $ref: '#/definitions/Error',
+ },
+ },
+ },
+ security: [
+ {
+ basic: [],
+ },
+ ],
+}
+
exports.paths = {
'/projects/{project_id}/import': { post: importSnapshot },
'/projects/{project_id}/legacy_import': { post: importSnapshot },
- '/projects/{project_id}/changes': { post: importChanges },
+ '/projects/{project_id}/changes': { get: getChanges, post: importChanges },
'/projects/{project_id}/legacy_changes': { post: importChanges },
+ '/projects/{project_id}/flush': { post: flushChanges },
+ '/projects/{project_id}/expire': { post: expireProject },
}
diff --git a/services/history-v1/api/swagger/projects.js b/services/history-v1/api/swagger/projects.js
index f4b8ed7352..cd4d2338fa 100644
--- a/services/history-v1/api/swagger/projects.js
+++ b/services/history-v1/api/swagger/projects.js
@@ -9,6 +9,7 @@ exports.paths = {
operationId: 'initializeProject',
tags: ['Project'],
description: 'Initialize project.',
+ consumes: ['application/json'],
parameters: [
{
name: 'body',
@@ -69,6 +70,52 @@ exports.paths = {
operationId: 'getProjectBlob',
tags: ['Project'],
description: 'Fetch blob content by its project id and hash.',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'project id',
+ required: true,
+ type: 'string',
+ },
+ {
+ name: 'hash',
+ in: 'path',
+ description: 'Hexadecimal SHA-1 hash',
+ required: true,
+ type: 'string',
+ pattern: Blob.HEX_HASH_RX_STRING,
+ },
+ {
+ name: 'range',
+ in: 'header',
+ description: 'HTTP Range header',
+ required: false,
+ type: 'string',
+ },
+ ],
+ produces: ['application/octet-stream'],
+ responses: {
+ 200: {
+ description: 'Success',
+ schema: {
+ type: 'file',
+ },
+ },
+ 404: {
+ description: 'Not Found',
+ schema: {
+ $ref: '#/definitions/Error',
+ },
+ },
+ },
+ security: [{ jwt: [] }, { token: [] }],
+ },
+ head: {
+ 'x-swagger-router-controller': 'projects',
+ operationId: 'headProjectBlob',
+ tags: ['Project'],
+ description: 'Fetch blob content-length by its project id and hash.',
parameters: [
{
name: 'project_id',
@@ -133,6 +180,42 @@ exports.paths = {
},
},
},
+ post: {
+ 'x-swagger-router-controller': 'projects',
+ operationId: 'copyProjectBlob',
+ tags: ['Project'],
+ description:
+ 'Copies a blob from a source project to a target project when duplicating a project',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'target project id',
+ required: true,
+ type: 'string',
+ },
+ {
+ name: 'hash',
+ in: 'path',
+ description: 'Hexadecimal SHA-1 hash',
+ required: true,
+ type: 'string',
+ pattern: Blob.HEX_HASH_RX_STRING,
+ },
+ {
+ name: 'copyFrom',
+ in: 'query',
+ description: 'source project id',
+ required: true,
+ type: 'string',
+ },
+ ],
+ responses: {
+ 201: {
+ description: 'Created',
+ },
+ },
+ },
},
'/projects/{project_id}/latest/content': {
get: {
@@ -238,6 +321,44 @@ exports.paths = {
},
},
},
+ '/projects/{project_id}/latest/history/raw': {
+ get: {
+ 'x-swagger-router-controller': 'projects',
+ operationId: 'getLatestHistoryRaw',
+ tags: ['Project'],
+ description: 'Get the metadata of latest sequence of changes.',
+ parameters: [
+ {
+ name: 'project_id',
+ in: 'path',
+ description: 'project id',
+ required: true,
+ type: 'string',
+ },
+ {
+ name: 'readOnly',
+ in: 'query',
+ description: 'use read only database connection',
+ required: false,
+ type: 'boolean',
+ },
+ ],
+ responses: {
+ 200: {
+ description: 'Success',
+ schema: {
+ $ref: '#/definitions/ChunkResponseRaw',
+ },
+ },
+ 404: {
+ description: 'Not Found',
+ schema: {
+ $ref: '#/definitions/Error',
+ },
+ },
+ },
+ },
+ },
'/projects/{project_id}/latest/persistedHistory': {
get: {
'x-swagger-router-controller': 'projects',
diff --git a/services/history-v1/app.js b/services/history-v1/app.js
index a10d69eba6..dd991c1a6d 100644
--- a/services/history-v1/app.js
+++ b/services/history-v1/app.js
@@ -6,7 +6,7 @@
require('@overleaf/metrics/initialize')
const config = require('config')
-const Events = require('events')
+const Events = require('node:events')
const BPromise = require('bluebird')
const express = require('express')
const helmet = require('helmet')
@@ -19,7 +19,7 @@ const swaggerDoc = require('./api/swagger')
const security = require('./api/app/security')
const healthChecks = require('./api/controllers/health_checks')
const { mongodb, loadGlobalBlobs } = require('./storage')
-const path = require('path')
+const path = require('node:path')
Events.setMaxListeners(20)
const app = express()
@@ -84,26 +84,29 @@ function setupErrorHandling() {
// Handle Swagger errors.
app.use(function (err, req, res, next) {
+ const projectId = req.swagger?.params?.project_id?.value
if (res.headersSent) {
return next(err)
}
if (err.code === 'SCHEMA_VALIDATION_FAILED') {
- logger.error(err)
+ logger.error({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json(err.results)
}
if (err.code === 'INVALID_TYPE' || err.code === 'PATTERN') {
- logger.error(err)
+ logger.error({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: 'invalid type: ' + err.paramName,
})
}
if (err.code === 'ENUM_MISMATCH') {
+ logger.warn({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: 'invalid enum value: ' + err.paramName,
})
}
if (err.code === 'REQUIRED') {
+ logger.warn({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: err.message,
})
@@ -112,7 +115,8 @@ function setupErrorHandling() {
})
app.use(function (err, req, res, next) {
- logger.error(err)
+ const projectId = req.swagger?.params?.project_id?.value
+ logger.error({ err, projectId }, err.message)
if (res.headersSent) {
return next(err)
diff --git a/services/history-v1/backup-deletion-app.mjs b/services/history-v1/backup-deletion-app.mjs
new file mode 100644
index 0000000000..81b2b5b8b9
--- /dev/null
+++ b/services/history-v1/backup-deletion-app.mjs
@@ -0,0 +1,81 @@
+// @ts-check
+// Metrics must be initialized before importing anything else
+import '@overleaf/metrics/initialize.js'
+import http from 'node:http'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import express from 'express'
+import logger from '@overleaf/logger'
+import Metrics from '@overleaf/metrics'
+import { hasValidBasicAuthCredentials } from './api/app/security.js'
+import {
+ deleteProjectBackupCb,
+ healthCheck,
+ healthCheckCb,
+ NotReadyToDelete,
+} from './storage/lib/backupDeletion.mjs'
+import { mongodb } from './storage/index.js'
+
+const app = express()
+
+logger.initialize('history-v1-backup-deletion')
+Metrics.open_sockets.monitor()
+Metrics.injectMetricsRoute(app)
+app.use(Metrics.http.monitor(logger))
+Metrics.leaked_sockets.monitor(logger)
+Metrics.event_loop.monitor(logger)
+Metrics.memory.monitor(logger)
+
+function basicAuth(req, res, next) {
+ if (hasValidBasicAuthCredentials(req)) return next()
+ res.setHeader('WWW-Authenticate', 'Basic realm="Application"')
+ res.sendStatus(401)
+}
+
+app.delete('/project/:projectId/backup', basicAuth, (req, res, next) => {
+ deleteProjectBackupCb(req.params.projectId, err => {
+ if (err) {
+ return next(err)
+ }
+ res.sendStatus(204)
+ })
+})
+
+app.get('/status', (req, res) => {
+ res.send('history-v1-backup-deletion is up')
+})
+
+app.get('/health_check', (req, res, next) => {
+ healthCheckCb(err => {
+ if (err) return next(err)
+ res.sendStatus(200)
+ })
+})
+
+app.use((err, req, res, next) => {
+ req.logger.addFields({ err })
+ if (err instanceof NotReadyToDelete) {
+ req.logger.setLevel('warn')
+ return res.status(422).send(err.message)
+ }
+ req.logger.setLevel('error')
+ next(err)
+})
+
+/**
+ * @param {number} port
+ * @return {Promise}
+ */
+export async function startApp(port) {
+ await mongodb.client.connect()
+ await healthCheck()
+ const server = http.createServer(app)
+ await promisify(server.listen.bind(server, port))()
+ return server
+}
+
+// Run this if we're called directly
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const PORT = parseInt(process.env.PORT || '3101', 10)
+ await startApp(PORT)
+}
diff --git a/services/history-v1/backup-verifier-app.mjs b/services/history-v1/backup-verifier-app.mjs
new file mode 100644
index 0000000000..856a15dd53
--- /dev/null
+++ b/services/history-v1/backup-verifier-app.mjs
@@ -0,0 +1,117 @@
+// @ts-check
+// Metrics must be initialized before importing anything else
+import '@overleaf/metrics/initialize.js'
+import http from 'node:http'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import { setTimeout } from 'node:timers/promises'
+import express from 'express'
+import logger from '@overleaf/logger'
+import Metrics from '@overleaf/metrics'
+import { healthCheck } from './backupVerifier/healthCheck.mjs'
+import {
+ BackupCorruptedError,
+ verifyBlob,
+} from './storage/lib/backupVerifier.mjs'
+import { mongodb } from './storage/index.js'
+import { expressify } from '@overleaf/promise-utils'
+import { Blob } from 'overleaf-editor-core'
+import { loadGlobalBlobs } from './storage/lib/blob_store/index.js'
+import { EventEmitter } from 'node:events'
+import {
+ loopRandomProjects,
+ setWriteMetrics,
+} from './backupVerifier/ProjectVerifier.mjs'
+
+const app = express()
+
+logger.initialize('history-v1-backup-verifier')
+Metrics.open_sockets.monitor()
+Metrics.injectMetricsRoute(app)
+app.use(Metrics.http.monitor(logger))
+Metrics.leaked_sockets.monitor(logger)
+Metrics.event_loop.monitor(logger)
+Metrics.memory.monitor(logger)
+
+app.get(
+ '/history/:historyId/blob/:hash/verify',
+ expressify(async (req, res) => {
+ const { historyId, hash } = req.params
+ try {
+ await verifyBlob(historyId, hash)
+ res.sendStatus(200)
+ } catch (err) {
+ logger.warn({ err, historyId, hash }, 'manual verify blob failed')
+ if (err instanceof Blob.NotFoundError) {
+ res.status(404).send(err.message)
+ } else if (err instanceof BackupCorruptedError) {
+ res.status(422).send(err.message)
+ } else {
+ throw err
+ }
+ }
+ })
+)
+
+app.get('/status', (req, res) => {
+ res.send('history-v1-backup-verifier is up')
+})
+
+app.get(
+ '/health_check',
+ expressify(async (req, res) => {
+ await healthCheck()
+ res.sendStatus(200)
+ })
+)
+
+app.use((err, req, res, next) => {
+ req.logger.addFields({ err })
+ req.logger.setLevel('error')
+ next(err)
+})
+
+const shutdownEmitter = new EventEmitter()
+
+shutdownEmitter.once('shutdown', async code => {
+ logger.info({ code }, 'shutting down')
+ await mongodb.client.close()
+ await setTimeout(100)
+ process.exit(code)
+})
+
+process.on('SIGTERM', () => {
+ shutdownEmitter.emit('shutdown', 0)
+})
+
+process.on('SIGINT', () => {
+ shutdownEmitter.emit('shutdown', 0)
+})
+
+/**
+ * @param {number} port
+ * @param {boolean} enableVerificationLoop
+ * @return {Promise}
+ */
+export async function startApp(port, enableVerificationLoop = true) {
+ await mongodb.client.connect()
+ await loadGlobalBlobs()
+ await healthCheck()
+ const server = http.createServer(app)
+ await promisify(server.listen.bind(server, port))()
+ enableVerificationLoop && loopRandomProjects(shutdownEmitter)
+ return server
+}
+
+setWriteMetrics(true)
+
+// Run this if we're called directly
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const PORT = parseInt(process.env.PORT || '3102', 10)
+ try {
+ await startApp(PORT)
+ } catch (error) {
+ shutdownEmitter.emit('shutdown', 1)
+ logger.error({ error }, 'error starting app')
+ }
+}
diff --git a/services/history-v1/backup-worker-app.mjs b/services/history-v1/backup-worker-app.mjs
new file mode 100644
index 0000000000..b21e55aafe
--- /dev/null
+++ b/services/history-v1/backup-worker-app.mjs
@@ -0,0 +1,70 @@
+// @ts-check
+// Metrics must be initialized before importing anything else
+import '@overleaf/metrics/initialize.js'
+import http from 'node:http'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import express from 'express'
+import logger from '@overleaf/logger'
+import Metrics from '@overleaf/metrics'
+import { expressify } from '@overleaf/promise-utils'
+import { drainQueue, healthCheck } from './storage/scripts/backup_worker.mjs'
+const app = express()
+
+logger.initialize('history-v1-backup-worker')
+Metrics.open_sockets.monitor()
+Metrics.injectMetricsRoute(app)
+app.use(Metrics.http.monitor(logger))
+Metrics.leaked_sockets.monitor(logger)
+Metrics.event_loop.monitor(logger)
+Metrics.memory.monitor(logger)
+
+app.get('/status', (req, res) => {
+ res.send('history-v1-backup-worker is up')
+})
+
+app.get(
+ '/health_check',
+ expressify(async (req, res) => {
+ await healthCheck()
+ res.sendStatus(200)
+ })
+)
+
+app.use((err, req, res, next) => {
+ req.logger.addFields({ err })
+ req.logger.setLevel('error')
+ next(err)
+})
+
+async function triggerGracefulShutdown(server, signal) {
+ logger.info({ signal }, 'graceful shutdown: started shutdown sequence')
+ await drainQueue()
+ server.close(function () {
+ logger.info({ signal }, 'graceful shutdown: closed server')
+ setTimeout(() => {
+ process.exit(0)
+ }, 1000)
+ })
+}
+
+/**
+ * @param {number} port
+ * @return {Promise}
+ */
+export async function startApp(port) {
+ await healthCheck()
+ const server = http.createServer(app)
+ await promisify(server.listen.bind(server, port))()
+ const signals = ['SIGINT', 'SIGTERM']
+ signals.forEach(signal => {
+ process.on(signal, () => triggerGracefulShutdown(server, signal))
+ })
+ return server
+}
+
+// Run this if we're called directly
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const PORT = parseInt(process.env.PORT || '3103', 10)
+ await startApp(PORT)
+}
diff --git a/services/history-v1/backupVerifier/ProjectMetrics.mjs b/services/history-v1/backupVerifier/ProjectMetrics.mjs
new file mode 100644
index 0000000000..ff37085787
--- /dev/null
+++ b/services/history-v1/backupVerifier/ProjectMetrics.mjs
@@ -0,0 +1,33 @@
+import Metrics from '@overleaf/metrics'
+import { objectIdFromDate } from './utils.mjs'
+import { db } from '../storage/lib/mongodb.js'
+
+const projectsCollection = db.collection('projects')
+
+/**
+ *
+ * @param {Date} beforeTime
+ * @return {Promise}
+ */
+export async function measurePendingChangesBeforeTime(beforeTime) {
+ const pendingChangeCount = await projectsCollection.countDocuments({
+ 'overleaf.backup.pendingChangeAt': {
+ $lt: beforeTime,
+ },
+ })
+
+ Metrics.gauge('backup_verification_pending_changes', pendingChangeCount)
+}
+
+/**
+ *
+ * @param {Date} graceTime
+ * @return {Promise}
+ */
+export async function measureNeverBackedUpProjects(graceTime) {
+ const neverBackedUpCount = await projectsCollection.countDocuments({
+ 'overleaf.backup.lastBackedUpVersion': null,
+ _id: { $lt: objectIdFromDate(graceTime) },
+ })
+ Metrics.gauge('backup_verification_never_backed_up', neverBackedUpCount)
+}
diff --git a/services/history-v1/backupVerifier/ProjectSampler.mjs b/services/history-v1/backupVerifier/ProjectSampler.mjs
new file mode 100644
index 0000000000..93d9a1a31f
--- /dev/null
+++ b/services/history-v1/backupVerifier/ProjectSampler.mjs
@@ -0,0 +1,79 @@
+// @ts-check
+import { objectIdFromDate } from './utils.mjs'
+import { db } from '../storage/lib/mongodb.js'
+import config from 'config'
+
+const projectsCollection = db.collection('projects')
+
+const HAS_PROJECTS_WITHOUT_HISTORY =
+ config.get('hasProjectsWithoutHistory') === 'true'
+
+/**
+ * @param {Date} start
+ * @param {Date} end
+ * @param {number} N
+ * @yields {string}
+ */
+export async function* getProjectsCreatedInDateRangeCursor(start, end, N) {
+ yield* getSampleProjectsCursor(N, [
+ {
+ $match: {
+ _id: {
+ $gt: objectIdFromDate(start),
+ $lte: objectIdFromDate(end),
+ },
+ },
+ },
+ ])
+}
+
+export async function* getProjectsUpdatedInDateRangeCursor(start, end, N) {
+ yield* getSampleProjectsCursor(N, [
+ {
+ $match: {
+ 'overleaf.history.updatedAt': {
+ $gt: start,
+ $lte: end,
+ },
+ },
+ },
+ ])
+}
+
+/**
+ * @typedef {import('mongodb').Document} Document
+ */
+
+/**
+ *
+ * @generator
+ * @param {number} N
+ * @param {Array} preSampleAggregationStages
+ * @yields {string}
+ */
+export async function* getSampleProjectsCursor(
+ N,
+ preSampleAggregationStages = []
+) {
+ const cursor = projectsCollection.aggregate([
+ ...preSampleAggregationStages,
+ { $sample: { size: N } },
+ { $project: { 'overleaf.history.id': 1 } },
+ ])
+
+ let validProjects = 0
+ let hasInvalidProject = false
+
+ for await (const project of cursor) {
+ if (HAS_PROJECTS_WITHOUT_HISTORY && !project.overleaf?.history?.id) {
+ hasInvalidProject = true
+ continue
+ }
+ validProjects++
+ yield project.overleaf.history.id.toString()
+ }
+
+ if (validProjects === 0 && hasInvalidProject) {
+ yield* getSampleProjectsCursor(N, preSampleAggregationStages)
+ }
+}
diff --git a/services/history-v1/backupVerifier/ProjectVerifier.mjs b/services/history-v1/backupVerifier/ProjectVerifier.mjs
new file mode 100644
index 0000000000..1e4086b700
--- /dev/null
+++ b/services/history-v1/backupVerifier/ProjectVerifier.mjs
@@ -0,0 +1,320 @@
+// @ts-check
+import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
+import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
+import logger from '@overleaf/logger'
+import metrics from '@overleaf/metrics'
+import {
+ getSampleProjectsCursor,
+ getProjectsCreatedInDateRangeCursor,
+ getProjectsUpdatedInDateRangeCursor,
+} from './ProjectSampler.mjs'
+import OError from '@overleaf/o-error'
+import { setTimeout } from 'node:timers/promises'
+
+const MS_PER_30_DAYS = 30 * 24 * 60 * 60 * 1000
+
+const failureCounter = new metrics.prom.Counter({
+ name: 'backup_project_verification_failed',
+ help: 'Number of projects that failed verification',
+ labelNames: ['name'],
+})
+
+const successCounter = new metrics.prom.Counter({
+ name: 'backup_project_verification_succeeded',
+ help: 'Number of projects that succeeded verification',
+})
+
+let WRITE_METRICS = false
+
+/**
+ * @typedef {import('node:events').EventEmitter} EventEmitter
+ */
+
+/**
+ * Allows writing metrics to be enabled or disabled.
+ * @param {Boolean} writeMetrics
+ */
+export function setWriteMetrics(writeMetrics) {
+ WRITE_METRICS = writeMetrics
+}
+
+/**
+ *
+ * @param {Error|unknown} error
+ * @param {string} historyId
+ */
+function handleVerificationError(error, historyId) {
+ const name = error instanceof Error ? error.name : 'UnknownError'
+ logger.error({ historyId, error, name }, 'error verifying project backup')
+
+ WRITE_METRICS && failureCounter.inc({ name })
+
+ return name
+}
+
+/**
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} interval
+ * @returns {Array}
+ */
+function splitJobs(startDate, endDate, interval) {
+ /** @type {Array} */
+ const jobs = []
+ while (startDate < endDate) {
+ const nextStart = new Date(
+ Math.min(startDate.getTime() + interval, endDate.getTime())
+ )
+ jobs.push({ startDate, endDate: nextStart })
+ startDate = nextStart
+ }
+ return jobs
+}
+
+/**
+ *
+ * @param {AsyncGenerator} historyIdCursor
+ * @param {EventEmitter} [eventEmitter]
+ * @param {number} [delay] - Allows a delay between each verification
+ * @return {Promise<{verified: number, total: number, errorTypes: *[], hasFailure: boolean}>}
+ */
+async function verifyProjectsFromCursor(
+ historyIdCursor,
+ eventEmitter,
+ delay = 0
+) {
+ const errorTypes = []
+ let verified = 0
+ let total = 0
+ let receivedShutdownSignal = false
+ if (eventEmitter) {
+ eventEmitter.once('shutdown', () => {
+ receivedShutdownSignal = true
+ })
+ }
+ for await (const historyId of historyIdCursor) {
+ if (receivedShutdownSignal) {
+ break
+ }
+ total++
+ try {
+ await verifyProjectWithErrorContext(historyId)
+ logger.debug({ historyId }, 'verified project backup successfully')
+ WRITE_METRICS && successCounter.inc()
+ verified++
+ } catch (error) {
+ const errorType = handleVerificationError(error, historyId)
+ errorTypes.push(errorType)
+ }
+ if (delay > 0) {
+ await setTimeout(delay)
+ }
+ }
+ return {
+ verified,
+ total,
+ errorTypes,
+ hasFailure: errorTypes.length > 0,
+ }
+}
+
+/**
+ *
+ * @param {number} nProjectsToSample
+ * @param {EventEmitter} [signal]
+ * @param {number} [delay]
+ * @return {Promise}
+ */
+export async function verifyRandomProjectSample(
+ nProjectsToSample,
+ signal,
+ delay = 0
+) {
+ const historyIds = await getSampleProjectsCursor(nProjectsToSample)
+ return await verifyProjectsFromCursor(historyIds, signal, delay)
+}
+
+/**
+ * Samples projects with history IDs between the specified dates and verifies them.
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} projectsPerRange
+ * @param {EventEmitter} [signal]
+ * @return {Promise}
+ */
+async function verifyRange(startDate, endDate, projectsPerRange, signal) {
+ logger.info({ startDate, endDate }, 'verifying range')
+
+ const results = await verifyProjectsFromCursor(
+ getProjectsCreatedInDateRangeCursor(startDate, endDate, projectsPerRange),
+ signal
+ )
+
+ if (results.total === 0) {
+ logger.debug(
+ { start: startDate, end: endDate },
+ 'No projects found in range'
+ )
+ }
+
+ const jobStatus = {
+ ...results,
+ startDate,
+ endDate,
+ }
+
+ logger.debug(
+ { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
+ 'Verified range'
+ )
+ return jobStatus
+}
+
+/**
+ * @typedef {Object} VerificationJobSpecification
+ * @property {Date} startDate
+ * @property {Date} endDate
+ */
+
+/**
+ * @typedef {import('./types.d.ts').VerificationJobStatus} VerificationJobStatus
+ */
+
+/**
+ * @typedef {Object} VerifyDateRangeOptions
+ * @property {Date} startDate
+ * @property {Date} endDate
+ * @property {number} [interval]
+ * @property {number} [projectsPerRange]
+ * @property {number} [concurrency]
+ * @property {EventEmitter} [signal]
+ */
+
+/**
+ *
+ * @param {VerifyDateRangeOptions} options
+ * @return {Promise}
+ */
+export async function verifyProjectsCreatedInDateRange({
+ concurrency = 0,
+ projectsPerRange = 10,
+ startDate,
+ endDate,
+ interval = MS_PER_30_DAYS,
+ signal,
+}) {
+ const jobs = splitJobs(startDate, endDate, interval)
+ if (jobs.length === 0) {
+ throw new OError('Time range could not be split into jobs', {
+ start: startDate,
+ end: endDate,
+ interval,
+ })
+ }
+ const settlements = await promiseMapSettledWithLimit(
+ concurrency,
+ jobs,
+ ({ startDate, endDate }) =>
+ verifyRange(startDate, endDate, projectsPerRange, signal)
+ )
+ return settlements.reduce(
+ /**
+ *
+ * @param {VerificationJobStatus} acc
+ * @param settlement
+ * @return {VerificationJobStatus}
+ */
+ (acc, settlement) => {
+ if (settlement.status !== 'rejected') {
+ if (settlement.value.hasFailure) {
+ acc.hasFailure = true
+ }
+ acc.total += settlement.value.total
+ acc.verified += settlement.value.verified
+ acc.errorTypes = acc.errorTypes.concat(settlement.value.errorTypes)
+ } else {
+ logger.error({ ...settlement.reason }, 'Error processing range')
+ }
+ return acc
+ },
+ /** @type {VerificationJobStatus} */
+ {
+ startDate,
+ endDate,
+ verified: 0,
+ total: 0,
+ hasFailure: false,
+ errorTypes: [],
+ }
+ )
+}
+
+/**
+ * Verifies that projects that have recently gone out of RPO have been updated.
+ *
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @param {number} nProjects
+ * @param {EventEmitter} [signal]
+ * @return {Promise}
+ */
+export async function verifyProjectsUpdatedInDateRange(
+ startDate,
+ endDate,
+ nProjects,
+ signal
+) {
+ logger.debug(
+ { startDate, endDate, nProjects },
+ 'Sampling projects updated in date range'
+ )
+ const results = await verifyProjectsFromCursor(
+ getProjectsUpdatedInDateRangeCursor(startDate, endDate, nProjects),
+ signal
+ )
+
+ if (results.total === 0) {
+ logger.debug(
+ { start: startDate, end: endDate },
+ 'No projects updated recently'
+ )
+ }
+
+ const jobStatus = {
+ ...results,
+ startDate,
+ endDate,
+ }
+
+ logger.debug(
+ { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
+ 'Verified recently updated projects'
+ )
+ return jobStatus
+}
+
+/**
+ *
+ * @param {EventEmitter} signal
+ * @return {void}
+ */
+export function loopRandomProjects(signal) {
+ let shutdown = false
+ signal.on('shutdown', function () {
+ shutdown = true
+ })
+ async function loop() {
+ do {
+ try {
+ const result = await verifyRandomProjectSample(100, signal, 2_000)
+ logger.debug({ result }, 'verified random project sample')
+ } catch (error) {
+ logger.error({ error }, 'error verifying random project sample')
+ }
+ // eslint-disable-next-line no-unmodified-loop-condition
+ } while (!shutdown)
+ }
+ loop()
+}
diff --git a/services/history-v1/backupVerifier/healthCheck.mjs b/services/history-v1/backupVerifier/healthCheck.mjs
new file mode 100644
index 0000000000..af998748b5
--- /dev/null
+++ b/services/history-v1/backupVerifier/healthCheck.mjs
@@ -0,0 +1,32 @@
+import config from 'config'
+import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
+import {
+ measureNeverBackedUpProjects,
+ measurePendingChangesBeforeTime,
+} from './ProjectMetrics.mjs'
+import { getEndDateForRPO, RPO } from './utils.mjs'
+
+/** @type {Array} */
+const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
+
+export async function healthCheck() {
+ if (!Array.isArray(HEALTH_CHECK_PROJECTS)) {
+ throw new Error('expected healthCheckProjects to be an array')
+ }
+ if (HEALTH_CHECK_PROJECTS.length !== 2) {
+ throw new Error('expected 2 healthCheckProjects')
+ }
+ if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
+ throw new Error('expected mongo id in healthCheckProjects')
+ }
+ if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
+ throw new Error('expected postgres id in healthCheckProjects')
+ }
+
+ for (const historyId of HEALTH_CHECK_PROJECTS) {
+ await verifyProjectWithErrorContext(historyId)
+ }
+
+ await measurePendingChangesBeforeTime(getEndDateForRPO(2))
+ await measureNeverBackedUpProjects(getEndDateForRPO(2))
+}
diff --git a/services/history-v1/backupVerifier/types.d.ts b/services/history-v1/backupVerifier/types.d.ts
new file mode 100644
index 0000000000..7bfa4a85ff
--- /dev/null
+++ b/services/history-v1/backupVerifier/types.d.ts
@@ -0,0 +1,8 @@
+export type VerificationJobStatus = {
+ verified: number
+ total: number
+ startDate?: Date
+ endDate?: Date
+ hasFailure: boolean
+ errorTypes: Array
+}
diff --git a/services/history-v1/backupVerifier/utils.mjs b/services/history-v1/backupVerifier/utils.mjs
new file mode 100644
index 0000000000..b2d7ed2d3c
--- /dev/null
+++ b/services/history-v1/backupVerifier/utils.mjs
@@ -0,0 +1,35 @@
+import { ObjectId } from 'mongodb'
+import config from 'config'
+
+export const RPO = parseInt(config.get('backupRPOInMS'), 10)
+
+/**
+ * @param {Date} time
+ * @return {ObjectId}
+ */
+export function objectIdFromDate(time) {
+ return ObjectId.createFromTime(time.getTime() / 1000)
+}
+
+/**
+ * @param {number} [factor] - Multiply RPO by this factor, default is 1
+ * @return {Date}
+ */
+export function getEndDateForRPO(factor = 1) {
+ return new Date(Date.now() - RPO * factor)
+}
+
+/**
+ * Creates a startDate, endDate pair that checks a period of time before the RPO horizon
+ *
+ * @param {number} offset - How many seconds we should check
+ * @return {{endDate: Date, startDate: Date}}
+ */
+export function getDatesBeforeRPO(offset) {
+ const now = new Date()
+ const endDate = new Date(now.getTime() - RPO)
+ return {
+ endDate,
+ startDate: new Date(endDate.getTime() - offset * 1000),
+ }
+}
diff --git a/services/history-v1/benchmarks/blob_store.js b/services/history-v1/benchmarks/blob_store.js
index 2efb90ffb2..9efad8747f 100644
--- a/services/history-v1/benchmarks/blob_store.js
+++ b/services/history-v1/benchmarks/blob_store.js
@@ -1,4 +1,4 @@
-const crypto = require('crypto')
+const crypto = require('node:crypto')
const benny = require('benny')
const { Blob } = require('overleaf-editor-core')
const mongoBackend = require('../storage/lib/blob_store/mongo')
diff --git a/services/history-v1/buildscript.txt b/services/history-v1/buildscript.txt
index 04430a155f..4ce8eb63c7 100644
--- a/services/history-v1/buildscript.txt
+++ b/services/history-v1/buildscript.txt
@@ -1,10 +1,10 @@
history-v1
---dependencies=postgres,gcs,mongo
+--dependencies=postgres,gcs,mongo,redis,s3
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=18.20.2
+--node-version=22.17.0
--public-repo=False
---script-version=4.5.0
---tsconfig-extra-includes=api/**/*,migrations/**/*,storage/**/*
+--script-version=4.7.0
+--tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,backup-worker-app.mjs,api/**/*,migrations/**/*,storage/**/*
diff --git a/services/history-v1/config/custom-environment-variables.json b/services/history-v1/config/custom-environment-variables.json
index 1822d33aa4..686ca25407 100644
--- a/services/history-v1/config/custom-environment-variables.json
+++ b/services/history-v1/config/custom-environment-variables.json
@@ -1,5 +1,6 @@
{
"databaseUrl": "HISTORY_CONNECTION_STRING",
+ "databaseUrlReadOnly": "HISTORY_FOLLOWER_CONNECTION_STRING",
"herokuDatabaseUrl": "DATABASE_URL",
"databasePoolMin": "DATABASE_POOL_MIN",
"databasePoolMax": "DATABASE_POOL_MAX",
@@ -8,6 +9,8 @@
"s3": {
"key": "AWS_ACCESS_KEY_ID",
"secret": "AWS_SECRET_ACCESS_KEY",
+ "endpoint": "AWS_S3_ENDPOINT",
+ "pathStyle": "AWS_S3_PATH_STYLE",
"maxRetries": "S3_MAX_RETRIES",
"httpOptions": {
"timeout": "S3_TIMEOUT"
@@ -30,6 +33,19 @@
"buckets": "PERSISTOR_BUCKET_MAPPING"
}
},
+ "backupPersistor": {
+ "keyEncryptionKeys": "BACKUP_KEY_ENCRYPTION_KEYS",
+ "s3SSEC": {
+ "key": "AWS_ACCESS_KEY_ID",
+ "secret": "AWS_SECRET_ACCESS_KEY",
+ "endpoint": "AWS_S3_ENDPOINT",
+ "pathStyle": "AWS_S3_PATH_STYLE",
+ "maxRetries": "BACKUP_S3_MAX_RETRIES",
+ "httpOptions": {
+ "timeout": "BACKUP_S3_TIMEOUT"
+ }
+ }
+ },
"blobStore": {
"globalBucket": "OVERLEAF_EDITOR_BLOBS_BUCKET",
"projectBucket": "OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET"
@@ -42,6 +58,16 @@
"bucket": "OVERLEAF_EDITOR_ZIPS_BUCKET",
"zipTimeoutMs": "ZIP_STORE_ZIP_TIMEOUT_MS"
},
+ "backupStore": {
+ "chunksBucket":"BACKUP_OVERLEAF_EDITOR_CHUNKS_BUCKET",
+ "deksBucket":"BACKUP_OVERLEAF_EDITOR_DEKS_BUCKET",
+ "globalBlobsBucket":"BACKUP_OVERLEAF_EDITOR_GLOBAL_BLOBS_BUCKET",
+ "projectBlobsBucket":"BACKUP_OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET"
+ },
+ "healthCheckBlobs": "HEALTH_CHECK_BLOBS",
+ "healthCheckProjects": "HEALTH_CHECK_PROJECTS",
+ "backupRPOInMS": "BACKUP_RPO_IN_MS",
+ "minSoftDeletionPeriodDays": "MIN_SOFT_DELETION_PERIOD_DAYS",
"mongo": {
"uri": "MONGO_CONNECTION_STRING"
},
@@ -57,5 +83,30 @@
"clusterWorkers": "CLUSTER_WORKERS",
"maxFileUploadSize": "MAX_FILE_UPLOAD_SIZE",
"httpsOnly": "HTTPS_ONLY",
- "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT"
+ "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT",
+ "historyBufferLevel": "HISTORY_BUFFER_LEVEL",
+ "forcePersistBuffer": "FORCE_PERSIST_BUFFER",
+ "nextHistoryBufferLevel": "NEXT_HISTORY_BUFFER_LEVEL",
+ "nextHistoryBufferLevelRolloutPercentage": "NEXT_HISTORY_BUFFER_LEVEL_ROLLOUT_PERCENTAGE",
+ "redis": {
+ "queue": {
+ "host": "QUEUES_REDIS_HOST",
+ "password": "QUEUES_REDIS_PASSWORD",
+ "port": "QUEUES_REDIS_PORT"
+ },
+ "history": {
+ "host": "HISTORY_REDIS_HOST",
+ "password": "HISTORY_REDIS_PASSWORD",
+ "port": "HISTORY_REDIS_PORT"
+ },
+ "lock": {
+ "host": "REDIS_HOST",
+ "password": "REDIS_PASSWORD",
+ "port": "REDIS_PORT"
+ }
+ },
+ "projectHistory": {
+ "host": "PROJECT_HISTORY_HOST",
+ "port": "PROJECT_HISTORY_PORT"
+ }
}
diff --git a/services/history-v1/config/default.json b/services/history-v1/config/default.json
index 84fd220789..e7732fe3f7 100644
--- a/services/history-v1/config/default.json
+++ b/services/history-v1/config/default.json
@@ -13,12 +13,25 @@
"deleteConcurrency": "50"
}
},
+ "backupPersistor": {
+ "backend": "s3SSEC",
+ "s3SSEC": {
+ "maxRetries": "1",
+ "pathStyle": false,
+ "httpOptions": {
+ "timeout": "120000"
+ }
+ }
+ },
+ "backupRPOInMS": "3600000",
"chunkStore": {
"historyStoreConcurrency": "4"
},
"zipStore": {
"zipTimeoutMs": "360000"
},
+ "hasProjectsWithoutHistory": false,
+ "minSoftDeletionPeriodDays": "90",
"maxDeleteKeys": "1000",
"useDeleteObjects": "true",
"clusterWorkers": "1",
@@ -26,5 +39,8 @@
"databasePoolMin": "2",
"databasePoolMax": "10",
"httpsOnly": "false",
- "httpRequestTimeout": "300000"
+ "httpRequestTimeout": "300000",
+ "projectHistory": {
+ "port": "3054"
+ }
}
diff --git a/services/history-v1/config/development.json b/services/history-v1/config/development.json
index f1423290b5..9cd73c62c1 100644
--- a/services/history-v1/config/development.json
+++ b/services/history-v1/config/development.json
@@ -23,6 +23,18 @@
"zipStore": {
"bucket": "overleaf-development-zips"
},
+ "backupStore": {
+ "chunksBucket":"overleaf-development-history-chunks",
+ "deksBucket":"overleaf-development-history-deks",
+ "globalBlobsBucket":"overleaf-development-history-global-blobs",
+ "projectBlobsBucket":"overleaf-development-history-project-blobs"
+ },
+ "backupPersistor": {
+ "keyEncryptionKeys": "[{\"key\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"salt\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}]",
+ "s3SSEC": {
+ "ca": "[\"/certs/public.crt\"]"
+ }
+ },
"useDeleteObjects": "false",
"mongo": {
"uri": "mongodb://mongo:27017/sharelatex"
diff --git a/services/history-v1/config/production.json b/services/history-v1/config/production.json
index ffcd4415b0..23f836b1f2 100644
--- a/services/history-v1/config/production.json
+++ b/services/history-v1/config/production.json
@@ -1 +1,5 @@
-{ }
+{
+ "backupPersistor": {
+ "tieringStorageClass": "INTELLIGENT_TIERING"
+ }
+}
diff --git a/services/history-v1/config/test.json b/services/history-v1/config/test.json
index 1e4ddd3a0b..c38e28e564 100644
--- a/services/history-v1/config/test.json
+++ b/services/history-v1/config/test.json
@@ -1,5 +1,6 @@
{
"databaseUrl": "postgres://overleaf:overleaf@postgres/overleaf-history-v1-test",
+ "databaseUrlReadOnly": "postgres://read_only:password@postgres/overleaf-history-v1-test",
"persistor": {
"backend": "gcs",
"gcs": {
@@ -20,6 +21,22 @@
"zipStore": {
"bucket": "overleaf-test-zips"
},
+ "backupStore": {
+ "chunksBucket":"overleaf-test-history-chunks",
+ "deksBucket":"overleaf-test-history-deks",
+ "globalBlobsBucket":"overleaf-test-history-global-blobs",
+ "projectBlobsBucket":"overleaf-test-history-project-blobs"
+ },
+ "backupPersistor": {
+ "keyEncryptionKeys": "[{\"key\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"salt\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}]",
+ "s3SSEC": {
+ "ca": "[\"/certs/public.crt\"]"
+ },
+ "tieringStorageClass": "REDUCED_REDUNDANCY"
+ },
+ "healthCheckBlobs": "[\"42/f70d7bba4ae1f07682e0358bd7a2068094fc023b\",\"000000000000000000000042/98d5521fe746bc2d11761edab5d0829bee286009\"]",
+ "healthCheckProjects": "[\"42\",\"000000000000000000000042\"]",
+ "backupRPOInMS": "360000",
"maxDeleteKeys": "3",
"useDeleteObjects": "false",
"mongo": {
diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml
index 4de133e440..cf6ec3357d 100644
--- a/services/history-v1/docker-compose.ci.yml
+++ b/services/history-v1/docker-compose.ci.yml
@@ -19,22 +19,44 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
ELASTIC_SEARCH_DSN: es:9200
+ REDIS_HOST: redis
+ QUEUES_REDIS_HOST: redis
+ HISTORY_REDIS_HOST: redis
+ ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
+ AWS_S3_ENDPOINT: https://minio:9000
+ AWS_S3_PATH_STYLE: 'true'
+ AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
+ volumes:
+ - ./test/acceptance/certs:/certs
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
+ condition: service_started
+ redis:
condition: service_healthy
postgres:
condition: service_healthy
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ minio_setup:
+ condition: service_completed_successfully
gcs:
condition: service_healthy
user: node
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -45,24 +67,169 @@ services:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
- mongo:
- image: mongo:6.0.13
- command: --replSet overleaf
+ redis:
+ image: redis:7.4.3
healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
+
+ mongo:
+ image: mongo:8.0.11
+ command: --replSet overleaf
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
postgres:
image: postgres:10
environment:
POSTGRES_USER: overleaf
POSTGRES_PASSWORD: overleaf
POSTGRES_DB: overleaf-history-v1-test
+ volumes:
+ - ./test/acceptance/pg-init/:/docker-entrypoint-initdb.d/
healthcheck:
test: pg_isready --quiet
interval: 1s
retries: 20
+ certs:
+ image: node:22.17.0
+ volumes:
+ - ./test/acceptance/certs:/certs
+ working_dir: /certs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ if [ ! -f ./certgen ]; then
+ wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+ chmod +x ./certgen
+ fi
+ if [ ! -f private.key ] || [ ! -f public.crt ]; then
+ ./certgen -host minio
+ fi
+
+ minio:
+ image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+ command: server /data
+ volumes:
+ - ./test/acceptance/certs:/root/.minio/certs
+ environment:
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+
+ minio_setup:
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+ volumes:
+ - ./test/acceptance/certs:/root/.mc/certs/CAs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ sleep 1
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+ mc mb --ignore-existing s3/overleaf-test-history-chunks
+ mc mb --ignore-existing s3/overleaf-test-history-deks
+ mc mb --ignore-existing s3/overleaf-test-history-global-blobs
+ mc mb --ignore-existing s3/overleaf-test-history-project-blobs
+ mc admin user add s3 \
+ OVERLEAF_HISTORY_S3_ACCESS_KEY_ID \
+ OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+ echo '
+ {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-chunks"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-deks"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
+ }
+ ]
+ }' > policy-history.json
+
+ mc admin policy create s3 overleaf-history policy-history.json
+ mc admin policy attach s3 overleaf-history \
+ --user=OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml
index f40ab5b940..3a33882d28 100644
--- a/services/history-v1/docker-compose.yml
+++ b/services/history-v1/docker-compose.yml
@@ -6,7 +6,10 @@ version: "2.3"
services:
test_unit:
- image: node:18.20.2
+ build:
+ context: ../..
+ dockerfile: services/history-v1/Dockerfile
+ target: base
volumes:
- .:/overleaf/services/history-v1
- ../../node_modules:/overleaf/node_modules
@@ -14,58 +17,228 @@ services:
working_dir: /overleaf/services/history-v1
environment:
MOCHA_GREP: ${MOCHA_GREP}
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:18.20.2
+ build:
+ context: ../..
+ dockerfile: services/history-v1/Dockerfile
+ target: base
volumes:
- .:/overleaf/services/history-v1
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
+ - ./test/acceptance/certs:/certs
+ - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/history-v1
environment:
ELASTIC_SEARCH_DSN: es:9200
+ REDIS_HOST: redis
+ HISTORY_REDIS_HOST: redis
+ QUEUES_REDIS_HOST: redis
+ ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
+ AWS_S3_ENDPOINT: https://minio:9000
+ AWS_S3_PATH_STYLE: 'true'
+ AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ERROR
+ LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
+ condition: service_started
+ redis:
condition: service_healthy
postgres:
condition: service_healthy
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ minio_setup:
+ condition: service_completed_successfully
gcs:
condition: service_healthy
+ entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
- mongo:
- image: mongo:6.0.13
- command: --replSet overleaf
+ redis:
+ image: redis:7.4.3
healthcheck:
- test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
+ mongo:
+ image: mongo:8.0.11
+ command: --replSet overleaf
+ volumes:
+ - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+ environment:
+ MONGO_INITDB_DATABASE: sharelatex
+ extra_hosts:
+ # Required when using the automatic database setup for initializing the
+ # replica set. This override is not needed when running the setup after
+ # starting up mongo.
+ - mongo:127.0.0.1
+
postgres:
image: postgres:10
environment:
POSTGRES_USER: overleaf
POSTGRES_PASSWORD: overleaf
POSTGRES_DB: overleaf-history-v1-test
+ volumes:
+ - ./test/acceptance/pg-init/:/docker-entrypoint-initdb.d/
healthcheck:
test: pg_isready --host=localhost --quiet
interval: 1s
retries: 20
+ certs:
+ image: node:22.17.0
+ volumes:
+ - ./test/acceptance/certs:/certs
+ working_dir: /certs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ if [ ! -f ./certgen ]; then
+ wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
+ chmod +x ./certgen
+ fi
+ if [ ! -f private.key ] || [ ! -f public.crt ]; then
+ ./certgen -host minio
+ fi
+
+ minio:
+ image: minio/minio:RELEASE.2024-10-13T13-34-11Z
+ command: server /data
+ volumes:
+ - ./test/acceptance/certs:/root/.minio/certs
+ environment:
+ MINIO_ROOT_USER: MINIO_ROOT_USER
+ MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+
+ minio_setup:
+ depends_on:
+ certs:
+ condition: service_completed_successfully
+ minio:
+ condition: service_started
+ image: minio/mc:RELEASE.2024-10-08T09-37-26Z
+ volumes:
+ - ./test/acceptance/certs:/root/.mc/certs/CAs
+ entrypoint: sh
+ command:
+ - '-cex'
+ - |
+ sleep 1
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
+ || sleep 3 && \
+ mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
+ mc mb --ignore-existing s3/overleaf-test-history-chunks
+ mc mb --ignore-existing s3/overleaf-test-history-deks
+ mc mb --ignore-existing s3/overleaf-test-history-global-blobs
+ mc mb --ignore-existing s3/overleaf-test-history-project-blobs
+ mc admin user add s3 \
+ OVERLEAF_HISTORY_S3_ACCESS_KEY_ID \
+ OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
+ echo '
+ {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-chunks"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-deks"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
+ }
+ ]
+ }' > policy-history.json
+
+ mc admin policy create s3 overleaf-history policy-history.json
+ mc admin policy attach s3 overleaf-history \
+ --user=OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/history-v1/install_deps.sh b/services/history-v1/install_deps.sh
new file mode 100755
index 0000000000..4ce7223b46
--- /dev/null
+++ b/services/history-v1/install_deps.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -ex
+
+apt-get update
+
+apt-get install jq parallel --yes
+
+rm -rf /var/lib/apt/lists/*
diff --git a/services/history-v1/migrations/20250415210802_add_chunks_closed.js b/services/history-v1/migrations/20250415210802_add_chunks_closed.js
new file mode 100644
index 0000000000..b5c1d577f9
--- /dev/null
+++ b/services/history-v1/migrations/20250415210802_add_chunks_closed.js
@@ -0,0 +1,27 @@
+// @ts-check
+
+/**
+ * @import { Knex } from "knex"
+ */
+
+/**
+ * @param { Knex } knex
+ * @returns { Promise }
+ */
+exports.up = async function (knex) {
+ await knex.raw(`
+ ALTER TABLE chunks
+ ADD COLUMN closed BOOLEAN NOT NULL DEFAULT FALSE
+ `)
+}
+
+/**
+ * @param { Knex } knex
+ * @returns { Promise }
+ */
+exports.down = async function (knex) {
+ await knex.raw(`
+ ALTER TABLE chunks
+ DROP COLUMN closed
+ `)
+}
diff --git a/services/history-v1/package.json b/services/history-v1/package.json
index 68c3dfd8aa..4796cafd03 100644
--- a/services/history-v1/package.json
+++ b/services/history-v1/package.json
@@ -6,10 +6,14 @@
"license": "Proprietary",
"private": true,
"dependencies": {
+ "@google-cloud/secret-manager": "^5.6.0",
+ "@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
+ "@overleaf/mongo-utils": "*",
"@overleaf/o-error": "*",
"@overleaf/object-persistor": "*",
+ "@overleaf/promise-utils": "*",
"@overleaf/redis-wrapper": "*",
"@overleaf/settings": "*",
"@overleaf/stream-utils": "^0.1.0",
@@ -17,11 +21,12 @@
"basic-auth": "^2.0.1",
"bluebird": "^3.7.2",
"body-parser": "^1.20.3",
+ "bull": "^4.16.5",
"bunyan": "^1.8.12",
"check-types": "^11.1.2",
"command-line-args": "^3.0.3",
- "config": "^1.19.0",
- "express": "^4.21.0",
+ "config": "^3.3.12",
+ "express": "^4.21.2",
"fs-extra": "^9.0.1",
"generic-pool": "^2.1.1",
"helmet": "^3.22.0",
@@ -29,9 +34,12 @@
"jsonwebtoken": "^9.0.0",
"knex": "^2.4.0",
"lodash": "^4.17.19",
- "mongodb": "^6.2.0",
+ "mongodb": "6.12.0",
"overleaf-editor-core": "*",
+ "p-limit": "^6.2.0",
+ "p-queue": "^8.1.0",
"pg": "^8.7.1",
+ "pg-query-stream": "^4.2.4",
"swagger-tools": "^0.10.4",
"temp": "^0.8.3",
"throng": "^4.0.0",
@@ -42,7 +50,8 @@
"benny": "^3.7.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^10.2.0",
+ "chai-exclude": "^2.1.1",
+ "mocha": "^11.1.0",
"node-fetch": "^2.7.0",
"sinon": "^9.0.2",
"swagger-client": "^3.10.0",
@@ -53,8 +62,8 @@
"start": "node app.js",
"lint": "eslint --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix .",
- "format": "prettier --list-different $PWD/'**/*.js'",
- "format:fix": "prettier --write $PWD/'**/*.js'",
+ "format": "prettier --list-different $PWD/'**/*.*js'",
+ "format:fix": "prettier --write $PWD/'**/*.*js'",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
diff --git a/services/history-v1/storage/index.js b/services/history-v1/storage/index.js
index a0fd471829..6bc81f60e8 100644
--- a/services/history-v1/storage/index.js
+++ b/services/history-v1/storage/index.js
@@ -2,11 +2,16 @@ exports.BatchBlobStore = require('./lib/batch_blob_store')
exports.blobHash = require('./lib/blob_hash')
exports.HashCheckBlobStore = require('./lib/hash_check_blob_store')
exports.chunkStore = require('./lib/chunk_store')
-exports.historyStore = require('./lib/history_store')
+exports.redisBuffer = require('./lib/chunk_store/redis')
+exports.historyStore = require('./lib/history_store').historyStore
exports.knex = require('./lib/knex')
exports.mongodb = require('./lib/mongodb')
+exports.redis = require('./lib/redis')
exports.persistChanges = require('./lib/persist_changes')
exports.persistor = require('./lib/persistor')
+exports.persistBuffer = require('./lib/persist_buffer')
+exports.commitChanges = require('./lib/commit_changes')
+exports.queueChanges = require('./lib/queue_changes')
exports.ProjectArchive = require('./lib/project_archive')
exports.streams = require('./lib/streams')
exports.temp = require('./lib/temp')
@@ -15,3 +20,9 @@ exports.zipStore = require('./lib/zip_store')
const { BlobStore, loadGlobalBlobs } = require('./lib/blob_store')
exports.BlobStore = BlobStore
exports.loadGlobalBlobs = loadGlobalBlobs
+
+const { InvalidChangeError } = require('./lib/errors')
+exports.InvalidChangeError = InvalidChangeError
+
+const { ChunkVersionConflictError } = require('./lib/chunk_store/errors')
+exports.ChunkVersionConflictError = ChunkVersionConflictError
diff --git a/services/history-v1/storage/lib/assert.js b/services/history-v1/storage/lib/assert.js
index 6f79086209..91f24da7e0 100644
--- a/services/history-v1/storage/lib/assert.js
+++ b/services/history-v1/storage/lib/assert.js
@@ -1,5 +1,7 @@
'use strict'
+const OError = require('@overleaf/o-error')
+
const check = require('check-types')
const { Blob } = require('overleaf-editor-core')
@@ -7,37 +9,58 @@ const assert = check.assert
const MONGO_ID_REGEXP = /^[0-9a-f]{24}$/
const POSTGRES_ID_REGEXP = /^[1-9][0-9]{0,9}$/
-const PROJECT_ID_REGEXP = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/
+const MONGO_OR_POSTGRES_ID_REGEXP = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/
function transaction(transaction, message) {
assert.function(transaction, message)
}
function blobHash(arg, message) {
- assert.match(arg, Blob.HEX_HASH_RX, message)
+ try {
+ assert.match(arg, Blob.HEX_HASH_RX, message)
+ } catch (error) {
+ throw OError.tag(error, message, { arg })
+ }
+}
+
+/**
+ * A project id is a string that contains either an integer (for projects stored in Postgres) or 24
+ * hex digits (for projects stored in Mongo)
+ */
+function projectId(arg, message) {
+ try {
+ assert.match(arg, MONGO_OR_POSTGRES_ID_REGEXP, message)
+ } catch (error) {
+ throw OError.tag(error, message, { arg })
+ }
}
/**
* A chunk id is a string that contains either an integer (for projects stored in Postgres) or 24
* hex digits (for projects stored in Mongo)
*/
-function projectId(arg, message) {
- assert.match(arg, PROJECT_ID_REGEXP, message)
-}
-
-/**
- * A chunk id is either a number (for projects stored in Postgres) or a 24
- * character string (for projects stored in Mongo)
- */
function chunkId(arg, message) {
- const valid = check.integer(arg) || check.match(arg, MONGO_ID_REGEXP)
- if (!valid) {
- throw new TypeError(message)
+ try {
+ assert.match(arg, MONGO_OR_POSTGRES_ID_REGEXP, message)
+ } catch (error) {
+ throw OError.tag(error, message, { arg })
}
}
function mongoId(arg, message) {
- assert.match(arg, MONGO_ID_REGEXP)
+ try {
+ assert.match(arg, MONGO_ID_REGEXP, message)
+ } catch (error) {
+ throw OError.tag(error, message, { arg })
+ }
+}
+
+function postgresId(arg, message) {
+ try {
+ assert.match(arg, POSTGRES_ID_REGEXP, message)
+ } catch (error) {
+ throw OError.tag(error, message, { arg })
+ }
}
module.exports = {
@@ -47,6 +70,7 @@ module.exports = {
projectId,
chunkId,
mongoId,
+ postgresId,
MONGO_ID_REGEXP,
POSTGRES_ID_REGEXP,
}
diff --git a/services/history-v1/storage/lib/backupArchiver.mjs b/services/history-v1/storage/lib/backupArchiver.mjs
new file mode 100644
index 0000000000..c6f0e3755d
--- /dev/null
+++ b/services/history-v1/storage/lib/backupArchiver.mjs
@@ -0,0 +1,474 @@
+// @ts-check
+import path from 'node:path'
+import projectKey from './project_key.js'
+import {
+ chunksBucket,
+ backupPersistor,
+ projectBlobsBucket,
+ globalBlobsBucket as backupGlobalBlobsBucket,
+} from './backupPersistor.mjs'
+import core, { Chunk, History } from 'overleaf-editor-core'
+import {
+ GLOBAL_BLOBS,
+ makeProjectKey,
+ getStringLengthOfFile,
+ makeGlobalKey,
+} from './blob_store/index.js'
+import streams from './streams.js'
+import objectPersistor from '@overleaf/object-persistor'
+import OError from '@overleaf/o-error'
+import chunkStore from './chunk_store/index.js'
+import logger from '@overleaf/logger'
+import fs from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import withTmpDir from '../../api/controllers/with_tmp_dir.js'
+import { loadChunk } from './backupVerifier.mjs'
+import globalBlobPersistor from './persistor.js'
+import config from 'config'
+import { NoKEKMatchedError } from '@overleaf/object-persistor/src/Errors.js'
+
+const globalBlobsBucket = config.get('blobStore.globalBucket')
+
+class BackupBlobStore {
+ /**
+ *
+ * @param {string} historyId
+ * @param {string} tmp
+ * @param {CachedPerProjectEncryptedS3Persistor} persistor
+ * @param {boolean} useBackupGlobalBlobs
+ */
+ constructor(historyId, tmp, persistor, useBackupGlobalBlobs) {
+ this.historyId = historyId
+ this.tmp = tmp
+ this.blobs = new Map()
+ this.persistor = persistor
+ this.useBackupGlobalBlobs = useBackupGlobalBlobs
+ }
+
+ /**
+ * Required for BlobStore interface - not supported.
+ *
+ * @template T
+ * @param {string} hash
+ * @return {Promise}
+ */
+ async getObject(hash) {
+ try {
+ const stream = await this.getStream(hash)
+ const buffer = await streams.readStreamToBuffer(stream)
+ return JSON.parse(buffer.toString())
+ } catch (err) {
+ logger.warn({ err, hash }, 'Failed to fetch chunk blob')
+ throw err
+ }
+ }
+
+ /**
+ *
+ * @param {Set} hashes
+ * @return {Promise}
+ */
+ async fetchBlobs(hashes) {
+ for await (const hash of hashes) {
+ if (this.blobs.has(hash)) return
+ const path = `${this.tmp}/${hash}`
+ /** @type {core.Blob} */
+ let blob
+ /** @type {NodeJS.ReadableStream} */
+ let blobStream
+ if (GLOBAL_BLOBS.has(hash)) {
+ try {
+ const blobData = await this.fetchGlobalBlob(hash)
+ await pipeline(blobData.stream, fs.createWriteStream(path))
+ blob = blobData.blob
+ } catch (err) {
+ logger.warn({ hash, err }, 'Failed to fetch global blob')
+ continue
+ }
+ } else {
+ try {
+ blobStream = await fetchBlob(this.historyId, hash, this.persistor)
+ await pipeline(blobStream, fs.createWriteStream(path))
+ blob = await this.makeBlob(hash, path)
+ } catch (err) {
+ logger.warn({ err, hash }, 'Failed to fetch chunk blob')
+ continue
+ }
+ }
+
+ this.blobs.set(hash, blob)
+ }
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @return {Promise<{ blob: core.Blob, stream: NodeJS.ReadableStream }>}
+ */
+ async fetchGlobalBlob(hash) {
+ const globalBlob = GLOBAL_BLOBS.get(hash)
+ if (!globalBlob) {
+ throw new Error('blob does not exist or is not a global blob')
+ }
+ let stream
+
+ const key = makeGlobalKey(hash)
+
+ if (this.useBackupGlobalBlobs) {
+ stream = await this.persistor.getObjectStream(
+ backupGlobalBlobsBucket,
+ key
+ )
+ } else {
+ stream = await globalBlobPersistor.getObjectStream(globalBlobsBucket, key)
+ }
+ return { blob: globalBlob.blob, stream }
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @param {string} pathname
+ * @return {Promise}
+ */
+ async makeBlob(hash, pathname) {
+ const stat = await fs.promises.stat(pathname)
+ const byteLength = stat.size
+ const stringLength = await getStringLengthOfFile(byteLength, pathname)
+ if (stringLength) {
+ return new core.Blob(hash, byteLength, stringLength)
+ }
+ return new core.Blob(hash, byteLength)
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @return {Promise}
+ */
+ async getString(hash) {
+ const stream = await this.getStream(hash)
+ const buffer = await streams.readStreamToBuffer(stream)
+ return buffer.toString()
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @return {Promise}
+ */
+ async getStream(hash) {
+ return fs.createReadStream(this.getBlobPathname(hash))
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @return {Promise}
+ */
+ async getBlob(hash) {
+ return this.blobs.get(hash)
+ }
+
+ /**
+ *
+ * @param {string} hash
+ * @return {string}
+ */
+ getBlobPathname(hash) {
+ return path.join(this.tmp, hash)
+ }
+}
+
+/**
+ * @typedef {(import('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js').CachedPerProjectEncryptedS3Persistor)} CachedPerProjectEncryptedS3Persistor
+ */
+
+/**
+ * @typedef {(import('archiver').Archiver)} Archiver
+ */
+
+/**
+ * @typedef {(import('overleaf-editor-core').FileMap)} FileMap
+ */
+
+/**
+ *
+ * @param historyId
+ * @return {Promise}
+ */
+async function getProjectPersistor(historyId) {
+ try {
+ return await backupPersistor.forProjectRO(
+ projectBlobsBucket,
+ makeProjectKey(historyId, '')
+ )
+ } catch (error) {
+ if (error instanceof NoKEKMatchedError) {
+ logger.info({}, 'no kek matched')
+ }
+ throw new BackupPersistorError(
+ 'Failed to get project persistor',
+ { historyId },
+ error instanceof Error ? error : undefined
+ )
+ }
+}
+
+/**
+ *
+ * @param persistor
+ * @param {string} key
+ * @return {Promise<{chunkData: any, buffer: Buffer}>}
+ */
+async function loadChunkByKey(persistor, key) {
+ try {
+ const buf = await streams.gunzipStreamToBuffer(
+ await persistor.getObjectStream(chunksBucket, key)
+ )
+ return { chunkData: JSON.parse(buf.toString('utf-8')), buffer: buf }
+ } catch (err) {
+ if (err instanceof objectPersistor.Errors.NotFoundError) {
+ throw new Chunk.NotPersistedError('chunk not found')
+ }
+ if (err instanceof Error) {
+ throw OError.tag(err, 'Failed to load chunk', { key })
+ }
+ throw err
+ }
+}
+
+/**
+ *
+ * @param {string} historyId
+ * @param {string} hash
+ * @param {CachedPerProjectEncryptedS3Persistor} persistor
+ * @return {Promise}
+ */
+async function fetchBlob(historyId, hash, persistor) {
+ const path = makeProjectKey(historyId, hash)
+ return await persistor.getObjectStream(projectBlobsBucket, path, {
+ autoGunzip: true,
+ })
+}
+
+/**
+ * @typedef {object} AddChunkOptions
+ * @property {string} [prefix] Should include trailing slash (if length > 0)
+ * @property {boolean} [useBackupGlobalBlobs]
+ */
+
+/**
+ *
+ * @param {History} history
+ * @param {Archiver} archive
+ * @param {CachedPerProjectEncryptedS3Persistor} projectCache
+ * @param {string} historyId
+ * @param {AddChunkOptions} [options]
+ * @returns {Promise}
+ */
+async function addChunkToArchive(
+ history,
+ archive,
+ projectCache,
+ historyId,
+ { prefix = '', useBackupGlobalBlobs = false } = {}
+) {
+ const chunkBlobs = new Set()
+ history.findBlobHashes(chunkBlobs)
+
+ await withTmpDir('recovery-blob-', async tmpDir => {
+ const blobStore = new BackupBlobStore(
+ historyId,
+ tmpDir,
+ projectCache,
+ useBackupGlobalBlobs
+ )
+ await blobStore.fetchBlobs(chunkBlobs)
+
+ await history.loadFiles('lazy', blobStore)
+
+ const snapshot = history.getSnapshot()
+ snapshot.applyAll(history.getChanges())
+
+ const filePaths = snapshot.getFilePathnames()
+
+ if (filePaths.length === 0) {
+ logger.warn(
+ { historyId, projectVersion: snapshot.projectVersion },
+ 'No files found in snapshot backup'
+ )
+ }
+ for (const filePath of filePaths) {
+ /** @type {core.File | null | undefined} */
+ const file = snapshot.getFile(filePath)
+ if (!file) {
+ logger.error({ filePath }, 'File not found in snapshot')
+ continue
+ }
+
+ try {
+ await file.load('eager', blobStore)
+ } catch (err) {
+ logger.error(
+ { filePath, err },
+ 'Failed to load file from snapshot, skipping'
+ )
+ continue
+ }
+
+ const hash = file.getHash()
+
+ /** @type {string | fs.ReadStream | null | undefined} */
+ let content = file.getContent({ filterTrackedDeletes: true })
+
+ if (content === null) {
+ if (!hash) {
+ logger.error({ filePath }, 'File does not have a hash')
+ continue
+ }
+ const blob = await blobStore.getBlob(hash)
+ if (!blob) {
+ logger.error({ filePath }, 'Blob not found in blob store')
+ continue
+ }
+ content = await blobStore.getStream(hash)
+ }
+ archive.append(content, {
+ name: `${prefix}${filePath}`,
+ })
+ }
+ })
+}
+
+/**
+ *
+ * @param {string} historyId
+ * @return {Promise}
+ */
+async function findStartVersionOfLatestChunk(historyId) {
+ const backend = chunkStore.getBackend(historyId)
+ const chunk = await backend.getLatestChunk(historyId, { readOnly: true })
+ if (!chunk) {
+ throw new Error('Latest chunk could not be loaded')
+ }
+ return chunk.startVersion
+}
+
+/**
+ * Restore a project from the latest snapshot
+ *
+ * There is an assumption that the database backup has been restored.
+ *
+ * @param {Archiver} archive
+ * @param {string} historyId
+ * @param {boolean} [useBackupGlobalBlobs]
+ * @return {Promise}
+ */
+export async function archiveLatestChunk(
+ archive,
+ historyId,
+ useBackupGlobalBlobs = false
+) {
+ logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
+
+ const projectCache = await getProjectPersistor(historyId)
+
+ const startVersion = await findStartVersionOfLatestChunk(historyId)
+
+ const backedUpChunkRaw = await loadChunk(
+ historyId,
+ startVersion,
+ projectCache
+ )
+
+ const backedUpChunk = History.fromRaw(backedUpChunkRaw)
+
+ await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
+ useBackupGlobalBlobs,
+ })
+
+ return archive
+}
+
+/**
+ * Fetches all raw blobs from the project and adds them to the archive.
+ *
+ * @param {string} historyId
+ * @param {Archiver} archive
+ * @param {CachedPerProjectEncryptedS3Persistor} projectCache
+ * @return {Promise}
+ */
+async function addRawBlobsToArchive(historyId, archive, projectCache) {
+ const key = projectKey.format(historyId)
+ const { contents } = await projectCache.listDirectory(projectBlobsBucket, key)
+ for (const blobRecord of contents) {
+ if (!blobRecord.Key) {
+ logger.debug({ blobRecord }, 'no key')
+ continue
+ }
+ const blobKey = blobRecord.Key
+ try {
+ const stream = await projectCache.getObjectStream(
+ projectBlobsBucket,
+ blobKey,
+ { autoGunzip: true }
+ )
+ archive.append(stream, {
+ name: path.join(historyId, 'blobs', blobKey),
+ })
+ } catch (err) {
+ logger.warn(
+ { err, path: blobRecord.Key },
+ 'Failed to append blob to archive'
+ )
+ }
+ }
+}
+
+/**
+ * Download raw files from the backup.
+ *
+ * This can work without the database being backed up.
+ *
+ * It will split the project into chunks per directory and download the blobs alongside the chunk.
+ *
+ * @param {Archiver} archive
+ * @param {string} historyId
+ * @param {boolean} [useBackupGlobalBlobs]
+ * @return {Promise}
+ */
+export async function archiveRawProject(
+ archive,
+ historyId,
+ useBackupGlobalBlobs = false
+) {
+ const projectCache = await getProjectPersistor(historyId)
+
+ const { contents: chunks } = await projectCache.listDirectory(
+ chunksBucket,
+ projectKey.format(historyId)
+ )
+
+ if (chunks.length === 0) {
+ throw new Error('No chunks found')
+ }
+
+ for (const chunkRecord of chunks) {
+ if (!chunkRecord.Key) {
+ logger.debug({ chunkRecord }, 'no key')
+ continue
+ }
+ const chunkId = chunkRecord.Key.split('/').pop()
+ logger.debug({ chunkId, key: chunkRecord.Key }, 'Processing chunk')
+
+ const { buffer } = await loadChunkByKey(projectCache, chunkRecord.Key)
+
+ archive.append(buffer, {
+ name: `${historyId}/chunks/${chunkId}/chunk.json`,
+ })
+ }
+ await addRawBlobsToArchive(historyId, archive, projectCache)
+}
+
+export class BackupPersistorError extends OError {}
diff --git a/services/history-v1/storage/lib/backupBlob.mjs b/services/history-v1/storage/lib/backupBlob.mjs
new file mode 100644
index 0000000000..8ae1a6a901
--- /dev/null
+++ b/services/history-v1/storage/lib/backupBlob.mjs
@@ -0,0 +1,251 @@
+// @ts-check
+import { backupPersistor, projectBlobsBucket } from './backupPersistor.mjs'
+import { GLOBAL_BLOBS, makeProjectKey, BlobStore } from './blob_store/index.js'
+import Stream from 'node:stream'
+import fs from 'node:fs'
+import Crypto from 'node:crypto'
+import assert from './assert.js'
+import { backedUpBlobs, projects } from './mongodb.js'
+import { Binary, ObjectId } from 'mongodb'
+import logger from '@overleaf/logger/logging-manager.js'
+import { AlreadyWrittenError } from '@overleaf/object-persistor/src/Errors.js'
+import metrics from '@overleaf/metrics'
+import zLib from 'node:zlib'
+import Path from 'node:path'
+
+const HIGHWATER_MARK = 1024 * 1024
+
+/**
+ * @typedef {import("overleaf-editor-core").Blob} Blob
+ */
+
+/**
+ * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
+ */
+
+/**
+ * Increment a metric to record the outcome of a backup operation.
+ *
+ * @param {"success"|"failure"|"skipped"} status
+ * @param {"global"|"already_backed_up"|"none"} reason
+ */
+function recordBackupConclusion(status, reason = 'none') {
+ metrics.inc('blob_backed_up', 1, { status, reason })
+}
+
+/**
+ * Downloads a blob to a specified directory
+ *
+ * @param {string} historyId - The history ID of the project the blob belongs to
+ * @param {Blob} blob - The blob to download
+ * @param {string} tmpDir - The directory path where the blob will be downloaded
+ * @returns {Promise} The full path where the blob was downloaded
+ */
+export async function downloadBlobToDir(historyId, blob, tmpDir) {
+ const blobStore = new BlobStore(historyId)
+ const blobHash = blob.getHash()
+ const src = await blobStore.getStream(blobHash)
+ const filePath = Path.join(tmpDir, `${historyId}-${blobHash}`)
+ try {
+ const dst = fs.createWriteStream(filePath, {
+ highWaterMark: HIGHWATER_MARK,
+ flags: 'wx',
+ })
+ await Stream.promises.pipeline(src, dst)
+ return filePath
+ } catch (error) {
+ try {
+ await fs.promises.unlink(filePath)
+ } catch {}
+ throw error
+ }
+}
+
+/**
+ * Performs the actual upload of the blob to the backup storage.
+ *
+ * @param {string} historyId - The history ID of the project the blob belongs to
+ * @param {Blob} blob - The blob being uploaded
+ * @param {string} path - The path to the file to upload (should have been stored on disk already)
+ * @return {Promise}
+ */
+export async function uploadBlobToBackup(historyId, blob, path, persistor) {
+ const md5 = Crypto.createHash('md5')
+ const filePathCompressed = path + '.gz'
+ let backupSource
+ let contentEncoding
+ let size
+ try {
+ if (blob.getStringLength()) {
+ backupSource = filePathCompressed
+ contentEncoding = 'gzip'
+ size = 0
+ await Stream.promises.pipeline(
+ fs.createReadStream(path, { highWaterMark: HIGHWATER_MARK }),
+ zLib.createGzip(),
+ async function* (source) {
+ for await (const chunk of source) {
+ size += chunk.byteLength
+ md5.update(chunk)
+ yield chunk
+ }
+ },
+ fs.createWriteStream(filePathCompressed, {
+ highWaterMark: HIGHWATER_MARK,
+ })
+ )
+ } else {
+ backupSource = path
+ size = blob.getByteLength()
+ await Stream.promises.pipeline(
+ fs.createReadStream(path, { highWaterMark: HIGHWATER_MARK }),
+ md5
+ )
+ }
+ const key = makeProjectKey(historyId, blob.getHash())
+ await persistor.sendStream(
+ projectBlobsBucket,
+ key,
+ fs.createReadStream(backupSource, { highWaterMark: HIGHWATER_MARK }),
+ {
+ contentEncoding,
+ contentType: 'application/octet-stream',
+ contentLength: size,
+ sourceMd5: md5.digest('hex'),
+ ifNoneMatch: '*',
+ }
+ )
+ } finally {
+ if (backupSource === filePathCompressed) {
+ try {
+ await fs.promises.rm(filePathCompressed, { force: true })
+ } catch {}
+ }
+ }
+}
+
+/**
+ * Converts a legacy (postgres) historyId to a mongo projectId
+ *
+ * @param {string} historyId
+ * @return {Promise}
+ * @private
+ */
+async function _convertLegacyHistoryIdToProjectId(historyId) {
+ const project = await projects.findOne(
+ { 'overleaf.history.id': parseInt(historyId) },
+ { projection: { _id: 1 } }
+ )
+
+ if (!project?._id) {
+ throw new Error('Did not find project for history id')
+ }
+
+ return project?._id?.toString()
+}
+
+/**
+ * Records that a blob was backed up for a project.
+ *
+ * @param {string} projectId - projectId for a project (mongo format)
+ * @param {string} hash
+ * @return {Promise}
+ */
+export async function storeBlobBackup(projectId, hash) {
+ await backedUpBlobs.updateOne(
+ { _id: new ObjectId(projectId) },
+ { $addToSet: { blobs: new Binary(Buffer.from(hash, 'hex')) } },
+ { upsert: true }
+ )
+}
+
+/**
+ * Determine whether a specific blob has been backed up in this project.
+ *
+ * @param {string} projectId
+ * @param {string} hash
+ * @return {Promise<*>}
+ * @private
+ */
+export async function _blobIsBackedUp(projectId, hash) {
+ const blobs = await backedUpBlobs.findOne(
+ {
+ _id: new ObjectId(projectId),
+ blobs: new Binary(Buffer.from(hash, 'hex')),
+ },
+ { projection: { _id: 1 } }
+ )
+ return blobs?._id
+}
+
+/**
+ * Back up a blob to the global storage and record that it was backed up.
+ *
+ * @param {string} historyId - history ID for a project (can be postgres format or mongo format)
+ * @param {Blob} blob - The blob that is being backed up
+ * @param {string} tmpPath - The path to a temporary file storing the contents of the blob.
+ * @param {CachedPerProjectEncryptedS3Persistor} [persistor] - The persistor to use (optional)
+ * @return {Promise}
+ */
+export async function backupBlob(historyId, blob, tmpPath, persistor) {
+ const hash = blob.getHash()
+
+ let projectId = historyId
+ if (assert.POSTGRES_ID_REGEXP.test(historyId)) {
+ projectId = await _convertLegacyHistoryIdToProjectId(historyId)
+ }
+
+ const globalBlob = GLOBAL_BLOBS.get(hash)
+
+ if (globalBlob && !globalBlob.demoted) {
+ recordBackupConclusion('skipped', 'global')
+ logger.debug({ projectId, hash }, 'Blob is global - skipping backup')
+ return
+ }
+
+ try {
+ if (await _blobIsBackedUp(projectId, hash)) {
+ recordBackupConclusion('skipped', 'already_backed_up')
+ logger.debug(
+ { projectId, hash },
+ 'Blob already backed up - skipping backup'
+ )
+ return
+ }
+ } catch (error) {
+ logger.warn({ error }, 'Failed to check if blob is backed up')
+ // We'll try anyway - we'll catch the error if it was backed up
+ }
+ // If we weren't passed a persistor for this project, create one.
+ // This will fetch the key from AWS, so it's prefereable to use
+ // the same persistor for all blobs in a project where possible.
+ if (!persistor) {
+ logger.debug(
+ { historyId, hash },
+ 'warning: persistor not passed to backupBlob'
+ )
+ }
+ persistor ??= await backupPersistor.forProject(
+ projectBlobsBucket,
+ makeProjectKey(historyId, '')
+ )
+ try {
+ logger.debug({ projectId, hash }, 'Starting blob backup')
+ await uploadBlobToBackup(historyId, blob, tmpPath, persistor)
+ await storeBlobBackup(projectId, hash)
+ recordBackupConclusion('success')
+ } catch (error) {
+ if (error instanceof AlreadyWrittenError) {
+ logger.debug({ error, projectId, hash }, 'Blob already backed up')
+ // record that we backed it up already
+ await storeBlobBackup(projectId, hash)
+ recordBackupConclusion('failure', 'already_backed_up')
+ return
+ }
+ // eventually queue this for retry - for now this will be fixed by running the script
+ recordBackupConclusion('failure')
+ logger.warn({ error, projectId, hash }, 'Failed to upload blob to backup')
+ } finally {
+ logger.debug({ projectId, hash }, 'Ended blob backup')
+ }
+}
diff --git a/services/history-v1/storage/lib/backupDeletion.mjs b/services/history-v1/storage/lib/backupDeletion.mjs
new file mode 100644
index 0000000000..ef50609753
--- /dev/null
+++ b/services/history-v1/storage/lib/backupDeletion.mjs
@@ -0,0 +1,93 @@
+// @ts-check
+import { callbackify } from 'util'
+import { ObjectId } from 'mongodb'
+import config from 'config'
+import OError from '@overleaf/o-error'
+import { db } from './mongodb.js'
+import projectKey from './project_key.js'
+import chunkStore from '../lib/chunk_store/index.js'
+import {
+ backupPersistor,
+ chunksBucket,
+ projectBlobsBucket,
+} from './backupPersistor.mjs'
+
+const MS_PER_DAY = 24 * 60 * 60 * 1000
+const EXPIRE_PROJECTS_AFTER_MS =
+ parseInt(config.get('minSoftDeletionPeriodDays'), 10) * MS_PER_DAY
+const deletedProjectsCollection = db.collection('deletedProjects')
+
+/**
+ * @param {string} historyId
+ * @return {Promise}
+ */
+async function projectHasLatestChunk(historyId) {
+ const chunk = await chunkStore.getBackend(historyId).getLatestChunk(historyId)
+ return chunk != null
+}
+
+export class NotReadyToDelete extends OError {}
+
+/**
+ * @param {string} projectId
+ * @return {Promise}
+ */
+async function deleteProjectBackup(projectId) {
+ const deletedProject = await deletedProjectsCollection.findOne(
+ { 'deleterData.deletedProjectId': new ObjectId(projectId) },
+ {
+ projection: {
+ 'deleterData.deletedProjectOverleafHistoryId': 1,
+ 'deleterData.deletedAt': 1,
+ },
+ }
+ )
+ if (!deletedProject) {
+ throw new NotReadyToDelete('refusing to delete non-deleted project')
+ }
+ const expiresAt =
+ deletedProject.deleterData.deletedAt.getTime() + EXPIRE_PROJECTS_AFTER_MS
+ if (expiresAt > Date.now()) {
+ throw new NotReadyToDelete('refusing to delete non-expired project')
+ }
+
+ const historyId =
+ deletedProject.deleterData.deletedProjectOverleafHistoryId?.toString()
+ if (!historyId) {
+ throw new NotReadyToDelete(
+ 'refusing to delete project with unknown historyId'
+ )
+ }
+
+ if (await projectHasLatestChunk(historyId)) {
+ throw new NotReadyToDelete(
+ 'refusing to delete project with remaining chunks'
+ )
+ }
+
+ const prefix = projectKey.format(historyId) + '/'
+ await backupPersistor.deleteDirectory(chunksBucket, prefix)
+ await backupPersistor.deleteDirectory(projectBlobsBucket, prefix)
+}
+
+export async function healthCheck() {
+ const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
+ if (HEALTH_CHECK_PROJECTS.length !== 2) {
+ throw new Error('expected 2 healthCheckProjects')
+ }
+ if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
+ throw new Error('expected mongo id in healthCheckProjects')
+ }
+ if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
+ throw new Error('expected postgres id in healthCheckProjects')
+ }
+
+ for (const historyId of HEALTH_CHECK_PROJECTS) {
+ if (!(await projectHasLatestChunk(historyId))) {
+ throw new Error(`project has no history: ${historyId}`)
+ }
+ }
+}
+
+export const healthCheckCb = callbackify(healthCheck)
+export const deleteProjectBackupCb = callbackify(deleteProjectBackup)
diff --git a/services/history-v1/storage/lib/backupGenerator.mjs b/services/history-v1/storage/lib/backupGenerator.mjs
new file mode 100644
index 0000000000..d8f1b0e99a
--- /dev/null
+++ b/services/history-v1/storage/lib/backupGenerator.mjs
@@ -0,0 +1,153 @@
+/**
+ * Provides a generator function to back up project chunks and blobs.
+ */
+
+import chunkStore from './chunk_store/index.js'
+
+import {
+ GLOBAL_BLOBS, // NOTE: must call loadGlobalBlobs() before using this
+ BlobStore,
+} from './blob_store/index.js'
+
+import assert from './assert.js'
+
+async function lookBehindForSeenBlobs(
+ projectId,
+ chunk,
+ lastBackedUpVersion,
+ seenBlobs
+) {
+ if (chunk.startVersion === 0) {
+ return // this is the first chunk, no need to check for blobs in the previous chunk
+ }
+ if (chunk.startVersion > 0 && lastBackedUpVersion > chunk.startVersion) {
+ return // the snapshot in this chunk has already been backed up
+ }
+ if (
+ chunk.startVersion > 0 &&
+ lastBackedUpVersion === chunk.startVersion // same as previousChunk.endVersion
+ ) {
+ // the snapshot in this chunk has not been backed up
+ // so we find the set of backed up blobs from the previous chunk
+ const previousChunk = await chunkStore.loadAtVersion(
+ projectId,
+ lastBackedUpVersion,
+ { persistedOnly: true }
+ )
+ const previousChunkHistory = previousChunk.getHistory()
+ previousChunkHistory.findBlobHashes(seenBlobs)
+ }
+}
+
+/**
+ * Records blob hashes that have been previously seen in a chunk's history.
+ *
+ * @param {Object} chunk - The chunk containing history data
+ * @param {number} currentBackedUpVersion - The version number that has been backed up
+ * @param {Set} seenBlobs - Set to collect previously seen blob hashes
+ * @returns {void}
+ */
+function recordPreviouslySeenBlobs(chunk, currentBackedUpVersion, seenBlobs) {
+ // We need to look at the chunk and decide how far we have backed up.
+ // If we have not backed up this chunk at all, we need to backup the blobs
+ // in the snapshot. Otherwise we need to backup the blobs in the changes
+ // that have occurred since the last backup.
+ const history = chunk.getHistory()
+ const startVersion = chunk.getStartVersion()
+ if (currentBackedUpVersion === 0) {
+ // If we have only backed up version 0 (i.e. the first change)
+ // then that includes the initial snapshot, so we consider
+ // the blobs of the initial snapshot as seen. If the project
+ // has not been backed up at all then currentBackedUpVersion
+ // will be undefined.
+ history.snapshot.findBlobHashes(seenBlobs)
+ } else if (currentBackedUpVersion > startVersion) {
+ history.snapshot.findBlobHashes(seenBlobs)
+ for (let i = 0; i < currentBackedUpVersion - startVersion; i++) {
+ history.changes[i].findBlobHashes(seenBlobs)
+ }
+ }
+}
+
+/**
+ * Collects new blob objects that need to be backed up from a given chunk.
+ *
+ * @param {Object} chunk - The chunk object containing history data
+ * @param {Object} blobStore - Storage interface for retrieving blobs
+ * @param {Set} seenBlobs - Set of blob hashes that have already been processed
+ * @returns {Promise} Array of blob objects that need to be backed up
+ * @throws {Error} If blob retrieval fails
+ */
+async function collectNewBlobsForBackup(chunk, blobStore, seenBlobs) {
+ /** @type {Set} */
+ const blobHashes = new Set()
+ const history = chunk.getHistory()
+ // Get all the blobs in this chunk, then exclude the seenBlobs and global blobs
+ history.findBlobHashes(blobHashes)
+ const blobsToBackup = await blobStore.getBlobs(
+ [...blobHashes].filter(
+ hash =>
+ hash &&
+ !seenBlobs.has(hash) &&
+ (!GLOBAL_BLOBS.has(hash) || GLOBAL_BLOBS.get(hash).demoted)
+ )
+ )
+ return blobsToBackup
+}
+
+/**
+ * Asynchronously generates backups for a project based on provided versions.
+ * @param {string} projectId - The ID of the project's history to back up.
+ * @param {number} lastBackedUpVersion - The last version that was successfully backed up.
+ * @yields {AsyncGenerator<{ chunkRecord: object, chunkToBackup: object, chunkBuffer: Buffer, blobsToBackup: object[] }>}
+ * Yields chunk records and corresponding data needed for backups.
+ */
+export async function* backupGenerator(projectId, lastBackedUpVersion) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.maybe.integer(lastBackedUpVersion, 'bad lastBackedUpVersion')
+
+ const blobStore = new BlobStore(projectId)
+
+ /** @type {Set} */
+ const seenBlobs = new Set() // records the blobs that are already backed up
+
+ const firstPendingVersion =
+ lastBackedUpVersion >= 0 ? lastBackedUpVersion + 1 : 0
+ let isStartingChunk = true
+ let currentBackedUpVersion = lastBackedUpVersion
+ const chunkRecordIterator = chunkStore.getProjectChunksFromVersion(
+ projectId,
+ firstPendingVersion
+ )
+
+ for await (const chunkRecord of chunkRecordIterator) {
+ const { chunk, chunkBuffer } = await chunkStore.loadByChunkRecord(
+ projectId,
+ chunkRecord
+ )
+
+ if (isStartingChunk) {
+ await lookBehindForSeenBlobs(
+ projectId,
+ chunkRecord,
+ lastBackedUpVersion,
+ seenBlobs
+ )
+ isStartingChunk = false
+ }
+
+ recordPreviouslySeenBlobs(chunk, currentBackedUpVersion, seenBlobs)
+
+ const blobsToBackup = await collectNewBlobsForBackup(
+ chunk,
+ blobStore,
+ seenBlobs
+ )
+
+ yield { chunkRecord, chunkToBackup: chunk, chunkBuffer, blobsToBackup }
+
+ // After we generate a backup of this chunk, mark the backed up blobs as seen
+ blobsToBackup.forEach(blob => seenBlobs.add(blob.getHash()))
+ currentBackedUpVersion = chunkRecord.endVersion
+ }
+}
diff --git a/services/history-v1/storage/lib/backupPersistor.mjs b/services/history-v1/storage/lib/backupPersistor.mjs
new file mode 100644
index 0000000000..8f80e5faaf
--- /dev/null
+++ b/services/history-v1/storage/lib/backupPersistor.mjs
@@ -0,0 +1,121 @@
+// @ts-check
+import fs from 'node:fs'
+import Path from 'node:path'
+import _ from 'lodash'
+import config from 'config'
+import { SecretManagerServiceClient } from '@google-cloud/secret-manager'
+import OError from '@overleaf/o-error'
+import {
+ PerProjectEncryptedS3Persistor,
+ RootKeyEncryptionKey,
+} from '@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js'
+import { HistoryStore } from './history_store.js'
+
+const persistorConfig = _.cloneDeep(config.get('backupPersistor'))
+const { chunksBucket, deksBucket, globalBlobsBucket, projectBlobsBucket } =
+ config.get('backupStore')
+
+export { chunksBucket, globalBlobsBucket, projectBlobsBucket }
+
+function convertKey(key, convertFn) {
+ if (_.has(persistorConfig, key)) {
+ _.update(persistorConfig, key, convertFn)
+ }
+}
+
+convertKey('s3SSEC.httpOptions.timeout', s => parseInt(s, 10))
+convertKey('s3SSEC.maxRetries', s => parseInt(s, 10))
+convertKey('s3SSEC.pathStyle', s => s === 'true')
+// array of CA, either inlined or on disk
+convertKey('s3SSEC.ca', s =>
+ JSON.parse(s).map(ca => (ca.startsWith('/') ? fs.readFileSync(ca) : ca))
+)
+
+/** @type {() => Promise} */
+let getRawRootKeyEncryptionKeys
+
+if ((process.env.NODE_ENV || 'production') === 'production') {
+ ;[persistorConfig.s3SSEC.key, persistorConfig.s3SSEC.secret] = (
+ await loadFromSecretsManager(
+ process.env.BACKUP_AWS_CREDENTIALS || '',
+ 'BACKUP_AWS_CREDENTIALS'
+ )
+ ).split(':')
+ getRawRootKeyEncryptionKeys = () =>
+ loadFromSecretsManager(
+ persistorConfig.keyEncryptionKeys,
+ 'BACKUP_KEY_ENCRYPTION_KEYS'
+ )
+} else {
+ getRawRootKeyEncryptionKeys = () => persistorConfig.keyEncryptionKeys
+}
+
+export const DELETION_ONLY = persistorConfig.keyEncryptionKeys === 'none'
+if (DELETION_ONLY) {
+ // For Backup-deleter; should not encrypt or read data; deleting does not need key.
+ getRawRootKeyEncryptionKeys = () => new Promise(_resolve => {})
+}
+
+const PROJECT_FOLDER_REGEX =
+ /^\d{3}\/\d{3}\/\d{3,}\/|[0-9a-f]{3}\/[0-9a-f]{3}\/[0-9a-f]{18}\/$/
+
+/**
+ * @param {string} bucketName
+ * @param {string} path
+ * @return {string}
+ */
+export function pathToProjectFolder(bucketName, path) {
+ switch (bucketName) {
+ case deksBucket:
+ case chunksBucket:
+ case projectBlobsBucket:
+ const projectFolder = Path.join(...path.split('/').slice(0, 3)) + '/'
+ if (!PROJECT_FOLDER_REGEX.test(projectFolder)) {
+ throw new OError('invalid project folder', { bucketName, path })
+ }
+ return projectFolder
+ default:
+ throw new Error(`${bucketName} does not store per-project files`)
+ }
+}
+
+/**
+ * @param {string} name
+ * @param {string} label
+ * @return {Promise}
+ */
+async function loadFromSecretsManager(name, label) {
+ const client = new SecretManagerServiceClient()
+ const [version] = await client.accessSecretVersion({ name })
+ if (!version.payload?.data) throw new Error(`empty secret: ${label}`)
+ return version.payload.data.toString()
+}
+
+async function getRootKeyEncryptionKeys() {
+ return JSON.parse(await getRawRootKeyEncryptionKeys()).map(
+ ({ key, salt }) => {
+ return new RootKeyEncryptionKey(
+ Buffer.from(key, 'base64'),
+ Buffer.from(salt, 'base64')
+ )
+ }
+ )
+}
+
+export const backupPersistor = new PerProjectEncryptedS3Persistor({
+ ...persistorConfig.s3SSEC,
+ disableMultiPartUpload: true,
+ dataEncryptionKeyBucketName: deksBucket,
+ pathToProjectFolder,
+ getRootKeyEncryptionKeys,
+ storageClass: {
+ [deksBucket]: 'STANDARD',
+ [chunksBucket]: persistorConfig.tieringStorageClass,
+ [projectBlobsBucket]: persistorConfig.tieringStorageClass,
+ },
+})
+
+export const backupHistoryStore = new HistoryStore(
+ backupPersistor,
+ chunksBucket
+)
diff --git a/services/history-v1/storage/lib/backupVerifier.mjs b/services/history-v1/storage/lib/backupVerifier.mjs
new file mode 100644
index 0000000000..6e767b21ba
--- /dev/null
+++ b/services/history-v1/storage/lib/backupVerifier.mjs
@@ -0,0 +1,220 @@
+// @ts-check
+import OError from '@overleaf/o-error'
+import chunkStore from '../lib/chunk_store/index.js'
+import {
+ backupPersistor,
+ chunksBucket,
+ projectBlobsBucket,
+} from './backupPersistor.mjs'
+import { Blob, Chunk, History } from 'overleaf-editor-core'
+import { BlobStore, GLOBAL_BLOBS, makeProjectKey } from './blob_store/index.js'
+import blobHash from './blob_hash.js'
+import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
+import logger from '@overleaf/logger'
+import path from 'node:path'
+import projectKey from './project_key.js'
+import streams from './streams.js'
+import objectPersistor from '@overleaf/object-persistor'
+import { getEndDateForRPO } from '../../backupVerifier/utils.mjs'
+
+/**
+ * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
+ */
+
+/**
+ * @param {string} historyId
+ * @param {string} hash
+ */
+export async function verifyBlob(historyId, hash) {
+ return await verifyBlobs(historyId, [hash])
+}
+
+/**
+ *
+ * @param {string} historyId
+ * @return {Promise}
+ */
+async function getProjectPersistor(historyId) {
+ try {
+ return await backupPersistor.forProjectRO(
+ projectBlobsBucket,
+ makeProjectKey(historyId, '')
+ )
+ } catch (err) {
+ if (err instanceof NotFoundError) {
+ throw new BackupCorruptedError('dek does not exist', {}, err)
+ }
+ throw err
+ }
+}
+
+/**
+ * @param {string} historyId
+ * @param {Array} hashes
+ * @param {CachedPerProjectEncryptedS3Persistor} [projectCache]
+ */
+export async function verifyBlobs(historyId, hashes, projectCache) {
+ if (hashes.length === 0) throw new Error('bug: empty hashes')
+
+ if (!projectCache) {
+ projectCache = await getProjectPersistor(historyId)
+ }
+ const blobStore = new BlobStore(historyId)
+ for (const hash of hashes) {
+ const path = makeProjectKey(historyId, hash)
+ const blob = await blobStore.getBlob(hash)
+ if (!blob) throw new Blob.NotFoundError(hash)
+ let stream
+ try {
+ stream = await projectCache.getObjectStream(projectBlobsBucket, path, {
+ autoGunzip: true,
+ })
+ } catch (err) {
+ if (err instanceof NotFoundError) {
+ throw new BackupCorruptedMissingBlobError('missing blob', {
+ path,
+ hash,
+ })
+ }
+ throw err
+ }
+ const backupHash = await blobHash.fromStream(blob.getByteLength(), stream)
+ if (backupHash !== hash) {
+ throw new BackupCorruptedInvalidBlobError(
+ 'hash mismatch for backed up blob',
+ {
+ path,
+ hash,
+ backupHash,
+ }
+ )
+ }
+ }
+}
+
+/**
+ * @param {string} historyId
+ * @param {Date} [endTimestamp]
+ */
+export async function verifyProjectWithErrorContext(
+ historyId,
+ endTimestamp = getEndDateForRPO()
+) {
+ try {
+ await verifyProject(historyId, endTimestamp)
+ } catch (err) {
+ // @ts-ignore err is Error instance
+ throw OError.tag(err, 'verifyProject', { historyId, endTimestamp })
+ }
+}
+
+/**
+ *
+ * @param {string} historyId
+ * @param {number} startVersion
+ * @param {CachedPerProjectEncryptedS3Persistor} backupPersistorForProject
+ * @return {Promise}
+ */
+export async function loadChunk(
+ historyId,
+ startVersion,
+ backupPersistorForProject
+) {
+ const key = path.join(
+ projectKey.format(historyId),
+ projectKey.pad(startVersion)
+ )
+ try {
+ const buf = await streams.gunzipStreamToBuffer(
+ await backupPersistorForProject.getObjectStream(chunksBucket, key)
+ )
+ return JSON.parse(buf.toString('utf-8'))
+ } catch (err) {
+ if (err instanceof objectPersistor.Errors.NotFoundError) {
+ throw new Chunk.NotPersistedError(historyId)
+ }
+ if (err instanceof Error) {
+ throw OError.tag(err, 'Failed to load chunk', { historyId, startVersion })
+ }
+ throw err
+ }
+}
+
+/**
+ * @param {string} historyId
+ * @param {Date} endTimestamp
+ */
+export async function verifyProject(historyId, endTimestamp) {
+ const backend = chunkStore.getBackend(historyId)
+ const [first, last] = await Promise.all([
+ backend.getFirstChunkBeforeTimestamp(historyId, endTimestamp),
+ backend.getLastActiveChunkBeforeTimestamp(historyId, endTimestamp),
+ ])
+
+ const chunksRecordsToVerify = [
+ {
+ chunkId: first.id,
+ chunkLabel: 'first',
+ },
+ ]
+ if (first.startVersion !== last.startVersion) {
+ chunksRecordsToVerify.push({
+ chunkId: last.id,
+ chunkLabel: 'last before RPO',
+ })
+ }
+
+ const projectCache = await getProjectPersistor(historyId)
+
+ const chunks = await Promise.all(
+ chunksRecordsToVerify.map(async chunk => {
+ try {
+ return History.fromRaw(
+ await loadChunk(historyId, chunk.startVersion, projectCache)
+ )
+ } catch (err) {
+ if (err instanceof Chunk.NotPersistedError) {
+ throw new BackupRPOViolationChunkNotBackedUpError(
+ 'BackupRPOviolation: chunk not backed up',
+ chunk
+ )
+ }
+ throw err
+ }
+ })
+ )
+ const seenBlobs = new Set()
+ const blobsToVerify = []
+ for (const chunk of chunks) {
+ /** @type {Set} */
+ const chunkBlobs = new Set()
+ chunk.findBlobHashes(chunkBlobs)
+ let hasAddedBlobFromThisChunk = false
+ for (const blobHash of chunkBlobs) {
+ if (seenBlobs.has(blobHash)) continue // old blob
+ if (GLOBAL_BLOBS.has(blobHash)) continue // global blob
+ seenBlobs.add(blobHash)
+ if (!hasAddedBlobFromThisChunk) {
+ blobsToVerify.push(blobHash)
+ hasAddedBlobFromThisChunk = true
+ }
+ }
+ }
+ if (blobsToVerify.length === 0) {
+ logger.debug(
+ {
+ historyId,
+ chunksRecordsToVerify: chunksRecordsToVerify.map(c => c.chunkId),
+ },
+ 'chunks contain no blobs to verify'
+ )
+ return
+ }
+ await verifyBlobs(historyId, blobsToVerify, projectCache)
+}
+
+export class BackupCorruptedError extends OError {}
+export class BackupRPOViolationError extends OError {}
+export class BackupCorruptedMissingBlobError extends BackupCorruptedError {}
+export class BackupCorruptedInvalidBlobError extends BackupCorruptedError {}
+export class BackupRPOViolationChunkNotBackedUpError extends OError {}
diff --git a/services/history-v1/storage/lib/backup_store/index.js b/services/history-v1/storage/lib/backup_store/index.js
new file mode 100644
index 0000000000..da7944786a
--- /dev/null
+++ b/services/history-v1/storage/lib/backup_store/index.js
@@ -0,0 +1,212 @@
+const { Binary, ObjectId } = require('mongodb')
+const { projects, backedUpBlobs } = require('../mongodb')
+const OError = require('@overleaf/o-error')
+
+// List projects with pending backups older than the specified interval
+function listPendingBackups(timeIntervalMs = 0, limit = null) {
+ const cutoffTime = new Date(Date.now() - timeIntervalMs)
+ const options = {
+ projection: { 'overleaf.backup.pendingChangeAt': 1 },
+ sort: { 'overleaf.backup.pendingChangeAt': 1 },
+ }
+
+ // Apply limit if provided
+ if (limit) {
+ options.limit = limit
+ }
+
+ const cursor = projects.find(
+ {
+ 'overleaf.backup.pendingChangeAt': {
+ $exists: true,
+ $lt: cutoffTime,
+ },
+ },
+ options
+ )
+ return cursor
+}
+
+// List projects that have never been backed up and are older than the specified interval
+function listUninitializedBackups(timeIntervalMs = 0, limit = null) {
+ const cutoffTimeInSeconds = (Date.now() - timeIntervalMs) / 1000
+ const options = {
+ projection: { _id: 1 },
+ sort: { _id: 1 },
+ }
+ // Apply limit if provided
+ if (limit) {
+ options.limit = limit
+ }
+ const cursor = projects.find(
+ {
+ 'overleaf.backup.lastBackedUpVersion': null,
+ _id: {
+ $lt: ObjectId.createFromTime(cutoffTimeInSeconds),
+ },
+ },
+ options
+ )
+ return cursor
+}
+
+// Retrieve the history ID for a given project without giving direct access to the
+// projects collection.
+
+async function getHistoryId(projectId) {
+ const project = await projects.findOne(
+ { _id: new ObjectId(projectId) },
+ {
+ projection: {
+ 'overleaf.history.id': 1,
+ },
+ }
+ )
+ if (!project) {
+ throw new Error('Project not found')
+ }
+ return project.overleaf.history.id
+}
+
+async function getBackupStatus(projectId) {
+ const project = await projects.findOne(
+ { _id: new ObjectId(projectId) },
+ {
+ projection: {
+ 'overleaf.history': 1,
+ 'overleaf.backup': 1,
+ },
+ }
+ )
+ if (!project) {
+ throw new Error('Project not found')
+ }
+ return {
+ backupStatus: project.overleaf.backup,
+ historyId: `${project.overleaf.history.id}`,
+ currentEndVersion: project.overleaf.history.currentEndVersion,
+ currentEndTimestamp: project.overleaf.history.currentEndTimestamp,
+ }
+}
+
+async function setBackupVersion(
+ projectId,
+ previousBackedUpVersion,
+ currentBackedUpVersion,
+ currentBackedUpAt
+) {
+ // FIXME: include a check to handle race conditions
+ // to make sure only one process updates the version numbers
+ const result = await projects.updateOne(
+ {
+ _id: new ObjectId(projectId),
+ 'overleaf.backup.lastBackedUpVersion': previousBackedUpVersion,
+ },
+ {
+ $set: {
+ 'overleaf.backup.lastBackedUpVersion': currentBackedUpVersion,
+ 'overleaf.backup.lastBackedUpAt': currentBackedUpAt,
+ },
+ }
+ )
+ if (result.matchedCount === 0 || result.modifiedCount === 0) {
+ throw new OError('Failed to update backup version', {
+ previousBackedUpVersion,
+ currentBackedUpVersion,
+ currentBackedUpAt,
+ result,
+ })
+ }
+}
+
+async function updateCurrentMetadataIfNotSet(projectId, latestChunkMetadata) {
+ await projects.updateOne(
+ {
+ _id: new ObjectId(projectId),
+ 'overleaf.history.currentEndVersion': { $exists: false },
+ 'overleaf.history.currentEndTimestamp': { $exists: false },
+ },
+ {
+ $set: {
+ 'overleaf.history.currentEndVersion': latestChunkMetadata.endVersion,
+ 'overleaf.history.currentEndTimestamp':
+ latestChunkMetadata.endTimestamp,
+ },
+ }
+ )
+}
+
+/**
+ * Updates the pending change timestamp for a project's backup status
+ * @param {string} projectId - The ID of the project to update
+ * @param {Date} backupStartTime - The timestamp to set for pending changes
+ * @returns {Promise}
+ *
+ * If the project's last backed up version matches the current end version,
+ * the pending change timestamp is removed. Otherwise, it's set to the provided
+ * backup start time.
+ */
+async function updatePendingChangeTimestamp(projectId, backupStartTime) {
+ await projects.updateOne({ _id: new ObjectId(projectId) }, [
+ {
+ $set: {
+ 'overleaf.backup.pendingChangeAt': {
+ $cond: {
+ if: {
+ $eq: [
+ '$overleaf.backup.lastBackedUpVersion',
+ '$overleaf.history.currentEndVersion',
+ ],
+ },
+ then: '$$REMOVE',
+ else: backupStartTime,
+ },
+ },
+ },
+ },
+ ])
+}
+
+async function getBackedUpBlobHashes(projectId) {
+ const result = await backedUpBlobs.findOne(
+ { _id: new ObjectId(projectId) },
+ { projection: { blobs: 1 } }
+ )
+ if (!result) {
+ return new Set()
+ }
+ const hashes = result.blobs.map(b => b.buffer.toString('hex'))
+ return new Set(hashes)
+}
+
+async function unsetBackedUpBlobHashes(projectId, hashes) {
+ const binaryHashes = hashes.map(h => new Binary(Buffer.from(h, 'hex')))
+ const result = await backedUpBlobs.findOneAndUpdate(
+ { _id: new ObjectId(projectId) },
+ {
+ $pullAll: {
+ blobs: binaryHashes,
+ },
+ },
+ { returnDocument: 'after' }
+ )
+ if (result && result.blobs.length === 0) {
+ await backedUpBlobs.deleteOne({
+ _id: new ObjectId(projectId),
+ blobs: { $size: 0 },
+ })
+ }
+ return result
+}
+
+module.exports = {
+ getHistoryId,
+ getBackupStatus,
+ setBackupVersion,
+ updateCurrentMetadataIfNotSet,
+ updatePendingChangeTimestamp,
+ listPendingBackups,
+ listUninitializedBackups,
+ getBackedUpBlobHashes,
+ unsetBackedUpBlobHashes,
+}
diff --git a/services/history-v1/storage/lib/blob_hash.js b/services/history-v1/storage/lib/blob_hash.js
index 5c2edac14f..10ac64b87b 100644
--- a/services/history-v1/storage/lib/blob_hash.js
+++ b/services/history-v1/storage/lib/blob_hash.js
@@ -2,9 +2,9 @@
'use strict'
const BPromise = require('bluebird')
-const fs = BPromise.promisifyAll(require('fs'))
-const crypto = require('crypto')
-const { pipeline } = require('stream')
+const fs = BPromise.promisifyAll(require('node:fs'))
+const crypto = require('node:crypto')
+const { pipeline } = require('node:stream')
const assert = require('./assert')
function getGitBlobHeader(byteLength) {
@@ -63,7 +63,7 @@ exports.fromString = function blobHashFromString(string) {
* Compute the git blob hash for the content of a file
*
* @param {string} filePath
- * @return {string} hexadecimal SHA-1 hash
+ * @return {Promise} hexadecimal SHA-1 hash
*/
exports.fromFile = function blobHashFromFile(pathname) {
assert.string(pathname, 'blobHash: bad pathname')
diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js
index 6f5632a595..033e288554 100644
--- a/services/history-v1/storage/lib/blob_store/index.js
+++ b/services/history-v1/storage/lib/blob_store/index.js
@@ -1,7 +1,7 @@
'use strict'
const config = require('config')
-const fs = require('fs')
+const fs = require('node:fs')
const isValidUtf8 = require('utf-8-validate')
const { ReadableString } = require('@overleaf/stream-utils')
@@ -24,6 +24,7 @@ const logger = require('@overleaf/logger')
/** @import { Readable } from 'stream' */
+/** @type {Map} */
const GLOBAL_BLOBS = new Map()
function makeGlobalKey(hash) {
@@ -79,23 +80,12 @@ function getBackend(projectId) {
}
async function makeBlobForFile(pathname) {
- async function getByteLengthOfFile() {
- const stat = await fs.promises.stat(pathname)
- return stat.size
- }
-
- async function getHashOfFile(blob) {
- const stream = fs.createReadStream(pathname)
- const hash = await blobHash.fromStream(blob.getByteLength(), stream)
- return hash
- }
-
- const blob = new Blob()
- const byteLength = await getByteLengthOfFile()
- blob.setByteLength(byteLength)
- const hash = await getHashOfFile(blob)
- blob.setHash(hash)
- return blob
+ const { size: byteLength } = await fs.promises.stat(pathname)
+ const hash = await blobHash.fromStream(
+ byteLength,
+ fs.createReadStream(pathname)
+ )
+ return new Blob(hash, byteLength)
}
async function getStringLengthOfFile(byteLength, pathname) {
@@ -136,6 +126,34 @@ async function loadGlobalBlobs() {
}
}
+/**
+ * Return metadata for all blobs in the given project
+ * @param {Array} projectIds
+ * @return {Promise<{nBlobs:number, blobs:Map>}>}
+ */
+async function getProjectBlobsBatch(projectIds) {
+ const mongoProjects = []
+ const postgresProjects = []
+ for (const projectId of projectIds) {
+ if (typeof projectId === 'number') {
+ postgresProjects.push(projectId)
+ } else {
+ mongoProjects.push(projectId)
+ }
+ }
+ const [
+ { nBlobs: nBlobsPostgres, blobs: blobsPostgres },
+ { nBlobs: nBlobsMongo, blobs: blobsMongo },
+ ] = await Promise.all([
+ postgresBackend.getProjectBlobsBatch(postgresProjects),
+ mongoBackend.getProjectBlobsBatch(mongoProjects),
+ ])
+ for (const [id, blobs] of blobsPostgres.entries()) {
+ blobsMongo.set(id.toString(), blobs)
+ }
+ return { nBlobs: nBlobsPostgres + nBlobsMongo, blobs: blobsMongo }
+}
+
/**
* @classdesc
* Fetch and store the content of files using content-addressable hashing. The
@@ -188,7 +206,7 @@ class BlobStore {
* temporary file).
*
* @param {string} pathname
- * @return {Promise.}
+ * @return {Promise}
*/
async putFile(pathname) {
assert.string(pathname, 'bad pathname')
@@ -202,11 +220,28 @@ class BlobStore {
pathname
)
newBlob.setStringLength(stringLength)
- await uploadBlob(this.projectId, newBlob, fs.createReadStream(pathname))
- await this.backend.insertBlob(this.projectId, newBlob)
+ await this.putBlob(pathname, newBlob)
return newBlob
}
+ /**
+ * Write a new blob, the stringLength must have been added already. It should
+ * have been checked that the blob does not exist yet. Consider using
+ * {@link putFile} instead of this lower-level method.
+ *
+ * @param {string} pathname
+ * @param {core.Blob} finializedBlob
+ * @return {Promise}
+ */
+ async putBlob(pathname, finializedBlob) {
+ await uploadBlob(
+ this.projectId,
+ finializedBlob,
+ fs.createReadStream(pathname)
+ )
+ await this.backend.insertBlob(this.projectId, finializedBlob)
+ }
+
/**
* Stores an object as a JSON string in a blob.
*
@@ -275,14 +310,15 @@ class BlobStore {
* failure, so the caller must be prepared to retry on errors, if appropriate.
*
* @param {string} hash hexadecimal SHA-1 hash
+ * @param {Object} opts
* @return {Promise.} a stream to read the file
*/
- async getStream(hash) {
+ async getStream(hash, opts = {}) {
assert.blobHash(hash, 'bad hash')
const { bucket, key } = getBlobLocation(this.projectId, hash)
try {
- const stream = await persistor.getObjectStream(bucket, key)
+ const stream = await persistor.getObjectStream(bucket, key, opts)
return stream
} catch (err) {
if (err instanceof objectPersistor.Errors.NotFoundError) {
@@ -308,6 +344,11 @@ class BlobStore {
return blob
}
+ /**
+ *
+ * @param {Array} hashes
+ * @return {Promise<*[]>}
+ */
async getBlobs(hashes) {
assert.array(hashes, 'bad hashes')
const nonGlobalHashes = []
@@ -320,6 +361,9 @@ class BlobStore {
nonGlobalHashes.push(hash)
}
}
+ if (nonGlobalHashes.length === 0) {
+ return blobs // to avoid unnecessary database lookup
+ }
const projectBlobs = await this.backend.findBlobs(
this.projectId,
nonGlobalHashes
@@ -328,6 +372,16 @@ class BlobStore {
return blobs
}
+ /**
+ * Retrieve all blobs associated with the project.
+ * @returns {Promise} A promise that resolves to an array of blobs.
+ */
+
+ async getProjectBlobs() {
+ const projectBlobs = await this.backend.getProjectBlobs(this.projectId)
+ return projectBlobs
+ }
+
/**
* Delete all blobs that belong to the project.
*/
@@ -346,6 +400,41 @@ class BlobStore {
const blob = await this.backend.findBlob(this.projectId, hash)
return blob
}
+
+ /**
+ * Copy an existing sourceBlob in this project to a target project.
+ * @param {Blob} sourceBlob
+ * @param {string} targetProjectId
+ * @return {Promise}
+ */
+ async copyBlob(sourceBlob, targetProjectId) {
+ assert.instance(sourceBlob, Blob, 'bad sourceBlob')
+ assert.projectId(targetProjectId, 'bad targetProjectId')
+ const hash = sourceBlob.getHash()
+ const sourceProjectId = this.projectId
+ const { bucket, key: sourceKey } = getBlobLocation(sourceProjectId, hash)
+ const destKey = makeProjectKey(targetProjectId, hash)
+ const targetBackend = getBackend(targetProjectId)
+ logger.debug({ sourceProjectId, targetProjectId, hash }, 'copyBlob started')
+ try {
+ await persistor.copyObject(bucket, sourceKey, destKey)
+ await targetBackend.insertBlob(targetProjectId, sourceBlob)
+ } finally {
+ logger.debug(
+ { sourceProjectId, targetProjectId, hash },
+ 'copyBlob finished'
+ )
+ }
+ }
}
-module.exports = { BlobStore, loadGlobalBlobs }
+module.exports = {
+ BlobStore,
+ getProjectBlobsBatch,
+ loadGlobalBlobs,
+ makeProjectKey,
+ makeGlobalKey,
+ makeBlobForFile,
+ getStringLengthOfFile,
+ GLOBAL_BLOBS,
+}
diff --git a/services/history-v1/storage/lib/blob_store/mongo.js b/services/history-v1/storage/lib/blob_store/mongo.js
index 6bd516addb..9117382148 100644
--- a/services/history-v1/storage/lib/blob_store/mongo.js
+++ b/services/history-v1/storage/lib/blob_store/mongo.js
@@ -1,3 +1,4 @@
+// @ts-check
/**
* Mongo backend for the blob store.
*
@@ -15,15 +16,20 @@
*/
const { Blob } = require('overleaf-editor-core')
-const { ObjectId, Binary } = require('mongodb')
+const { ObjectId, Binary, MongoError, ReadPreference } = require('mongodb')
const assert = require('../assert')
const mongodb = require('../mongodb')
const MAX_BLOBS_IN_BUCKET = 8
const DUPLICATE_KEY_ERROR_CODE = 11000
+/**
+ * @typedef {import('mongodb').ReadPreferenceLike} ReadPreferenceLike
+ */
+
/**
* Set up the data structures for a given project.
+ * @param {string} projectId
*/
async function initialize(projectId) {
assert.mongoId(projectId, 'bad projectId')
@@ -33,14 +39,18 @@ async function initialize(projectId) {
blobs: {},
})
} catch (err) {
- if (err.code !== DUPLICATE_KEY_ERROR_CODE) {
- throw err
+ if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) {
+ return // ignore already initialized case
}
+ throw err
}
}
/**
* Return blob metadata for the given project and hash.
+ * @param {string} projectId
+ * @param {string} hash
+ * @return {Promise}
*/
async function findBlob(projectId, hash) {
assert.mongoId(projectId, 'bad projectId')
@@ -69,6 +79,9 @@ async function findBlob(projectId, hash) {
/**
* Search in the sharded collection for blob metadata
+ * @param {string} projectId
+ * @param {string} hash
+ * @return {Promise}
*/
async function findBlobSharded(projectId, hash) {
const [shard, bucket] = getShardedBucket(hash)
@@ -81,11 +94,15 @@ async function findBlobSharded(projectId, hash) {
return null
}
const record = result.blobs.find(blob => blob.h.toString('hex') === hash)
+ if (!record) return null
return recordToBlob(record)
}
/**
* Read multiple blob metadata records by hexadecimal hashes.
+ * @param {string} projectId
+ * @param {Array} hashes
+ * @return {Promise>}
*/
async function findBlobs(projectId, hashes) {
assert.mongoId(projectId, 'bad projectId')
@@ -135,6 +152,9 @@ async function findBlobs(projectId, hashes) {
/**
* Search in the sharded collection for blob metadata.
+ * @param {string} projectId
+ * @param {Set} hashSet
+ * @return {Promise>}
*/
async function findBlobsSharded(projectId, hashSet) {
// Build a map of buckets by shard key
@@ -181,8 +201,113 @@ async function findBlobsSharded(projectId, hashSet) {
return blobs
}
+/**
+ * Return metadata for all blobs in the given project
+ */
+async function getProjectBlobs(projectId) {
+ assert.mongoId(projectId, 'bad projectId')
+
+ const result = await mongodb.blobs.findOne(
+ { _id: new ObjectId(projectId) },
+ { projection: { _id: 0 } }
+ )
+
+ if (!result) {
+ return []
+ }
+
+ // Build blobs from the query results
+ const blobs = []
+ for (const bucket of Object.values(result.blobs)) {
+ for (const record of bucket) {
+ blobs.push(recordToBlob(record))
+ }
+ }
+
+ // Look for all possible sharded blobs
+
+ const minShardedId = makeShardedId(projectId, '0')
+ const maxShardedId = makeShardedId(projectId, 'f')
+ // @ts-ignore We are using a custom _id here.
+ const shardedRecords = mongodb.shardedBlobs.find(
+ {
+ _id: { $gte: minShardedId, $lte: maxShardedId },
+ },
+ { projection: { _id: 0 } }
+ )
+
+ for await (const shardedRecord of shardedRecords) {
+ if (shardedRecord.blobs == null) {
+ continue
+ }
+ for (const bucket of Object.values(shardedRecord.blobs)) {
+ for (const record of bucket) {
+ blobs.push(recordToBlob(record))
+ }
+ }
+ }
+
+ return blobs
+}
+
+/**
+ * Return metadata for all blobs in the given project
+ * @param {Array} projectIds
+ * @return {Promise<{ nBlobs: number, blobs: Map> }>}
+ */
+async function getProjectBlobsBatch(projectIds) {
+ for (const project of projectIds) {
+ assert.mongoId(project, 'bad projectId')
+ }
+ let nBlobs = 0
+ const blobs = new Map()
+ if (projectIds.length === 0) return { nBlobs, blobs }
+
+ // blobs
+ {
+ const cursor = await mongodb.blobs.find(
+ { _id: { $in: projectIds.map(projectId => new ObjectId(projectId)) } },
+ { readPreference: ReadPreference.secondaryPreferred }
+ )
+ for await (const record of cursor) {
+ const projectBlobs = Object.values(record.blobs).flat().map(recordToBlob)
+ blobs.set(record._id.toString(), projectBlobs)
+ nBlobs += projectBlobs.length
+ }
+ }
+
+ // sharded blobs
+ {
+ // @ts-ignore We are using a custom _id here.
+ const cursor = await mongodb.shardedBlobs.find(
+ {
+ _id: {
+ $gte: makeShardedId(projectIds[0], '0'),
+ $lte: makeShardedId(projectIds[projectIds.length - 1], 'f'),
+ },
+ },
+ { readPreference: ReadPreference.secondaryPreferred }
+ )
+ for await (const record of cursor) {
+ const recordIdHex = record._id.toString('hex')
+ const recordProjectId = recordIdHex.slice(0, 24)
+ const projectBlobs = Object.values(record.blobs).flat().map(recordToBlob)
+ const found = blobs.get(recordProjectId)
+ if (found) {
+ found.push(...projectBlobs)
+ } else {
+ blobs.set(recordProjectId, projectBlobs)
+ }
+ nBlobs += projectBlobs.length
+ }
+ }
+ return { nBlobs, blobs }
+}
+
/**
* Add a blob's metadata to the blobs collection after it has been uploaded.
+ * @param {string} projectId
+ * @param {Blob} blob
*/
async function insertBlob(projectId, blob) {
assert.mongoId(projectId, 'bad projectId')
@@ -208,6 +333,10 @@ async function insertBlob(projectId, blob) {
/**
* Add a blob's metadata to the sharded blobs collection.
+ * @param {string} projectId
+ * @param {string} hash
+ * @param {Record} record
+ * @return {Promise}
*/
async function insertRecordSharded(projectId, hash, record) {
const [shard, bucket] = getShardedBucket(hash)
@@ -221,6 +350,7 @@ async function insertRecordSharded(projectId, hash, record) {
/**
* Delete all blobs for a given project.
+ * @param {string} projectId
*/
async function deleteBlobs(projectId) {
assert.mongoId(projectId, 'bad projectId')
@@ -228,12 +358,15 @@ async function deleteBlobs(projectId) {
const minShardedId = makeShardedId(projectId, '0')
const maxShardedId = makeShardedId(projectId, 'f')
await mongodb.shardedBlobs.deleteMany({
+ // @ts-ignore We are using a custom _id here.
_id: { $gte: minShardedId, $lte: maxShardedId },
})
}
/**
* Return the Mongo path to the bucket for the given hash.
+ * @param {string} hash
+ * @return {string}
*/
function getBucket(hash) {
return `blobs.${hash.slice(0, 3)}`
@@ -242,6 +375,8 @@ function getBucket(hash) {
/**
* Return the shard key and Mongo path to the bucket for the given hash in the
* sharded collection.
+ * @param {string} hash
+ * @return {[string, string]}
*/
function getShardedBucket(hash) {
const shard = hash.slice(0, 1)
@@ -251,13 +386,25 @@ function getShardedBucket(hash) {
/**
* Create an _id key for the sharded collection.
+ * @param {string} projectId
+ * @param {string} shard
+ * @return {Binary}
*/
function makeShardedId(projectId, shard) {
return new Binary(Buffer.from(`${projectId}0${shard}`, 'hex'))
}
+/**
+ * @typedef {Object} Record
+ * @property {Binary} h
+ * @property {number} b
+ * @property {number} [s]
+ */
+
/**
* Return the Mongo record for the given blob.
+ * @param {Blob} blob
+ * @return {Record}
*/
function blobToRecord(blob) {
const hash = blob.getHash()
@@ -272,11 +419,10 @@ function blobToRecord(blob) {
/**
* Create a blob from the given Mongo record.
+ * @param {Record} record
+ * @return {Blob}
*/
function recordToBlob(record) {
- if (record == null) {
- return
- }
return new Blob(record.h.toString('hex'), record.b, record.s)
}
@@ -284,6 +430,8 @@ module.exports = {
initialize,
findBlob,
findBlobs,
+ getProjectBlobs,
+ getProjectBlobsBatch,
insertBlob,
deleteBlobs,
}
diff --git a/services/history-v1/storage/lib/blob_store/postgres.js b/services/history-v1/storage/lib/blob_store/postgres.js
index 9e40c255da..1cedeec5d7 100644
--- a/services/history-v1/storage/lib/blob_store/postgres.js
+++ b/services/history-v1/storage/lib/blob_store/postgres.js
@@ -13,8 +13,8 @@ async function initialize(projectId) {
* Return blob metadata for the given project and hash
*/
async function findBlob(projectId, hash) {
+ assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
assert.blobHash(hash, 'bad hash')
const binaryHash = hashToBuffer(hash)
@@ -35,8 +35,8 @@ async function findBlob(projectId, hash) {
* @return {Promise.>} no guarantee on order
*/
async function findBlobs(projectId, hashes) {
+ assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
assert.array(hashes, 'bad hashes: not array')
hashes.forEach(function (hash) {
assert.blobHash(hash, 'bad hash')
@@ -53,12 +53,58 @@ async function findBlobs(projectId, hashes) {
return blobs
}
+/**
+ * Return metadata for all blobs in the given project
+ */
+async function getProjectBlobs(projectId) {
+ assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+
+ const records = await knex('project_blobs')
+ .select('hash_bytes', 'byte_length', 'string_length')
+ .where({
+ project_id: projectId,
+ })
+
+ const blobs = records.map(recordToBlob)
+ return blobs
+}
+
+/**
+ * Return metadata for all blobs in the given project
+ * @param {Array} projectIds
+ * @return {Promise<{ nBlobs: number, blobs: Map> }>}
+ */
+async function getProjectBlobsBatch(projectIds) {
+ for (const projectId of projectIds) {
+ assert.integer(projectId, 'bad projectId')
+ }
+ let nBlobs = 0
+ const blobs = new Map()
+ if (projectIds.length === 0) return { nBlobs, blobs }
+
+ const cursor = knex('project_blobs')
+ .select('project_id', 'hash_bytes', 'byte_length', 'string_length')
+ .whereIn('project_id', projectIds)
+ .stream()
+ for await (const record of cursor) {
+ const found = blobs.get(record.project_id)
+ if (found) {
+ found.push(recordToBlob(record))
+ } else {
+ blobs.set(record.project_id, [recordToBlob(record)])
+ }
+ nBlobs++
+ }
+ return { nBlobs, blobs }
+}
+
/**
* Add a blob's metadata to the blobs table after it has been uploaded.
*/
async function insertBlob(projectId, blob) {
+ assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
await knex('project_blobs')
.insert(blobToRecord(projectId, blob))
@@ -70,8 +116,8 @@ async function insertBlob(projectId, blob) {
* Deletes all blobs for a given project
*/
async function deleteBlobs(projectId) {
+ assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
await knex('project_blobs').where('project_id', projectId).delete()
}
@@ -108,6 +154,8 @@ module.exports = {
initialize,
findBlob,
findBlobs,
+ getProjectBlobs,
+ getProjectBlobsBatch,
insertBlob,
deleteBlobs,
}
diff --git a/services/history-v1/storage/lib/chunk_store/errors.js b/services/history-v1/storage/lib/chunk_store/errors.js
index 5f0eba6aac..75b830f9a0 100644
--- a/services/history-v1/storage/lib/chunk_store/errors.js
+++ b/services/history-v1/storage/lib/chunk_store/errors.js
@@ -1,7 +1,15 @@
const OError = require('@overleaf/o-error')
class ChunkVersionConflictError extends OError {}
+class BaseVersionConflictError extends OError {}
+class JobNotFoundError extends OError {}
+class JobNotReadyError extends OError {}
+class VersionOutOfBoundsError extends OError {}
module.exports = {
ChunkVersionConflictError,
+ BaseVersionConflictError,
+ JobNotFoundError,
+ JobNotReadyError,
+ VersionOutOfBoundsError,
}
diff --git a/services/history-v1/storage/lib/chunk_store/index.js b/services/history-v1/storage/lib/chunk_store/index.js
index eb3c8ba48d..f387b68d90 100644
--- a/services/history-v1/storage/lib/chunk_store/index.js
+++ b/services/history-v1/storage/lib/chunk_store/index.js
@@ -1,3 +1,5 @@
+// @ts-check
+
'use strict'
/**
@@ -27,10 +29,18 @@ const { Chunk, History, Snapshot } = require('overleaf-editor-core')
const assert = require('../assert')
const BatchBlobStore = require('../batch_blob_store')
const { BlobStore } = require('../blob_store')
-const historyStore = require('../history_store')
+const { historyStore } = require('../history_store')
const mongoBackend = require('./mongo')
const postgresBackend = require('./postgres')
-const { ChunkVersionConflictError } = require('./errors')
+const redisBackend = require('./redis')
+const {
+ ChunkVersionConflictError,
+ VersionOutOfBoundsError,
+} = require('./errors')
+
+/**
+ * @import { Change } from 'overleaf-editor-core'
+ */
const DEFAULT_DELETE_BATCH_SIZE = parseInt(config.get('maxDeleteKeys'), 10)
const DEFAULT_DELETE_TIMEOUT_SECS = 3000 // 50 minutes
@@ -81,49 +91,120 @@ async function lazyLoadHistoryFiles(history, batchBlobStore) {
/**
* Load the latest Chunk stored for a project, including blob metadata.
*
- * @param {number} projectId
- * @return {Promise.}
+ * @param {string} projectId
+ * @param {Object} [opts]
+ * @param {boolean} [opts.readOnly]
+ * @return {Promise<{id: string, startVersion: number, endVersion: number, endTimestamp: Date}>}
*/
-async function loadLatest(projectId) {
+async function getLatestChunkMetadata(projectId, opts) {
assert.projectId(projectId, 'bad projectId')
const backend = getBackend(projectId)
- const blobStore = new BlobStore(projectId)
- const batchBlobStore = new BatchBlobStore(blobStore)
- const chunkRecord = await backend.getLatestChunk(projectId)
- if (chunkRecord == null) {
+ const chunkMetadata = await backend.getLatestChunk(projectId, opts)
+ if (chunkMetadata == null) {
throw new Chunk.NotFoundError(projectId)
}
+ return chunkMetadata
+}
- const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
+/**
+ * Load the latest Chunk stored for a project, including blob metadata.
+ *
+ * @param {string} projectId
+ * @param {object} [opts]
+ * @param {boolean} [opts.persistedOnly] - only include persisted changes
+ * @return {Promise}
+ */
+async function loadLatest(projectId, opts = {}) {
+ const chunkMetadata = await getLatestChunkMetadata(projectId)
+ const rawHistory = await historyStore.loadRaw(projectId, chunkMetadata.id)
const history = History.fromRaw(rawHistory)
+
+ if (!opts.persistedOnly) {
+ const nonPersistedChanges = await getChunkExtension(
+ projectId,
+ chunkMetadata.endVersion
+ )
+ history.pushChanges(nonPersistedChanges)
+ }
+
+ const blobStore = new BlobStore(projectId)
+ const batchBlobStore = new BatchBlobStore(blobStore)
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, chunkRecord.startVersion)
+ return new Chunk(history, chunkMetadata.startVersion)
}
/**
* Load the the chunk that contains the given version, including blob metadata.
+ *
+ * @param {string} projectId
+ * @param {number} version
+ * @param {object} [opts]
+ * @param {boolean} [opts.persistedOnly] - only include persisted changes
+ * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
+ * two chunks, return the newer chunk.
*/
-async function loadAtVersion(projectId, version) {
+async function loadAtVersion(projectId, version, opts = {}) {
assert.projectId(projectId, 'bad projectId')
assert.integer(version, 'bad version')
const backend = getBackend(projectId)
const blobStore = new BlobStore(projectId)
const batchBlobStore = new BatchBlobStore(blobStore)
+ const latestChunkMetadata = await getLatestChunkMetadata(projectId)
- const chunkRecord = await backend.getChunkForVersion(projectId, version)
+ // When loading a chunk for a version there are three cases to consider:
+ // 1. If `persistedOnly` is true, we always use the requested version
+ // to fetch the chunk.
+ // 2. If `persistedOnly` is false and the requested version is in the
+ // persisted chunk version range, we use the requested version.
+ // 3. If `persistedOnly` is false and the requested version is ahead of
+ // the persisted chunk versions, we fetch the latest chunk and see if
+ // the non-persisted changes include the requested version.
+ const targetChunkVersion = opts.persistedOnly
+ ? version
+ : Math.min(latestChunkMetadata.endVersion, version)
+
+ const chunkRecord = await backend.getChunkForVersion(
+ projectId,
+ targetChunkVersion,
+ {
+ preferNewer: opts.preferNewer,
+ }
+ )
const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
const history = History.fromRaw(rawHistory)
+ const startVersion = chunkRecord.endVersion - history.countChanges()
+
+ if (!opts.persistedOnly) {
+ // Try to extend the chunk with any non-persisted changes that
+ // follow the chunk's end version.
+ const nonPersistedChanges = await getChunkExtension(
+ projectId,
+ chunkRecord.endVersion
+ )
+ history.pushChanges(nonPersistedChanges)
+
+ // Check that the changes do actually contain the requested version
+ if (version > chunkRecord.endVersion + nonPersistedChanges.length) {
+ throw new Chunk.VersionNotFoundError(projectId, version)
+ }
+ }
+
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, chunkRecord.endVersion - history.countChanges())
+ return new Chunk(history, startVersion)
}
/**
* Load the chunk that contains the version that was current at the given
* timestamp, including blob metadata.
+ *
+ * @param {string} projectId
+ * @param {Date} timestamp
+ * @param {object} [opts]
+ * @param {boolean} [opts.persistedOnly] - only include persisted changes
*/
-async function loadAtTimestamp(projectId, timestamp) {
+async function loadAtTimestamp(projectId, timestamp, opts = {}) {
assert.projectId(projectId, 'bad projectId')
assert.date(timestamp, 'bad timestamp')
@@ -134,24 +215,58 @@ async function loadAtTimestamp(projectId, timestamp) {
const chunkRecord = await backend.getChunkForTimestamp(projectId, timestamp)
const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
const history = History.fromRaw(rawHistory)
+ const startVersion = chunkRecord.endVersion - history.countChanges()
+
+ if (!opts.persistedOnly) {
+ const nonPersistedChanges = await getChunkExtension(
+ projectId,
+ chunkRecord.endVersion
+ )
+ history.pushChanges(nonPersistedChanges)
+ }
+
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, chunkRecord.endVersion - history.countChanges())
+ return new Chunk(history, startVersion)
}
/**
* Store the chunk and insert corresponding records in the database.
*
- * @param {number} projectId
+ * @param {string} projectId
* @param {Chunk} chunk
- * @return {Promise.} for the chunkId of the inserted chunk
+ * @param {Date} [earliestChangeTimestamp]
*/
-async function create(projectId, chunk) {
+async function create(projectId, chunk, earliestChangeTimestamp) {
assert.projectId(projectId, 'bad projectId')
assert.instance(chunk, Chunk, 'bad chunk')
+ assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
const backend = getBackend(projectId)
+ const chunkStart = chunk.getStartVersion()
+
+ const opts = {}
+ if (chunkStart > 0) {
+ const oldChunk = await backend.getChunkForVersion(projectId, chunkStart)
+
+ if (oldChunk.endVersion !== chunkStart) {
+ throw new ChunkVersionConflictError(
+ 'unexpected end version on chunk to be updated',
+ {
+ projectId,
+ expectedVersion: chunkStart,
+ actualVersion: oldChunk.endVersion,
+ }
+ )
+ }
+
+ opts.oldChunkId = oldChunk.id
+ }
+ if (earliestChangeTimestamp != null) {
+ opts.earliestChangeTimestamp = earliestChangeTimestamp
+ }
+
const chunkId = await uploadChunk(projectId, chunk)
- await backend.confirmCreate(projectId, chunk, chunkId)
+ await backend.confirmCreate(projectId, chunk, chunkId, opts)
}
/**
@@ -180,29 +295,67 @@ async function uploadChunk(projectId, chunk) {
* Extend the project's history by replacing the latest chunk with a new
* chunk.
*
- * @param {number} projectId
- * @param {number} oldEndVersion
+ * @param {string} projectId
* @param {Chunk} newChunk
+ * @param {Date} [earliestChangeTimestamp]
* @return {Promise}
*/
-async function update(projectId, oldEndVersion, newChunk) {
+async function update(projectId, newChunk, earliestChangeTimestamp) {
assert.projectId(projectId, 'bad projectId')
- assert.integer(oldEndVersion, 'bad oldEndVersion')
assert.instance(newChunk, Chunk, 'bad newChunk')
+ assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
const backend = getBackend(projectId)
- const oldChunkId = await getChunkIdForVersion(projectId, oldEndVersion)
+ const oldChunk = await backend.getChunkForVersion(
+ projectId,
+ newChunk.getStartVersion(),
+ { preferNewer: true }
+ )
+
+ if (oldChunk.startVersion !== newChunk.getStartVersion()) {
+ throw new ChunkVersionConflictError(
+ 'unexpected start version on chunk to be updated',
+ {
+ projectId,
+ expectedVersion: newChunk.getStartVersion(),
+ actualVersion: oldChunk.startVersion,
+ }
+ )
+ }
+
+ if (oldChunk.endVersion > newChunk.getEndVersion()) {
+ throw new ChunkVersionConflictError(
+ 'chunk update would decrease chunk version',
+ {
+ projectId,
+ currentVersion: oldChunk.endVersion,
+ newVersion: newChunk.getEndVersion(),
+ }
+ )
+ }
+
const newChunkId = await uploadChunk(projectId, newChunk)
- await backend.confirmUpdate(projectId, oldChunkId, newChunk, newChunkId)
+ const opts = {}
+ if (earliestChangeTimestamp != null) {
+ opts.earliestChangeTimestamp = earliestChangeTimestamp
+ }
+
+ await backend.confirmUpdate(
+ projectId,
+ oldChunk.id,
+ newChunk,
+ newChunkId,
+ opts
+ )
}
/**
* Find the chunk ID for a given version of a project.
*
- * @param {number} projectId
+ * @param {string} projectId
* @param {number} version
- * @return {Promise.}
+ * @return {Promise.}
*/
async function getChunkIdForVersion(projectId, version) {
const backend = getBackend(projectId)
@@ -210,6 +363,19 @@ async function getChunkIdForVersion(projectId, version) {
return chunkRecord.id
}
+/**
+ * Find the chunk metadata for a given version of a project.
+ *
+ * @param {string} projectId
+ * @param {number} version
+ * @return {Promise.<{id: string|number, startVersion: number, endVersion: number}>}
+ */
+async function getChunkMetadataForVersion(projectId, version) {
+ const backend = getBackend(projectId)
+ const chunkRecord = await backend.getChunkForVersion(projectId, version)
+ return chunkRecord
+}
+
/**
* Get all of a project's chunk ids
*/
@@ -219,6 +385,62 @@ async function getProjectChunkIds(projectId) {
return chunkIds
}
+/**
+ * Get all of a projects chunks directly
+ */
+async function getProjectChunks(projectId) {
+ const backend = getBackend(projectId)
+ const chunkIds = await backend.getProjectChunks(projectId)
+ return chunkIds
+}
+
+/**
+ * Load the chunk for a given chunk record, including blob metadata.
+ */
+async function loadByChunkRecord(projectId, chunkRecord) {
+ const blobStore = new BlobStore(projectId)
+ const batchBlobStore = new BatchBlobStore(blobStore)
+ const { raw: rawHistory, buffer: chunkBuffer } =
+ await historyStore.loadRawWithBuffer(projectId, chunkRecord.id)
+ const history = History.fromRaw(rawHistory)
+ await lazyLoadHistoryFiles(history, batchBlobStore)
+ return {
+ chunk: new Chunk(history, chunkRecord.endVersion - history.countChanges()),
+ chunkBuffer,
+ }
+}
+
+/**
+ * Asynchronously retrieves project chunks starting from a specific version.
+ *
+ * This generator function yields chunk records for a given project starting from the specified version (inclusive).
+ * It continues to fetch and yield subsequent chunk records until the end version of the latest chunk metadata is reached.
+ * If you want to fetch all the chunks *after* a version V, call this function with V+1.
+ *
+ * @param {string} projectId - The ID of the project.
+ * @param {number} version - The starting version to retrieve chunks from.
+ * @returns {AsyncGenerator} An async generator that yields chunk records.
+ */
+async function* getProjectChunksFromVersion(projectId, version) {
+ const backend = getBackend(projectId)
+ const latestChunkMetadata = await getLatestChunkMetadata(projectId)
+ if (!latestChunkMetadata || version > latestChunkMetadata.endVersion) {
+ return
+ }
+ let chunkRecord = await backend.getChunkForVersion(projectId, version)
+ while (chunkRecord != null) {
+ yield chunkRecord
+ if (chunkRecord.endVersion >= latestChunkMetadata.endVersion) {
+ break
+ } else {
+ chunkRecord = await backend.getChunkForVersion(
+ projectId,
+ chunkRecord.endVersion + 1
+ )
+ }
+ }
+}
+
/**
* Delete the given chunk from the database.
*
@@ -242,10 +464,14 @@ async function deleteProjectChunks(projectId) {
* Delete a given number of old chunks from both the database
* and from object storage.
*
- * @param {number} count - number of chunks to delete
- * @param {number} minAgeSecs - how many seconds ago must chunks have been
- * deleted
- * @return {Promise}
+ * @param {object} options
+ * @param {number} [options.batchSize] - number of chunks to delete in each
+ * batch
+ * @param {number} [options.maxBatches] - maximum number of batches to process
+ * @param {number} [options.minAgeSecs] - minimum age of chunks to delete
+ * @param {number} [options.timeout] - maximum time to spend deleting chunks
+ *
+ * @return {Promise} number of chunks deleted
*/
async function deleteOldChunks(options = {}) {
const batchSize = options.batchSize ?? DEFAULT_DELETE_BATCH_SIZE
@@ -308,6 +534,31 @@ function getBackend(projectId) {
}
}
+/**
+ * Gets non-persisted changes that could extend a chunk
+ *
+ * @param {string} projectId
+ * @param {number} chunkEndVersion - end version of the chunk to extend
+ *
+ * @return {Promise}
+ */
+async function getChunkExtension(projectId, chunkEndVersion) {
+ try {
+ const changes = await redisBackend.getNonPersistedChanges(
+ projectId,
+ chunkEndVersion
+ )
+ return changes
+ } catch (err) {
+ if (err instanceof VersionOutOfBoundsError) {
+ // If we can't extend the chunk, simply return an empty list
+ return []
+ } else {
+ throw err
+ }
+ }
+}
+
class AlreadyInitialized extends OError {
constructor(projectId) {
super('Project is already initialized', { projectId })
@@ -315,15 +566,21 @@ class AlreadyInitialized extends OError {
}
module.exports = {
+ getBackend,
initializeProject,
loadLatest,
+ getLatestChunkMetadata,
loadAtVersion,
loadAtTimestamp,
+ loadByChunkRecord,
create,
update,
destroy,
getChunkIdForVersion,
+ getChunkMetadataForVersion,
getProjectChunkIds,
+ getProjectChunks,
+ getProjectChunksFromVersion,
deleteProjectChunks,
deleteOldChunks,
AlreadyInitialized,
diff --git a/services/history-v1/storage/lib/chunk_store/mongo.js b/services/history-v1/storage/lib/chunk_store/mongo.js
index f56131a25b..49020c6be4 100644
--- a/services/history-v1/storage/lib/chunk_store/mongo.js
+++ b/services/history-v1/storage/lib/chunk_store/mongo.js
@@ -1,6 +1,9 @@
-const { ObjectId } = require('mongodb')
+// @ts-check
+
+const { ObjectId, ReadPreference, MongoError } = require('mongodb')
const { Chunk } = require('overleaf-editor-core')
const OError = require('@overleaf/o-error')
+const config = require('config')
const assert = require('../assert')
const mongodb = require('../mongodb')
const { ChunkVersionConflictError } = require('./errors')
@@ -8,14 +11,30 @@ const { ChunkVersionConflictError } = require('./errors')
const DUPLICATE_KEY_ERROR_CODE = 11000
/**
- * Get the latest chunk's metadata from the database
+ * @import { ClientSession } from 'mongodb'
*/
-async function getLatestChunk(projectId) {
+
+/**
+ * Get the latest chunk's metadata from the database
+ * @param {string} projectId
+ * @param {Object} [opts]
+ * @param {boolean} [opts.readOnly]
+ */
+async function getLatestChunk(projectId, opts = {}) {
assert.mongoId(projectId, 'bad projectId')
+ const { readOnly = false } = opts
const record = await mongodb.chunks.findOne(
- { projectId: new ObjectId(projectId), state: 'active' },
- { sort: { startVersion: -1 } }
+ {
+ projectId: new ObjectId(projectId),
+ state: { $in: ['active', 'closed'] },
+ },
+ {
+ sort: { startVersion: -1 },
+ readPreference: readOnly
+ ? ReadPreference.secondaryPreferred
+ : ReadPreference.primary,
+ }
)
if (record == null) {
return null
@@ -25,19 +44,25 @@ async function getLatestChunk(projectId) {
/**
* Get the metadata for the chunk that contains the given version.
+ *
+ * @param {string} projectId
+ * @param {number} version
+ * @param {object} [opts]
+ * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
+ * two chunks, return the newer chunk.
*/
-async function getChunkForVersion(projectId, version) {
+async function getChunkForVersion(projectId, version, opts = {}) {
assert.mongoId(projectId, 'bad projectId')
assert.integer(version, 'bad version')
const record = await mongodb.chunks.findOne(
{
projectId: new ObjectId(projectId),
- state: 'active',
+ state: { $in: ['active', 'closed'] },
startVersion: { $lte: version },
endVersion: { $gte: version },
},
- { sort: { startVersion: 1 } }
+ { sort: { startVersion: opts.preferNewer ? -1 : 1 } }
)
if (record == null) {
throw new Chunk.VersionNotFoundError(projectId, version)
@@ -45,6 +70,35 @@ async function getChunkForVersion(projectId, version) {
return chunkFromRecord(record)
}
+/**
+ * Get the metadata for the chunk that contains the given version before the endTime.
+ */
+async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
+ assert.mongoId(projectId, 'bad projectId')
+ assert.date(timestamp, 'bad timestamp')
+
+ const recordActive = await getChunkForVersion(projectId, 0)
+ if (recordActive && recordActive.endTimestamp <= timestamp) {
+ return recordActive
+ }
+
+ // fallback to deleted chunk
+ const recordDeleted = await mongodb.chunks.findOne(
+ {
+ projectId: new ObjectId(projectId),
+ state: 'deleted',
+ startVersion: 0,
+ updatedAt: { $lte: timestamp }, // indexed for state=deleted
+ endTimestamp: { $lte: timestamp },
+ },
+ { sort: { updatedAt: -1 } }
+ )
+ if (recordDeleted) {
+ return chunkFromRecord(recordDeleted)
+ }
+ throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+}
+
/**
* Get the metadata for the chunk that contains the version that was current at
* the given timestamp.
@@ -56,7 +110,7 @@ async function getChunkForTimestamp(projectId, timestamp) {
const record = await mongodb.chunks.findOne(
{
projectId: new ObjectId(projectId),
- state: 'active',
+ state: { $in: ['active', 'closed'] },
endTimestamp: { $gte: timestamp },
},
// We use the index on the startVersion for sorting records. This assumes
@@ -77,6 +131,39 @@ async function getChunkForTimestamp(projectId, timestamp) {
return chunkFromRecord(record)
}
+/**
+ * Get the metadata for the chunk that contains the version that was current before
+ * the given timestamp.
+ */
+async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
+ assert.mongoId(projectId, 'bad projectId')
+ assert.date(timestamp, 'bad timestamp')
+
+ const record = await mongodb.chunks.findOne(
+ {
+ projectId: new ObjectId(projectId),
+ state: { $in: ['active', 'closed'] },
+ $or: [
+ {
+ endTimestamp: {
+ $lte: timestamp,
+ },
+ },
+ {
+ endTimestamp: null,
+ },
+ ],
+ },
+ // We use the index on the startVersion for sorting records. This assumes
+ // that timestamps go up with each version.
+ { sort: { startVersion: -1 } }
+ )
+ if (record == null) {
+ throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+ }
+ return chunkFromRecord(record)
+}
+
/**
* Get all of a project's chunk ids
*/
@@ -84,12 +171,33 @@ async function getProjectChunkIds(projectId) {
assert.mongoId(projectId, 'bad projectId')
const cursor = mongodb.chunks.find(
- { projectId: new ObjectId(projectId), state: 'active' },
+ {
+ projectId: new ObjectId(projectId),
+ state: { $in: ['active', 'closed'] },
+ },
{ projection: { _id: 1 } }
)
return await cursor.map(record => record._id).toArray()
}
+/**
+ * Get all of a projects chunks directly
+ */
+async function getProjectChunks(projectId) {
+ assert.mongoId(projectId, 'bad projectId')
+
+ const cursor = mongodb.chunks
+ .find(
+ {
+ projectId: new ObjectId(projectId),
+ state: { $in: ['active', 'closed'] },
+ },
+ { projection: { state: 0 } }
+ )
+ .sort({ startVersion: 1 })
+ return await cursor.map(chunkFromRecord).toArray()
+}
+
/**
* Insert a pending chunk before sending it to object storage.
*/
@@ -112,10 +220,141 @@ async function insertPendingChunk(projectId, chunk) {
/**
* Record that a new chunk was created.
+ *
+ * @param {string} projectId
+ * @param {Chunk} chunk
+ * @param {string} chunkId
+ * @param {object} opts
+ * @param {Date} [opts.earliestChangeTimestamp]
+ * @param {string} [opts.oldChunkId]
*/
-async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
+async function confirmCreate(projectId, chunk, chunkId, opts = {}) {
+ assert.mongoId(projectId, 'bad projectId')
+ assert.instance(chunk, Chunk, 'bad newChunk')
+ assert.mongoId(chunkId, 'bad newChunkId')
+
+ await mongodb.client.withSession(async session => {
+ await session.withTransaction(async () => {
+ if (opts.oldChunkId != null) {
+ await closeChunk(projectId, opts.oldChunkId, { session })
+ }
+
+ await activateChunk(projectId, chunkId, { session })
+
+ await updateProjectRecord(
+ projectId,
+ chunk,
+ opts.earliestChangeTimestamp,
+ { session }
+ )
+ })
+ })
+}
+
+/**
+ * Write the metadata to the project record
+ */
+async function updateProjectRecord(
+ projectId,
+ chunk,
+ earliestChangeTimestamp,
+ mongoOpts = {}
+) {
+ if (!config.has('backupStore')) {
+ return
+ }
+ // record the end version against the project
+ await mongodb.projects.updateOne(
+ {
+ 'overleaf.history.id': projectId, // string for Object ids, number for postgres ids
+ },
+ {
+ // always store the latest end version and timestamp for the chunk
+ $max: {
+ 'overleaf.history.currentEndVersion': chunk.getEndVersion(),
+ 'overleaf.history.currentEndTimestamp': chunk.getEndTimestamp(),
+ 'overleaf.history.updatedAt': new Date(),
+ },
+ // store the first pending change timestamp for the chunk, this will
+ // be cleared every time a backup is completed.
+ $min: {
+ 'overleaf.backup.pendingChangeAt':
+ earliestChangeTimestamp || chunk.getEndTimestamp() || new Date(),
+ },
+ },
+ mongoOpts
+ )
+}
+
+/**
+ * @param {number} historyId
+ * @return {Promise}
+ */
+async function lookupMongoProjectIdFromHistoryId(historyId) {
+ const project = await mongodb.projects.findOne(
+ // string for Object ids, number for postgres ids
+ { 'overleaf.history.id': historyId },
+ { projection: { _id: 1 } }
+ )
+ if (!project) {
+ // should not happen: We flush before allowing a project to be soft-deleted.
+ throw new OError('mongo project not found by history id', { historyId })
+ }
+ return project._id.toString()
+}
+
+async function resolveHistoryIdToMongoProjectId(projectId) {
+ return projectId
+}
+
+/**
+ * Record that a chunk was replaced by a new one.
+ *
+ * @param {string} projectId
+ * @param {string} oldChunkId
+ * @param {Chunk} newChunk
+ * @param {string} newChunkId
+ * @param {object} [opts]
+ * @param {Date} [opts.earliestChangeTimestamp]
+ */
+async function confirmUpdate(
+ projectId,
+ oldChunkId,
+ newChunk,
+ newChunkId,
+ opts = {}
+) {
+ assert.mongoId(projectId, 'bad projectId')
+ assert.mongoId(oldChunkId, 'bad oldChunkId')
+ assert.instance(newChunk, Chunk, 'bad newChunk')
+ assert.mongoId(newChunkId, 'bad newChunkId')
+
+ await mongodb.client.withSession(async session => {
+ await session.withTransaction(async () => {
+ await deleteActiveChunk(projectId, oldChunkId, { session })
+
+ await activateChunk(projectId, newChunkId, { session })
+
+ await updateProjectRecord(
+ projectId,
+ newChunk,
+ opts.earliestChangeTimestamp,
+ { session }
+ )
+ })
+ })
+}
+
+/**
+ * Activate a pending chunk
+ *
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {object} [opts]
+ * @param {ClientSession} [opts.session]
+ */
+async function activateChunk(projectId, chunkId, opts = {}) {
assert.mongoId(projectId, 'bad projectId')
- assert.instance(chunk, Chunk, 'bad chunk')
assert.mongoId(chunkId, 'bad chunkId')
let result
@@ -127,10 +366,10 @@ async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
state: 'pending',
},
{ $set: { state: 'active', updatedAt: new Date() } },
- mongoOpts
+ opts
)
} catch (err) {
- if (err.code === DUPLICATE_KEY_ERROR_CODE) {
+ if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) {
throw new ChunkVersionConflictError('chunk start version is not unique', {
projectId,
chunkId,
@@ -145,30 +384,70 @@ async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
}
/**
- * Record that a chunk was replaced by a new one.
+ * Close a chunk
+ *
+ * A closed chunk is one that can't be extended anymore.
+ *
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {object} [opts]
+ * @param {ClientSession} [opts.session]
*/
-async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
- assert.mongoId(projectId, 'bad projectId')
- assert.mongoId(oldChunkId, 'bad oldChunkId')
- assert.instance(newChunk, Chunk, 'bad newChunk')
- assert.mongoId(newChunkId, 'bad newChunkId')
+async function closeChunk(projectId, chunkId, opts = {}) {
+ const result = await mongodb.chunks.updateOne(
+ {
+ _id: new ObjectId(chunkId),
+ projectId: new ObjectId(projectId),
+ state: 'active',
+ },
+ { $set: { state: 'closed' } },
+ opts
+ )
- const session = mongodb.client.startSession()
- try {
- await session.withTransaction(async () => {
- await deleteChunk(projectId, oldChunkId, { session })
- await confirmCreate(projectId, newChunk, newChunkId, { session })
+ if (result.matchedCount === 0) {
+ throw new ChunkVersionConflictError('unable to close chunk', {
+ projectId,
+ chunkId,
+ })
+ }
+}
+
+/**
+ * Delete an active chunk
+ *
+ * This is used to delete chunks that are in the process of being extended. It
+ * will refuse to delete chunks that are already closed and can therefore not be
+ * extended.
+ *
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {object} [opts]
+ * @param {ClientSession} [opts.session]
+ */
+async function deleteActiveChunk(projectId, chunkId, opts = {}) {
+ const updateResult = await mongodb.chunks.updateOne(
+ {
+ _id: new ObjectId(chunkId),
+ projectId: new ObjectId(projectId),
+ state: 'active',
+ },
+ { $set: { state: 'deleted', updatedAt: new Date() } },
+ opts
+ )
+
+ if (updateResult.matchedCount === 0) {
+ throw new ChunkVersionConflictError('unable to delete active chunk', {
+ projectId,
+ chunkId,
})
- } finally {
- await session.endSession()
}
}
/**
* Delete a chunk.
*
- * @param {number} projectId
- * @param {number} chunkId
+ * @param {string} projectId
+ * @param {string} chunkId
* @return {Promise}
*/
async function deleteChunk(projectId, chunkId, mongoOpts = {}) {
@@ -189,7 +468,10 @@ async function deleteProjectChunks(projectId) {
assert.mongoId(projectId, 'bad projectId')
await mongodb.chunks.updateMany(
- { projectId: new ObjectId(projectId), state: 'active' },
+ {
+ projectId: new ObjectId(projectId),
+ state: { $in: ['active', 'closed'] },
+ },
{ $set: { state: 'deleted', updatedAt: new Date() } }
)
}
@@ -252,19 +534,26 @@ function chunkFromRecord(record) {
id: record._id.toString(),
startVersion: record.startVersion,
endVersion: record.endVersion,
+ endTimestamp: record.endTimestamp,
}
}
module.exports = {
getLatestChunk,
+ getFirstChunkBeforeTimestamp,
+ getLastActiveChunkBeforeTimestamp,
getChunkForVersion,
getChunkForTimestamp,
getProjectChunkIds,
+ getProjectChunks,
insertPendingChunk,
confirmCreate,
confirmUpdate,
+ updateProjectRecord,
deleteChunk,
deleteProjectChunks,
getOldChunksBatch,
deleteOldChunks,
+ lookupMongoProjectIdFromHistoryId,
+ resolveHistoryIdToMongoProjectId,
}
diff --git a/services/history-v1/storage/lib/chunk_store/postgres.js b/services/history-v1/storage/lib/chunk_store/postgres.js
index f6eead7354..8906db38e1 100644
--- a/services/history-v1/storage/lib/chunk_store/postgres.js
+++ b/services/history-v1/storage/lib/chunk_store/postgres.js
@@ -1,19 +1,33 @@
+// @ts-check
+
const { Chunk } = require('overleaf-editor-core')
const assert = require('../assert')
const knex = require('../knex')
+const knexReadOnly = require('../knex_read_only')
const { ChunkVersionConflictError } = require('./errors')
+const {
+ updateProjectRecord,
+ lookupMongoProjectIdFromHistoryId,
+} = require('./mongo')
const DUPLICATE_KEY_ERROR_CODE = '23505'
/**
- * Get the latest chunk's metadata from the database
+ * @import { Knex } from 'knex'
*/
-async function getLatestChunk(projectId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
- const record = await knex('chunks')
- .where('doc_id', projectId)
+/**
+ * Get the latest chunk's metadata from the database
+ * @param {string} projectId
+ * @param {Object} [opts]
+ * @param {boolean} [opts.readOnly]
+ */
+async function getLatestChunk(projectId, opts = {}) {
+ assert.postgresId(projectId, 'bad projectId')
+ const { readOnly = false } = opts
+
+ const record = await (readOnly ? knexReadOnly : knex)('chunks')
+ .where('doc_id', parseInt(projectId, 10))
.orderBy('end_version', 'desc')
.first()
if (record == null) {
@@ -24,15 +38,21 @@ async function getLatestChunk(projectId) {
/**
* Get the metadata for the chunk that contains the given version.
+ *
+ * @param {string} projectId
+ * @param {number} version
+ * @param {object} [opts]
+ * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
+ * two chunks, return the newer chunk.
*/
-async function getChunkForVersion(projectId, version) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+async function getChunkForVersion(projectId, version, opts = {}) {
+ assert.postgresId(projectId, 'bad projectId')
const record = await knex('chunks')
- .where('doc_id', projectId)
+ .where('doc_id', parseInt(projectId, 10))
+ .where('start_version', '<=', version)
.where('end_version', '>=', version)
- .orderBy('end_version')
+ .orderBy('end_version', opts.preferNewer ? 'desc' : 'asc')
.first()
if (!record) {
throw new Chunk.VersionNotFoundError(projectId, version)
@@ -40,13 +60,73 @@ async function getChunkForVersion(projectId, version) {
return chunkFromRecord(record)
}
+/**
+ * Get the metadata for the chunk that contains the given version.
+ *
+ * @param {string} projectId
+ * @param {Date} timestamp
+ */
+async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
+ assert.date(timestamp, 'bad timestamp')
+
+ const recordActive = await getChunkForVersion(projectId, 0)
+
+ // projectId must be valid if getChunkForVersion did not throw
+ if (recordActive && recordActive.endTimestamp <= timestamp) {
+ return recordActive
+ }
+
+ // fallback to deleted chunk
+ const recordDeleted = await knex('old_chunks')
+ .where('doc_id', parseInt(projectId, 10))
+ .where('start_version', '=', 0)
+ .where('end_timestamp', '<=', timestamp)
+ .orderBy('end_version', 'desc')
+ .first()
+ if (recordDeleted) {
+ return chunkFromRecord(recordDeleted)
+ }
+ throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+}
+
/**
* Get the metadata for the chunk that contains the version that was current at
* the given timestamp.
+ *
+ * @param {string} projectId
+ * @param {Date} timestamp
+ */
+async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
+ assert.date(timestamp, 'bad timestamp')
+ assert.postgresId(projectId, 'bad projectId')
+
+ const query = knex('chunks')
+ .where('doc_id', parseInt(projectId, 10))
+ .where(function () {
+ this.where('end_timestamp', '<=', timestamp).orWhere(
+ 'end_timestamp',
+ null
+ )
+ })
+ .orderBy('end_version', 'desc', 'last')
+
+ const record = await query.first()
+
+ if (!record) {
+ throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
+ }
+ return chunkFromRecord(record)
+}
+
+/**
+ * Get the metadata for the chunk that contains the version that was current at
+ * the given timestamp.
+ *
+ * @param {string} projectId
+ * @param {Date} timestamp
*/
async function getChunkForTimestamp(projectId, timestamp) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+ assert.postgresId(projectId, 'bad projectId')
// This query will find the latest chunk after the timestamp (query orders
// in reverse chronological order), OR the latest chunk
@@ -59,11 +139,11 @@ async function getChunkForTimestamp(projectId, timestamp) {
'WHERE doc_id = ? ' +
'ORDER BY end_version desc LIMIT 1' +
')',
- [timestamp, projectId]
+ [timestamp, parseInt(projectId, 10)]
)
const record = await knex('chunks')
- .where('doc_id', projectId)
+ .where('doc_id', parseInt(projectId, 10))
.where(whereAfterEndTimestampOrLatestChunk)
.orderBy('end_version')
.first()
@@ -78,29 +158,50 @@ async function getChunkForTimestamp(projectId, timestamp) {
*/
function chunkFromRecord(record) {
return {
- id: record.id,
+ id: record.id.toString(),
startVersion: record.start_version,
endVersion: record.end_version,
+ endTimestamp: record.end_timestamp,
}
}
/**
* Get all of a project's chunk ids
+ *
+ * @param {string} projectId
*/
async function getProjectChunkIds(projectId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+ assert.postgresId(projectId, 'bad projectId')
- const records = await knex('chunks').select('id').where('doc_id', projectId)
+ const records = await knex('chunks')
+ .select('id')
+ .where('doc_id', parseInt(projectId, 10))
return records.map(record => record.id)
}
+/**
+ * Get all of a projects chunks directly
+ *
+ * @param {string} projectId
+ */
+async function getProjectChunks(projectId) {
+ assert.postgresId(projectId, 'bad projectId')
+
+ const records = await knex('chunks')
+ .select()
+ .where('doc_id', parseInt(projectId, 10))
+ .orderBy('end_version')
+ return records.map(chunkFromRecord)
+}
+
/**
* Insert a pending chunk before sending it to object storage.
+ *
+ * @param {string} projectId
+ * @param {Chunk} chunk
*/
async function insertPendingChunk(projectId, chunk) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+ assert.postgresId(projectId, 'bad projectId')
const result = await knex.first(
knex.raw("nextval('chunks_id_seq'::regclass)::integer as chunkid")
@@ -108,67 +209,119 @@ async function insertPendingChunk(projectId, chunk) {
const chunkId = result.chunkid
await knex('pending_chunks').insert({
id: chunkId,
- doc_id: projectId,
+ doc_id: parseInt(projectId, 10),
end_version: chunk.getEndVersion(),
start_version: chunk.getStartVersion(),
end_timestamp: chunk.getEndTimestamp(),
})
- return chunkId
+ return chunkId.toString()
}
/**
* Record that a new chunk was created.
+ *
+ * @param {string} projectId
+ * @param {Chunk} chunk
+ * @param {string} chunkId
+ * @param {object} opts
+ * @param {Date} [opts.earliestChangeTimestamp]
+ * @param {string} [opts.oldChunkId]
*/
-async function confirmCreate(projectId, chunk, chunkId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+async function confirmCreate(projectId, chunk, chunkId, opts = {}) {
+ assert.postgresId(projectId, 'bad projectId')
await knex.transaction(async tx => {
+ if (opts.oldChunkId != null) {
+ await _assertChunkIsNotClosed(tx, projectId, opts.oldChunkId)
+ await _closeChunk(tx, projectId, opts.oldChunkId)
+ }
await Promise.all([
_deletePendingChunk(tx, projectId, chunkId),
_insertChunk(tx, projectId, chunk, chunkId),
])
+ await updateProjectRecord(
+ // The history id in Mongo is an integer for Postgres projects
+ parseInt(projectId, 10),
+ chunk,
+ opts.earliestChangeTimestamp
+ )
})
}
/**
* Record that a chunk was replaced by a new one.
+ *
+ * @param {string} projectId
+ * @param {string} oldChunkId
+ * @param {Chunk} newChunk
+ * @param {string} newChunkId
*/
-async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+async function confirmUpdate(
+ projectId,
+ oldChunkId,
+ newChunk,
+ newChunkId,
+ opts = {}
+) {
+ assert.postgresId(projectId, 'bad projectId')
await knex.transaction(async tx => {
+ await _assertChunkIsNotClosed(tx, projectId, oldChunkId)
await _deleteChunks(tx, { doc_id: projectId, id: oldChunkId })
await Promise.all([
_deletePendingChunk(tx, projectId, newChunkId),
_insertChunk(tx, projectId, newChunk, newChunkId),
])
+ await updateProjectRecord(
+ // The history id in Mongo is an integer for Postgres projects
+ parseInt(projectId, 10),
+ newChunk,
+ opts.earliestChangeTimestamp
+ )
})
}
+/**
+ * Delete a pending chunk
+ *
+ * @param {Knex} tx
+ * @param {string} projectId
+ * @param {string} chunkId
+ */
async function _deletePendingChunk(tx, projectId, chunkId) {
await tx('pending_chunks')
.where({
- doc_id: projectId,
- id: chunkId,
+ doc_id: parseInt(projectId, 10),
+ id: parseInt(chunkId, 10),
})
.del()
}
+/**
+ * Adds an active chunk
+ *
+ * @param {Knex} tx
+ * @param {string} projectId
+ * @param {Chunk} chunk
+ * @param {string} chunkId
+ */
async function _insertChunk(tx, projectId, chunk, chunkId) {
const startVersion = chunk.getStartVersion()
const endVersion = chunk.getEndVersion()
try {
await tx('chunks').insert({
- id: chunkId,
- doc_id: projectId,
+ id: parseInt(chunkId, 10),
+ doc_id: parseInt(projectId, 10),
start_version: startVersion,
end_version: endVersion,
end_timestamp: chunk.getEndTimestamp(),
})
} catch (err) {
- if (err.code === DUPLICATE_KEY_ERROR_CODE) {
+ if (
+ err instanceof Error &&
+ 'code' in err &&
+ err.code === DUPLICATE_KEY_ERROR_CODE
+ ) {
throw new ChunkVersionConflictError(
'chunk start or end version is not unique',
{ projectId, chunkId, startVersion, endVersion }
@@ -178,35 +331,92 @@ async function _insertChunk(tx, projectId, chunk, chunkId) {
}
}
+/**
+ * Check that a chunk is not closed
+ *
+ * This is used to synchronize chunk creations and extensions.
+ *
+ * @param {Knex} tx
+ * @param {string} projectId
+ * @param {string} chunkId
+ */
+async function _assertChunkIsNotClosed(tx, projectId, chunkId) {
+ const record = await tx('chunks')
+ .forUpdate()
+ .select('closed')
+ .where('doc_id', parseInt(projectId, 10))
+ .where('id', parseInt(chunkId, 10))
+ .first()
+ if (!record) {
+ throw new ChunkVersionConflictError('unable to close chunk: not found', {
+ projectId,
+ chunkId,
+ })
+ }
+ if (record.closed) {
+ throw new ChunkVersionConflictError(
+ 'unable to close chunk: already closed',
+ {
+ projectId,
+ chunkId,
+ }
+ )
+ }
+}
+
+/**
+ * Close a chunk
+ *
+ * A closed chunk can no longer be extended.
+ *
+ * @param {Knex} tx
+ * @param {string} projectId
+ * @param {string} chunkId
+ */
+async function _closeChunk(tx, projectId, chunkId) {
+ await tx('chunks')
+ .update({ closed: true })
+ .where('doc_id', parseInt(projectId, 10))
+ .where('id', parseInt(chunkId, 10))
+}
+
/**
* Delete a chunk.
*
- * @param {number} projectId
- * @param {number} chunkId
- * @return {Promise}
+ * @param {string} projectId
+ * @param {string} chunkId
*/
async function deleteChunk(projectId, chunkId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+ assert.postgresId(projectId, 'bad projectId')
assert.integer(chunkId, 'bad chunkId')
- await _deleteChunks(knex, { doc_id: projectId, id: chunkId })
+ await _deleteChunks(knex, {
+ doc_id: parseInt(projectId, 10),
+ id: parseInt(chunkId, 10),
+ })
}
/**
* Delete all of a project's chunks
+ *
+ * @param {string} projectId
*/
async function deleteProjectChunks(projectId) {
- projectId = parseInt(projectId, 10)
- assert.integer(projectId, 'bad projectId')
+ assert.postgresId(projectId, 'bad projectId')
await knex.transaction(async tx => {
- await _deleteChunks(knex, { doc_id: projectId })
+ await _deleteChunks(knex, { doc_id: parseInt(projectId, 10) })
})
}
+/**
+ * Delete many chunks
+ *
+ * @param {Knex} tx
+ * @param {any} whereClause
+ */
async function _deleteChunks(tx, whereClause) {
- const rows = await tx('chunks').returning('*').where(whereClause).del()
+ const rows = await tx('chunks').where(whereClause).del().returning('*')
if (rows.length === 0) {
return
}
@@ -224,6 +434,9 @@ async function _deleteChunks(tx, whereClause) {
/**
* Get a batch of old chunks for deletion
+ *
+ * @param {number} count
+ * @param {number} minAgeSecs
*/
async function getOldChunksBatch(count, minAgeSecs) {
const maxDeletedAt = new Date(Date.now() - minAgeSecs * 1000)
@@ -234,15 +447,22 @@ async function getOldChunksBatch(count, minAgeSecs) {
.limit(count)
return records.map(oldChunk => ({
projectId: oldChunk.doc_id.toString(),
- chunkId: oldChunk.chunk_id,
+ chunkId: oldChunk.chunk_id.toString(),
}))
}
/**
* Delete a batch of old chunks from the database
+ *
+ * @param {string[]} chunkIds
*/
async function deleteOldChunks(chunkIds) {
- await knex('old_chunks').whereIn('chunk_id', chunkIds).del()
+ await knex('old_chunks')
+ .whereIn(
+ 'chunk_id',
+ chunkIds.map(id => parseInt(id, 10))
+ )
+ .del()
}
/**
@@ -255,11 +475,18 @@ async function generateProjectId() {
return record.doc_id.toString()
}
+async function resolveHistoryIdToMongoProjectId(projectId) {
+ return await lookupMongoProjectIdFromHistoryId(parseInt(projectId, 10))
+}
+
module.exports = {
getLatestChunk,
+ getFirstChunkBeforeTimestamp,
+ getLastActiveChunkBeforeTimestamp,
getChunkForVersion,
getChunkForTimestamp,
getProjectChunkIds,
+ getProjectChunks,
insertPendingChunk,
confirmCreate,
confirmUpdate,
@@ -268,4 +495,5 @@ module.exports = {
getOldChunksBatch,
deleteOldChunks,
generateProjectId,
+ resolveHistoryIdToMongoProjectId,
}
diff --git a/services/history-v1/storage/lib/chunk_store/redis.js b/services/history-v1/storage/lib/chunk_store/redis.js
new file mode 100644
index 0000000000..b8a79b498d
--- /dev/null
+++ b/services/history-v1/storage/lib/chunk_store/redis.js
@@ -0,0 +1,854 @@
+// @ts-check
+
+const metrics = require('@overleaf/metrics')
+const OError = require('@overleaf/o-error')
+const { Change, Snapshot } = require('overleaf-editor-core')
+const redis = require('../redis')
+const rclient = redis.rclientHistory
+const {
+ BaseVersionConflictError,
+ JobNotFoundError,
+ JobNotReadyError,
+ VersionOutOfBoundsError,
+} = require('./errors')
+
+const MAX_PERSISTED_CHANGES = 100 // Maximum number of persisted changes to keep in the buffer for clients that need to catch up.
+const PROJECT_TTL_MS = 3600 * 1000 // Amount of time a project can stay inactive before it gets expired
+const MAX_PERSIST_DELAY_MS = 300 * 1000 // Maximum amount of time before a change is persisted
+const RETRY_DELAY_MS = 120 * 1000 // Time before a claimed job is considered stale and a worker can retry it.
+
+const keySchema = {
+ head({ projectId }) {
+ return `head:{${projectId}}`
+ },
+ headVersion({ projectId }) {
+ return `head-version:{${projectId}}`
+ },
+ persistedVersion({ projectId }) {
+ return `persisted-version:{${projectId}}`
+ },
+ expireTime({ projectId }) {
+ return `expire-time:{${projectId}}`
+ },
+ persistTime({ projectId }) {
+ return `persist-time:{${projectId}}`
+ },
+ changes({ projectId }) {
+ return `changes:{${projectId}}`
+ },
+}
+
+rclient.defineCommand('get_head_snapshot', {
+ numberOfKeys: 2,
+ lua: `
+ local headSnapshotKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+
+ -- Check if the head version exists. If not, consider it a cache miss.
+ local version = redis.call('GET', headVersionKey)
+ if not version then
+ return nil
+ end
+
+ -- Retrieve the snapshot value
+ local snapshot = redis.call('GET', headSnapshotKey)
+ return {snapshot, version}
+ `,
+})
+
+/**
+ * Retrieves the head snapshot from Redis storage
+ * @param {string} projectId - The unique identifier of the project
+ * @returns {Promise<{version: number, snapshot: Snapshot}|null>} A Promise that resolves to an object containing the version and Snapshot,
+ * or null if retrieval fails or cache miss
+ * @throws {Error} If Redis operations fail
+ */
+async function getHeadSnapshot(projectId) {
+ try {
+ const result = await rclient.get_head_snapshot(
+ keySchema.head({ projectId }),
+ keySchema.headVersion({ projectId })
+ )
+ if (!result) {
+ metrics.inc('chunk_store.redis.get_head_snapshot', 1, {
+ status: 'cache-miss',
+ })
+ return null // cache-miss
+ }
+ const snapshot = Snapshot.fromRaw(JSON.parse(result[0]))
+ const version = parseInt(result[1], 10)
+ metrics.inc('chunk_store.redis.get_head_snapshot', 1, {
+ status: 'success',
+ })
+ return { version, snapshot }
+ } catch (err) {
+ metrics.inc('chunk_store.redis.get_head_snapshot', 1, { status: 'error' })
+ throw err
+ }
+}
+
+rclient.defineCommand('queue_changes', {
+ numberOfKeys: 5,
+ lua: `
+ local headSnapshotKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+ local changesKey = KEYS[3]
+ local expireTimeKey = KEYS[4]
+ local persistTimeKey = KEYS[5]
+
+ local baseVersion = tonumber(ARGV[1])
+ local head = ARGV[2]
+ local persistTime = tonumber(ARGV[3])
+ local expireTime = tonumber(ARGV[4])
+ local onlyIfExists = ARGV[5]
+ local changesIndex = 6 -- Changes start here
+
+ local headVersion = tonumber(redis.call('GET', headVersionKey))
+
+ -- Check if updates should only be queued if the project already exists (used for gradual rollouts)
+ if not headVersion and onlyIfExists == 'true' then
+ return 'ignore'
+ end
+
+ -- Check that the supplied baseVersion matches the head version
+ -- If headVersion is nil, it means the project does not exist yet and will be created.
+ if headVersion and headVersion ~= baseVersion then
+ return 'conflict'
+ end
+
+ -- Check if there are any changes to queue
+ if #ARGV < changesIndex then
+ return 'no_changes_provided'
+ end
+
+ -- Store the changes
+ -- RPUSH changesKey change1 change2 ...
+ redis.call('RPUSH', changesKey, unpack(ARGV, changesIndex, #ARGV))
+
+ -- Update head snapshot only if changes were successfully pushed
+ redis.call('SET', headSnapshotKey, head)
+
+ -- Update the head version
+ local numChanges = #ARGV - changesIndex + 1
+ local newHeadVersion = baseVersion + numChanges
+ redis.call('SET', headVersionKey, newHeadVersion)
+
+ -- Update the persist time if the new time is sooner
+ local currentPersistTime = tonumber(redis.call('GET', persistTimeKey))
+ if not currentPersistTime or persistTime < currentPersistTime then
+ redis.call('SET', persistTimeKey, persistTime)
+ end
+
+ -- Update the expire time
+ redis.call('SET', expireTimeKey, expireTime)
+
+ return 'ok'
+ `,
+})
+
+/**
+ * Atomically queues changes to the project history in Redis if the baseVersion matches.
+ * Updates head snapshot, version, persist time, and expire time.
+ *
+ * @param {string} projectId - The project identifier.
+ * @param {Snapshot} headSnapshot - The new head snapshot after applying changes.
+ * @param {number} baseVersion - The expected current head version.
+ * @param {Change[]} changes - An array of Change objects to queue.
+ * @param {object} [opts]
+ * @param {number} [opts.persistTime] - Timestamp (ms since epoch) when the
+ * oldest change in the buffer should be persisted.
+ * @param {number} [opts.expireTime] - Timestamp (ms since epoch) when the
+ * project buffer should expire if inactive.
+ * @param {boolean} [opts.onlyIfExists] - If true, only queue changes if the
+ * project already exists in Redis, otherwise ignore.
+ * @returns {Promise} Resolves on success to either 'ok' or 'ignore'.
+ * @throws {BaseVersionConflictError} If the baseVersion does not match the current head version in Redis.
+ * @throws {Error} If changes array is empty or if Redis operations fail.
+ */
+async function queueChanges(
+ projectId,
+ headSnapshot,
+ baseVersion,
+ changes,
+ opts = {}
+) {
+ if (!changes || changes.length === 0) {
+ throw new Error('Cannot queue empty changes array')
+ }
+
+ const persistTime = opts.persistTime ?? Date.now() + MAX_PERSIST_DELAY_MS
+ const expireTime = opts.expireTime ?? Date.now() + PROJECT_TTL_MS
+ const onlyIfExists = Boolean(opts.onlyIfExists)
+
+ try {
+ const keys = [
+ keySchema.head({ projectId }),
+ keySchema.headVersion({ projectId }),
+ keySchema.changes({ projectId }),
+ keySchema.expireTime({ projectId }),
+ keySchema.persistTime({ projectId }),
+ ]
+
+ const args = [
+ baseVersion.toString(),
+ JSON.stringify(headSnapshot.toRaw()),
+ persistTime.toString(),
+ expireTime.toString(),
+ onlyIfExists.toString(), // Only queue changes if the snapshot already exists
+ ...changes.map(change => JSON.stringify(change.toRaw())), // Serialize changes
+ ]
+
+ const status = await rclient.queue_changes(keys, args)
+ metrics.inc('chunk_store.redis.queue_changes', 1, { status })
+ if (status === 'ok') {
+ return status
+ }
+ if (status === 'ignore') {
+ return status // skip changes when project does not exist and onlyIfExists is true
+ }
+ if (status === 'conflict') {
+ throw new BaseVersionConflictError('base version mismatch', {
+ projectId,
+ baseVersion,
+ })
+ } else {
+ throw new Error(`unexpected result queuing changes: ${status}`)
+ }
+ } catch (err) {
+ if (err instanceof BaseVersionConflictError) {
+ // Re-throw conflict errors directly
+ throw err
+ }
+ metrics.inc('chunk_store.redis.queue_changes', 1, { status: 'error' })
+ throw err
+ }
+}
+
+rclient.defineCommand('get_state', {
+ numberOfKeys: 6, // Number of keys defined in keySchema
+ lua: `
+ local headSnapshotKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+ local persistedVersionKey = KEYS[3]
+ local expireTimeKey = KEYS[4]
+ local persistTimeKey = KEYS[5]
+ local changesKey = KEYS[6]
+
+ local headSnapshot = redis.call('GET', headSnapshotKey)
+ local headVersion = redis.call('GET', headVersionKey)
+ local persistedVersion = redis.call('GET', persistedVersionKey)
+ local expireTime = redis.call('GET', expireTimeKey)
+ local persistTime = redis.call('GET', persistTimeKey)
+ local changes = redis.call('LRANGE', changesKey, 0, -1) -- Get all changes in the list
+
+ return {headSnapshot, headVersion, persistedVersion, expireTime, persistTime, changes}
+ `,
+})
+
+/**
+ * Retrieves the entire state associated with a project from Redis atomically.
+ * @param {string} projectId - The unique identifier of the project.
+ * @returns {Promise} A Promise that resolves to an object containing the project state,
+ * or null if the project state does not exist (e.g., head version is missing).
+ * @throws {Error} If Redis operations fail.
+ */
+async function getState(projectId) {
+ const keys = [
+ keySchema.head({ projectId }),
+ keySchema.headVersion({ projectId }),
+ keySchema.persistedVersion({ projectId }),
+ keySchema.expireTime({ projectId }),
+ keySchema.persistTime({ projectId }),
+ keySchema.changes({ projectId }),
+ ]
+
+ // Pass keys individually, not as an array
+ const result = await rclient.get_state(...keys)
+
+ const [
+ rawHeadSnapshot,
+ rawHeadVersion,
+ rawPersistedVersion,
+ rawExpireTime,
+ rawPersistTime,
+ rawChanges,
+ ] = result
+
+ // Safely parse values, providing defaults or nulls if necessary
+ const headSnapshot = rawHeadSnapshot
+ ? JSON.parse(rawHeadSnapshot)
+ : rawHeadSnapshot
+ const headVersion = rawHeadVersion ? parseInt(rawHeadVersion, 10) : null // Should always exist if result is not null
+ const persistedVersion = rawPersistedVersion
+ ? parseInt(rawPersistedVersion, 10)
+ : null
+ const expireTime = rawExpireTime ? parseInt(rawExpireTime, 10) : null
+ const persistTime = rawPersistTime ? parseInt(rawPersistTime, 10) : null
+ const changes = rawChanges ? rawChanges.map(JSON.parse) : null
+
+ return {
+ headSnapshot,
+ headVersion,
+ persistedVersion,
+ expireTime,
+ persistTime,
+ changes,
+ }
+}
+
+rclient.defineCommand('get_changes_since_version', {
+ numberOfKeys: 2,
+ lua: `
+ local headVersionKey = KEYS[1]
+ local changesKey = KEYS[2]
+
+ local requestedVersion = tonumber(ARGV[1])
+
+ -- Check if head version exists
+ local headVersion = tonumber(redis.call('GET', headVersionKey))
+ if not headVersion then
+ return {'not_found'}
+ end
+
+ -- If requested version equals head version, return empty array
+ if requestedVersion == headVersion then
+ return {'ok', {}}
+ end
+
+ -- If requested version is greater than head version, return error
+ if requestedVersion > headVersion then
+ return {'out_of_bounds'}
+ end
+
+ -- Get length of changes list
+ local changesCount = redis.call('LLEN', changesKey)
+
+ -- Check if requested version is too old (changes already removed from buffer)
+ if requestedVersion < (headVersion - changesCount) then
+ return {'out_of_bounds'}
+ end
+
+ -- Calculate the starting index, using negative indexing to count backwards
+ -- from the end of the list
+ local startIndex = requestedVersion - headVersion
+
+ -- Get changes using LRANGE
+ local changes = redis.call('LRANGE', changesKey, startIndex, -1)
+
+ return {'ok', changes}
+ `,
+})
+
+/**
+ * Retrieves changes since a specific version for a project from Redis.
+ *
+ * @param {string} projectId - The unique identifier of the project.
+ * @param {number} version - The version number to retrieve changes since.
+ * @returns {Promise<{status: string, changes?: Array}>} A Promise that resolves to an object containing:
+ * - status: 'OK', 'NOT_FOUND', or 'OUT_OF_BOUNDS'
+ * - changes: Array of Change objects (only when status is 'OK')
+ * @throws {Error} If Redis operations fail.
+ */
+async function getChangesSinceVersion(projectId, version) {
+ try {
+ const keys = [
+ keySchema.headVersion({ projectId }),
+ keySchema.changes({ projectId }),
+ ]
+
+ const args = [version.toString()]
+
+ const result = await rclient.get_changes_since_version(keys, args)
+ const status = result[0]
+
+ if (status === 'ok') {
+ // If status is OK, parse the changes
+ const changes = result[1]
+ ? result[1].map(rawChange =>
+ typeof rawChange === 'string' ? JSON.parse(rawChange) : rawChange
+ )
+ : []
+
+ metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
+ status: 'success',
+ })
+ return { status, changes }
+ } else {
+ // For other statuses, just return the status
+ metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
+ status,
+ })
+ return { status }
+ }
+ } catch (err) {
+ metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
+ status: 'error',
+ })
+ throw err
+ }
+}
+
+rclient.defineCommand('get_non_persisted_changes', {
+ numberOfKeys: 3,
+ lua: `
+ local headVersionKey = KEYS[1]
+ local persistedVersionKey = KEYS[2]
+ local changesKey = KEYS[3]
+ local baseVersion = tonumber(ARGV[1])
+ local maxChanges = tonumber(ARGV[2])
+
+ -- Check if head version exists
+ local headVersion = tonumber(redis.call('GET', headVersionKey))
+ if not headVersion then
+ return {'not_found'}
+ end
+
+ -- Check if persisted version exists
+ local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
+ if not persistedVersion then
+ local changesCount = tonumber(redis.call('LLEN', changesKey))
+ persistedVersion = headVersion - changesCount
+ end
+
+ if baseVersion < persistedVersion or baseVersion > headVersion then
+ return {'out_of_bounds'}
+ elseif baseVersion == headVersion then
+ return {'ok', {}}
+ else
+ local numChanges = headVersion - baseVersion
+
+ local endIndex, expectedChanges
+ if maxChanges > 0 and maxChanges < numChanges then
+ -- return only the first maxChanges changes; the end index is inclusive
+ endIndex = -numChanges + maxChanges - 1
+ expectedChanges = maxChanges
+ else
+ endIndex = -1
+ expectedChanges = numChanges
+ end
+
+ local changes = redis.call('LRANGE', changesKey, -numChanges, endIndex)
+
+ if #changes < expectedChanges then
+ -- We didn't get as many changes as we expected
+ return {'out_of_bounds'}
+ end
+
+ return {'ok', changes}
+ end
+ `,
+})
+
+/**
+ * Retrieves non-persisted changes for a project from Redis.
+ *
+ * @param {string} projectId - The unique identifier of the project.
+ * @param {number} baseVersion - The version on top of which the changes should
+ * be applied.
+ * @param {object} [opts]
+ * @param {number} [opts.maxChanges] - The maximum number of changes to return.
+ * Defaults to 0, meaning no limit.
+ * @returns {Promise} Changes that can be applied on top of
+ * baseVersion. An empty array means that the project doesn't have
+ * changes to persist. A null value means that the non-persisted
+ * changes can't be applied to the given base version.
+ *
+ * @throws {Error} If Redis operations fail.
+ */
+async function getNonPersistedChanges(projectId, baseVersion, opts = {}) {
+ let result
+ try {
+ result = await rclient.get_non_persisted_changes(
+ keySchema.headVersion({ projectId }),
+ keySchema.persistedVersion({ projectId }),
+ keySchema.changes({ projectId }),
+ baseVersion.toString(),
+ opts.maxChanges ?? 0
+ )
+ } catch (err) {
+ metrics.inc('chunk_store.redis.get_non_persisted_changes', 1, {
+ status: 'error',
+ })
+ throw err
+ }
+
+ const status = result[0]
+ metrics.inc('chunk_store.redis.get_non_persisted_changes', 1, {
+ status,
+ })
+
+ if (status === 'ok') {
+ return result[1].map(json => Change.fromRaw(JSON.parse(json)))
+ } else if (status === 'not_found') {
+ return []
+ } else if (status === 'out_of_bounds') {
+ throw new VersionOutOfBoundsError(
+ "Non-persisted changes can't be applied to base version",
+ { projectId, baseVersion }
+ )
+ } else {
+ throw new OError('unknown status for get_non_persisted_changes', {
+ projectId,
+ baseVersion,
+ status,
+ })
+ }
+}
+
+rclient.defineCommand('set_persisted_version', {
+ numberOfKeys: 4,
+ lua: `
+ local headVersionKey = KEYS[1]
+ local persistedVersionKey = KEYS[2]
+ local persistTimeKey = KEYS[3]
+ local changesKey = KEYS[4]
+
+ local newPersistedVersion = tonumber(ARGV[1])
+ local maxPersistedChanges = tonumber(ARGV[2])
+
+ -- Check if head version exists
+ local headVersion = tonumber(redis.call('GET', headVersionKey))
+ if not headVersion then
+ return 'not_found'
+ end
+
+ -- Get current persisted version
+ local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
+ if persistedVersion and persistedVersion > newPersistedVersion then
+ return 'too_low'
+ end
+
+ -- Refuse to set a persisted version that is higher than the head version
+ if newPersistedVersion > headVersion then
+ return 'too_high'
+ end
+
+ -- Set the persisted version
+ redis.call('SET', persistedVersionKey, newPersistedVersion)
+
+ -- Clear the persist time if the persisted version now matches the head version
+ if newPersistedVersion == headVersion then
+ redis.call('DEL', persistTimeKey)
+ end
+
+ -- Calculate the starting index, to keep only maxPersistedChanges beyond the persisted version
+ -- Using negative indexing to count backwards from the end of the list
+ local startIndex = newPersistedVersion - headVersion - maxPersistedChanges
+
+ -- Trim the changes list to keep only the specified number of changes beyond persisted version
+ if startIndex < 0 then
+ redis.call('LTRIM', changesKey, startIndex, -1)
+ end
+
+ return 'ok'
+ `,
+})
+
+/**
+ * Sets the persisted version for a project in Redis and trims the changes list.
+ *
+ * @param {string} projectId - The unique identifier of the project.
+ * @param {number} persistedVersion - The version number to set as persisted.
+ * @returns {Promise} A Promise that resolves to 'OK' or 'NOT_FOUND'.
+ * @throws {Error} If Redis operations fail.
+ */
+async function setPersistedVersion(projectId, persistedVersion) {
+ try {
+ const keys = [
+ keySchema.headVersion({ projectId }),
+ keySchema.persistedVersion({ projectId }),
+ keySchema.persistTime({ projectId }),
+ keySchema.changes({ projectId }),
+ ]
+
+ const args = [persistedVersion.toString(), MAX_PERSISTED_CHANGES.toString()]
+
+ const status = await rclient.set_persisted_version(keys, args)
+
+ metrics.inc('chunk_store.redis.set_persisted_version', 1, {
+ status,
+ })
+
+ if (status === 'too_high') {
+ throw new VersionOutOfBoundsError(
+ 'Persisted version cannot be higher than head version',
+ { projectId, persistedVersion }
+ )
+ }
+
+ return status
+ } catch (err) {
+ metrics.inc('chunk_store.redis.set_persisted_version', 1, {
+ status: 'error',
+ })
+ throw err
+ }
+}
+
+rclient.defineCommand('hard_delete_project', {
+ numberOfKeys: 6,
+ lua: `
+ local headKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+ local persistedVersionKey = KEYS[3]
+ local expireTimeKey = KEYS[4]
+ local persistTimeKey = KEYS[5]
+ local changesKey = KEYS[6]
+ -- Delete all keys associated with the project
+ redis.call('DEL',
+ headKey,
+ headVersionKey,
+ persistedVersionKey,
+ expireTimeKey,
+ persistTimeKey,
+ changesKey
+ )
+ return 'ok'
+ `,
+})
+
+/** Hard delete a project from Redis by removing all keys associated with it.
+ * This is only to be used when a project is **permanently** deleted.
+ * DO NOT USE THIS FOR ANY OTHER PURPOSES AS IT WILL REMOVE NON-PERSISTED CHANGES.
+ * @param {string} projectId - The unique identifier of the project to delete.
+ * @returns {Promise} A Promise that resolves to 'ok' on success.
+ * @throws {Error} If Redis operations fail.
+ */
+async function hardDeleteProject(projectId) {
+ try {
+ const status = await rclient.hard_delete_project(
+ keySchema.head({ projectId }),
+ keySchema.headVersion({ projectId }),
+ keySchema.persistedVersion({ projectId }),
+ keySchema.expireTime({ projectId }),
+ keySchema.persistTime({ projectId }),
+ keySchema.changes({ projectId })
+ )
+ metrics.inc('chunk_store.redis.hard_delete_project', 1, { status })
+ return status
+ } catch (err) {
+ metrics.inc('chunk_store.redis.hard_delete_project', 1, { status: 'error' })
+ throw err
+ }
+}
+
+rclient.defineCommand('set_expire_time', {
+ numberOfKeys: 2,
+ lua: `
+ local expireTimeKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+ local expireTime = tonumber(ARGV[1])
+
+ -- Only set the expire time if the project is loaded in Redis
+ local headVersion = redis.call('GET', headVersionKey)
+ if headVersion then
+ redis.call('SET', expireTimeKey, expireTime)
+ end
+ `,
+})
+
+/**
+ * Sets the expire version for a project in Redis
+ *
+ * @param {string} projectId
+ * @param {number} expireTime - Timestamp (ms since epoch) when the project
+ * buffer should expire if inactive
+ */
+async function setExpireTime(projectId, expireTime) {
+ try {
+ await rclient.set_expire_time(
+ keySchema.expireTime({ projectId }),
+ keySchema.headVersion({ projectId }),
+ expireTime.toString()
+ )
+ metrics.inc('chunk_store.redis.set_expire_time', 1, { status: 'success' })
+ } catch (err) {
+ metrics.inc('chunk_store.redis.set_expire_time', 1, { status: 'error' })
+ throw err
+ }
+}
+
+rclient.defineCommand('expire_project', {
+ numberOfKeys: 6,
+ lua: `
+ local headKey = KEYS[1]
+ local headVersionKey = KEYS[2]
+ local changesKey = KEYS[3]
+ local persistedVersionKey = KEYS[4]
+ local persistTimeKey = KEYS[5]
+ local expireTimeKey = KEYS[6]
+
+ local headVersion = tonumber(redis.call('GET', headVersionKey))
+ if not headVersion then
+ return 'not-found'
+ end
+
+ local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
+ if not persistedVersion or persistedVersion ~= headVersion then
+ return 'not-persisted'
+ end
+
+ redis.call('DEL',
+ headKey,
+ headVersionKey,
+ changesKey,
+ persistedVersionKey,
+ persistTimeKey,
+ expireTimeKey
+ )
+ return 'success'
+ `,
+})
+
+async function expireProject(projectId) {
+ try {
+ const status = await rclient.expire_project(
+ keySchema.head({ projectId }),
+ keySchema.headVersion({ projectId }),
+ keySchema.changes({ projectId }),
+ keySchema.persistedVersion({ projectId }),
+ keySchema.persistTime({ projectId }),
+ keySchema.expireTime({ projectId })
+ )
+ metrics.inc('chunk_store.redis.expire_project', 1, {
+ status,
+ })
+ return status
+ } catch (err) {
+ metrics.inc('chunk_store.redis.expire_project', 1, {
+ status: 'error',
+ })
+ throw err
+ }
+}
+
+rclient.defineCommand('claim_job', {
+ numberOfKeys: 1,
+ lua: `
+ local jobTimeKey = KEYS[1]
+ local currentTime = tonumber(ARGV[1])
+ local retryDelay = tonumber(ARGV[2])
+
+ local jobTime = tonumber(redis.call('GET', jobTimeKey))
+ if not jobTime then
+ return {'no-job'}
+ end
+
+ local msUntilReady = jobTime - currentTime
+ if msUntilReady <= 0 then
+ local retryTime = currentTime + retryDelay
+ redis.call('SET', jobTimeKey, retryTime)
+ return {'ok', retryTime}
+ else
+ return {'wait', msUntilReady}
+ end
+ `,
+})
+
+rclient.defineCommand('close_job', {
+ numberOfKeys: 1,
+ lua: `
+ local jobTimeKey = KEYS[1]
+ local expectedJobTime = tonumber(ARGV[1])
+
+ local jobTime = tonumber(redis.call('GET', jobTimeKey))
+ if jobTime and jobTime == expectedJobTime then
+ redis.call('DEL', jobTimeKey)
+ end
+ `,
+})
+
+/**
+ * Claim an expire job
+ *
+ * @param {string} projectId
+ * @return {Promise}
+ */
+async function claimExpireJob(projectId) {
+ return await claimJob(keySchema.expireTime({ projectId }))
+}
+
+/**
+ * Claim a persist job
+ *
+ * @param {string} projectId
+ * @return {Promise}
+ */
+async function claimPersistJob(projectId) {
+ return await claimJob(keySchema.persistTime({ projectId }))
+}
+
+/**
+ * Claim a persist or expire job
+ *
+ * @param {string} jobKey - the Redis key containing the time at which the job
+ * is ready
+ * @return {Promise}
+ */
+async function claimJob(jobKey) {
+ let result, status
+ try {
+ result = await rclient.claim_job(jobKey, Date.now(), RETRY_DELAY_MS)
+ status = result[0]
+ metrics.inc('chunk_store.redis.claim_job', 1, { status })
+ } catch (err) {
+ metrics.inc('chunk_store.redis.claim_job', 1, { status: 'error' })
+ throw err
+ }
+
+ if (status === 'ok') {
+ return new Job(jobKey, parseInt(result[1], 10))
+ } else if (status === 'wait') {
+ throw new JobNotReadyError('job not ready', {
+ jobKey,
+ retryTime: result[1],
+ })
+ } else if (status === 'no-job') {
+ throw new JobNotFoundError('job not found', { jobKey })
+ } else {
+ throw new OError('unknown status for claim_job', { jobKey, status })
+ }
+}
+
+/**
+ * Handle for a claimed job
+ */
+class Job {
+ /**
+ * @param {string} redisKey
+ * @param {number} claimTimestamp
+ */
+ constructor(redisKey, claimTimestamp) {
+ this.redisKey = redisKey
+ this.claimTimestamp = claimTimestamp
+ }
+
+ async close() {
+ try {
+ await rclient.close_job(this.redisKey, this.claimTimestamp.toString())
+ metrics.inc('chunk_store.redis.close_job', 1, { status: 'success' })
+ } catch (err) {
+ metrics.inc('chunk_store.redis.close_job', 1, { status: 'error' })
+ throw err
+ }
+ }
+}
+
+module.exports = {
+ getHeadSnapshot,
+ queueChanges,
+ getState,
+ getChangesSinceVersion,
+ getNonPersistedChanges,
+ setPersistedVersion,
+ hardDeleteProject,
+ setExpireTime,
+ expireProject,
+ claimExpireJob,
+ claimPersistJob,
+ MAX_PERSISTED_CHANGES,
+ MAX_PERSIST_DELAY_MS,
+ PROJECT_TTL_MS,
+ RETRY_DELAY_MS,
+ keySchema,
+}
diff --git a/services/history-v1/storage/lib/commit_changes.js b/services/history-v1/storage/lib/commit_changes.js
new file mode 100644
index 0000000000..5749e5fc0e
--- /dev/null
+++ b/services/history-v1/storage/lib/commit_changes.js
@@ -0,0 +1,159 @@
+// @ts-check
+
+'use strict'
+
+const metrics = require('@overleaf/metrics')
+const redisBackend = require('./chunk_store/redis')
+const logger = require('@overleaf/logger')
+const queueChanges = require('./queue_changes')
+const persistChanges = require('./persist_changes')
+const persistBuffer = require('./persist_buffer')
+
+/**
+ * @typedef {import('overleaf-editor-core').Change} Change
+ */
+
+/**
+ * Handle incoming changes by processing them according to the specified options.
+ * @param {string} projectId
+ * @param {Change[]} changes
+ * @param {Object} limits
+ * @param {number} endVersion
+ * @param {Object} options
+ * @param {number} [options.historyBufferLevel] - The history buffer level to use for processing changes.
+ * @param {Boolean} [options.forcePersistBuffer] - If true, forces the buffer to be persisted before any operation.
+ * @return {Promise.}
+ */
+
+async function commitChanges(
+ projectId,
+ changes,
+ limits,
+ endVersion,
+ options = {}
+) {
+ const { historyBufferLevel, forcePersistBuffer } = options
+
+ // Force the buffer to be persisted if specified.
+ if (forcePersistBuffer) {
+ try {
+ const status = await redisBackend.expireProject(projectId) // clear the project from Redis if it is persisted, returns 'not-persisted' if it was not persisted
+ if (status === 'not-persisted') {
+ await persistBuffer(projectId, limits)
+ await redisBackend.expireProject(projectId) // clear the project from Redis after persisting
+ metrics.inc('persist_buffer_force', 1, { status: 'persisted' })
+ }
+ } catch (err) {
+ metrics.inc('persist_buffer_force', 1, { status: 'error' })
+ logger.error(
+ { err, projectId },
+ 'failed to persist buffer before committing changes'
+ )
+ }
+ }
+
+ metrics.inc('commit_changes', 1, {
+ history_buffer_level: historyBufferLevel || 0,
+ })
+
+ // Now handle the changes based on the configured history buffer level.
+ switch (historyBufferLevel) {
+ case 4: // Queue changes and only persist them in the background
+ await queueChanges(projectId, changes, endVersion)
+ return {}
+ case 3: // Queue changes and immediately persist with persistBuffer
+ await queueChanges(projectId, changes, endVersion)
+ return await persistBuffer(projectId, limits)
+ case 2: // Equivalent to queueChangesInRedis:true
+ await queueChangesFake(projectId, changes, endVersion)
+ return await persistChanges(projectId, changes, limits, endVersion)
+ case 1: // Queue changes with fake persist only for projects in redis already
+ await queueChangesFakeOnlyIfExists(projectId, changes, endVersion)
+ return await persistChanges(projectId, changes, limits, endVersion)
+ case 0: // Persist changes directly to the chunk store
+ return await persistChanges(projectId, changes, limits, endVersion)
+ default:
+ throw new Error(`Invalid history buffer level: ${historyBufferLevel}`)
+ }
+}
+
+/**
+ * Queues a set of changes in redis as if they had been persisted, ignoring any errors.
+ * @param {string} projectId
+ * @param {Change[]} changes
+ * @param {number} endVersion
+ * @param {Object} [options]
+ * @param {boolean} [options.onlyIfExists] - If true, only queue changes if the project
+ * already exists in Redis.
+ */
+
+async function queueChangesFake(projectId, changes, endVersion, options = {}) {
+ try {
+ await queueChanges(projectId, changes, endVersion)
+ await fakePersistRedisChanges(projectId, changes, endVersion)
+ } catch (err) {
+ logger.error({ err }, 'Chunk buffer verification failed')
+ }
+}
+
+/**
+ * Queues changes in Redis, simulating persistence, but only if the project already exists.
+ * @param {string} projectId - The ID of the project.
+ * @param {Change[]} changes - An array of changes to be queued.
+ * @param {number} endVersion - The expected version of the project before these changes are applied.
+ */
+
+async function queueChangesFakeOnlyIfExists(projectId, changes, endVersion) {
+ await queueChangesFake(projectId, changes, endVersion, {
+ onlyIfExists: true,
+ })
+}
+
+/**
+ * Simulates the persistence of changes by verifying a given set of changes against
+ * what is currently stored as non-persisted in Redis, and then updates the
+ * persisted version number in Redis.
+ *
+ * @async
+ * @param {string} projectId - The ID of the project.
+ * @param {Change[]} changesToPersist - An array of changes that are expected to be
+ * persisted. These are used for verification
+ * against the changes currently in Redis.
+ * @param {number} baseVersion - The base version number from which to calculate
+ * the new persisted version.
+ * @returns {Promise} A promise that resolves when the persisted version
+ * in Redis has been updated.
+ */
+async function fakePersistRedisChanges(
+ projectId,
+ changesToPersist,
+ baseVersion
+) {
+ const nonPersistedChanges = await redisBackend.getNonPersistedChanges(
+ projectId,
+ baseVersion
+ )
+
+ if (
+ serializeChanges(nonPersistedChanges) === serializeChanges(changesToPersist)
+ ) {
+ metrics.inc('persist_redis_changes_verification', 1, { status: 'match' })
+ } else {
+ logger.warn({ projectId }, 'mismatch of non-persisted changes from Redis')
+ metrics.inc('persist_redis_changes_verification', 1, {
+ status: 'mismatch',
+ })
+ }
+
+ const persistedVersion = baseVersion + nonPersistedChanges.length
+ await redisBackend.setPersistedVersion(projectId, persistedVersion)
+}
+
+/**
+ * @param {Change[]} changes
+ */
+function serializeChanges(changes) {
+ return JSON.stringify(changes.map(change => change.toRaw()))
+}
+
+module.exports = commitChanges
diff --git a/services/history-v1/storage/lib/content_hash.js b/services/history-v1/storage/lib/content_hash.js
new file mode 100644
index 0000000000..a381babc04
--- /dev/null
+++ b/services/history-v1/storage/lib/content_hash.js
@@ -0,0 +1,18 @@
+// @ts-check
+
+const { createHash } = require('node:crypto')
+
+/**
+ * Compute a SHA-1 hash of the content
+ *
+ * This is used to validate incoming updates.
+ *
+ * @param {string} content
+ */
+function getContentHash(content) {
+ const hash = createHash('sha-1')
+ hash.update(content)
+ return hash.digest('hex')
+}
+
+module.exports = { getContentHash }
diff --git a/services/history-v1/storage/lib/errors.js b/services/history-v1/storage/lib/errors.js
new file mode 100644
index 0000000000..626536b079
--- /dev/null
+++ b/services/history-v1/storage/lib/errors.js
@@ -0,0 +1,5 @@
+const OError = require('@overleaf/o-error')
+
+class InvalidChangeError extends OError {}
+
+module.exports = { InvalidChangeError }
diff --git a/services/history-v1/storage/lib/history_store.js b/services/history-v1/storage/lib/history_store.js
index 0e7c11ffb5..e51bdc25c5 100644
--- a/services/history-v1/storage/lib/history_store.js
+++ b/services/history-v1/storage/lib/history_store.js
@@ -1,10 +1,13 @@
+// @ts-check
'use strict'
-const BPromise = require('bluebird')
const core = require('overleaf-editor-core')
const config = require('config')
-const path = require('path')
+const path = require('node:path')
+const Stream = require('node:stream')
+const { promisify } = require('node:util')
+const zlib = require('node:zlib')
const OError = require('@overleaf/o-error')
const objectPersistor = require('@overleaf/object-persistor')
@@ -17,26 +20,48 @@ const streams = require('./streams')
const Chunk = core.Chunk
-const BUCKET = config.get('chunkStore.bucket')
+const gzip = promisify(zlib.gzip)
+const gunzip = promisify(zlib.gunzip)
class LoadError extends OError {
- constructor(projectId, chunkId) {
- super('HistoryStore: failed to load chunk history', { projectId, chunkId })
+ /**
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {any} cause
+ */
+ constructor(projectId, chunkId, cause) {
+ super(
+ 'HistoryStore: failed to load chunk history',
+ { projectId, chunkId },
+ cause
+ )
this.projectId = projectId
this.chunkId = chunkId
}
}
-HistoryStore.LoadError = LoadError
class StoreError extends OError {
- constructor(projectId, chunkId) {
- super('HistoryStore: failed to store chunk history', { projectId, chunkId })
+ /**
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {any} cause
+ */
+ constructor(projectId, chunkId, cause) {
+ super(
+ 'HistoryStore: failed to store chunk history',
+ { projectId, chunkId },
+ cause
+ )
this.projectId = projectId
this.chunkId = chunkId
}
}
-HistoryStore.StoreError = StoreError
+/**
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @return {string}
+ */
function getKey(projectId, chunkId) {
return path.join(projectKey.format(projectId), projectKey.pad(chunkId))
}
@@ -53,86 +78,125 @@ function getKey(projectId, chunkId) {
*
* @class
*/
-function HistoryStore() {}
+class HistoryStore {
+ #persistor
+ #bucket
+ constructor(persistor, bucket) {
+ this.#persistor = persistor
+ this.#bucket = bucket
+ }
-/**
- * Load the raw object for a History.
- *
- * @param {number} projectId
- * @param {number} chunkId
- * @return {Promise.}
- */
-HistoryStore.prototype.loadRaw = function historyStoreLoadRaw(
- projectId,
- chunkId
-) {
- assert.projectId(projectId, 'bad projectId')
- assert.chunkId(chunkId, 'bad chunkId')
+ /**
+ * Load the raw object for a History.
+ *
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @return {Promise}
+ */
+ async loadRaw(projectId, chunkId) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.chunkId(chunkId, 'bad chunkId')
- const key = getKey(projectId, chunkId)
+ const key = getKey(projectId, chunkId)
- logger.debug({ projectId, chunkId }, 'loadRaw started')
- return BPromise.resolve()
- .then(() => persistor.getObjectStream(BUCKET, key))
- .then(streams.gunzipStreamToBuffer)
- .then(buffer => JSON.parse(buffer))
- .catch(err => {
+ logger.debug({ projectId, chunkId }, 'loadRaw started')
+ try {
+ const buf = await streams.gunzipStreamToBuffer(
+ await this.#persistor.getObjectStream(this.#bucket, key)
+ )
+ return JSON.parse(buf.toString('utf-8'))
+ } catch (err) {
if (err instanceof objectPersistor.Errors.NotFoundError) {
throw new Chunk.NotPersistedError(projectId)
}
- throw new HistoryStore.LoadError(projectId, chunkId).withCause(err)
- })
- .finally(() => logger.debug({ projectId, chunkId }, 'loadRaw finished'))
+ throw new LoadError(projectId, chunkId, err)
+ } finally {
+ logger.debug({ projectId, chunkId }, 'loadRaw finished')
+ }
+ }
+
+ async loadRawWithBuffer(projectId, chunkId) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.chunkId(chunkId, 'bad chunkId')
+
+ const key = getKey(projectId, chunkId)
+
+ logger.debug({ projectId, chunkId }, 'loadBuffer started')
+ try {
+ const buf = await streams.readStreamToBuffer(
+ await this.#persistor.getObjectStream(this.#bucket, key)
+ )
+ const unzipped = await gunzip(buf)
+ return {
+ buffer: buf,
+ raw: JSON.parse(unzipped.toString('utf-8')),
+ }
+ } catch (err) {
+ if (err instanceof objectPersistor.Errors.NotFoundError) {
+ throw new Chunk.NotPersistedError(projectId)
+ }
+ throw new LoadError(projectId, chunkId, err)
+ } finally {
+ logger.debug({ projectId, chunkId }, 'loadBuffer finished')
+ }
+ }
+
+ /**
+ * Compress and store a {@link History}.
+ *
+ * @param {string} projectId
+ * @param {string} chunkId
+ * @param {import('overleaf-editor-core/lib/types').RawHistory} rawHistory
+ */
+ async storeRaw(projectId, chunkId, rawHistory) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.chunkId(chunkId, 'bad chunkId')
+ assert.object(rawHistory, 'bad rawHistory')
+
+ const key = getKey(projectId, chunkId)
+
+ logger.debug({ projectId, chunkId }, 'storeRaw started')
+
+ const buf = await gzip(JSON.stringify(rawHistory))
+ try {
+ await this.#persistor.sendStream(
+ this.#bucket,
+ key,
+ Stream.Readable.from([buf]),
+ {
+ contentType: 'application/json',
+ contentEncoding: 'gzip',
+ contentLength: buf.byteLength,
+ }
+ )
+ } catch (err) {
+ throw new StoreError(projectId, chunkId, err)
+ } finally {
+ logger.debug({ projectId, chunkId }, 'storeRaw finished')
+ }
+ }
+
+ /**
+ * Delete multiple chunks from bucket. Expects an Array of objects with
+ * projectId and chunkId properties
+ * @param {Array<{projectId: string,chunkId:string}>} chunks
+ */
+ async deleteChunks(chunks) {
+ logger.debug({ chunks }, 'deleteChunks started')
+ try {
+ await Promise.all(
+ chunks.map(chunk => {
+ const key = getKey(chunk.projectId, chunk.chunkId)
+ return this.#persistor.deleteObject(this.#bucket, key)
+ })
+ )
+ } finally {
+ logger.debug({ chunks }, 'deleteChunks finished')
+ }
+ }
}
-/**
- * Compress and store a {@link History}.
- *
- * @param {number} projectId
- * @param {number} chunkId
- * @param {Object} rawHistory
- * @return {Promise}
- */
-HistoryStore.prototype.storeRaw = function historyStoreStoreRaw(
- projectId,
- chunkId,
- rawHistory
-) {
- assert.projectId(projectId, 'bad projectId')
- assert.chunkId(chunkId, 'bad chunkId')
- assert.object(rawHistory, 'bad rawHistory')
-
- const key = getKey(projectId, chunkId)
-
- logger.debug({ projectId, chunkId }, 'storeRaw started')
- return BPromise.resolve()
- .then(() => streams.gzipStringToStream(JSON.stringify(rawHistory)))
- .then(stream =>
- persistor.sendStream(BUCKET, key, stream, {
- contentType: 'application/json',
- contentEncoding: 'gzip',
- })
- )
- .catch(err => {
- throw new HistoryStore.StoreError(projectId, chunkId).withCause(err)
- })
- .finally(() => logger.debug({ projectId, chunkId }, 'storeRaw finished'))
+module.exports = {
+ HistoryStore,
+ historyStore: new HistoryStore(persistor, config.get('chunkStore.bucket')),
}
-
-/**
- * Delete multiple chunks from bucket. Expects an Array of objects with
- * projectId and chunkId properties
- * @param {Array} chunks
- * @return {Promise}
- */
-HistoryStore.prototype.deleteChunks = function historyDeleteChunks(chunks) {
- logger.debug({ chunks }, 'deleteChunks started')
- return BPromise.all(
- chunks.map(chunk => {
- const key = getKey(chunk.projectId, chunk.chunkId)
- return persistor.deleteObject(BUCKET, key)
- })
- ).finally(() => logger.debug({ chunks }, 'deleteChunks finished'))
-}
-
-module.exports = new HistoryStore()
diff --git a/services/history-v1/storage/lib/knex.js b/services/history-v1/storage/lib/knex.js
index 5cdc85e2ab..7000fe034c 100644
--- a/services/history-v1/storage/lib/knex.js
+++ b/services/history-v1/storage/lib/knex.js
@@ -1,6 +1,8 @@
+// @ts-check
+
'use strict'
const env = process.env.NODE_ENV || 'development'
const knexfile = require('../../knexfile')
-module.exports = require('knex')(knexfile[env])
+module.exports = require('knex').default(knexfile[env])
diff --git a/services/history-v1/storage/lib/knex_read_only.js b/services/history-v1/storage/lib/knex_read_only.js
new file mode 100644
index 0000000000..a78c4689a4
--- /dev/null
+++ b/services/history-v1/storage/lib/knex_read_only.js
@@ -0,0 +1,19 @@
+'use strict'
+
+const config = require('config')
+const knexfile = require('../../knexfile')
+
+const env = process.env.NODE_ENV || 'development'
+
+if (config.databaseUrlReadOnly) {
+ module.exports = require('knex')({
+ ...knexfile[env],
+ pool: {
+ ...knexfile[env].pool,
+ min: 0,
+ },
+ connection: config.databaseUrlReadOnly,
+ })
+} else {
+ module.exports = require('./knex')
+}
diff --git a/services/history-v1/storage/lib/mongodb.js b/services/history-v1/storage/lib/mongodb.js
index 53b1837a8f..e887bc25a5 100644
--- a/services/history-v1/storage/lib/mongodb.js
+++ b/services/history-v1/storage/lib/mongodb.js
@@ -10,7 +10,21 @@ const chunks = db.collection('projectHistoryChunks')
const blobs = db.collection('projectHistoryBlobs')
const globalBlobs = db.collection('projectHistoryGlobalBlobs')
const shardedBlobs = db.collection('projectHistoryShardedBlobs')
+const projects = db.collection('projects')
+// Temporary collection for tracking progress of backed up old blobs (without a hash).
+// The initial sync process will be able to skip over these.
+// Schema: _id: projectId, blobs: [Binary]
+const backedUpBlobs = db.collection('projectHistoryBackedUpBlobs')
Metrics.mongodb.monitor(client)
-module.exports = { client, db, chunks, blobs, globalBlobs, shardedBlobs }
+module.exports = {
+ client,
+ db,
+ chunks,
+ blobs,
+ globalBlobs,
+ projects,
+ shardedBlobs,
+ backedUpBlobs,
+}
diff --git a/services/history-v1/storage/lib/persist_buffer.js b/services/history-v1/storage/lib/persist_buffer.js
new file mode 100644
index 0000000000..68b71e148f
--- /dev/null
+++ b/services/history-v1/storage/lib/persist_buffer.js
@@ -0,0 +1,237 @@
+// @ts-check
+'use strict'
+
+const logger = require('@overleaf/logger')
+const metrics = require('@overleaf/metrics')
+const OError = require('@overleaf/o-error')
+const assert = require('./assert')
+const chunkStore = require('./chunk_store')
+const { BlobStore } = require('./blob_store')
+const BatchBlobStore = require('./batch_blob_store')
+const persistChanges = require('./persist_changes')
+const resyncProject = require('./resync_project')
+const redisBackend = require('./chunk_store/redis')
+
+const PERSIST_BATCH_SIZE = 50
+
+/**
+ * Persist the changes from Redis buffer to the main storage
+ *
+ * Algorithm Outline:
+ * 1. Get the latest chunk's endVersion from the database
+ * 2. Get non-persisted changes from Redis that are after this endVersion.
+ * 3. If no such changes, exit.
+ * 4. Load file blobs for these Redis changes.
+ * 5. Run the persistChanges() algorithm to store these changes into a new chunk(s) in GCS.
+ * - This must not decrease the endVersion. If changes were processed, it must advance.
+ * 6. Set the new persisted version (endVersion of the latest persisted chunk) in Redis.
+ *
+ * @param {string} projectId
+ * @param {Object} limits
+ * @throws {Error | OError} If a critical error occurs during persistence.
+ */
+async function persistBuffer(projectId, limits) {
+ assert.projectId(projectId)
+ logger.debug({ projectId }, 'starting persistBuffer operation')
+
+ // 1. Get the latest chunk's endVersion from GCS/main store
+ let endVersion
+ const latestChunkMetadata = await chunkStore.getLatestChunkMetadata(projectId)
+
+ if (latestChunkMetadata) {
+ endVersion = latestChunkMetadata.endVersion
+ } else {
+ endVersion = 0 // No chunks found, start from version 0
+ logger.debug({ projectId }, 'no existing chunks found in main storage')
+ }
+ const originalEndVersion = endVersion
+
+ logger.debug({ projectId, endVersion }, 'got latest persisted chunk')
+
+ // Process changes in batches
+ let numberOfChangesPersisted = 0
+ let currentChunk = null
+ let resyncNeeded = false
+ let resyncChangesWerePersisted = false
+ while (true) {
+ // 2. Get non-persisted changes from Redis
+ const changesToPersist = await redisBackend.getNonPersistedChanges(
+ projectId,
+ endVersion,
+ { maxChanges: PERSIST_BATCH_SIZE }
+ )
+
+ if (changesToPersist.length === 0) {
+ break
+ }
+
+ logger.debug(
+ {
+ projectId,
+ endVersion,
+ count: changesToPersist.length,
+ },
+ 'found changes in Redis to persist'
+ )
+
+ // 4. Load file blobs for these Redis changes. Errors will propagate.
+ const blobStore = new BlobStore(projectId)
+ const batchBlobStore = new BatchBlobStore(blobStore)
+
+ const blobHashes = new Set()
+ for (const change of changesToPersist) {
+ change.findBlobHashes(blobHashes)
+ }
+ if (blobHashes.size > 0) {
+ await batchBlobStore.preload(Array.from(blobHashes))
+ }
+ for (const change of changesToPersist) {
+ await change.loadFiles('lazy', blobStore)
+ }
+
+ // 5. Run the persistChanges() algorithm. Errors will propagate.
+ logger.debug(
+ {
+ projectId,
+ endVersion,
+ changeCount: changesToPersist.length,
+ },
+ 'calling persistChanges'
+ )
+
+ const persistResult = await persistChanges(
+ projectId,
+ changesToPersist,
+ limits,
+ endVersion
+ )
+
+ if (!persistResult || !persistResult.currentChunk) {
+ metrics.inc('persist_buffer', 1, { status: 'no-chunk-error' })
+ throw new OError(
+ 'persistChanges did not produce a new chunk for non-empty changes',
+ {
+ projectId,
+ endVersion,
+ changeCount: changesToPersist.length,
+ }
+ )
+ }
+
+ currentChunk = persistResult.currentChunk
+ const newEndVersion = currentChunk.getEndVersion()
+
+ if (newEndVersion <= endVersion) {
+ metrics.inc('persist_buffer', 1, { status: 'chunk-version-mismatch' })
+ throw new OError(
+ 'persisted chunk endVersion must be greater than current persisted chunk end version for non-empty changes',
+ {
+ projectId,
+ newEndVersion,
+ endVersion,
+ changeCount: changesToPersist.length,
+ }
+ )
+ }
+
+ logger.debug(
+ {
+ projectId,
+ oldVersion: endVersion,
+ newVersion: newEndVersion,
+ },
+ 'successfully persisted changes from Redis to main storage'
+ )
+
+ // 6. Set the persisted version in Redis. Errors will propagate.
+ const status = await redisBackend.setPersistedVersion(
+ projectId,
+ newEndVersion
+ )
+
+ if (status !== 'ok') {
+ metrics.inc('persist_buffer', 1, { status: 'error-on-persisted-version' })
+ throw new OError('failed to update persisted version in Redis', {
+ projectId,
+ newEndVersion,
+ status,
+ })
+ }
+
+ logger.debug(
+ { projectId, newEndVersion },
+ 'updated persisted version in Redis'
+ )
+ numberOfChangesPersisted += persistResult.numberOfChangesPersisted
+ endVersion = newEndVersion
+
+ // Check if a resync might be needed
+ if (persistResult.resyncNeeded) {
+ resyncNeeded = true
+ }
+
+ if (
+ changesToPersist.some(
+ change => change.getOrigin()?.getKind() === 'history-resync'
+ )
+ ) {
+ resyncChangesWerePersisted = true
+ }
+
+ if (persistResult.numberOfChangesPersisted < PERSIST_BATCH_SIZE) {
+ // We reached the end of available changes
+ break
+ }
+ }
+
+ if (numberOfChangesPersisted === 0) {
+ logger.debug(
+ { projectId, endVersion },
+ 'no new changes in Redis buffer to persist'
+ )
+ metrics.inc('persist_buffer', 1, { status: 'no_changes' })
+ // No changes to persist, update the persisted version in Redis
+ // to match the current endVersion. This shouldn't be needed
+ // unless a worker failed to update the persisted version.
+ await redisBackend.setPersistedVersion(projectId, endVersion)
+ } else {
+ logger.debug(
+ { projectId, finalPersistedVersion: endVersion },
+ 'persistBuffer operation completed successfully'
+ )
+ metrics.inc('persist_buffer', 1, { status: 'persisted' })
+ }
+
+ if (limits.autoResync && resyncNeeded) {
+ if (resyncChangesWerePersisted) {
+ // To avoid an infinite loop, do not resync if the current batch of
+ // changes contains a history resync.
+ logger.warn(
+ { projectId },
+ 'content hash validation failed while persisting a history resync, skipping additional resync'
+ )
+ } else {
+ const backend = chunkStore.getBackend(projectId)
+ const mongoProjectId =
+ await backend.resolveHistoryIdToMongoProjectId(projectId)
+ await resyncProject(mongoProjectId)
+ }
+ }
+
+ if (currentChunk == null) {
+ const { chunk } = await chunkStore.loadByChunkRecord(
+ projectId,
+ latestChunkMetadata
+ )
+ currentChunk = chunk
+ }
+
+ return {
+ numberOfChangesPersisted,
+ originalEndVersion,
+ currentChunk,
+ resyncNeeded,
+ }
+}
+
+module.exports = persistBuffer
diff --git a/services/history-v1/storage/lib/persist_changes.js b/services/history-v1/storage/lib/persist_changes.js
index b661a4818c..d2ca00053f 100644
--- a/services/history-v1/storage/lib/persist_changes.js
+++ b/services/history-v1/storage/lib/persist_changes.js
@@ -1,8 +1,9 @@
-/** @module */
+// @ts-check
+
'use strict'
const _ = require('lodash')
-const BPromise = require('bluebird')
+const logger = require('@overleaf/logger')
const core = require('overleaf-editor-core')
const Chunk = core.Chunk
@@ -10,6 +11,9 @@ const History = core.History
const assert = require('./assert')
const chunkStore = require('./chunk_store')
+const { BlobStore } = require('./blob_store')
+const { InvalidChangeError } = require('./errors')
+const { getContentHash } = require('./content_hash')
function countChangeBytes(change) {
// Note: This is not quite accurate, because the raw change may contain raw
@@ -48,28 +52,35 @@ Timer.prototype.elapsed = function () {
* endVersion may be better suited to the metadata record.
*
* @param {string} projectId
- * @param {Array.} allChanges
+ * @param {core.Change[]} allChanges
* @param {Object} limits
* @param {number} clientEndVersion
* @return {Promise.}
*/
-module.exports = function persistChanges(
- projectId,
- allChanges,
- limits,
- clientEndVersion
-) {
+async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
assert.projectId(projectId)
assert.array(allChanges)
assert.maybe.object(limits)
assert.integer(clientEndVersion)
+ const blobStore = new BlobStore(projectId)
+
+ const earliestChangeTimestamp =
+ allChanges.length > 0 ? allChanges[0].getTimestamp() : null
+
let currentChunk
- // currentSnapshot tracks the latest change that we're applying; we use it to
- // check that the changes we are persisting are valid.
+
+ /**
+ * currentSnapshot tracks the latest change that we're applying; we use it to
+ * check that the changes we are persisting are valid.
+ *
+ * @type {core.Snapshot}
+ */
let currentSnapshot
+
let originalEndVersion
let changesToPersist
+ let resyncNeeded = false
limits = limits || {}
_.defaults(limits, {
@@ -88,57 +99,135 @@ module.exports = function persistChanges(
}
}
- function fillChunk(chunk, changes) {
+ /**
+ * Add changes to a chunk until the chunk is full
+ *
+ * The chunk is full if it reaches a certain number of changes or a certain
+ * size in bytes
+ *
+ * @param {core.Chunk} chunk
+ * @param {core.Change[]} changes
+ */
+ async function fillChunk(chunk, changes) {
let totalBytes = totalChangeBytes(chunk.getChanges())
let changesPushed = false
while (changes.length > 0) {
- if (chunk.getChanges().length >= limits.maxChunkChanges) break
- const changeBytes = countChangeBytes(changes[0])
- if (totalBytes + changeBytes > limits.maxChunkChangeBytes) break
- const changesToFill = changes.splice(0, 1)
- currentSnapshot.applyAll(changesToFill, { strict: true })
- chunk.pushChanges(changesToFill)
+ if (chunk.getChanges().length >= limits.maxChunkChanges) {
+ break
+ }
+
+ const change = changes[0]
+ const changeBytes = countChangeBytes(change)
+
+ if (totalBytes + changeBytes > limits.maxChunkChangeBytes) {
+ break
+ }
+
+ for (const operation of change.iterativelyApplyTo(currentSnapshot, {
+ strict: true,
+ })) {
+ await validateContentHash(operation)
+ }
+
+ chunk.pushChanges([change])
+ changes.shift()
totalBytes += changeBytes
changesPushed = true
}
return changesPushed
}
- function extendLastChunkIfPossible() {
- return chunkStore.loadLatest(projectId).then(function (latestChunk) {
- currentChunk = latestChunk
- originalEndVersion = latestChunk.getEndVersion()
- if (originalEndVersion !== clientEndVersion) {
- throw new Chunk.ConflictingEndVersion(
- clientEndVersion,
- originalEndVersion
- )
+ /**
+ * Check that the operation is valid and can be incorporated to the history.
+ *
+ * For now, this checks content hashes when they are provided.
+ *
+ * @param {core.Operation} operation
+ */
+ async function validateContentHash(operation) {
+ if (operation instanceof core.EditFileOperation) {
+ const editOperation = operation.getOperation()
+ if (
+ editOperation instanceof core.TextOperation &&
+ editOperation.contentHash != null
+ ) {
+ const path = operation.getPathname()
+ const file = currentSnapshot.getFile(path)
+ if (file == null) {
+ throw new InvalidChangeError('file not found for hash validation', {
+ projectId,
+ path,
+ })
+ }
+ await file.load('eager', blobStore)
+ const content = file.getContent({ filterTrackedDeletes: true })
+ const expectedHash = editOperation.contentHash
+ const actualHash = content != null ? getContentHash(content) : null
+ logger.debug({ expectedHash, actualHash }, 'validating content hash')
+ if (actualHash !== expectedHash) {
+ // only log a warning on the first mismatch in each persistChanges call
+ if (!resyncNeeded) {
+ logger.warn(
+ { projectId, path, expectedHash, actualHash },
+ 'content hash mismatch'
+ )
+ }
+ resyncNeeded = true
+ }
+
+ // Remove the content hash from the change before storing it in the chunk.
+ // It was only useful for validation.
+ editOperation.contentHash = null
}
-
- currentSnapshot = latestChunk.getSnapshot().clone()
- const timer = new Timer()
- currentSnapshot.applyAll(latestChunk.getChanges())
-
- if (!fillChunk(currentChunk, changesToPersist)) return
- checkElapsedTime(timer)
-
- return chunkStore.update(projectId, originalEndVersion, currentChunk)
- })
+ }
}
- function createNewChunksAsNeeded() {
- if (changesToPersist.length === 0) return
+ async function loadLatestChunk() {
+ const latestChunk = await chunkStore.loadLatest(projectId, {
+ persistedOnly: true,
+ })
- const endVersion = currentChunk.getEndVersion()
- const history = new History(currentSnapshot.clone(), [])
- const chunk = new Chunk(history, endVersion)
- const timer = new Timer()
- if (fillChunk(chunk, changesToPersist)) {
- checkElapsedTime(timer)
- currentChunk = chunk
- return chunkStore.create(projectId, chunk).then(createNewChunksAsNeeded)
+ currentChunk = latestChunk
+ originalEndVersion = latestChunk.getEndVersion()
+ if (originalEndVersion !== clientEndVersion) {
+ throw new Chunk.ConflictingEndVersion(
+ clientEndVersion,
+ originalEndVersion
+ )
+ }
+
+ currentSnapshot = latestChunk.getSnapshot().clone()
+ currentSnapshot.applyAll(currentChunk.getChanges())
+ }
+
+ async function extendLastChunkIfPossible() {
+ const timer = new Timer()
+ const changesPushed = await fillChunk(currentChunk, changesToPersist)
+ if (!changesPushed) {
+ return
+ }
+
+ checkElapsedTime(timer)
+
+ await chunkStore.update(projectId, currentChunk, earliestChangeTimestamp)
+ }
+
+ async function createNewChunksAsNeeded() {
+ while (changesToPersist.length > 0) {
+ const endVersion = currentChunk.getEndVersion()
+ const history = new History(currentSnapshot.clone(), [])
+ const chunk = new Chunk(history, endVersion)
+ const timer = new Timer()
+
+ const changesPushed = await fillChunk(chunk, changesToPersist)
+ if (changesPushed) {
+ checkElapsedTime(timer)
+ currentChunk = chunk
+ await chunkStore.create(projectId, chunk, earliestChangeTimestamp)
+ } else {
+ throw new Error('failed to fill empty chunk')
+ }
}
- throw new Error('failed to fill empty chunk')
}
function isOlderThanMinChangeTimestamp(change) {
@@ -157,15 +246,20 @@ module.exports = function persistChanges(
if (anyTooOld || tooManyChanges || tooManyBytes) {
changesToPersist = oldChanges
const numberOfChangesToPersist = oldChanges.length
- return extendLastChunkIfPossible()
- .then(createNewChunksAsNeeded)
- .then(function () {
- return {
- numberOfChangesPersisted: numberOfChangesToPersist,
- originalEndVersion,
- currentChunk,
- }
- })
+
+ await loadLatestChunk()
+ await extendLastChunkIfPossible()
+ await createNewChunksAsNeeded()
+
+ return {
+ numberOfChangesPersisted: numberOfChangesToPersist,
+ originalEndVersion,
+ currentChunk,
+ resyncNeeded,
+ }
+ } else {
+ return null
}
- return BPromise.resolve(null)
}
+
+module.exports = persistChanges
diff --git a/services/history-v1/storage/lib/project_archive.js b/services/history-v1/storage/lib/project_archive.js
index d6680ae33c..8a8e93f1c9 100644
--- a/services/history-v1/storage/lib/project_archive.js
+++ b/services/history-v1/storage/lib/project_archive.js
@@ -8,8 +8,8 @@
const Archive = require('archiver')
const BPromise = require('bluebird')
-const fs = require('fs')
-const { pipeline } = require('stream')
+const fs = require('node:fs')
+const { pipeline } = require('node:stream')
const core = require('overleaf-editor-core')
@@ -49,7 +49,7 @@ class ProjectArchive {
/**
* @constructor
* @param {Snapshot} snapshot
- * @param {?number} timeout in ms
+ * @param {number} [timeout] in ms
* @classdesc
* Writes the project snapshot to a zip file.
*/
diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js
index e84f024e06..03fb2a5141 100644
--- a/services/history-v1/storage/lib/project_key.js
+++ b/services/history-v1/storage/lib/project_key.js
@@ -1,5 +1,6 @@
+// Keep in sync with services/web/app/src/Features/History/project_key.js
const _ = require('lodash')
-const path = require('path')
+const path = require('node:path')
//
// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
diff --git a/services/history-v1/storage/lib/queue_changes.js b/services/history-v1/storage/lib/queue_changes.js
new file mode 100644
index 0000000000..6b8d4b22b4
--- /dev/null
+++ b/services/history-v1/storage/lib/queue_changes.js
@@ -0,0 +1,75 @@
+// @ts-check
+
+'use strict'
+
+const redisBackend = require('./chunk_store/redis')
+const { BlobStore } = require('./blob_store')
+const chunkStore = require('./chunk_store')
+const core = require('overleaf-editor-core')
+const Chunk = core.Chunk
+
+/**
+ * Queues an incoming set of changes after validating them against the current snapshot.
+ *
+ * @async
+ * @function queueChanges
+ * @param {string} projectId - The project to queue changes for.
+ * @param {Array} changesToQueue - An array of change objects to be applied and queued.
+ * @param {number} endVersion - The expected version of the project before these changes are applied.
+ * This is used for optimistic concurrency control.
+ * @param {Object} [opts] - Additional options for queuing changes.
+ * @throws {Chunk.ConflictingEndVersion} If the provided `endVersion` does not match the
+ * current version of the project.
+ * @returns {Promise} A promise that resolves with the status returned by the
+ * `redisBackend.queueChanges` operation.
+ */
+async function queueChanges(projectId, changesToQueue, endVersion, opts) {
+ const result = await redisBackend.getHeadSnapshot(projectId)
+ let currentSnapshot = null
+ let currentVersion = null
+ if (result) {
+ // If we have a snapshot in redis, we can use it to check the current state
+ // of the project and apply changes to it.
+ currentSnapshot = result.snapshot
+ currentVersion = result.version
+ } else {
+ // Otherwise, load the latest chunk from the chunk store.
+ const latestChunk = await chunkStore.loadLatest(projectId, {
+ persistedOnly: true,
+ })
+ // Throw an error if no latest chunk is found, indicating the project has not been initialised.
+ if (!latestChunk) {
+ throw new Chunk.NotFoundError(projectId)
+ }
+ currentSnapshot = latestChunk.getSnapshot()
+ currentSnapshot.applyAll(latestChunk.getChanges())
+ currentVersion = latestChunk.getEndVersion()
+ }
+
+ // Ensure the endVersion matches the current version of the project.
+ if (endVersion !== currentVersion) {
+ throw new Chunk.ConflictingEndVersion(endVersion, currentVersion)
+ }
+
+ // Compute the new hollow snapshot to be saved to redis.
+ const hollowSnapshot = currentSnapshot
+ const blobStore = new BlobStore(projectId)
+ await hollowSnapshot.loadFiles('hollow', blobStore)
+ // Clone the changes to avoid modifying the original ones when computing the hollow snapshot.
+ const hollowChanges = changesToQueue.map(change => change.clone())
+ for (const change of hollowChanges) {
+ await change.loadFiles('hollow', blobStore)
+ }
+ hollowSnapshot.applyAll(hollowChanges, { strict: true })
+ const baseVersion = currentVersion
+ const status = await redisBackend.queueChanges(
+ projectId,
+ hollowSnapshot,
+ baseVersion,
+ changesToQueue,
+ opts
+ )
+ return status
+}
+
+module.exports = queueChanges
diff --git a/services/history-v1/storage/lib/redis.js b/services/history-v1/storage/lib/redis.js
new file mode 100644
index 0000000000..9b00cc0a26
--- /dev/null
+++ b/services/history-v1/storage/lib/redis.js
@@ -0,0 +1,19 @@
+const config = require('config')
+const redis = require('@overleaf/redis-wrapper')
+
+const historyRedisOptions = config.get('redis.history')
+const rclientHistory = redis.createClient(historyRedisOptions)
+
+const lockRedisOptions = config.get('redis.history')
+const rclientLock = redis.createClient(lockRedisOptions)
+
+async function disconnect() {
+ await Promise.all([rclientHistory.disconnect(), rclientLock.disconnect()])
+}
+
+module.exports = {
+ rclientHistory,
+ rclientLock,
+ redis,
+ disconnect,
+}
diff --git a/services/history-v1/storage/lib/resync_project.js b/services/history-v1/storage/lib/resync_project.js
new file mode 100644
index 0000000000..3ec680bb5b
--- /dev/null
+++ b/services/history-v1/storage/lib/resync_project.js
@@ -0,0 +1,14 @@
+// @ts-check
+
+const config = require('config')
+const { fetchNothing } = require('@overleaf/fetch-utils')
+
+const PROJECT_HISTORY_URL = `http://${config.projectHistory.host}:${config.projectHistory.port}`
+
+async function resyncProject(projectId) {
+ await fetchNothing(`${PROJECT_HISTORY_URL}/project/${projectId}/resync`, {
+ method: 'POST',
+ })
+}
+
+module.exports = resyncProject
diff --git a/services/history-v1/storage/lib/scan.js b/services/history-v1/storage/lib/scan.js
new file mode 100644
index 0000000000..d55f5362c1
--- /dev/null
+++ b/services/history-v1/storage/lib/scan.js
@@ -0,0 +1,202 @@
+// @ts-check
+
+'use strict'
+
+const logger = require('@overleaf/logger')
+const { JobNotFoundError, JobNotReadyError } = require('./chunk_store/errors')
+const BATCH_SIZE = 1000 // Default batch size for SCAN
+
+/**
+ * Asynchronously scans a Redis instance or cluster for keys matching a pattern.
+ *
+ * This function handles both standalone Redis instances and Redis clusters.
+ * For clusters, it iterates over all master nodes. It yields keys in batches
+ * as they are found by the SCAN command.
+ *
+ * @param {object} redisClient - The Redis client instance (from @overleaf/redis-wrapper).
+ * @param {string} pattern - The pattern to match keys against (e.g., 'user:*').
+ * @param {number} [count=BATCH_SIZE] - Optional hint for Redis SCAN count per iteration.
+ * @yields {string[]} A batch of matching keys.
+ */
+async function* scanRedisCluster(redisClient, pattern, count = BATCH_SIZE) {
+ const nodes = redisClient.nodes ? redisClient.nodes('master') : [redisClient]
+
+ for (const node of nodes) {
+ let cursor = '0'
+ do {
+ // redisClient from @overleaf/redis-wrapper uses ioredis style commands
+ const [nextCursor, keys] = await node.scan(
+ cursor,
+ 'MATCH',
+ pattern,
+ 'COUNT',
+ count
+ )
+ cursor = nextCursor
+ if (keys.length > 0) {
+ yield keys
+ }
+ } while (cursor !== '0')
+ }
+}
+
+/**
+ * Extracts the content within the first pair of curly braces {} from a string.
+ * This is used to extract a user ID or project ID from a Redis key.
+ *
+ * @param {string} key - The input string containing content within curly braces.
+ * @returns {string | null} The extracted content (the key ID) if found, otherwise null.
+ */
+function extractKeyId(key) {
+ const match = key.match(/\{(.*?)\}/)
+ if (match && match[1]) {
+ return match[1]
+ }
+ return null
+}
+
+/**
+ * Fetches timestamps for a list of project IDs based on a given key name.
+ *
+ * @param {string[]} projectIds - Array of project identifiers.
+ * @param {object} rclient - The Redis client instance.
+ * @param {string} keyName - The base name for the Redis keys storing the timestamps (e.g., "expire-time", "persist-time").
+ * @param {number} currentTime - The current time (timestamp in milliseconds) to compare against.
+ * @returns {Promise>}
+ * A promise that resolves to an array of objects, each containing a projectId and
+ * its corresponding timestampValue, for due projects only.
+ */
+async function fetchOverdueProjects(projectIds, rclient, keyName, currentTime) {
+ if (!projectIds || projectIds.length === 0) {
+ return []
+ }
+ const timestampKeys = projectIds.map(id => `${keyName}:{${id}}`)
+ const timestamps = await rclient.mget(timestampKeys)
+
+ const dueProjects = []
+ for (let i = 0; i < projectIds.length; i++) {
+ const projectId = projectIds[i]
+ const timestampValue = timestamps[i]
+
+ if (timestampValue !== null) {
+ const timestamp = parseInt(timestampValue, 10)
+ if (!isNaN(timestamp) && currentTime > timestamp) {
+ dueProjects.push({ projectId, timestampValue })
+ }
+ }
+ }
+ return dueProjects
+}
+
+/**
+ * Scans Redis for keys matching a pattern derived from keyName, identifies items that are "due" based on a timestamp,
+ * and performs a specified action on them.
+ *
+ * @param {object} rclient - The Redis client instance.
+ * @param {string} taskName - A descriptive name for the task (used in logging).
+ * @param {string} keyName - The base name for the Redis keys (e.g., "expire-time", "persist-time").
+ * The function will derive the key prefix as `${keyName}:` and scan pattern as `${keyName}:{*}`.
+ * @param {function(string): Promise} actionFn - An async function that takes a projectId and performs an action.
+ * @param {boolean} DRY_RUN - If true, logs actions that would be taken without performing them.
+ * @returns {Promise<{scannedKeyCount: number, processedKeyCount: number}>} Counts of scanned and processed keys.
+ */
+async function scanAndProcessDueItems(
+ rclient,
+ taskName,
+ keyName,
+ actionFn,
+ DRY_RUN
+) {
+ let scannedKeyCount = 0
+ let processedKeyCount = 0
+ const START_TIME = Date.now()
+ const logContext = { taskName, dryRun: DRY_RUN }
+
+ const scanPattern = `${keyName}:{*}`
+
+ if (DRY_RUN) {
+ logger.info(logContext, `Starting ${taskName} scan in DRY RUN mode`)
+ } else {
+ logger.info(logContext, `Starting ${taskName} scan`)
+ }
+
+ for await (const keysBatch of scanRedisCluster(rclient, scanPattern)) {
+ scannedKeyCount += keysBatch.length
+ const projectIds = keysBatch.map(extractKeyId).filter(id => id != null)
+
+ if (projectIds.length === 0) {
+ continue
+ }
+
+ const currentTime = Date.now()
+ const overdueProjects = await fetchOverdueProjects(
+ projectIds,
+ rclient,
+ keyName,
+ currentTime
+ )
+
+ for (const project of overdueProjects) {
+ const { projectId } = project
+ if (DRY_RUN) {
+ logger.info(
+ { ...logContext, projectId },
+ `[Dry Run] Would perform ${taskName} for project`
+ )
+ } else {
+ try {
+ await actionFn(projectId)
+ logger.debug(
+ { ...logContext, projectId },
+ `Successfully performed ${taskName} for project`
+ )
+ } catch (err) {
+ if (err instanceof JobNotReadyError) {
+ // the project has been touched since the job was created
+ logger.info(
+ { ...logContext, projectId },
+ `Job not ready for ${taskName} for project`
+ )
+ } else if (err instanceof JobNotFoundError) {
+ // the project has been expired already by another worker
+ logger.info(
+ { ...logContext, projectId },
+ `Job not found for ${taskName} for project`
+ )
+ } else {
+ logger.error(
+ { ...logContext, projectId, err },
+ `Error performing ${taskName} for project`
+ )
+ }
+ continue
+ }
+ }
+ processedKeyCount++
+
+ if (processedKeyCount % 1000 === 0 && processedKeyCount > 0) {
+ logger.info(
+ { ...logContext, scannedKeyCount, processedKeyCount },
+ `${taskName} scan progress`
+ )
+ }
+ }
+ }
+
+ logger.info(
+ {
+ ...logContext,
+ scannedKeyCount,
+ processedKeyCount,
+ elapsedTimeInSeconds: Math.floor((Date.now() - START_TIME) / 1000),
+ },
+ `${taskName} scan complete`
+ )
+ return { scannedKeyCount, processedKeyCount }
+}
+
+module.exports = {
+ scanRedisCluster,
+ extractKeyId,
+ scanAndProcessDueItems,
+}
diff --git a/services/history-v1/storage/lib/streams.js b/services/history-v1/storage/lib/streams.js
index 9e216c3f97..e60e5aa725 100644
--- a/services/history-v1/storage/lib/streams.js
+++ b/services/history-v1/storage/lib/streams.js
@@ -1,3 +1,4 @@
+// @ts-check
/**
* Promises are promises and streams are streams, and ne'er the twain shall
* meet.
@@ -5,52 +6,20 @@
*/
'use strict'
-const BPromise = require('bluebird')
-const zlib = require('zlib')
-const { WritableBuffer, ReadableString } = require('@overleaf/stream-utils')
-const { pipeline } = require('stream')
-
-/**
- * Pipe a read stream to a write stream. The promise resolves when the write
- * stream finishes.
- *
- * @function
- * @param {stream.Readable} readStream
- * @param {stream.Writable} writeStream
- * @return {Promise}
- */
-function promisePipe(readStream, writeStream) {
- return new BPromise(function (resolve, reject) {
- pipeline(readStream, writeStream, function (err) {
- if (err) {
- reject(err)
- } else {
- resolve()
- }
- })
- })
-}
-
-exports.promisePipe = promisePipe
+const Stream = require('node:stream')
+const zlib = require('node:zlib')
+const { WritableBuffer } = require('@overleaf/stream-utils')
/**
* Create a promise for the result of reading a stream to a buffer.
*
- * @function
- * @param {stream.Readable} readStream
- * @return {Promise.}
+ * @param {Stream.Readable} readStream
+ * @return {Promise}
*/
-function readStreamToBuffer(readStream) {
- return new BPromise(function (resolve, reject) {
- const bufferStream = new WritableBuffer()
- pipeline(readStream, bufferStream, function (err) {
- if (err) {
- reject(err)
- } else {
- resolve(bufferStream.contents())
- }
- })
- })
+async function readStreamToBuffer(readStream) {
+ const bufferStream = new WritableBuffer()
+ await Stream.promises.pipeline(readStream, bufferStream)
+ return bufferStream.contents()
}
exports.readStreamToBuffer = readStreamToBuffer
@@ -58,43 +27,14 @@ exports.readStreamToBuffer = readStreamToBuffer
/**
* Create a promise for the result of un-gzipping a stream to a buffer.
*
- * @function
- * @param {stream.Readable} readStream
- * @return {Promise.}
+ * @param {NodeJS.ReadableStream} readStream
+ * @return {Promise}
*/
-function gunzipStreamToBuffer(readStream) {
+async function gunzipStreamToBuffer(readStream) {
const gunzip = zlib.createGunzip()
const bufferStream = new WritableBuffer()
- return new BPromise(function (resolve, reject) {
- pipeline(readStream, gunzip, bufferStream, function (err) {
- if (err) {
- reject(err)
- } else {
- resolve(bufferStream.contents())
- }
- })
- })
+ await Stream.promises.pipeline(readStream, gunzip, bufferStream)
+ return bufferStream.contents()
}
exports.gunzipStreamToBuffer = gunzipStreamToBuffer
-
-/**
- * Create a write stream that gzips the given string.
- *
- * @function
- * @param {string} string
- * @return {Promise.}
- */
-function gzipStringToStream(string) {
- return new BPromise(function (resolve, reject) {
- zlib.gzip(Buffer.from(string), function (error, result) {
- if (error) {
- reject(error)
- } else {
- resolve(new ReadableString(result))
- }
- })
- })
-}
-
-exports.gzipStringToStream = gzipStringToStream
diff --git a/services/history-v1/storage/lib/temp.js b/services/history-v1/storage/lib/temp.js
index 719e0767a6..1aab3c1df4 100644
--- a/services/history-v1/storage/lib/temp.js
+++ b/services/history-v1/storage/lib/temp.js
@@ -7,7 +7,7 @@
*/
const BPromise = require('bluebird')
-const fs = BPromise.promisifyAll(require('fs'))
+const fs = BPromise.promisifyAll(require('node:fs'))
const temp = BPromise.promisifyAll(require('temp'))
exports.open = function (affixes) {
diff --git a/services/history-v1/storage/lib/zip_store.js b/services/history-v1/storage/lib/zip_store.js
index b3cf7c2663..0741829303 100644
--- a/services/history-v1/storage/lib/zip_store.js
+++ b/services/history-v1/storage/lib/zip_store.js
@@ -2,8 +2,8 @@
const BPromise = require('bluebird')
const config = require('config')
-const fs = require('fs')
-const path = require('path')
+const fs = require('node:fs')
+const path = require('node:path')
const OError = require('@overleaf/o-error')
const objectPersistor = require('@overleaf/object-persistor')
diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
new file mode 100644
index 0000000000..2e12328e5c
--- /dev/null
+++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
@@ -0,0 +1,1158 @@
+// @ts-check
+import Events from 'node:events'
+import fs from 'node:fs'
+import Path from 'node:path'
+import { performance } from 'node:perf_hooks'
+import Stream from 'node:stream'
+import { setTimeout } from 'node:timers/promises'
+import { ObjectId } from 'mongodb'
+import pLimit from 'p-limit'
+import logger from '@overleaf/logger'
+import {
+ batchedUpdate,
+ objectIdFromInput,
+ renderObjectId,
+} from '@overleaf/mongo-utils/batchedUpdate.js'
+import OError from '@overleaf/o-error'
+import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
+import {
+ BlobStore,
+ GLOBAL_BLOBS,
+ loadGlobalBlobs,
+ getProjectBlobsBatch,
+ getStringLengthOfFile,
+ makeBlobForFile,
+} from '../lib/blob_store/index.js'
+import { db } from '../lib/mongodb.js'
+import commandLineArgs from 'command-line-args'
+import readline from 'node:readline'
+
+// Silence warning.
+Events.setMaxListeners(20)
+
+// Enable caching for ObjectId.toString()
+ObjectId.cacheHexString = true
+
+/**
+ * @typedef {import("overleaf-editor-core").Blob} Blob
+ * @typedef {import("perf_hooks").EventLoopUtilization} EventLoopUtilization
+ * @typedef {import("mongodb").Collection} Collection
+ * @typedef {import("mongodb").Collection} ProjectsCollection
+ * @typedef {import("mongodb").Collection<{project:Project}>} DeletedProjectsCollection
+ * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
+ */
+
+/**
+ * @typedef {Object} FileRef
+ * @property {ObjectId} _id
+ * @property {string} hash
+ */
+
+/**
+ * @typedef {Object} Folder
+ * @property {Array} folders
+ * @property {Array} fileRefs
+ */
+
+/**
+ * @typedef {Object} DeletedFileRef
+ * @property {ObjectId} _id
+ * @property {ObjectId} projectId
+ * @property {string} hash
+ */
+
+/**
+ * @typedef {Object} Project
+ * @property {ObjectId} _id
+ * @property {Array} rootFolder
+ * @property {{history: {id: (number|string)}}} overleaf
+ */
+
+/**
+ * @typedef {Object} QueueEntry
+ * @property {ProjectContext} ctx
+ * @property {string} cacheKey
+ * @property {string} [fileId]
+ * @property {string} path
+ * @property {string} [hash]
+ * @property {Blob} [blob]
+ */
+
+/**
+ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean}}
+ */
+function parseArgs() {
+ const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z')
+ const args = commandLineArgs([
+ { name: 'processNonDeletedProjects', type: String, defaultValue: 'false' },
+ { name: 'processDeletedProjects', type: String, defaultValue: 'false' },
+ { name: 'processHashedFiles', type: String, defaultValue: 'false' },
+ { name: 'processBlobs', type: String, defaultValue: 'true' },
+ { name: 'projectIdsFrom', type: String, defaultValue: '' },
+ {
+ name: 'BATCH_RANGE_START',
+ type: String,
+ defaultValue: PUBLIC_LAUNCH_DATE.toISOString(),
+ },
+ {
+ name: 'BATCH_RANGE_END',
+ type: String,
+ defaultValue: new Date().toISOString(),
+ },
+ { name: 'LOGGING_IDENTIFIER', type: String, defaultValue: '' },
+ ])
+ /**
+ * commandLineArgs cannot handle --foo=false, so go the long way
+ * @param {string} name
+ * @return {boolean}
+ */
+ function boolVal(name) {
+ const v = args[name]
+ if (['true', 'false'].includes(v)) return v === 'true'
+ throw new Error(`expected "true" or "false" for boolean option ${name}`)
+ }
+ const BATCH_RANGE_START = objectIdFromInput(
+ args['BATCH_RANGE_START']
+ ).toString()
+ const BATCH_RANGE_END = objectIdFromInput(args['BATCH_RANGE_END']).toString()
+ return {
+ PROCESS_NON_DELETED_PROJECTS: boolVal('processNonDeletedProjects'),
+ PROCESS_DELETED_PROJECTS: boolVal('processDeletedProjects'),
+ PROCESS_BLOBS: boolVal('processBlobs'),
+ PROCESS_HASHED_FILES: boolVal('processHashedFiles'),
+ BATCH_RANGE_START,
+ BATCH_RANGE_END,
+ LOGGING_IDENTIFIER: args['LOGGING_IDENTIFIER'] || BATCH_RANGE_START,
+ PROJECT_IDS_FROM: args['projectIdsFrom'],
+ }
+}
+
+const {
+ PROCESS_NON_DELETED_PROJECTS,
+ PROCESS_DELETED_PROJECTS,
+ PROCESS_BLOBS,
+ PROCESS_HASHED_FILES,
+ BATCH_RANGE_START,
+ BATCH_RANGE_END,
+ LOGGING_IDENTIFIER,
+ PROJECT_IDS_FROM,
+} = parseArgs()
+
+// We need to handle the start and end differently as ids of deleted projects are created at time of deletion.
+if (process.env.BATCH_RANGE_START || process.env.BATCH_RANGE_END) {
+ throw new Error('use --BATCH_RANGE_START and --BATCH_RANGE_END')
+}
+
+// Concurrency for downloading from GCS and updating hashes in mongo
+const CONCURRENCY = parseInt(process.env.CONCURRENCY || '100', 10)
+const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10)
+// Retries for processing a given file
+const RETRIES = parseInt(process.env.RETRIES || '10', 10)
+const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10)
+
+const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true'
+const BUFFER_DIR = fs.mkdtempSync(
+ process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-'
+)
+// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode
+const STREAM_HIGH_WATER_MARK = parseInt(
+ process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(),
+ 10
+)
+const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10)
+const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10)
+
+// Filestore endpoint location
+const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1'
+const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009'
+
+async function fetchFromFilestore(projectId, fileId) {
+ const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}`
+ const response = await fetch(url)
+ if (!response.ok) {
+ if (response.status === 404) {
+ throw new NotFoundError('file not found in filestore', {
+ status: response.status,
+ })
+ }
+ const body = await response.text()
+ throw new OError('fetchFromFilestore failed', {
+ projectId,
+ fileId,
+ status: response.status,
+ body,
+ })
+ }
+ if (!response.body) {
+ throw new OError('fetchFromFilestore response has no body', {
+ projectId,
+ fileId,
+ status: response.status,
+ })
+ }
+ return response.body
+}
+
+const projectsCollection = db.collection('projects')
+/** @type {ProjectsCollection} */
+const typedProjectsCollection = db.collection('projects')
+const deletedProjectsCollection = db.collection('deletedProjects')
+/** @type {DeletedProjectsCollection} */
+const typedDeletedProjectsCollection = db.collection('deletedProjects')
+
+const concurrencyLimit = pLimit(CONCURRENCY)
+
+/**
+ * @template T
+ * @template V
+ * @param {Array} array
+ * @param {(arg: T) => Promise} fn
+ * @return {Promise>>}
+ */
+async function processConcurrently(array, fn) {
+ return await Promise.all(array.map(x => concurrencyLimit(() => fn(x))))
+}
+
+const STATS = {
+ projects: 0,
+ blobs: 0,
+ filesWithHash: 0,
+ filesWithoutHash: 0,
+ filesDuplicated: 0,
+ filesRetries: 0,
+ filesFailed: 0,
+ fileTreeUpdated: 0,
+ badFileTrees: 0,
+ globalBlobsCount: 0,
+ globalBlobsEgress: 0,
+ projectDeleted: 0,
+ projectHardDeleted: 0,
+ fileHardDeleted: 0,
+ mongoUpdates: 0,
+ readFromGCSCount: 0,
+ readFromGCSIngress: 0,
+ writeToGCSCount: 0,
+ writeToGCSEgress: 0,
+}
+
+const processStart = performance.now()
+let lastLogTS = processStart
+let lastLog = Object.assign({}, STATS)
+let lastEventLoopStats = performance.eventLoopUtilization()
+
+/**
+ * @param {number} v
+ * @param {number} ms
+ */
+function toMiBPerSecond(v, ms) {
+ const ONE_MiB = 1024 * 1024
+ return v / ONE_MiB / (ms / 1000)
+}
+
+/**
+ * @param {any} stats
+ * @param {number} ms
+ * @return {{readFromGCSThroughputMiBPerSecond: number}}
+ */
+function bandwidthStats(stats, ms) {
+ return {
+ readFromGCSThroughputMiBPerSecond: toMiBPerSecond(
+ stats.readFromGCSIngress,
+ ms
+ ),
+ }
+}
+
+/**
+ * @param {EventLoopUtilization} nextEventLoopStats
+ * @param {number} now
+ * @return {Object}
+ */
+function computeDiff(nextEventLoopStats, now) {
+ const ms = now - lastLogTS
+ lastLogTS = now
+ const diff = {
+ eventLoop: performance.eventLoopUtilization(
+ nextEventLoopStats,
+ lastEventLoopStats
+ ),
+ }
+ for (const [name, v] of Object.entries(STATS)) {
+ diff[name] = v - lastLog[name]
+ }
+ return Object.assign(diff, bandwidthStats(diff, ms))
+}
+
+/**
+ * @param {boolean} isLast
+ */
+function printStats(isLast = false) {
+ const now = performance.now()
+ const nextEventLoopStats = performance.eventLoopUtilization()
+ const logLine = JSON.stringify({
+ time: new Date(),
+ LOGGING_IDENTIFIER,
+ ...STATS,
+ ...bandwidthStats(STATS, now - processStart),
+ eventLoop: nextEventLoopStats,
+ diff: computeDiff(nextEventLoopStats, now),
+ deferredBatches: Array.from(deferredBatches.keys()),
+ })
+ if (isLast) {
+ console.warn(logLine)
+ } else {
+ console.log(logLine)
+ }
+ lastEventLoopStats = nextEventLoopStats
+ lastLog = Object.assign({}, STATS)
+}
+
+setInterval(printStats, LOGGING_INTERVAL)
+
+let gracefulShutdownInitiated = false
+
+process.on('SIGINT', handleSignal)
+process.on('SIGTERM', handleSignal)
+
+function handleSignal() {
+ gracefulShutdownInitiated = true
+ console.warn('graceful shutdown initiated, draining queue')
+}
+
+/**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+async function processFileWithCleanup(entry) {
+ const {
+ ctx: { projectId },
+ cacheKey,
+ } = entry
+ const filePath = Path.join(BUFFER_DIR, projectId.toString() + cacheKey)
+ try {
+ return await processFile(entry, filePath)
+ } finally {
+ await Promise.all([
+ fs.promises.rm(filePath, { force: true }),
+ fs.promises.rm(filePath + GZ_SUFFIX, { force: true }),
+ ])
+ }
+}
+
+/**
+ * @param {QueueEntry} entry
+ * @param {string} filePath
+ * @return {Promise}
+ */
+async function processFile(entry, filePath) {
+ for (let attempt = 0; attempt < RETRIES; attempt++) {
+ try {
+ return await processFileOnce(entry, filePath)
+ } catch (err) {
+ if (gracefulShutdownInitiated) throw err
+ if (err instanceof NotFoundError) {
+ if (!RETRY_FILESTORE_404) {
+ throw err // disable retries for not found in filestore bucket case
+ }
+ }
+ STATS.filesRetries++
+ const {
+ ctx: { projectId },
+ fileId,
+ hash,
+ path,
+ } = entry
+ logger.warn(
+ { err, projectId, fileId, hash, path, attempt },
+ 'failed to process file, trying again'
+ )
+ const jitter = Math.random() * RETRY_DELAY_MS
+ await setTimeout(RETRY_DELAY_MS + jitter)
+ }
+ }
+ return await processFileOnce(entry, filePath)
+}
+
+/**
+ * @param {QueueEntry} entry
+ * @param {string} filePath
+ * @return {Promise}
+ */
+async function processFileOnce(entry, filePath) {
+ const {
+ ctx: { projectId, historyId },
+ fileId,
+ } = entry
+ if (entry.hash && entry.ctx.hasCompletedBlob(entry.hash)) {
+ // We can enter this case for two identical files in the same project,
+ // one with hash, the other without. When the one without hash gets
+ // processed first, we can skip downloading the other one we already
+ // know the hash of.
+ return entry.hash
+ }
+ const blobStore = new BlobStore(historyId)
+ STATS.readFromGCSCount++
+ // make a fetch request to filestore itself
+ const src = await fetchFromFilestore(projectId, fileId)
+ const dst = fs.createWriteStream(filePath, {
+ highWaterMark: STREAM_HIGH_WATER_MARK,
+ })
+ try {
+ await Stream.promises.pipeline(src, dst)
+ } finally {
+ STATS.readFromGCSIngress += dst.bytesWritten
+ }
+ const blob = await makeBlobForFile(filePath)
+ blob.setStringLength(
+ await getStringLengthOfFile(blob.getByteLength(), filePath)
+ )
+ const hash = blob.getHash()
+ if (entry.hash && hash !== entry.hash) {
+ throw new OError('hash mismatch', { entry, hash })
+ }
+
+ if (GLOBAL_BLOBS.has(hash)) {
+ STATS.globalBlobsCount++
+ STATS.globalBlobsEgress += estimateBlobSize(blob)
+ return hash
+ }
+ if (entry.ctx.hasCompletedBlob(hash)) {
+ return hash
+ }
+ entry.ctx.recordPendingBlob(hash)
+
+ try {
+ await uploadBlobToGCS(blobStore, entry, blob, hash, filePath)
+ entry.ctx.recordCompletedBlob(hash) // mark upload as completed
+ } catch (err) {
+ entry.ctx.recordFailedBlob(hash)
+ throw err
+ }
+ return hash
+}
+
+/**
+ * @param {BlobStore} blobStore
+ * @param {QueueEntry} entry
+ * @param {Blob} blob
+ * @param {string} hash
+ * @param {string} filePath
+ * @return {Promise}
+ */
+async function uploadBlobToGCS(blobStore, entry, blob, hash, filePath) {
+ if (entry.ctx.getCachedHistoryBlob(hash)) {
+ return // fast-path using hint from pre-fetched blobs
+ }
+ if (!PROCESS_BLOBS) {
+ // round trip to postgres/mongo when not pre-fetched
+ const blob = await blobStore.getBlob(hash)
+ if (blob) {
+ entry.ctx.recordHistoryBlob(blob)
+ return
+ }
+ }
+ // blob missing in history-v1, create in GCS and persist in postgres/mongo
+ STATS.writeToGCSCount++
+ STATS.writeToGCSEgress += blob.getByteLength()
+ await blobStore.putBlob(filePath, blob)
+ entry.ctx.recordHistoryBlob(blob)
+}
+
+const GZ_SUFFIX = '.gz'
+
+/**
+ * @param {Array} files
+ * @return {Promise}
+ */
+async function processFiles(files) {
+ await processConcurrently(
+ files,
+ /**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+ async function (entry) {
+ if (gracefulShutdownInitiated) return
+ try {
+ await entry.ctx.processFile(entry)
+ } catch (err) {
+ STATS.filesFailed++
+ const {
+ ctx: { projectId },
+ fileId,
+ hash,
+ path,
+ } = entry
+ logger.error(
+ { err, projectId, fileId, hash, path },
+ 'failed to process file'
+ )
+ }
+ }
+ )
+}
+
+/** @type {Map} */
+const deferredBatches = new Map()
+
+async function waitForDeferredQueues() {
+ // Wait for ALL pending batches to finish, especially wait for their mongo
+ // writes to finish to avoid extra work when resuming the batch.
+ const all = await Promise.allSettled(deferredBatches.values())
+ // Now that all batches finished, we can throw if needed.
+ for (const res of all) {
+ if (res.status === 'rejected') {
+ throw res.reason
+ }
+ }
+}
+
+/**
+ * @param {Array} batch
+ * @param {string} prefix
+ */
+async function queueNextBatch(batch, prefix = 'rootFolder.0') {
+ if (gracefulShutdownInitiated) {
+ throw new Error('graceful shutdown: aborting batch processing')
+ }
+
+ // Read ids now, the batch will get trimmed by processBatch shortly.
+ const start = renderObjectId(batch[0]._id)
+ const end = renderObjectId(batch[batch.length - 1]._id)
+ const deferred = processBatch(batch, prefix)
+ .then(() => {
+ console.error(`Actually completed batch ending ${end}`)
+ })
+ .catch(err => {
+ logger.error({ err, start, end }, 'fatal error processing batch')
+ throw err
+ })
+ .finally(() => {
+ deferredBatches.delete(end)
+ })
+ deferredBatches.set(end, deferred)
+
+ if (deferredBatches.size >= CONCURRENT_BATCHES) {
+ // Wait for any of the deferred batches to finish before fetching the next.
+ // We should never have more than CONCURRENT_BATCHES batches in memory.
+ await Promise.race(deferredBatches.values())
+ }
+}
+
+/**
+ * @param {Array} batch
+ * @param {string} prefix
+ * @return {Promise}
+ */
+async function processBatch(batch, prefix = 'rootFolder.0') {
+ const { nBlobs, blobs } = await collectProjectBlobs(batch)
+ const files = Array.from(findFileInBatch(batch, prefix, blobs))
+ STATS.projects += batch.length
+ STATS.blobs += nBlobs
+
+ // GC
+ batch.length = 0
+ blobs.clear()
+
+ // The files are currently ordered by project-id.
+ // Order them by file-id ASC then hash ASC to
+ // increase the hit rate on the "already processed
+ // hash for project" checks.
+ files.sort(
+ /**
+ * @param {QueueEntry} a
+ * @param {QueueEntry} b
+ * @return {number}
+ */
+ function (a, b) {
+ if (a.fileId && b.fileId) return a.fileId > b.fileId ? 1 : -1
+ if (a.hash && b.hash) return a.hash > b.hash ? 1 : -1
+ if (a.fileId) return -1
+ return 1
+ }
+ )
+ await processFiles(files)
+ await processConcurrently(
+ files,
+ /**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+ async function (entry) {
+ await entry.ctx.flushMongoQueues()
+ }
+ )
+}
+
+/**
+ * @param {Array<{project: Project}>} batch
+ * @return {Promise}
+ */
+async function handleDeletedFileTreeBatch(batch) {
+ await queueNextBatch(
+ batch.map(d => d.project),
+ 'project.rootFolder.0'
+ )
+}
+
+/**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+async function tryUpdateFileRefInMongo(entry) {
+ if (entry.path.startsWith('project.')) {
+ return await tryUpdateFileRefInMongoInDeletedProject(entry)
+ }
+
+ STATS.mongoUpdates++
+ const result = await projectsCollection.updateOne(
+ {
+ _id: entry.ctx.projectId,
+ [`${entry.path}._id`]: new ObjectId(entry.fileId),
+ },
+ {
+ $set: { [`${entry.path}.hash`]: entry.hash },
+ }
+ )
+ return result.matchedCount === 1
+}
+
+/**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+async function tryUpdateFileRefInMongoInDeletedProject(entry) {
+ STATS.mongoUpdates++
+ const result = await deletedProjectsCollection.updateOne(
+ {
+ 'deleterData.deletedProjectId': entry.ctx.projectId,
+ [`${entry.path}._id`]: new ObjectId(entry.fileId),
+ },
+ {
+ $set: { [`${entry.path}.hash`]: entry.hash },
+ }
+ )
+ return result.matchedCount === 1
+}
+
+const RETRY_UPDATE_HASH = 100
+
+/**
+ * @param {QueueEntry} entry
+ * @return {Promise}
+ */
+async function updateFileRefInMongo(entry) {
+ if (await tryUpdateFileRefInMongo(entry)) return
+
+ const { fileId } = entry
+ const { projectId } = entry.ctx
+ for (let i = 0; i < RETRY_UPDATE_HASH; i++) {
+ let prefix = 'rootFolder.0'
+ let p = await projectsCollection.findOne(
+ { _id: projectId },
+ { projection: { rootFolder: 1 } }
+ )
+ if (!p) {
+ STATS.projectDeleted++
+ prefix = 'project.rootFolder.0'
+ const deletedProject = await deletedProjectsCollection.findOne(
+ {
+ 'deleterData.deletedProjectId': projectId,
+ project: { $exists: true },
+ },
+ { projection: { 'project.rootFolder': 1 } }
+ )
+ p = deletedProject?.project
+ if (!p) {
+ STATS.projectHardDeleted++
+ console.warn(
+ 'bug: project hard-deleted while processing',
+ projectId,
+ fileId
+ )
+ return
+ }
+ }
+ let found = false
+ for (const e of findFiles(entry.ctx, p.rootFolder[0], prefix)) {
+ found = e.fileId === fileId
+ if (!found) continue
+ if (await tryUpdateFileRefInMongo(e)) return
+ break
+ }
+ if (!found) {
+ STATS.fileHardDeleted++
+ console.warn('bug: file hard-deleted while processing', projectId, fileId)
+ return
+ }
+
+ STATS.fileTreeUpdated++
+ }
+ throw new OError(
+ 'file-tree updated repeatedly while trying to add hash',
+ entry
+ )
+}
+
+/**
+ * @param {ProjectContext} ctx
+ * @param {Folder} folder
+ * @param {string} path
+ * @param {boolean} isInputLoop
+ * @return Generator
+ */
+function* findFiles(ctx, folder, path, isInputLoop = false) {
+ if (!folder || typeof folder !== 'object') {
+ ctx.fileTreeBroken = true
+ logger.warn({ projectId: ctx.projectId, path }, 'bad file-tree, bad folder')
+ return
+ }
+ if (!Array.isArray(folder.folders)) {
+ folder.folders = []
+ ctx.fileTreeBroken = true
+ logger.warn(
+ { projectId: ctx.projectId, path: `${path}.folders` },
+ 'bad file-tree, bad folders'
+ )
+ }
+ let i = 0
+ for (const child of folder.folders) {
+ const idx = i++
+ yield* findFiles(ctx, child, `${path}.folders.${idx}`, isInputLoop)
+ }
+ if (!Array.isArray(folder.fileRefs)) {
+ folder.fileRefs = []
+ ctx.fileTreeBroken = true
+ logger.warn(
+ { projectId: ctx.projectId, path: `${path}.fileRefs` },
+ 'bad file-tree, bad fileRefs'
+ )
+ }
+ i = 0
+ for (const fileRef of folder.fileRefs) {
+ const idx = i++
+ const fileRefPath = `${path}.fileRefs.${idx}`
+ if (!fileRef._id || !(fileRef._id instanceof ObjectId)) {
+ ctx.fileTreeBroken = true
+ logger.warn(
+ { projectId: ctx.projectId, path: fileRefPath },
+ 'bad file-tree, bad fileRef id'
+ )
+ continue
+ }
+ const fileId = fileRef._id.toString()
+ if (PROCESS_HASHED_FILES && fileRef.hash) {
+ if (ctx.canSkipProcessingHashedFile(fileRef.hash)) continue
+ if (isInputLoop) {
+ ctx.remainingQueueEntries++
+ STATS.filesWithHash++
+ }
+ yield {
+ ctx,
+ cacheKey: fileRef.hash,
+ fileId,
+ path: MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE,
+ hash: fileRef.hash,
+ }
+ }
+ if (!fileRef.hash) {
+ if (isInputLoop) {
+ ctx.remainingQueueEntries++
+ STATS.filesWithoutHash++
+ }
+ yield {
+ ctx,
+ cacheKey: fileId,
+ fileId,
+ path: fileRefPath,
+ }
+ }
+ }
+}
+
+/**
+ * @param {Array} projects
+ * @param {string} prefix
+ * @param {Map>} blobs
+ * @return Generator
+ */
+function* findFileInBatch(projects, prefix, blobs) {
+ for (const project of projects) {
+ const projectIdS = project._id.toString()
+ const historyIdS = project.overleaf.history.id.toString()
+ const projectBlobs = blobs.get(historyIdS) || []
+ const ctx = new ProjectContext(project._id, historyIdS, projectBlobs)
+ try {
+ yield* findFiles(ctx, project.rootFolder?.[0], prefix, true)
+ } catch (err) {
+ logger.error(
+ { err, projectId: projectIdS },
+ 'bad file-tree, processing error'
+ )
+ } finally {
+ if (ctx.fileTreeBroken) STATS.badFileTrees++
+ }
+ }
+}
+
+/**
+ * @param {Array} batch
+ * @return {Promise<{nBlobs: number, blobs: Map>}>}
+ */
+async function collectProjectBlobs(batch) {
+ if (!PROCESS_BLOBS) return { nBlobs: 0, blobs: new Map() }
+ return await getProjectBlobsBatch(batch.map(p => p.overleaf.history.id))
+}
+
+const BATCH_FILE_UPDATES = 100
+
+const MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE = 'skip-write-to-file-tree'
+
+class ProjectContext {
+ /** @type {Map} */
+ #historyBlobs
+
+ /** @type {number} */
+ remainingQueueEntries = 0
+
+ /** @type {boolean} */
+ fileTreeBroken = false
+
+ /**
+ * @param {ObjectId} projectId
+ * @param {string} historyId
+ * @param {Array} blobs
+ */
+ constructor(projectId, historyId, blobs) {
+ this.projectId = projectId
+ this.historyId = historyId
+ this.#historyBlobs = new Map(blobs.map(b => [b.getHash(), b]))
+ }
+
+ /**
+ * @param {string} hash
+ * @return {Blob | undefined}
+ */
+ getCachedHistoryBlob(hash) {
+ return this.#historyBlobs.get(hash)
+ }
+
+ /**
+ * @param {Blob} blob
+ */
+ recordHistoryBlob(blob) {
+ this.#historyBlobs.set(blob.getHash(), blob)
+ }
+
+ /**
+ * @param {string} hash
+ * @return {boolean}
+ */
+ canSkipProcessingHashedFile(hash) {
+ if (this.#historyBlobs.has(hash)) return true // This file will be processed as blob.
+ if (GLOBAL_BLOBS.has(hash)) return true // global blob
+ return false
+ }
+
+ async flushMongoQueuesIfNeeded() {
+ if (this.remainingQueueEntries === 0) {
+ await this.flushMongoQueues()
+ }
+
+ if (this.#pendingFileWrites.length > BATCH_FILE_UPDATES) {
+ await this.#storeFileHashes()
+ }
+ }
+
+ async flushMongoQueues() {
+ await this.#storeFileHashes()
+ }
+
+ /** @type {Set} */
+ #pendingBlobs = new Set()
+ /** @type {Set} */
+ #completedBlobs = new Set()
+
+ /**
+ * @param {string} hash
+ */
+ recordPendingBlob(hash) {
+ this.#pendingBlobs.add(hash)
+ }
+
+ /**
+ * @param {string} hash
+ */
+ recordFailedBlob(hash) {
+ this.#pendingBlobs.delete(hash)
+ }
+
+ /**
+ * @param {string} hash
+ */
+ recordCompletedBlob(hash) {
+ this.#completedBlobs.add(hash)
+ this.#pendingBlobs.delete(hash)
+ }
+
+ /**
+ * @param {string} hash
+ * @return {boolean}
+ */
+ hasCompletedBlob(hash) {
+ return this.#pendingBlobs.has(hash) || this.#completedBlobs.has(hash)
+ }
+
+ /** @type {Array} */
+ #pendingFileWrites = []
+
+ /**
+ * @param {QueueEntry} entry
+ */
+ queueFileForWritingHash(entry) {
+ if (entry.path === MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE) return
+ this.#pendingFileWrites.push(entry)
+ }
+
+ /**
+ * @param {Collection} collection
+ * @param {Array} entries
+ * @param {Object} query
+ * @return {Promise 
+